summaryrefslogtreecommitdiff
path: root/src/buildtool/execution_api/local
diff options
context:
space:
mode:
Diffstat (limited to 'src/buildtool/execution_api/local')
-rw-r--r--src/buildtool/execution_api/local/local_action.cpp121
-rw-r--r--src/buildtool/execution_api/local/local_action.hpp63
-rw-r--r--src/buildtool/execution_api/local/local_api.cpp28
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp24
4 files changed, 224 insertions, 12 deletions
diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp
index d9733f18..8050c3d7 100644
--- a/src/buildtool/execution_api/local/local_action.cpp
+++ b/src/buildtool/execution_api/local/local_action.cpp
@@ -134,8 +134,11 @@ auto LocalAction::Execute(Logger const* logger) noexcept
if (auto result =
local_context_.storage->ActionCache().CachedResult(*action)) {
if (result->exit_code() == 0 and
- ActionResultContainsExpectedOutputs(
- *result, output_files_, output_dirs_)) {
+ (mode_ == RequestMode::kV2_1
+ ? ActionResultContainsExpectedOutputs(*result,
+ output_paths_)
+ : ActionResultContainsExpectedOutputs(
+ *result, output_files_, output_dirs_))) {
return create_response(
logger,
action->hash(),
@@ -376,6 +379,16 @@ auto LocalAction::CreateDirectoryStructure(
}
return true;
};
+
+ if (mode_ == RequestMode::kV2_1) {
+ return std::all_of(
+ output_paths_.begin(),
+ output_paths_.end(),
+ [this, &exec_path, &create_dir](auto const& local_path) {
+ auto dir = (exec_path / cwd_ / local_path).parent_path();
+ return create_dir(dir);
+ });
+ }
return std::all_of(output_files_.begin(),
output_files_.end(),
[this, &exec_path, &create_dir](auto const& local_path) {
@@ -487,11 +500,109 @@ auto LocalAction::CollectOutputDirOrSymlink(
return std::nullopt;
}
+auto LocalAction::CollectOutputPath(
+ std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept -> std::optional<OutputPath> {
+ auto out_path = exec_path / local_path;
+ auto type = FileSystemManager::Type(out_path, /*allow_upwards=*/true);
+ if (not type) {
+ logger_.Emit(LogLevel::Error, "expected known type at {}", local_path);
+ return std::nullopt;
+ }
+ if (IsSymlinkObject(*type)) {
+ if (auto content = FileSystemManager::ReadSymlink(out_path)) {
+ // in native mode: check validity of symlink
+ if (ProtocolTraits::IsNative(
+ local_context_.storage->GetHashFunction().GetType()) and
+ not PathIsNonUpwards(*content)) {
+ logger_.Emit(
+ LogLevel::Error, "found invalid symlink at {}", local_path);
+ return std::nullopt;
+ }
+ if (local_context_.storage->CAS().StoreBlob(*content)) {
+ auto out_symlink = bazel_re::OutputSymlink{};
+ out_symlink.set_path(local_path);
+ out_symlink.set_target(*content);
+ return out_symlink;
+ }
+ }
+ }
+ else if (IsFileObject(*type)) {
+ bool is_executable = IsExecutableObject(*type);
+ auto digest = local_context_.storage->CAS().StoreBlob</*kOwner=*/true>(
+ out_path, is_executable);
+ if (digest) {
+ auto out_file = bazel_re::OutputFile{};
+ out_file.set_path(local_path);
+ *out_file.mutable_digest() =
+ ArtifactDigestFactory::ToBazel(*digest);
+ out_file.set_is_executable(is_executable);
+ return out_file;
+ }
+ }
+ else if (IsTreeObject(*type)) {
+ if (auto digest = CreateDigestFromLocalOwnedTree(
+ *local_context_.storage, out_path)) {
+ auto out_dir = bazel_re::OutputDirectory{};
+ out_dir.set_path(local_path);
+ (*out_dir.mutable_tree_digest()) =
+ ArtifactDigestFactory::ToBazel(*digest);
+ return out_dir;
+ }
+ logger_.Emit(LogLevel::Error,
+ "found invalid entries in directory at {}",
+ local_path);
+ }
+ else {
+ logger_.Emit(LogLevel::Error,
+ "expected file, directory, or symlink at {}",
+ local_path);
+ }
+ return std::nullopt;
+}
+
auto LocalAction::CollectAndStoreOutputs(
bazel_re::ActionResult* result,
std::filesystem::path const& exec_path) const noexcept -> bool {
try {
logger_.Emit(LogLevel::Trace, "collecting outputs:");
+ if (mode_ == RequestMode::kV2_1) {
+ for (auto const& path : output_paths_) {
+ auto out = CollectOutputPath(exec_path, path);
+ if (not out) {
+ logger_.Emit(LogLevel::Error,
+ "could not collect output path {}",
+ path);
+ return false;
+ }
+ if (std::holds_alternative<bazel_re::OutputSymlink>(*out)) {
+ auto out_symlink = std::get<bazel_re::OutputSymlink>(*out);
+ logger_.Emit(LogLevel::Trace,
+ " - symlink {}: {}",
+ path,
+ out_symlink.target());
+ result->mutable_output_symlinks()->Add(
+ std::move(out_symlink));
+ }
+ else if (std::holds_alternative<bazel_re::OutputFile>(*out)) {
+ auto out_file = std::get<bazel_re::OutputFile>(*out);
+ auto const& digest = out_file.digest().hash();
+ logger_.Emit(
+ LogLevel::Trace, " - file {}: {}", path, digest);
+ result->mutable_output_files()->Add(std::move(out_file));
+ }
+ else {
+ auto out_dir = std::get<bazel_re::OutputDirectory>(*out);
+ auto const& digest = out_dir.tree_digest().hash();
+ logger_.Emit(
+ LogLevel::Trace, " - dir {}: {}", path, digest);
+ result->mutable_output_directories()->Add(
+ std::move(out_dir));
+ }
+ }
+ return true;
+ }
+ // if mode_ is RequestMode::kV2_0 or RequestMode::kBestEffort
for (auto const& path : output_files_) {
auto out = CollectOutputFileOrSymlink(exec_path, path);
if (not out) {
@@ -506,6 +617,9 @@ auto LocalAction::CollectAndStoreOutputs(
" - symlink {}: {}",
path,
out_symlink.target());
+ if (mode_ == RequestMode::kBestEffort) {
+ *result->mutable_output_symlinks()->Add() = out_symlink;
+ }
result->mutable_output_file_symlinks()->Add(
std::move(out_symlink));
}
@@ -530,6 +644,9 @@ auto LocalAction::CollectAndStoreOutputs(
" - symlink {}: {}",
path,
out_symlink.target());
+ if (mode_ == RequestMode::kBestEffort) {
+ *result->mutable_output_symlinks()->Add() = out_symlink;
+ }
result->mutable_output_directory_symlinks()->Add(
std::move(out_symlink));
}
diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp
index 7610def0..47f912d8 100644
--- a/src/buildtool/execution_api/local/local_action.hpp
+++ b/src/buildtool/execution_api/local/local_action.hpp
@@ -18,6 +18,7 @@
#include <algorithm>
#include <chrono>
#include <compare>
+#include <cstdint>
#include <filesystem>
#include <functional> // IWYU pragma: keep
#include <map>
@@ -45,6 +46,12 @@
class LocalAction final : public IExecutionAction {
friend class LocalApi;
+ enum class RequestMode : std::uint8_t {
+ kV2_0, // RBEv2.0
+ kV2_1, // >=RBEv2.1
+ kBestEffort // construct both, let the server pick
+ };
+
public:
struct Output {
bazel_re::ActionResult action;
@@ -56,6 +63,9 @@ class LocalAction final : public IExecutionAction {
std::variant<bazel_re::OutputFile, bazel_re::OutputSymlink>;
using OutputDirOrSymlink =
std::variant<bazel_re::OutputDirectory, bazel_re::OutputSymlink>;
+ using OutputPath = std::variant<bazel_re::OutputFile,
+ bazel_re::OutputDirectory,
+ bazel_re::OutputSymlink>;
using FileCopies = std::unordered_map<Artifact::ObjectInfo, TmpDir::Ptr>;
@@ -76,31 +86,65 @@ class LocalAction final : public IExecutionAction {
std::string const cwd_;
std::vector<std::string> output_files_;
std::vector<std::string> output_dirs_;
+ std::vector<std::string> output_paths_;
std::map<std::string, std::string> const env_vars_;
std::vector<bazel_re::Platform_Property> const properties_;
std::chrono::milliseconds timeout_{kDefaultTimeout};
CacheFlag cache_flag_{CacheFlag::CacheOutput};
+ RequestMode mode_{};
+
+ explicit LocalAction(gsl::not_null<LocalContext const*> local_context,
+ ArtifactDigest root_digest,
+ std::vector<std::string> command,
+ std::string cwd,
+ std::vector<std::string> output_files,
+ std::vector<std::string> output_dirs,
+ std::map<std::string, std::string> env_vars,
+ std::map<std::string, std::string> const& properties,
+ bool best_effort) noexcept
+ : local_context_{*local_context},
+ root_digest_{std::move(root_digest)},
+ cmdline_{std::move(command)},
+ cwd_{std::move(cwd)},
+ output_files_{std::move(output_files)},
+ output_dirs_{std::move(output_dirs)},
+ env_vars_{std::move(env_vars)},
+ properties_{BazelMsgFactory::CreateMessageVectorFromMap<
+ bazel_re::Platform_Property>(properties)},
+ mode_{best_effort ? RequestMode::kBestEffort : RequestMode::kV2_0} {
+ std::sort(output_files_.begin(), output_files_.end());
+ std::sort(output_dirs_.begin(), output_dirs_.end());
+ if (best_effort) {
+ output_paths_.reserve(output_files_.size() + output_dirs_.size());
+ output_paths_.insert(output_paths_.end(),
+ output_files_.begin(),
+ output_files_.end());
+ output_paths_.insert(
+ output_paths_.end(), output_dirs_.begin(), output_dirs_.end());
+ std::sort(output_paths_.begin(), output_paths_.end());
+ }
+ }
+ // Alternative constructor with combined output_paths for files and dirs, as
+ // it is used by RBEv2.1 and above.
explicit LocalAction(
gsl::not_null<LocalContext const*> local_context,
ArtifactDigest root_digest,
std::vector<std::string> command,
std::string cwd,
- std::vector<std::string> output_files,
- std::vector<std::string> output_dirs,
+ std::vector<std::string> output_paths,
std::map<std::string, std::string> env_vars,
std::map<std::string, std::string> const& properties) noexcept
: local_context_{*local_context},
root_digest_{std::move(root_digest)},
cmdline_{std::move(command)},
cwd_{std::move(cwd)},
- output_files_{std::move(output_files)},
- output_dirs_{std::move(output_dirs)},
+ output_paths_{std::move(output_paths)},
env_vars_{std::move(env_vars)},
properties_{BazelMsgFactory::CreateMessageVectorFromMap<
- bazel_re::Platform_Property>(properties)} {
- std::sort(output_files_.begin(), output_files_.end());
- std::sort(output_dirs_.begin(), output_dirs_.end());
+ bazel_re::Platform_Property>(properties)},
+ mode_{RequestMode::kV2_1} {
+ std::sort(output_paths_.begin(), output_paths_.end());
}
[[nodiscard]] auto CreateActionDigest(ArtifactDigest const& exec_dir,
@@ -114,6 +158,7 @@ class LocalAction final : public IExecutionAction {
.cwd = &cwd_,
.output_files = &output_files_,
.output_dirs = &output_dirs_,
+ .output_paths = &output_paths_,
.env_vars = &env_vars,
.properties = &properties_,
.exec_dir = &exec_dir,
@@ -153,6 +198,10 @@ class LocalAction final : public IExecutionAction {
std::string const& local_path) const noexcept
-> std::optional<OutputDirOrSymlink>;
+ [[nodiscard]] auto CollectOutputPath(std::filesystem::path const& exec_path,
+ std::string const& local_path)
+ const noexcept -> std::optional<OutputPath>;
+
[[nodiscard]] auto CollectAndStoreOutputs(
bazel_re::ActionResult* result,
std::filesystem::path const& exec_path) const noexcept -> bool;
diff --git a/src/buildtool/execution_api/local/local_api.cpp b/src/buildtool/execution_api/local/local_api.cpp
index 3eede672..c0a3a30e 100644
--- a/src/buildtool/execution_api/local/local_api.cpp
+++ b/src/buildtool/execution_api/local/local_api.cpp
@@ -59,6 +59,7 @@ LocalApi::LocalApi(gsl::not_null<LocalContext const*> const& local_context,
: local_context_{*local_context},
git_api_{CreateFallbackApi(*local_context->storage, repo_config)} {}
+// NOLINTNEXTLINE(google-default-arguments)
auto LocalApi::CreateAction(
ArtifactDigest const& root_digest,
std::vector<std::string> const& command,
@@ -66,8 +67,13 @@ auto LocalApi::CreateAction(
std::vector<std::string> const& output_files,
std::vector<std::string> const& output_dirs,
std::map<std::string, std::string> const& env_vars,
- std::map<std::string, std::string> const& properties) const noexcept
- -> IExecutionAction::Ptr {
+ std::map<std::string, std::string> const& properties,
+ bool force_legacy) const noexcept -> IExecutionAction::Ptr {
+ if (ProtocolTraits::IsNative(GetHashType())) {
+ // fall back to legacy for native
+ force_legacy = true;
+ }
+ bool best_effort = not force_legacy;
return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_,
root_digest,
command,
@@ -75,6 +81,24 @@ auto LocalApi::CreateAction(
output_files,
output_dirs,
env_vars,
+ properties,
+ best_effort}};
+}
+
+auto LocalApi::CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::string const& cwd,
+ std::vector<std::string> const& output_paths,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) const noexcept
+ -> IExecutionAction::Ptr {
+ return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_,
+ root_digest,
+ command,
+ cwd,
+ output_paths,
+ env_vars,
properties}};
}
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index e2374258..1b6ddeab 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -40,6 +40,7 @@ class LocalApi final : public IExecutionApi {
explicit LocalApi(gsl::not_null<LocalContext const*> const& local_context,
RepositoryConfig const* repo_config = nullptr) noexcept;
+ // NOLINTNEXTLINE(google-default-arguments)
[[nodiscard]] auto CreateAction(
ArtifactDigest const& root_digest,
std::vector<std::string> const& command,
@@ -47,9 +48,30 @@ class LocalApi final : public IExecutionApi {
std::vector<std::string> const& output_files,
std::vector<std::string> const& output_dirs,
std::map<std::string, std::string> const& env_vars,
- std::map<std::string, std::string> const& properties) const noexcept
+ std::map<std::string, std::string> const& properties,
+ bool force_legacy = false) const noexcept
-> IExecutionAction::Ptr final;
+ /// \brief Create a new action (>=RBEv2.1).
+ /// \param[in] root_digest Digest of the build root.
+ /// \param[in] command Command as argv vector
+ /// \param[in] cwd Working directory, relative to execution root
+ /// \param[in] output_paths List of output file/dir paths, relative to cwd
+ /// \param[in] env_vars The environment variables to set.
+ /// \param[in] properties Platform properties to set.
+ /// \returns The new action.
+ /// Note that, due to missing file/dir separation, the execution response
+ /// will not report output file symlinks and directory symlinks separately.
+ /// (see \ref LocalResponse::DirectorySymlinks)
+ [[nodiscard]] auto CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::string const& cwd,
+ std::vector<std::string> const& output_paths,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) const noexcept
+ -> IExecutionAction::Ptr;
+
// NOLINTNEXTLINE(google-default-arguments)
[[nodiscard]] auto RetrieveToPaths(
std::vector<Artifact::ObjectInfo> const& artifacts_info,