diff options
Diffstat (limited to 'src/buildtool/execution_api/local')
-rw-r--r-- | src/buildtool/execution_api/local/local_action.cpp | 121 | ||||
-rw-r--r-- | src/buildtool/execution_api/local/local_action.hpp | 63 | ||||
-rw-r--r-- | src/buildtool/execution_api/local/local_api.cpp | 28 | ||||
-rw-r--r-- | src/buildtool/execution_api/local/local_api.hpp | 24 |
4 files changed, 224 insertions, 12 deletions
diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index d9733f18..8050c3d7 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -134,8 +134,11 @@ auto LocalAction::Execute(Logger const* logger) noexcept if (auto result = local_context_.storage->ActionCache().CachedResult(*action)) { if (result->exit_code() == 0 and - ActionResultContainsExpectedOutputs( - *result, output_files_, output_dirs_)) { + (mode_ == RequestMode::kV2_1 + ? ActionResultContainsExpectedOutputs(*result, + output_paths_) + : ActionResultContainsExpectedOutputs( + *result, output_files_, output_dirs_))) { return create_response( logger, action->hash(), @@ -376,6 +379,16 @@ auto LocalAction::CreateDirectoryStructure( } return true; }; + + if (mode_ == RequestMode::kV2_1) { + return std::all_of( + output_paths_.begin(), + output_paths_.end(), + [this, &exec_path, &create_dir](auto const& local_path) { + auto dir = (exec_path / cwd_ / local_path).parent_path(); + return create_dir(dir); + }); + } return std::all_of(output_files_.begin(), output_files_.end(), [this, &exec_path, &create_dir](auto const& local_path) { @@ -487,11 +500,109 @@ auto LocalAction::CollectOutputDirOrSymlink( return std::nullopt; } +auto LocalAction::CollectOutputPath( + std::filesystem::path const& exec_path, + std::string const& local_path) const noexcept -> std::optional<OutputPath> { + auto out_path = exec_path / local_path; + auto type = FileSystemManager::Type(out_path, /*allow_upwards=*/true); + if (not type) { + logger_.Emit(LogLevel::Error, "expected known type at {}", local_path); + return std::nullopt; + } + if (IsSymlinkObject(*type)) { + if (auto content = FileSystemManager::ReadSymlink(out_path)) { + // in native mode: check validity of symlink + if (ProtocolTraits::IsNative( + local_context_.storage->GetHashFunction().GetType()) and + not PathIsNonUpwards(*content)) { + logger_.Emit( + LogLevel::Error, "found invalid symlink at {}", local_path); + return std::nullopt; + } + if (local_context_.storage->CAS().StoreBlob(*content)) { + auto out_symlink = bazel_re::OutputSymlink{}; + out_symlink.set_path(local_path); + out_symlink.set_target(*content); + return out_symlink; + } + } + } + else if (IsFileObject(*type)) { + bool is_executable = IsExecutableObject(*type); + auto digest = local_context_.storage->CAS().StoreBlob</*kOwner=*/true>( + out_path, is_executable); + if (digest) { + auto out_file = bazel_re::OutputFile{}; + out_file.set_path(local_path); + *out_file.mutable_digest() = + ArtifactDigestFactory::ToBazel(*digest); + out_file.set_is_executable(is_executable); + return out_file; + } + } + else if (IsTreeObject(*type)) { + if (auto digest = CreateDigestFromLocalOwnedTree( + *local_context_.storage, out_path)) { + auto out_dir = bazel_re::OutputDirectory{}; + out_dir.set_path(local_path); + (*out_dir.mutable_tree_digest()) = + ArtifactDigestFactory::ToBazel(*digest); + return out_dir; + } + logger_.Emit(LogLevel::Error, + "found invalid entries in directory at {}", + local_path); + } + else { + logger_.Emit(LogLevel::Error, + "expected file, directory, or symlink at {}", + local_path); + } + return std::nullopt; +} + auto LocalAction::CollectAndStoreOutputs( bazel_re::ActionResult* result, std::filesystem::path const& exec_path) const noexcept -> bool { try { logger_.Emit(LogLevel::Trace, "collecting outputs:"); + if (mode_ == RequestMode::kV2_1) { + for (auto const& path : output_paths_) { + auto out = CollectOutputPath(exec_path, path); + if (not out) { + logger_.Emit(LogLevel::Error, + "could not collect output path {}", + path); + return false; + } + if (std::holds_alternative<bazel_re::OutputSymlink>(*out)) { + auto out_symlink = std::get<bazel_re::OutputSymlink>(*out); + logger_.Emit(LogLevel::Trace, + " - symlink {}: {}", + path, + out_symlink.target()); + result->mutable_output_symlinks()->Add( + std::move(out_symlink)); + } + else if (std::holds_alternative<bazel_re::OutputFile>(*out)) { + auto out_file = std::get<bazel_re::OutputFile>(*out); + auto const& digest = out_file.digest().hash(); + logger_.Emit( + LogLevel::Trace, " - file {}: {}", path, digest); + result->mutable_output_files()->Add(std::move(out_file)); + } + else { + auto out_dir = std::get<bazel_re::OutputDirectory>(*out); + auto const& digest = out_dir.tree_digest().hash(); + logger_.Emit( + LogLevel::Trace, " - dir {}: {}", path, digest); + result->mutable_output_directories()->Add( + std::move(out_dir)); + } + } + return true; + } + // if mode_ is RequestMode::kV2_0 or RequestMode::kBestEffort for (auto const& path : output_files_) { auto out = CollectOutputFileOrSymlink(exec_path, path); if (not out) { @@ -506,6 +617,9 @@ auto LocalAction::CollectAndStoreOutputs( " - symlink {}: {}", path, out_symlink.target()); + if (mode_ == RequestMode::kBestEffort) { + *result->mutable_output_symlinks()->Add() = out_symlink; + } result->mutable_output_file_symlinks()->Add( std::move(out_symlink)); } @@ -530,6 +644,9 @@ auto LocalAction::CollectAndStoreOutputs( " - symlink {}: {}", path, out_symlink.target()); + if (mode_ == RequestMode::kBestEffort) { + *result->mutable_output_symlinks()->Add() = out_symlink; + } result->mutable_output_directory_symlinks()->Add( std::move(out_symlink)); } diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp index 7610def0..47f912d8 100644 --- a/src/buildtool/execution_api/local/local_action.hpp +++ b/src/buildtool/execution_api/local/local_action.hpp @@ -18,6 +18,7 @@ #include <algorithm> #include <chrono> #include <compare> +#include <cstdint> #include <filesystem> #include <functional> // IWYU pragma: keep #include <map> @@ -45,6 +46,12 @@ class LocalAction final : public IExecutionAction { friend class LocalApi; + enum class RequestMode : std::uint8_t { + kV2_0, // RBEv2.0 + kV2_1, // >=RBEv2.1 + kBestEffort // construct both, let the server pick + }; + public: struct Output { bazel_re::ActionResult action; @@ -56,6 +63,9 @@ class LocalAction final : public IExecutionAction { std::variant<bazel_re::OutputFile, bazel_re::OutputSymlink>; using OutputDirOrSymlink = std::variant<bazel_re::OutputDirectory, bazel_re::OutputSymlink>; + using OutputPath = std::variant<bazel_re::OutputFile, + bazel_re::OutputDirectory, + bazel_re::OutputSymlink>; using FileCopies = std::unordered_map<Artifact::ObjectInfo, TmpDir::Ptr>; @@ -76,31 +86,65 @@ class LocalAction final : public IExecutionAction { std::string const cwd_; std::vector<std::string> output_files_; std::vector<std::string> output_dirs_; + std::vector<std::string> output_paths_; std::map<std::string, std::string> const env_vars_; std::vector<bazel_re::Platform_Property> const properties_; std::chrono::milliseconds timeout_{kDefaultTimeout}; CacheFlag cache_flag_{CacheFlag::CacheOutput}; + RequestMode mode_{}; + + explicit LocalAction(gsl::not_null<LocalContext const*> local_context, + ArtifactDigest root_digest, + std::vector<std::string> command, + std::string cwd, + std::vector<std::string> output_files, + std::vector<std::string> output_dirs, + std::map<std::string, std::string> env_vars, + std::map<std::string, std::string> const& properties, + bool best_effort) noexcept + : local_context_{*local_context}, + root_digest_{std::move(root_digest)}, + cmdline_{std::move(command)}, + cwd_{std::move(cwd)}, + output_files_{std::move(output_files)}, + output_dirs_{std::move(output_dirs)}, + env_vars_{std::move(env_vars)}, + properties_{BazelMsgFactory::CreateMessageVectorFromMap< + bazel_re::Platform_Property>(properties)}, + mode_{best_effort ? RequestMode::kBestEffort : RequestMode::kV2_0} { + std::sort(output_files_.begin(), output_files_.end()); + std::sort(output_dirs_.begin(), output_dirs_.end()); + if (best_effort) { + output_paths_.reserve(output_files_.size() + output_dirs_.size()); + output_paths_.insert(output_paths_.end(), + output_files_.begin(), + output_files_.end()); + output_paths_.insert( + output_paths_.end(), output_dirs_.begin(), output_dirs_.end()); + std::sort(output_paths_.begin(), output_paths_.end()); + } + } + // Alternative constructor with combined output_paths for files and dirs, as + // it is used by RBEv2.1 and above. explicit LocalAction( gsl::not_null<LocalContext const*> local_context, ArtifactDigest root_digest, std::vector<std::string> command, std::string cwd, - std::vector<std::string> output_files, - std::vector<std::string> output_dirs, + std::vector<std::string> output_paths, std::map<std::string, std::string> env_vars, std::map<std::string, std::string> const& properties) noexcept : local_context_{*local_context}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, cwd_{std::move(cwd)}, - output_files_{std::move(output_files)}, - output_dirs_{std::move(output_dirs)}, + output_paths_{std::move(output_paths)}, env_vars_{std::move(env_vars)}, properties_{BazelMsgFactory::CreateMessageVectorFromMap< - bazel_re::Platform_Property>(properties)} { - std::sort(output_files_.begin(), output_files_.end()); - std::sort(output_dirs_.begin(), output_dirs_.end()); + bazel_re::Platform_Property>(properties)}, + mode_{RequestMode::kV2_1} { + std::sort(output_paths_.begin(), output_paths_.end()); } [[nodiscard]] auto CreateActionDigest(ArtifactDigest const& exec_dir, @@ -114,6 +158,7 @@ class LocalAction final : public IExecutionAction { .cwd = &cwd_, .output_files = &output_files_, .output_dirs = &output_dirs_, + .output_paths = &output_paths_, .env_vars = &env_vars, .properties = &properties_, .exec_dir = &exec_dir, @@ -153,6 +198,10 @@ class LocalAction final : public IExecutionAction { std::string const& local_path) const noexcept -> std::optional<OutputDirOrSymlink>; + [[nodiscard]] auto CollectOutputPath(std::filesystem::path const& exec_path, + std::string const& local_path) + const noexcept -> std::optional<OutputPath>; + [[nodiscard]] auto CollectAndStoreOutputs( bazel_re::ActionResult* result, std::filesystem::path const& exec_path) const noexcept -> bool; diff --git a/src/buildtool/execution_api/local/local_api.cpp b/src/buildtool/execution_api/local/local_api.cpp index 3eede672..c0a3a30e 100644 --- a/src/buildtool/execution_api/local/local_api.cpp +++ b/src/buildtool/execution_api/local/local_api.cpp @@ -59,6 +59,7 @@ LocalApi::LocalApi(gsl::not_null<LocalContext const*> const& local_context, : local_context_{*local_context}, git_api_{CreateFallbackApi(*local_context->storage, repo_config)} {} +// NOLINTNEXTLINE(google-default-arguments) auto LocalApi::CreateAction( ArtifactDigest const& root_digest, std::vector<std::string> const& command, @@ -66,8 +67,13 @@ auto LocalApi::CreateAction( std::vector<std::string> const& output_files, std::vector<std::string> const& output_dirs, std::map<std::string, std::string> const& env_vars, - std::map<std::string, std::string> const& properties) const noexcept - -> IExecutionAction::Ptr { + std::map<std::string, std::string> const& properties, + bool force_legacy) const noexcept -> IExecutionAction::Ptr { + if (ProtocolTraits::IsNative(GetHashType())) { + // fall back to legacy for native + force_legacy = true; + } + bool best_effort = not force_legacy; return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_, root_digest, command, @@ -75,6 +81,24 @@ auto LocalApi::CreateAction( output_files, output_dirs, env_vars, + properties, + best_effort}}; +} + +auto LocalApi::CreateAction( + ArtifactDigest const& root_digest, + std::vector<std::string> const& command, + std::string const& cwd, + std::vector<std::string> const& output_paths, + std::map<std::string, std::string> const& env_vars, + std::map<std::string, std::string> const& properties) const noexcept + -> IExecutionAction::Ptr { + return IExecutionAction::Ptr{new (std::nothrow) LocalAction{&local_context_, + root_digest, + command, + cwd, + output_paths, + env_vars, properties}}; } diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index e2374258..1b6ddeab 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -40,6 +40,7 @@ class LocalApi final : public IExecutionApi { explicit LocalApi(gsl::not_null<LocalContext const*> const& local_context, RepositoryConfig const* repo_config = nullptr) noexcept; + // NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector<std::string> const& command, @@ -47,9 +48,30 @@ class LocalApi final : public IExecutionApi { std::vector<std::string> const& output_files, std::vector<std::string> const& output_dirs, std::map<std::string, std::string> const& env_vars, - std::map<std::string, std::string> const& properties) const noexcept + std::map<std::string, std::string> const& properties, + bool force_legacy = false) const noexcept -> IExecutionAction::Ptr final; + /// \brief Create a new action (>=RBEv2.1). + /// \param[in] root_digest Digest of the build root. + /// \param[in] command Command as argv vector + /// \param[in] cwd Working directory, relative to execution root + /// \param[in] output_paths List of output file/dir paths, relative to cwd + /// \param[in] env_vars The environment variables to set. + /// \param[in] properties Platform properties to set. + /// \returns The new action. + /// Note that, due to missing file/dir separation, the execution response + /// will not report output file symlinks and directory symlinks separately. + /// (see \ref LocalResponse::DirectorySymlinks) + [[nodiscard]] auto CreateAction( + ArtifactDigest const& root_digest, + std::vector<std::string> const& command, + std::string const& cwd, + std::vector<std::string> const& output_paths, + std::map<std::string, std::string> const& env_vars, + std::map<std::string, std::string> const& properties) const noexcept + -> IExecutionAction::Ptr; + // NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] auto RetrieveToPaths( std::vector<Artifact::ObjectInfo> const& artifacts_info, |