From 03f948f4b10e916052a2234448b6658b80ee9143 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 1 Aug 2024 10:26:13 +0200 Subject: Execution API: support cwd ... following the remote-execution standard that all output paths (but none of the input paths) are relative to the working directory. Therefore, the executor has to do the path translation. For our implementation of the API interface - the local API now handles cwd correctly, - the remote API forwards cwd correctly, and - the git API continues to report actions as not implemented. --- .../execution_api/bazel_msg/bazel_msg_factory.cpp | 1 + .../execution_api/bazel_msg/bazel_msg_factory.hpp | 3 +++ src/buildtool/execution_api/common/execution_api.hpp | 4 +++- .../execution_api/execution_service/execution_server.cpp | 1 + src/buildtool/execution_api/git/git_api.hpp | 1 + src/buildtool/execution_api/local/local_action.cpp | 14 ++++++++------ src/buildtool/execution_api/local/local_action.hpp | 4 ++++ src/buildtool/execution_api/local/local_api.hpp | 2 ++ src/buildtool/execution_api/remote/bazel/bazel_action.cpp | 3 +++ src/buildtool/execution_api/remote/bazel/bazel_action.hpp | 2 ++ src/buildtool/execution_api/remote/bazel/bazel_api.cpp | 2 ++ src/buildtool/execution_api/remote/bazel/bazel_api.hpp | 1 + 12 files changed, 31 insertions(+), 7 deletions(-) (limited to 'src/buildtool/execution_api') diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp index 09462bec..d4a096fb 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp @@ -210,6 +210,7 @@ template // [Action][build.bazel.remote.execution.v2.Action] for migration. // (https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L646) msg.set_allocated_platform(CreatePlatform(*request.properties).release()); + msg.set_working_directory(*request.cwd); std::copy(request.command_line->begin(), request.command_line->end(), pb::back_inserter(msg.mutable_arguments())); diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp index c48e605d..0b006273 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp @@ -144,6 +144,9 @@ struct BazelMsgFactory::ActionDigestRequest final { /// \brief The command line. VectorPtr const command_line; + /// \brief The workingg direcotry + gsl::not_null const cwd; + /// \brief The paths of output files. VectorPtr const output_files; diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp index 2108c68d..c9ee276b 100644 --- a/src/buildtool/execution_api/common/execution_api.hpp +++ b/src/buildtool/execution_api/common/execution_api.hpp @@ -46,7 +46,8 @@ class IExecutionApi { /// \brief Create a new action. /// \param[in] root_digest Digest of the build root. /// \param[in] command Command as argv vector - /// \param[in] output_files List of paths to output files. + /// \param[in] cwd Working direcotry, relative to execution root + /// \param[in] output_files List of paths to output files, relative to cwd /// \param[in] output_dirs List of paths to output directories. /// \param[in] env_vars The environment variables to set. /// \param[in] properties Platform properties to set. @@ -54,6 +55,7 @@ class IExecutionApi { [[nodiscard]] virtual auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, + std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, diff --git a/src/buildtool/execution_api/execution_service/execution_server.cpp b/src/buildtool/execution_api/execution_service/execution_server.cpp index b77ad9d3..2e56bd04 100644 --- a/src/buildtool/execution_api/execution_service/execution_server.cpp +++ b/src/buildtool/execution_api/execution_service/execution_server.cpp @@ -136,6 +136,7 @@ auto ExecutionServiceImpl::GetIExecutionAction( auto i_execution_action = api_.CreateAction( ArtifactDigest{action.input_root_digest()}, {c->arguments().begin(), c->arguments().end()}, + c->working_directory(), {c->output_files().begin(), c->output_files().end()}, {c->output_directories().begin(), c->output_directories().end()}, env_vars, diff --git a/src/buildtool/execution_api/git/git_api.hpp b/src/buildtool/execution_api/git/git_api.hpp index 139291ef..43afc14a 100644 --- a/src/buildtool/execution_api/git/git_api.hpp +++ b/src/buildtool/execution_api/git/git_api.hpp @@ -42,6 +42,7 @@ class GitApi final : public IExecutionApi { [[nodiscard]] auto CreateAction( ArtifactDigest const& /*root_digest*/, std::vector const& /*command*/, + std::string const& /*cwd*/, std::vector const& /*output_files*/, std::vector const& /*output_dirs*/, std::map const& /*env_vars*/, diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index aac6bb39..8cc1d242 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -82,6 +82,7 @@ class BuildCleanupAnchor { auto LocalAction::Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr { + auto do_cache = CacheEnabled(cache_flag_); auto action = CreateActionDigest(root_digest_, not do_cache); if (not action) { @@ -174,7 +175,7 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept SystemCommand system{"LocalExecution"}; auto const exit_code = - system.Execute(cmdline, env_vars_, build_root, *exec_path); + system.Execute(cmdline, env_vars_, build_root / cwd_, *exec_path); if (exit_code.has_value()) { Output result{}; result.action.set_exit_code(*exit_code); @@ -187,7 +188,7 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept result.action.set_allocated_stderr_digest(digest_ptr); } - if (CollectAndStoreOutputs(&result.action, build_root)) { + if (CollectAndStoreOutputs(&result.action, build_root / cwd_)) { if (cache_flag_ == CacheFlag::CacheOutput) { if (not local_context_.storage->ActionCache().StoreResult( action_id, result.action)) { @@ -343,14 +344,15 @@ auto LocalAction::CreateDirectoryStructure( }; return std::all_of(output_files_.begin(), output_files_.end(), - [&exec_path, &create_dir](auto const& local_path) { - auto dir = (exec_path / local_path).parent_path(); + [this, &exec_path, &create_dir](auto const& local_path) { + auto dir = + (exec_path / cwd_ / local_path).parent_path(); return create_dir(dir); }) and std::all_of(output_dirs_.begin(), output_dirs_.end(), - [&exec_path, &create_dir](auto const& local_path) { - return create_dir(exec_path / local_path); + [this, &exec_path, &create_dir](auto const& local_path) { + return create_dir(exec_path / cwd_ / local_path); }); } diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp index b2016ed7..4d256302 100644 --- a/src/buildtool/execution_api/local/local_action.hpp +++ b/src/buildtool/execution_api/local/local_action.hpp @@ -65,6 +65,7 @@ class LocalAction final : public IExecutionAction { LocalContext const& local_context_; ArtifactDigest const root_digest_{}; std::vector const cmdline_{}; + std::string const cwd_{}; std::vector output_files_{}; std::vector output_dirs_{}; std::map const env_vars_{}; @@ -76,6 +77,7 @@ class LocalAction final : public IExecutionAction { gsl::not_null local_context, ArtifactDigest root_digest, std::vector command, + std::string cwd, std::vector output_files, std::vector output_dirs, std::map env_vars, @@ -83,6 +85,7 @@ class LocalAction final : public IExecutionAction { : local_context_{*local_context}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, + cwd_{std::move(cwd)}, output_files_{std::move(output_files)}, output_dirs_{std::move(output_dirs)}, env_vars_{std::move(env_vars)}, @@ -100,6 +103,7 @@ class LocalAction final : public IExecutionAction { BazelMsgFactory::ActionDigestRequest request{ .command_line = &cmdline_, + .cwd = &cwd_, .output_files = &output_files_, .output_dirs = &output_dirs_, .env_vars = &env_vars, diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 3a40272e..eadf45b7 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -61,6 +61,7 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, + std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, @@ -69,6 +70,7 @@ class LocalApi final : public IExecutionApi { return IExecutionAction::Ptr{new LocalAction{&local_context_, root_digest, command, + cwd, output_files, output_dirs, env_vars, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.cpp b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp index 4190ea19..dcbddb8d 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_action.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp @@ -26,6 +26,7 @@ BazelAction::BazelAction( std::shared_ptr network, bazel_re::Digest root_digest, std::vector command, + std::string cwd, std::vector output_files, std::vector output_dirs, std::map const& env_vars, @@ -33,6 +34,7 @@ BazelAction::BazelAction( : network_{std::move(network)}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, + cwd_{std::move(cwd)}, output_files_{std::move(output_files)}, output_dirs_{std::move(output_dirs)}, env_vars_{BazelMsgFactory::CreateMessageVectorFromMap< @@ -122,6 +124,7 @@ auto BazelAction::CreateBundlesForAction(BazelBlobContainer* blobs, } BazelMsgFactory::ActionDigestRequest request{ .command_line = &cmdline_, + .cwd = &cwd_, .output_files = &output_files_, .output_dirs = &output_dirs_, .env_vars = &env_vars_, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.hpp b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp index 8316cec1..7bf62a45 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_action.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp @@ -44,6 +44,7 @@ class BazelAction final : public IExecutionAction { std::shared_ptr const network_; bazel_re::Digest const root_digest_; std::vector const cmdline_; + std::string const cwd_; std::vector output_files_; std::vector output_dirs_; std::vector const env_vars_; @@ -55,6 +56,7 @@ class BazelAction final : public IExecutionAction { std::shared_ptr network, bazel_re::Digest root_digest, std::vector command, + std::string cwd, std::vector output_files, std::vector output_dirs, std::map const& env_vars, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index d3dc1b2f..2f54a377 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -213,6 +213,7 @@ BazelApi::~BazelApi() = default; auto BazelApi::CreateAction( ArtifactDigest const& root_digest, std::vector const& command, + std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, @@ -221,6 +222,7 @@ auto BazelApi::CreateAction( return std::unique_ptr{new BazelAction{network_, root_digest, command, + cwd, output_files, output_dirs, env_vars, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index 5946f0f7..6b2a6f17 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -58,6 +58,7 @@ class BazelApi final : public IExecutionApi { [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, std::vector const& command, + std::string const& cwd, std::vector const& output_files, std::vector const& output_dirs, std::map const& env_vars, -- cgit v1.2.3