diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2025-02-21 15:18:58 +0100 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2025-03-24 09:25:05 +0100 |
commit | f192705cb4834252507d228e7d6e0a38095976e3 (patch) | |
tree | 83040a418bc8c700cd5234d8d60f0497e2b7631f | |
parent | fc0c842eb2e938c7de405e365ff320eb28e04bc7 (diff) | |
download | justbuild-f192705cb4834252507d228e7d6e0a38095976e3.tar.gz |
ExecutionApi: Return TmpDir
20 files changed, 186 insertions, 64 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 3ffd508f..b13ac8b5 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -23,6 +23,7 @@ , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] + , ["src/utils/cpp", "tmp_dir"] ] , "private-deps": [ ["@", "json", "", "json"] diff --git a/src/buildtool/execution_api/common/api_bundle.cpp b/src/buildtool/execution_api/common/api_bundle.cpp index 7ab06e81..391e1343 100644 --- a/src/buildtool/execution_api/common/api_bundle.cpp +++ b/src/buildtool/execution_api/common/api_bundle.cpp @@ -43,7 +43,8 @@ auto ApiBundle::Create( remote_context->auth, remote_context->retry_config, config, - local_context->storage_config->hash_function); + local_context->storage_config->hash_function, + local_api->GetTempSpace()); } return ApiBundle{.local = std::move(local_api), .remote = std::move(remote_api)}; @@ -63,7 +64,8 @@ auto ApiBundle::MakeRemote( authentication, retry_config, config, - HashFunction{local->GetHashType()}); + HashFunction{local->GetHashType()}, + local->GetTempSpace()); } return local; } diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp index 43a7e581..001c9189 100644 --- a/src/buildtool/execution_api/common/execution_api.hpp +++ b/src/buildtool/execution_api/common/execution_api.hpp @@ -30,6 +30,7 @@ #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" +#include "src/utils/cpp/tmp_dir.hpp" /// \brief Abstract remote execution API /// Can be used to create actions. @@ -156,6 +157,8 @@ class IExecutionApi { [[nodiscard]] virtual auto GetHashType() const noexcept -> HashFunction::Type = 0; + + [[nodiscard]] virtual auto GetTempSpace() const noexcept -> TmpDir::Ptr = 0; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP diff --git a/src/buildtool/execution_api/local/local_api.cpp b/src/buildtool/execution_api/local/local_api.cpp index 81fb4bed..eef05021 100644 --- a/src/buildtool/execution_api/local/local_api.cpp +++ b/src/buildtool/execution_api/local/local_api.cpp @@ -333,3 +333,7 @@ auto LocalApi::SpliceBlob(ArtifactDigest const& blob_digest, auto LocalApi::GetHashType() const noexcept -> HashFunction::Type { return local_context_.storage_config->hash_function.GetType(); } + +auto LocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr { + return local_context_.storage_config->CreateTypedTmpDir("api_space"); +} diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 142da6fb..e2374258 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -33,6 +33,7 @@ #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" +#include "src/utils/cpp/tmp_dir.hpp" class LocalApi final : public IExecutionApi { public: @@ -105,6 +106,8 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; + private: LocalContext const& local_context_; std::optional<GitApi> const git_api_; diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index c6a8c4c4..7fccc2c6 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -85,6 +85,7 @@ , ["src/buildtool/execution_api/bazel_msg", "execution_config"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_engine/dag", "dag"] + , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index b2f145b0..08fc8e99 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -144,7 +144,8 @@ BazelApi::BazelApi(std::string const& instance_name, gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, ExecutionConfiguration const& exec_config, - HashFunction hash_function) noexcept { + HashFunction hash_function, + TmpDir::Ptr temp_space) noexcept { network_ = std::make_shared<BazelNetwork>(instance_name, host, port, @@ -152,7 +153,7 @@ BazelApi::BazelApi(std::string const& instance_name, retry_config, exec_config, hash_function, - /*temp_space=*/nullptr); + std::move(temp_space)); } // implement move constructor in cpp, where all members are complete types @@ -605,3 +606,7 @@ auto BazelApi::CreateAction( -> HashFunction::Type { return network_->GetHashFunction().GetType(); } + +[[nodiscard]] auto BazelApi::GetTempSpace() const noexcept -> TmpDir::Ptr { + return network_->GetTempSpace(); +} diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index 633d9d85..df60204f 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -36,6 +36,7 @@ #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" +#include "src/utils/cpp/tmp_dir.hpp" // forward declaration for actual implementations class BazelNetwork; @@ -49,7 +50,8 @@ class BazelApi final : public IExecutionApi { gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, ExecutionConfiguration const& exec_config, - HashFunction hash_function) noexcept; + HashFunction hash_function, + TmpDir::Ptr temp_space) noexcept; BazelApi(BazelApi const&) = delete; BazelApi(BazelApi&& other) noexcept; auto operator=(BazelApi const&) -> BazelApi& = delete; @@ -124,6 +126,8 @@ class BazelApi final : public IExecutionApi { [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; + private: std::shared_ptr<BazelNetwork> network_; diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS index eadea902..5f562fac 100644 --- a/src/buildtool/execution_api/serve/TARGETS +++ b/src/buildtool/execution_api/serve/TARGETS @@ -32,6 +32,7 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "context"] , ["src/buildtool/execution_engine/dag", "dag"] + , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp index 3394c270..9c58d73c 100644 --- a/src/buildtool/execution_api/serve/mr_local_api.cpp +++ b/src/buildtool/execution_api/serve/mr_local_api.cpp @@ -152,3 +152,7 @@ auto MRLocalApi::GetHashType() const noexcept -> HashFunction::Type { return compat_local_api_ == nullptr ? native_local_api_->GetHashType() : compat_local_api_->GetHashType(); } + +auto MRLocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr { + return native_local_api_->GetTempSpace(); +} diff --git a/src/buildtool/execution_api/serve/mr_local_api.hpp b/src/buildtool/execution_api/serve/mr_local_api.hpp index 3ad19475..d9f323cf 100644 --- a/src/buildtool/execution_api/serve/mr_local_api.hpp +++ b/src/buildtool/execution_api/serve/mr_local_api.hpp @@ -31,6 +31,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/context.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" +#include "src/utils/cpp/tmp_dir.hpp" /// \brief Multi-repo-specific implementation of the abstract Execution API. /// Handles interaction between a native storage and a remote, irrespective of @@ -133,6 +134,8 @@ class MRLocalApi final : public IExecutionApi { [[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final; + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final; + private: // retain local context references to have direct access to storages gsl::not_null<LocalContext const*> native_context_; diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS index 049efe44..2ac64635 100644 --- a/src/buildtool/execution_engine/executor/TARGETS +++ b/src/buildtool/execution_engine/executor/TARGETS @@ -38,6 +38,7 @@ , ["src/utils/cpp", "hex_string"] , ["src/utils/cpp", "path_rebase"] , ["src/utils/cpp", "prefix"] + , ["src/utils/cpp", "tmp_dir"] ] , "stage": ["src", "buildtool", "execution_engine", "executor"] } diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp index e030d2d6..12721b01 100644 --- a/src/buildtool/execution_engine/executor/executor.hpp +++ b/src/buildtool/execution_engine/executor/executor.hpp @@ -72,6 +72,7 @@ #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/path_rebase.hpp" #include "src/utils/cpp/prefix.hpp" +#include "src/utils/cpp/tmp_dir.hpp" /// \brief Implementations for executing actions and uploading artifacts. class ExecutorImpl { @@ -133,8 +134,10 @@ class ExecutorImpl { } // get the alternative endpoint - auto alternative_api = GetAlternativeEndpoint( - merged_properties, remote_context, api.GetHashType()); + auto alternative_api = GetAlternativeEndpoint(merged_properties, + remote_context, + api.GetHashType(), + api.GetTempSpace()); if (alternative_api) { if (not api.ParallelRetrieveToCas( std::vector<Artifact::ObjectInfo>{Artifact::ObjectInfo{ @@ -758,7 +761,8 @@ class ExecutorImpl { [[nodiscard]] static auto GetAlternativeEndpoint( const ExecutionProperties& properties, const gsl::not_null<RemoteContext const*>& remote_context, - HashFunction::Type hash_type) -> std::unique_ptr<BazelApi> { + HashFunction::Type hash_type, + TmpDir::Ptr temp_space) -> std::unique_ptr<BazelApi> { for (auto const& [pred, endpoint] : remote_context->exec_config->dispatch) { bool match = true; @@ -781,7 +785,8 @@ class ExecutorImpl { remote_context->auth, remote_context->retry_config, config, - HashFunction{hash_type}); + HashFunction{hash_type}, + std::move(temp_space)); } } return nullptr; diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 7fafada8..0dd3a155 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -426,7 +426,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, &*auth_config, &*retry_config, config, - hash_fct); + hash_fct, + mr_local_api->GetTempSpace()); } bool const has_remote_api = remote_api != nullptr; diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 57ed0fd9..ee9781c9 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -234,7 +234,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &*auth_config, &*retry_config, config, - hash_fct); + hash_fct, + mr_local_api->GetTempSpace()); } bool const has_remote_api = remote_api != nullptr; diff --git a/test/buildtool/execution_api/bazel/TARGETS b/test/buildtool/execution_api/bazel/TARGETS index c2898d1c..acbee81f 100644 --- a/test/buildtool/execution_api/bazel/TARGETS +++ b/test/buildtool/execution_api/bazel/TARGETS @@ -140,11 +140,13 @@ , ["@", "src", "src/buildtool/execution_api/common", "common"] , ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] + , ["@", "src", "src/buildtool/storage", "config"] + , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["buildtool/execution_api/common", "api_test"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] - , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] + , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } diff --git a/test/buildtool/execution_api/bazel/bazel_api.test.cpp b/test/buildtool/execution_api/bazel/bazel_api.test.cpp index 230ee2dc..8fb50006 100644 --- a/test/buildtool/execution_api/bazel/bazel_api.test.cpp +++ b/test/buildtool/execution_api/bazel/bazel_api.test.cpp @@ -15,6 +15,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include <optional> +#include <utility> #include "catch2/catch_test_macros.hpp" #include "gsl/gsl" @@ -24,8 +25,10 @@ #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/utils/cpp/tmp_dir.hpp" #include "test/buildtool/execution_api/common/api_test.hpp" -#include "test/utils/hermeticity/test_hash_function_type.hpp" +#include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" @@ -36,10 +39,12 @@ class FactoryApi final { explicit FactoryApi( gsl::not_null<ServerAddress const*> const& server_address, gsl::not_null<Auth const*> const& auth, - HashFunction hash_function) noexcept + HashFunction hash_function, + TmpDir::Ptr temp_space) noexcept : address_{*server_address}, auth_{*auth}, - hash_function_{hash_function} {} + hash_function_{hash_function}, + temp_space_{std::move(temp_space)} {} [[nodiscard]] auto operator()() const -> IExecutionApi::Ptr { static RetryConfig retry_config{}; // default retry config @@ -49,20 +54,22 @@ class FactoryApi final { &auth_, &retry_config, {}, - hash_function_}}; + hash_function_, + temp_space_}}; } private: ServerAddress const& address_; Auth const& auth_; HashFunction const hash_function_; + TmpDir::Ptr temp_space_; }; } // namespace TEST_CASE("BazelAPI: No input, no output", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -70,13 +77,16 @@ TEST_CASE("BazelAPI: No input, no output", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestNoInputNoOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: No input, create output", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -84,13 +94,16 @@ TEST_CASE("BazelAPI: No input, create output", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestNoInputCreateOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: One input copied to output", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -98,13 +111,16 @@ TEST_CASE("BazelAPI: One input copied to output", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestOneInputCopiedToOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Non-zero exit code, create output", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -112,14 +128,17 @@ TEST_CASE("BazelAPI: Non-zero exit code, create output", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestNonZeroExitCodeCreateOutput(api_factory, remote_config->platform_properties); } TEST_CASE("BazelAPI: Retrieve two identical trees to path", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -127,15 +146,18 @@ TEST_CASE("BazelAPI: Retrieve two identical trees to path", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveTwoIdenticalTreesToPath( api_factory, remote_config->platform_properties, "two_trees"); } TEST_CASE("BazelAPI: Retrieve file and symlink with same content to path", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -143,14 +165,17 @@ TEST_CASE("BazelAPI: Retrieve file and symlink with same content to path", REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveFileAndSymlinkWithSameContentToPath( api_factory, remote_config->platform_properties, "file_and_symlink"); } TEST_CASE("BazelAPI: Retrieve mixed blobs and trees", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -158,14 +183,17 @@ TEST_CASE("BazelAPI: Retrieve mixed blobs and trees", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestRetrieveMixedBlobsAndTrees( api_factory, remote_config->platform_properties, "blobs_and_trees"); } TEST_CASE("BazelAPI: Create directory prior to execution", "[execution_api]") { + auto storage_config = TestStorageConfig::Create(); auto remote_config = TestRemoteConfig::ReadFromEnvironment(); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; REQUIRE(remote_config); REQUIRE(remote_config->remote_address); @@ -173,7 +201,10 @@ TEST_CASE("BazelAPI: Create directory prior to execution", "[execution_api]") { REQUIRE(auth); FactoryApi api_factory{ - &*remote_config->remote_address, &*auth, hash_function}; + &*remote_config->remote_address, + &*auth, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; TestCreateDirPriorToExecution(api_factory, remote_config->platform_properties); } diff --git a/test/buildtool/execution_engine/executor/TARGETS b/test/buildtool/execution_engine/executor/TARGETS index a27e8dac..184c9fe1 100644 --- a/test/buildtool/execution_engine/executor/TARGETS +++ b/test/buildtool/execution_engine/executor/TARGETS @@ -59,10 +59,13 @@ , ["@", "src", "src/buildtool/file_system", "object_type"] , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] + , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] + , ["@", "src", "src/utils/cpp", "tmp_dir"] , ["", "catch-main"] , ["utils", "test_api_bundle"] , ["utils", "test_hash_function_type"] + , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } @@ -108,10 +111,11 @@ , ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] + , ["@", "src", "src/buildtool/storage", "config"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] - , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] + , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } diff --git a/test/buildtool/execution_engine/executor/executor.test.cpp b/test/buildtool/execution_engine/executor/executor.test.cpp index a9f91ade..93dd10b6 100644 --- a/test/buildtool/execution_engine/executor/executor.test.cpp +++ b/test/buildtool/execution_engine/executor/executor.test.cpp @@ -57,9 +57,12 @@ #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" +#include "src/utils/cpp/tmp_dir.hpp" #include "test/utils/executor/test_api_bundle.hpp" #include "test/utils/hermeticity/test_hash_function_type.hpp" +#include "test/utils/hermeticity/test_storage_config.hpp" /// \brief Mockup API test config. struct TestApiConfig { @@ -190,8 +193,11 @@ class TestAction : public IExecutionAction { class TestApi : public IExecutionApi { public: explicit TestApi(TestApiConfig config, - HashFunction::Type hash_type) noexcept - : config_{std::move(config)}, hash_type_{hash_type} {} + HashFunction::Type hash_type, + TmpDir::Ptr temp_space) noexcept + : config_{std::move(config)}, + hash_type_{hash_type}, + temp_space_{std::move(temp_space)} {} [[nodiscard]] auto CreateAction( ArtifactDigest const& /*unused*/, @@ -284,9 +290,14 @@ class TestApi : public IExecutionApi { return hash_type_; } + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final { + return temp_space_; + } + private: TestApiConfig config_{}; HashFunction::Type hash_type_; + TmpDir::Ptr temp_space_; }; [[nodiscard]] auto SetupConfig(std::filesystem::path const& ws) @@ -334,12 +345,13 @@ class TestApi : public IExecutionApi { } TEST_CASE("Executor: Process artifact", "[executor]") { + auto const storage_config = TestStorageConfig::Create(); std::filesystem::path workspace_path{ "test/buildtool/execution_engine/executor"}; DependencyGraph g; auto [config, repo_config] = CreateTest(&g, workspace_path); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + HashFunction const hash_function = storage_config.Get().hash_function; auto const local_cpp_id = ArtifactDescription::CreateLocal("local.cpp", "").Id(); @@ -356,7 +368,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") { .exec_config = &remote_config}; SECTION("Processing succeeds for valid config") { - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -375,7 +390,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") { SECTION("Processing fails if uploading local artifact failed") { config.artifacts[NamedDigest("local.cpp").hash()].uploads = false; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -394,7 +412,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") { SECTION("Processing fails if known artifact is not available") { config.artifacts[NamedDigest("known.cpp").hash()].available = false; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -412,13 +433,14 @@ TEST_CASE("Executor: Process artifact", "[executor]") { } TEST_CASE("Executor: Process action", "[executor]") { + auto const storage_config = TestStorageConfig::Create(); std::filesystem::path workspace_path{ "test/buildtool/execution_engine/executor"}; DependencyGraph g; auto [config, repo_config] = CreateTest(&g, workspace_path); - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + HashFunction const hash_function = storage_config.Get().hash_function; auto const local_cpp_id = ArtifactDescription::CreateLocal("local.cpp", "").Id(); @@ -442,7 +464,10 @@ TEST_CASE("Executor: Process action", "[executor]") { .exec_config = &remote_config}; SECTION("Processing succeeds for valid config") { - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -464,7 +489,10 @@ TEST_CASE("Executor: Process action", "[executor]") { SECTION("Processing succeeds even if result was is not cached") { config.response.cached = false; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -486,7 +514,10 @@ TEST_CASE("Executor: Process action", "[executor]") { SECTION("Processing succeeds even if output is not available in CAS") { config.artifacts[NamedDigest("output2.exe").hash()].available = false; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -511,7 +542,10 @@ TEST_CASE("Executor: Process action", "[executor]") { SECTION("Processing fails if execution failed") { config.execution.failed = true; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -533,7 +567,10 @@ TEST_CASE("Executor: Process action", "[executor]") { SECTION("Processing fails if exit code is non-zero") { config.response.exit_code = 1; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); @@ -558,7 +595,10 @@ TEST_CASE("Executor: Process action", "[executor]") { SECTION("Processing fails if any output is missing") { config.execution.outputs = {"output1.exe" /*, "output2.exe"*/}; - auto api = std::make_shared<TestApi>(config, hash_function.GetType()); + auto api = std::make_shared<TestApi>( + config, + hash_function.GetType(), + storage_config.Get().CreateTypedTmpDir("temp_space")); Statistics stats{}; Progress progress{}; auto const apis = CreateTestApiBundle(api); diff --git a/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp b/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp index a809d04f..c92f3aef 100644 --- a/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp +++ b/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp @@ -21,13 +21,13 @@ #include "src/buildtool/common/remote/retry_config.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" -#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/execution_config.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/config.hpp" #include "test/buildtool/execution_engine/executor/executor_api.test.hpp" -#include "test/utils/hermeticity/test_hash_function_type.hpp" +#include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" @@ -44,16 +44,18 @@ TEST_CASE("Executor<BazelApi>: Upload blob", "[executor]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto storage_config = TestStorageConfig::Create(); TestBlobUpload(&repo_config, [&] { - return std::make_shared<BazelApi>("remote-execution", - remote_config->remote_address->host, - remote_config->remote_address->port, - &*auth_config, - &retry_config, - config, - hash_function); + return std::make_shared<BazelApi>( + "remote-execution", + remote_config->remote_address->host, + remote_config->remote_address->port, + &*auth_config, + &retry_config, + config, + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")); }); } @@ -73,7 +75,7 @@ TEST_CASE("Executor<BazelApi>: Compile hello world", "[executor]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto storage_config = TestStorageConfig::Create(); TestHelloWorldCompilation( &repo_config, @@ -87,7 +89,8 @@ TEST_CASE("Executor<BazelApi>: Compile hello world", "[executor]") { &*auth_config, &retry_config, config, - hash_function); + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); @@ -109,7 +112,7 @@ TEST_CASE("Executor<BazelApi>: Compile greeter", "[executor]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto storage_config = TestStorageConfig::Create(); TestGreeterCompilation( &repo_config, @@ -123,7 +126,8 @@ TEST_CASE("Executor<BazelApi>: Compile greeter", "[executor]") { &*auth_config, &retry_config, config, - hash_function); + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); @@ -145,7 +149,7 @@ TEST_CASE("Executor<BazelApi>: Upload and download trees", "[executor]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto storage_config = TestStorageConfig::Create(); TestUploadAndDownloadTrees( &repo_config, @@ -159,7 +163,8 @@ TEST_CASE("Executor<BazelApi>: Upload and download trees", "[executor]") { &*auth_config, &retry_config, config, - hash_function); + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); @@ -181,7 +186,7 @@ TEST_CASE("Executor<BazelApi>: Retrieve output directories", "[executor]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto storage_config = TestStorageConfig::Create(); TestRetrieveOutputDirectories( &repo_config, @@ -195,7 +200,8 @@ TEST_CASE("Executor<BazelApi>: Retrieve output directories", "[executor]") { &*auth_config, &retry_config, config, - hash_function); + storage_config.Get().hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")); }, &*auth_config, false /* not hermetic */); |