summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaksim Denisov <denisov.maksim@huawei.com>2025-02-21 15:18:58 +0100
committerMaksim Denisov <denisov.maksim@huawei.com>2025-03-24 09:25:05 +0100
commitf192705cb4834252507d228e7d6e0a38095976e3 (patch)
tree83040a418bc8c700cd5234d8d60f0497e2b7631f
parentfc0c842eb2e938c7de405e365ff320eb28e04bc7 (diff)
downloadjustbuild-f192705cb4834252507d228e7d6e0a38095976e3.tar.gz
ExecutionApi: Return TmpDir
-rw-r--r--src/buildtool/execution_api/common/TARGETS1
-rw-r--r--src/buildtool/execution_api/common/api_bundle.cpp6
-rw-r--r--src/buildtool/execution_api/common/execution_api.hpp3
-rw-r--r--src/buildtool/execution_api/local/local_api.cpp4
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp3
-rw-r--r--src/buildtool/execution_api/remote/TARGETS1
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.cpp9
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.hpp6
-rw-r--r--src/buildtool/execution_api/serve/TARGETS1
-rw-r--r--src/buildtool/execution_api/serve/mr_local_api.cpp4
-rw-r--r--src/buildtool/execution_api/serve/mr_local_api.hpp3
-rw-r--r--src/buildtool/execution_engine/executor/TARGETS1
-rw-r--r--src/buildtool/execution_engine/executor/executor.hpp13
-rw-r--r--src/other_tools/just_mr/fetch.cpp3
-rw-r--r--src/other_tools/just_mr/setup.cpp3
-rw-r--r--test/buildtool/execution_api/bazel/TARGETS4
-rw-r--r--test/buildtool/execution_api/bazel/bazel_api.test.cpp71
-rw-r--r--test/buildtool/execution_engine/executor/TARGETS6
-rw-r--r--test/buildtool/execution_engine/executor/executor.test.cpp66
-rw-r--r--test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp42
20 files changed, 186 insertions, 64 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS
index 3ffd508f..b13ac8b5 100644
--- a/src/buildtool/execution_api/common/TARGETS
+++ b/src/buildtool/execution_api/common/TARGETS
@@ -23,6 +23,7 @@
, ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/logging", "logging"]
, ["src/utils/cpp", "expected"]
+ , ["src/utils/cpp", "tmp_dir"]
]
, "private-deps":
[ ["@", "json", "", "json"]
diff --git a/src/buildtool/execution_api/common/api_bundle.cpp b/src/buildtool/execution_api/common/api_bundle.cpp
index 7ab06e81..391e1343 100644
--- a/src/buildtool/execution_api/common/api_bundle.cpp
+++ b/src/buildtool/execution_api/common/api_bundle.cpp
@@ -43,7 +43,8 @@ auto ApiBundle::Create(
remote_context->auth,
remote_context->retry_config,
config,
- local_context->storage_config->hash_function);
+ local_context->storage_config->hash_function,
+ local_api->GetTempSpace());
}
return ApiBundle{.local = std::move(local_api),
.remote = std::move(remote_api)};
@@ -63,7 +64,8 @@ auto ApiBundle::MakeRemote(
authentication,
retry_config,
config,
- HashFunction{local->GetHashType()});
+ HashFunction{local->GetHashType()},
+ local->GetTempSpace());
}
return local;
}
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp
index 43a7e581..001c9189 100644
--- a/src/buildtool/execution_api/common/execution_api.hpp
+++ b/src/buildtool/execution_api/common/execution_api.hpp
@@ -30,6 +30,7 @@
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/execution_action.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
/// \brief Abstract remote execution API
/// Can be used to create actions.
@@ -156,6 +157,8 @@ class IExecutionApi {
[[nodiscard]] virtual auto GetHashType() const noexcept
-> HashFunction::Type = 0;
+
+ [[nodiscard]] virtual auto GetTempSpace() const noexcept -> TmpDir::Ptr = 0;
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
diff --git a/src/buildtool/execution_api/local/local_api.cpp b/src/buildtool/execution_api/local/local_api.cpp
index 81fb4bed..eef05021 100644
--- a/src/buildtool/execution_api/local/local_api.cpp
+++ b/src/buildtool/execution_api/local/local_api.cpp
@@ -333,3 +333,7 @@ auto LocalApi::SpliceBlob(ArtifactDigest const& blob_digest,
auto LocalApi::GetHashType() const noexcept -> HashFunction::Type {
return local_context_.storage_config->hash_function.GetType();
}
+
+auto LocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr {
+ return local_context_.storage_config->CreateTypedTmpDir("api_space");
+}
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 142da6fb..e2374258 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -33,6 +33,7 @@
#include "src/buildtool/execution_api/git/git_api.hpp"
#include "src/buildtool/execution_api/local/context.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
class LocalApi final : public IExecutionApi {
public:
@@ -105,6 +106,8 @@ class LocalApi final : public IExecutionApi {
[[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final;
+ [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final;
+
private:
LocalContext const& local_context_;
std::optional<GitApi> const git_api_;
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS
index c6a8c4c4..7fccc2c6 100644
--- a/src/buildtool/execution_api/remote/TARGETS
+++ b/src/buildtool/execution_api/remote/TARGETS
@@ -85,6 +85,7 @@
, ["src/buildtool/execution_api/bazel_msg", "execution_config"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_engine/dag", "dag"]
+ , ["src/utils/cpp", "tmp_dir"]
]
, "stage": ["src", "buildtool", "execution_api", "remote"]
, "private-deps":
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
index b2f145b0..08fc8e99 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
@@ -144,7 +144,8 @@ BazelApi::BazelApi(std::string const& instance_name,
gsl::not_null<Auth const*> const& auth,
gsl::not_null<RetryConfig const*> const& retry_config,
ExecutionConfiguration const& exec_config,
- HashFunction hash_function) noexcept {
+ HashFunction hash_function,
+ TmpDir::Ptr temp_space) noexcept {
network_ = std::make_shared<BazelNetwork>(instance_name,
host,
port,
@@ -152,7 +153,7 @@ BazelApi::BazelApi(std::string const& instance_name,
retry_config,
exec_config,
hash_function,
- /*temp_space=*/nullptr);
+ std::move(temp_space));
}
// implement move constructor in cpp, where all members are complete types
@@ -605,3 +606,7 @@ auto BazelApi::CreateAction(
-> HashFunction::Type {
return network_->GetHashFunction().GetType();
}
+
+[[nodiscard]] auto BazelApi::GetTempSpace() const noexcept -> TmpDir::Ptr {
+ return network_->GetTempSpace();
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
index 633d9d85..df60204f 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
@@ -36,6 +36,7 @@
#include "src/buildtool/execution_api/common/execution_action.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
// forward declaration for actual implementations
class BazelNetwork;
@@ -49,7 +50,8 @@ class BazelApi final : public IExecutionApi {
gsl::not_null<Auth const*> const& auth,
gsl::not_null<RetryConfig const*> const& retry_config,
ExecutionConfiguration const& exec_config,
- HashFunction hash_function) noexcept;
+ HashFunction hash_function,
+ TmpDir::Ptr temp_space) noexcept;
BazelApi(BazelApi const&) = delete;
BazelApi(BazelApi&& other) noexcept;
auto operator=(BazelApi const&) -> BazelApi& = delete;
@@ -124,6 +126,8 @@ class BazelApi final : public IExecutionApi {
[[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final;
+ [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final;
+
private:
std::shared_ptr<BazelNetwork> network_;
diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS
index eadea902..5f562fac 100644
--- a/src/buildtool/execution_api/serve/TARGETS
+++ b/src/buildtool/execution_api/serve/TARGETS
@@ -32,6 +32,7 @@
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_api/local", "context"]
, ["src/buildtool/execution_engine/dag", "dag"]
+ , ["src/utils/cpp", "tmp_dir"]
]
, "stage": ["src", "buildtool", "execution_api", "serve"]
, "private-deps":
diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp
index 3394c270..9c58d73c 100644
--- a/src/buildtool/execution_api/serve/mr_local_api.cpp
+++ b/src/buildtool/execution_api/serve/mr_local_api.cpp
@@ -152,3 +152,7 @@ auto MRLocalApi::GetHashType() const noexcept -> HashFunction::Type {
return compat_local_api_ == nullptr ? native_local_api_->GetHashType()
: compat_local_api_->GetHashType();
}
+
+auto MRLocalApi::GetTempSpace() const noexcept -> TmpDir::Ptr {
+ return native_local_api_->GetTempSpace();
+}
diff --git a/src/buildtool/execution_api/serve/mr_local_api.hpp b/src/buildtool/execution_api/serve/mr_local_api.hpp
index 3ad19475..d9f323cf 100644
--- a/src/buildtool/execution_api/serve/mr_local_api.hpp
+++ b/src/buildtool/execution_api/serve/mr_local_api.hpp
@@ -31,6 +31,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_api/local/context.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
/// \brief Multi-repo-specific implementation of the abstract Execution API.
/// Handles interaction between a native storage and a remote, irrespective of
@@ -133,6 +134,8 @@ class MRLocalApi final : public IExecutionApi {
[[nodiscard]] auto GetHashType() const noexcept -> HashFunction::Type final;
+ [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final;
+
private:
// retain local context references to have direct access to storages
gsl::not_null<LocalContext const*> native_context_;
diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS
index 049efe44..2ac64635 100644
--- a/src/buildtool/execution_engine/executor/TARGETS
+++ b/src/buildtool/execution_engine/executor/TARGETS
@@ -38,6 +38,7 @@
, ["src/utils/cpp", "hex_string"]
, ["src/utils/cpp", "path_rebase"]
, ["src/utils/cpp", "prefix"]
+ , ["src/utils/cpp", "tmp_dir"]
]
, "stage": ["src", "buildtool", "execution_engine", "executor"]
}
diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp
index e030d2d6..12721b01 100644
--- a/src/buildtool/execution_engine/executor/executor.hpp
+++ b/src/buildtool/execution_engine/executor/executor.hpp
@@ -72,6 +72,7 @@
#include "src/utils/cpp/hex_string.hpp"
#include "src/utils/cpp/path_rebase.hpp"
#include "src/utils/cpp/prefix.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
/// \brief Implementations for executing actions and uploading artifacts.
class ExecutorImpl {
@@ -133,8 +134,10 @@ class ExecutorImpl {
}
// get the alternative endpoint
- auto alternative_api = GetAlternativeEndpoint(
- merged_properties, remote_context, api.GetHashType());
+ auto alternative_api = GetAlternativeEndpoint(merged_properties,
+ remote_context,
+ api.GetHashType(),
+ api.GetTempSpace());
if (alternative_api) {
if (not api.ParallelRetrieveToCas(
std::vector<Artifact::ObjectInfo>{Artifact::ObjectInfo{
@@ -758,7 +761,8 @@ class ExecutorImpl {
[[nodiscard]] static auto GetAlternativeEndpoint(
const ExecutionProperties& properties,
const gsl::not_null<RemoteContext const*>& remote_context,
- HashFunction::Type hash_type) -> std::unique_ptr<BazelApi> {
+ HashFunction::Type hash_type,
+ TmpDir::Ptr temp_space) -> std::unique_ptr<BazelApi> {
for (auto const& [pred, endpoint] :
remote_context->exec_config->dispatch) {
bool match = true;
@@ -781,7 +785,8 @@ class ExecutorImpl {
remote_context->auth,
remote_context->retry_config,
config,
- HashFunction{hash_type});
+ HashFunction{hash_type},
+ std::move(temp_space));
}
}
return nullptr;
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 7fafada8..0dd3a155 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -426,7 +426,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
&*auth_config,
&*retry_config,
config,
- hash_fct);
+ hash_fct,
+ mr_local_api->GetTempSpace());
}
bool const has_remote_api = remote_api != nullptr;
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 57ed0fd9..ee9781c9 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -234,7 +234,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&*auth_config,
&*retry_config,
config,
- hash_fct);
+ hash_fct,
+ mr_local_api->GetTempSpace());
}
bool const has_remote_api = remote_api != nullptr;
diff --git a/test/buildtool/execution_api/bazel/TARGETS b/test/buildtool/execution_api/bazel/TARGETS
index c2898d1c..acbee81f 100644
--- a/test/buildtool/execution_api/bazel/TARGETS
+++ b/test/buildtool/execution_api/bazel/TARGETS
@@ -140,11 +140,13 @@
, ["@", "src", "src/buildtool/execution_api/common", "common"]
, ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"]
, ["@", "src", "src/buildtool/execution_api/remote", "config"]
+ , ["@", "src", "src/buildtool/storage", "config"]
+ , ["@", "src", "src/utils/cpp", "tmp_dir"]
, ["buildtool/execution_api/common", "api_test"]
, ["utils", "catch-main-remote-execution"]
, ["utils", "test_auth_config"]
- , ["utils", "test_hash_function_type"]
, ["utils", "test_remote_config"]
+ , ["utils", "test_storage_config"]
]
, "stage": ["test", "buildtool", "execution_api", "bazel"]
}
diff --git a/test/buildtool/execution_api/bazel/bazel_api.test.cpp b/test/buildtool/execution_api/bazel/bazel_api.test.cpp
index 230ee2dc..8fb50006 100644
--- a/test/buildtool/execution_api/bazel/bazel_api.test.cpp
+++ b/test/buildtool/execution_api/bazel/bazel_api.test.cpp
@@ -15,6 +15,7 @@
#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
#include <optional>
+#include <utility>
#include "catch2/catch_test_macros.hpp"
#include "gsl/gsl"
@@ -24,8 +25,10 @@
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
+#include "src/buildtool/storage/config.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
#include "test/buildtool/execution_api/common/api_test.hpp"
-#include "test/utils/hermeticity/test_hash_function_type.hpp"
+#include "test/utils/hermeticity/test_storage_config.hpp"
#include "test/utils/remote_execution/test_auth_config.hpp"
#include "test/utils/remote_execution/test_remote_config.hpp"
@@ -36,10 +39,12 @@ class FactoryApi final {
explicit FactoryApi(
gsl::not_null<ServerAddress const*> const& server_address,
gsl::not_null<Auth const*> const& auth,
- HashFunction hash_function) noexcept
+ HashFunction hash_function,
+ TmpDir::Ptr temp_space) noexcept
: address_{*server_address},
auth_{*auth},
- hash_function_{hash_function} {}
+ hash_function_{hash_function},
+ temp_space_{std::move(temp_space)} {}
[[nodiscard]] auto operator()() const -> IExecutionApi::Ptr {
static RetryConfig retry_config{}; // default retry config
@@ -49,20 +54,22 @@ class FactoryApi final {
&auth_,
&retry_config,
{},
- hash_function_}};
+ hash_function_,
+ temp_space_}};
}
private:
ServerAddress const& address_;
Auth const& auth_;
HashFunction const hash_function_;
+ TmpDir::Ptr temp_space_;
};
} // namespace
TEST_CASE("BazelAPI: No input, no output", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -70,13 +77,16 @@ TEST_CASE("BazelAPI: No input, no output", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestNoInputNoOutput(api_factory, remote_config->platform_properties);
}
TEST_CASE("BazelAPI: No input, create output", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -84,13 +94,16 @@ TEST_CASE("BazelAPI: No input, create output", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestNoInputCreateOutput(api_factory, remote_config->platform_properties);
}
TEST_CASE("BazelAPI: One input copied to output", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -98,13 +111,16 @@ TEST_CASE("BazelAPI: One input copied to output", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestOneInputCopiedToOutput(api_factory, remote_config->platform_properties);
}
TEST_CASE("BazelAPI: Non-zero exit code, create output", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -112,14 +128,17 @@ TEST_CASE("BazelAPI: Non-zero exit code, create output", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestNonZeroExitCodeCreateOutput(api_factory,
remote_config->platform_properties);
}
TEST_CASE("BazelAPI: Retrieve two identical trees to path", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -127,15 +146,18 @@ TEST_CASE("BazelAPI: Retrieve two identical trees to path", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestRetrieveTwoIdenticalTreesToPath(
api_factory, remote_config->platform_properties, "two_trees");
}
TEST_CASE("BazelAPI: Retrieve file and symlink with same content to path",
"[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -143,14 +165,17 @@ TEST_CASE("BazelAPI: Retrieve file and symlink with same content to path",
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestRetrieveFileAndSymlinkWithSameContentToPath(
api_factory, remote_config->platform_properties, "file_and_symlink");
}
TEST_CASE("BazelAPI: Retrieve mixed blobs and trees", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -158,14 +183,17 @@ TEST_CASE("BazelAPI: Retrieve mixed blobs and trees", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestRetrieveMixedBlobsAndTrees(
api_factory, remote_config->platform_properties, "blobs_and_trees");
}
TEST_CASE("BazelAPI: Create directory prior to execution", "[execution_api]") {
+ auto storage_config = TestStorageConfig::Create();
auto remote_config = TestRemoteConfig::ReadFromEnvironment();
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
REQUIRE(remote_config);
REQUIRE(remote_config->remote_address);
@@ -173,7 +201,10 @@ TEST_CASE("BazelAPI: Create directory prior to execution", "[execution_api]") {
REQUIRE(auth);
FactoryApi api_factory{
- &*remote_config->remote_address, &*auth, hash_function};
+ &*remote_config->remote_address,
+ &*auth,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space")};
TestCreateDirPriorToExecution(api_factory,
remote_config->platform_properties);
}
diff --git a/test/buildtool/execution_engine/executor/TARGETS b/test/buildtool/execution_engine/executor/TARGETS
index a27e8dac..184c9fe1 100644
--- a/test/buildtool/execution_engine/executor/TARGETS
+++ b/test/buildtool/execution_engine/executor/TARGETS
@@ -59,10 +59,13 @@
, ["@", "src", "src/buildtool/file_system", "object_type"]
, ["@", "src", "src/buildtool/logging", "logging"]
, ["@", "src", "src/buildtool/progress_reporting", "progress"]
+ , ["@", "src", "src/buildtool/storage", "config"]
, ["@", "src", "src/utils/cpp", "expected"]
+ , ["@", "src", "src/utils/cpp", "tmp_dir"]
, ["", "catch-main"]
, ["utils", "test_api_bundle"]
, ["utils", "test_hash_function_type"]
+ , ["utils", "test_storage_config"]
]
, "stage": ["test", "buildtool", "execution_engine", "executor"]
}
@@ -108,10 +111,11 @@
, ["@", "src", "src/buildtool/execution_api/remote", "bazel_api"]
, ["@", "src", "src/buildtool/execution_api/remote", "config"]
, ["@", "src", "src/buildtool/progress_reporting", "progress"]
+ , ["@", "src", "src/buildtool/storage", "config"]
, ["utils", "catch-main-remote-execution"]
, ["utils", "test_auth_config"]
- , ["utils", "test_hash_function_type"]
, ["utils", "test_remote_config"]
+ , ["utils", "test_storage_config"]
]
, "stage": ["test", "buildtool", "execution_engine", "executor"]
}
diff --git a/test/buildtool/execution_engine/executor/executor.test.cpp b/test/buildtool/execution_engine/executor/executor.test.cpp
index a9f91ade..93dd10b6 100644
--- a/test/buildtool/execution_engine/executor/executor.test.cpp
+++ b/test/buildtool/execution_engine/executor/executor.test.cpp
@@ -57,9 +57,12 @@
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/buildtool/progress_reporting/progress.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "src/utils/cpp/expected.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
#include "test/utils/executor/test_api_bundle.hpp"
#include "test/utils/hermeticity/test_hash_function_type.hpp"
+#include "test/utils/hermeticity/test_storage_config.hpp"
/// \brief Mockup API test config.
struct TestApiConfig {
@@ -190,8 +193,11 @@ class TestAction : public IExecutionAction {
class TestApi : public IExecutionApi {
public:
explicit TestApi(TestApiConfig config,
- HashFunction::Type hash_type) noexcept
- : config_{std::move(config)}, hash_type_{hash_type} {}
+ HashFunction::Type hash_type,
+ TmpDir::Ptr temp_space) noexcept
+ : config_{std::move(config)},
+ hash_type_{hash_type},
+ temp_space_{std::move(temp_space)} {}
[[nodiscard]] auto CreateAction(
ArtifactDigest const& /*unused*/,
@@ -284,9 +290,14 @@ class TestApi : public IExecutionApi {
return hash_type_;
}
+ [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr final {
+ return temp_space_;
+ }
+
private:
TestApiConfig config_{};
HashFunction::Type hash_type_;
+ TmpDir::Ptr temp_space_;
};
[[nodiscard]] auto SetupConfig(std::filesystem::path const& ws)
@@ -334,12 +345,13 @@ class TestApi : public IExecutionApi {
}
TEST_CASE("Executor: Process artifact", "[executor]") {
+ auto const storage_config = TestStorageConfig::Create();
std::filesystem::path workspace_path{
"test/buildtool/execution_engine/executor"};
DependencyGraph g;
auto [config, repo_config] = CreateTest(&g, workspace_path);
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ HashFunction const hash_function = storage_config.Get().hash_function;
auto const local_cpp_id =
ArtifactDescription::CreateLocal("local.cpp", "").Id();
@@ -356,7 +368,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") {
.exec_config = &remote_config};
SECTION("Processing succeeds for valid config") {
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -375,7 +390,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") {
SECTION("Processing fails if uploading local artifact failed") {
config.artifacts[NamedDigest("local.cpp").hash()].uploads = false;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -394,7 +412,10 @@ TEST_CASE("Executor: Process artifact", "[executor]") {
SECTION("Processing fails if known artifact is not available") {
config.artifacts[NamedDigest("known.cpp").hash()].available = false;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -412,13 +433,14 @@ TEST_CASE("Executor: Process artifact", "[executor]") {
}
TEST_CASE("Executor: Process action", "[executor]") {
+ auto const storage_config = TestStorageConfig::Create();
std::filesystem::path workspace_path{
"test/buildtool/execution_engine/executor"};
DependencyGraph g;
auto [config, repo_config] = CreateTest(&g, workspace_path);
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ HashFunction const hash_function = storage_config.Get().hash_function;
auto const local_cpp_id =
ArtifactDescription::CreateLocal("local.cpp", "").Id();
@@ -442,7 +464,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
.exec_config = &remote_config};
SECTION("Processing succeeds for valid config") {
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -464,7 +489,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
SECTION("Processing succeeds even if result was is not cached") {
config.response.cached = false;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -486,7 +514,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
SECTION("Processing succeeds even if output is not available in CAS") {
config.artifacts[NamedDigest("output2.exe").hash()].available = false;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -511,7 +542,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
SECTION("Processing fails if execution failed") {
config.execution.failed = true;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -533,7 +567,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
SECTION("Processing fails if exit code is non-zero") {
config.response.exit_code = 1;
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
@@ -558,7 +595,10 @@ TEST_CASE("Executor: Process action", "[executor]") {
SECTION("Processing fails if any output is missing") {
config.execution.outputs = {"output1.exe" /*, "output2.exe"*/};
- auto api = std::make_shared<TestApi>(config, hash_function.GetType());
+ auto api = std::make_shared<TestApi>(
+ config,
+ hash_function.GetType(),
+ storage_config.Get().CreateTypedTmpDir("temp_space"));
Statistics stats{};
Progress progress{};
auto const apis = CreateTestApiBundle(api);
diff --git a/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp b/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp
index a809d04f..c92f3aef 100644
--- a/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp
+++ b/test/buildtool/execution_engine/executor/executor_api_remote_bazel.test.cpp
@@ -21,13 +21,13 @@
#include "src/buildtool/common/remote/retry_config.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/common/statistics.hpp"
-#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/bazel_msg/execution_config.hpp"
#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/progress_reporting/progress.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "test/buildtool/execution_engine/executor/executor_api.test.hpp"
-#include "test/utils/hermeticity/test_hash_function_type.hpp"
+#include "test/utils/hermeticity/test_storage_config.hpp"
#include "test/utils/remote_execution/test_auth_config.hpp"
#include "test/utils/remote_execution/test_remote_config.hpp"
@@ -44,16 +44,18 @@ TEST_CASE("Executor<BazelApi>: Upload blob", "[executor]") {
RetryConfig retry_config{}; // default retry config
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ auto storage_config = TestStorageConfig::Create();
TestBlobUpload(&repo_config, [&] {
- return std::make_shared<BazelApi>("remote-execution",
- remote_config->remote_address->host,
- remote_config->remote_address->port,
- &*auth_config,
- &retry_config,
- config,
- hash_function);
+ return std::make_shared<BazelApi>(
+ "remote-execution",
+ remote_config->remote_address->host,
+ remote_config->remote_address->port,
+ &*auth_config,
+ &retry_config,
+ config,
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space"));
});
}
@@ -73,7 +75,7 @@ TEST_CASE("Executor<BazelApi>: Compile hello world", "[executor]") {
RetryConfig retry_config{}; // default retry config
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ auto storage_config = TestStorageConfig::Create();
TestHelloWorldCompilation(
&repo_config,
@@ -87,7 +89,8 @@ TEST_CASE("Executor<BazelApi>: Compile hello world", "[executor]") {
&*auth_config,
&retry_config,
config,
- hash_function);
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space"));
},
&*auth_config,
false /* not hermetic */);
@@ -109,7 +112,7 @@ TEST_CASE("Executor<BazelApi>: Compile greeter", "[executor]") {
RetryConfig retry_config{}; // default retry config
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ auto storage_config = TestStorageConfig::Create();
TestGreeterCompilation(
&repo_config,
@@ -123,7 +126,8 @@ TEST_CASE("Executor<BazelApi>: Compile greeter", "[executor]") {
&*auth_config,
&retry_config,
config,
- hash_function);
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space"));
},
&*auth_config,
false /* not hermetic */);
@@ -145,7 +149,7 @@ TEST_CASE("Executor<BazelApi>: Upload and download trees", "[executor]") {
RetryConfig retry_config{}; // default retry config
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ auto storage_config = TestStorageConfig::Create();
TestUploadAndDownloadTrees(
&repo_config,
@@ -159,7 +163,8 @@ TEST_CASE("Executor<BazelApi>: Upload and download trees", "[executor]") {
&*auth_config,
&retry_config,
config,
- hash_function);
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space"));
},
&*auth_config,
false /* not hermetic */);
@@ -181,7 +186,7 @@ TEST_CASE("Executor<BazelApi>: Retrieve output directories", "[executor]") {
RetryConfig retry_config{}; // default retry config
- HashFunction const hash_function{TestHashType::ReadFromEnvironment()};
+ auto storage_config = TestStorageConfig::Create();
TestRetrieveOutputDirectories(
&repo_config,
@@ -195,7 +200,8 @@ TEST_CASE("Executor<BazelApi>: Retrieve output directories", "[executor]") {
&*auth_config,
&retry_config,
config,
- hash_function);
+ storage_config.Get().hash_function,
+ storage_config.Get().CreateTypedTmpDir("test_space"));
},
&*auth_config,
false /* not hermetic */);