diff options
10 files changed, 70 insertions, 39 deletions
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index e275dead..c6a8c4c4 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -38,6 +38,7 @@ , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "incremental_reader"] + , ["src/utils/cpp", "tmp_dir"] ] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index eb140b3c..b2f145b0 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -151,7 +151,8 @@ BazelApi::BazelApi(std::string const& instance_name, auth, retry_config, exec_config, - hash_function); + hash_function, + /*temp_space=*/nullptr); } // implement move constructor in cpp, where all members are complete types diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index a02b8d4c..6f07d322 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -220,10 +220,12 @@ BazelCasClient::BazelCasClient( Port port, gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, - gsl::not_null<BazelCapabilitiesClient const*> const& capabilities) noexcept + gsl::not_null<BazelCapabilitiesClient const*> const& capabilities, + TmpDir::Ptr temp_space) noexcept : stream_{std::make_unique<ByteStreamClient>(server, port, auth)}, retry_config_{*retry_config}, - capabilities_{*capabilities} { + capabilities_{*capabilities}, + temp_space_{std::move(temp_space)} { stub_ = bazel_re::ContentAddressableStorage::NewStub( CreateChannelWithCredentials(server, port, auth)); } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp index 7275a352..3f83d712 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp @@ -39,6 +39,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bytestream_client.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/tmp_dir.hpp" /// Implements client side for serivce defined here: /// https://github.com/bazelbuild/remote-apis/blob/e1fe21be4c9ae76269a5a63215bb3c72ed9ab3f0/build/bazel/remote/execution/v2/remote_execution.proto#L317 @@ -49,8 +50,8 @@ class BazelCasClient { Port port, gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, - gsl::not_null<BazelCapabilitiesClient const*> const& - capabilities) noexcept; + gsl::not_null<BazelCapabilitiesClient const*> const& capabilities, + TmpDir::Ptr temp_space) noexcept; /// \brief Find missing blobs /// \param[in] instance_name Name of the CAS instance @@ -146,10 +147,15 @@ class BazelCasClient { [[nodiscard]] auto GetMaxBatchTransferSize( std::string const& instance_name) const noexcept -> std::size_t; + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr { + return temp_space_; + } + private: std::unique_ptr<ByteStreamClient> stream_; RetryConfig const& retry_config_; BazelCapabilitiesClient const& capabilities_; + TmpDir::Ptr temp_space_; std::unique_ptr<bazel_re::ContentAddressableStorage::Stub> stub_; Logger logger_{"RemoteCasClient"}; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 1a8e6d8f..f8b31871 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -29,7 +29,8 @@ BazelNetwork::BazelNetwork( gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, ExecutionConfiguration const& exec_config, - HashFunction hash_function) noexcept + HashFunction hash_function, + TmpDir::Ptr temp_space) noexcept : instance_name_{std::move(instance_name)}, capabilities_{std::make_unique<BazelCapabilitiesClient>(host, port, @@ -39,7 +40,8 @@ BazelNetwork::BazelNetwork( port, auth, retry_config, - capabilities_.get())}, + capabilities_.get(), + std::move(temp_space))}, ac_{std::make_unique<BazelAcClient>(host, port, auth, retry_config)}, exec_{std::make_unique<BazelExecutionClient>(host, port, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index 3af4bccd..4dbf1467 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -35,6 +35,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" +#include "src/utils/cpp/tmp_dir.hpp" /// \brief Contains all network clients and is responsible for all network IO. class BazelNetwork { @@ -45,7 +46,8 @@ class BazelNetwork { gsl::not_null<Auth const*> const& auth, gsl::not_null<RetryConfig const*> const& retry_config, ExecutionConfiguration const& exec_config, - HashFunction hash_function) noexcept; + HashFunction hash_function, + TmpDir::Ptr temp_space) noexcept; /// \brief Check if digest exists in CAS /// \param[in] digest The digest to look up @@ -86,6 +88,10 @@ class BazelNetwork { return hash_function_; } + [[nodiscard]] auto GetTempSpace() const noexcept -> TmpDir::Ptr { + return cas_->GetTempSpace(); + } + [[nodiscard]] auto GetCachedActionResult( bazel_re::Digest const& action, std::vector<std::string> const& output_files) const noexcept diff --git a/test/buildtool/execution_api/bazel/TARGETS b/test/buildtool/execution_api/bazel/TARGETS index 230c6f48..c2898d1c 100644 --- a/test/buildtool/execution_api/bazel/TARGETS +++ b/test/buildtool/execution_api/bazel/TARGETS @@ -13,11 +13,12 @@ , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] + , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "test_auth_config"] - , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] + , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } @@ -88,12 +89,13 @@ , ["@", "src", "src/buildtool/execution_api/remote", "bazel_network"] , ["@", "src", "src/buildtool/execution_api/remote", "config"] , ["@", "src", "src/buildtool/file_system", "object_type"] + , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/utils/cpp", "expected"] , ["utils", "catch-main-remote-execution"] , ["utils", "execution_bazel"] , ["utils", "test_auth_config"] - , ["utils", "test_hash_function_type"] , ["utils", "test_remote_config"] + , ["utils", "test_storage_config"] ] , "stage": ["test", "buildtool", "execution_api", "bazel"] } diff --git a/test/buildtool/execution_api/bazel/bazel_cas_client.test.cpp b/test/buildtool/execution_api/bazel/bazel_cas_client.test.cpp index 814de567..4b129920 100644 --- a/test/buildtool/execution_api/bazel/bazel_cas_client.test.cpp +++ b/test/buildtool/execution_api/bazel/bazel_cas_client.test.cpp @@ -30,8 +30,9 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_capabilities_client.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" -#include "test/utils/hermeticity/test_hash_function_type.hpp" +#include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" @@ -39,6 +40,8 @@ TEST_CASE("Bazel internals: CAS Client", "[execution_api]") { std::string instance_name{"remote-execution"}; std::string content("test"); + auto const storage_config = TestStorageConfig::Create(); + auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); @@ -51,18 +54,18 @@ TEST_CASE("Bazel internals: CAS Client", "[execution_api]") { remote_config->remote_address->port, &*auth_config, &retry_config); - BazelCasClient cas_client(remote_config->remote_address->host, - remote_config->remote_address->port, - &*auth_config, - &retry_config, - &capabilities); + BazelCasClient cas_client( + remote_config->remote_address->host, + remote_config->remote_address->port, + &*auth_config, + &retry_config, + &capabilities, + storage_config.Get().CreateTypedTmpDir("test_space")); SECTION("Valid digest and blob") { - // digest of "test" - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; // Valid blob - auto const blob = - ArtifactBlob::FromMemory(hash_function, ObjectType::File, content); + auto const blob = ArtifactBlob::FromMemory( + storage_config.Get().hash_function, ObjectType::File, content); REQUIRE(blob.has_value()); // Search blob via digest diff --git a/test/buildtool/execution_api/bazel/bazel_network.test.cpp b/test/buildtool/execution_api/bazel/bazel_network.test.cpp index b902473b..6b8f7940 100644 --- a/test/buildtool/execution_api/bazel/bazel_network.test.cpp +++ b/test/buildtool/execution_api/bazel/bazel_network.test.cpp @@ -36,8 +36,9 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/expected.hpp" -#include "test/utils/hermeticity/test_hash_function_type.hpp" +#include "test/utils/hermeticity/test_storage_config.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" #include "test/utils/remote_execution/test_remote_config.hpp" @@ -46,6 +47,8 @@ constexpr std::size_t kLargeSize = GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH + 1; TEST_CASE("Bazel network: write/read blobs", "[execution_api]") { std::string instance_name{"remote-execution"}; + auto const storage_config = TestStorageConfig::Create(); + auto auth_config = TestAuthConfig::ReadFromEnvironment(); REQUIRE(auth_config); @@ -55,15 +58,17 @@ TEST_CASE("Bazel network: write/read blobs", "[execution_api]") { RetryConfig retry_config{}; // default retry config - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + HashFunction const hash_function = storage_config.Get().hash_function; - auto network = BazelNetwork{instance_name, - remote_config->remote_address->host, - remote_config->remote_address->port, - &*auth_config, - &retry_config, - {}, - hash_function}; + auto network = + BazelNetwork{instance_name, + remote_config->remote_address->host, + remote_config->remote_address->port, + &*auth_config, + &retry_config, + {}, + hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; std::string content_foo("foo"); std::string content_bar("bar"); @@ -107,7 +112,8 @@ TEST_CASE("Bazel network: write/read blobs", "[execution_api]") { } TEST_CASE("Bazel network: read blobs with unknown size", "[execution_api]") { - HashFunction const hash_function{TestHashType::ReadFromEnvironment()}; + auto const storage_config = TestStorageConfig::Create(); + HashFunction const hash_function = storage_config.Get().hash_function; if (not ProtocolTraits::IsNative(hash_function.GetType())) { // only supported in native mode return; @@ -124,13 +130,15 @@ TEST_CASE("Bazel network: read blobs with unknown size", "[execution_api]") { RetryConfig retry_config{}; // default retry config - auto network = BazelNetwork{instance_name, - remote_config->remote_address->host, - remote_config->remote_address->port, - &*auth_config, - &retry_config, - {}, - hash_function}; + auto network = + BazelNetwork{instance_name, + remote_config->remote_address->host, + remote_config->remote_address->port, + &*auth_config, + &retry_config, + {}, + hash_function, + storage_config.Get().CreateTypedTmpDir("test_space")}; std::string content_foo("foo"); std::string content_bar(kLargeSize, 'x'); // single larger blob diff --git a/test/utils/remote_execution/bazel_action_creator.hpp b/test/utils/remote_execution/bazel_action_creator.hpp index 69a5b172..b340dccb 100644 --- a/test/utils/remote_execution/bazel_action_creator.hpp +++ b/test/utils/remote_execution/bazel_action_creator.hpp @@ -121,8 +121,8 @@ remote_config->remote_address->port, &*auth_config, &retry_config, - &capabilities); - + &capabilities, + /*temp_space=*/nullptr); if (cas_client.BatchUpdateBlobs(instance_name, blobs) == blobs.size()) { return std::make_unique<bazel_re::Digest>( ArtifactDigestFactory::ToBazel(action_blob->GetDigest())); |