From 24c900225902337feee7a8cc4399fdffdecd945f Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 9 Jul 2024 17:44:45 +0200 Subject: TargetCache: Use StorageConfig instance for sharding Instead of computing the shard based on the RemoteExecutionConfig singleton, use the already computed hash stored in the passed StorageConfig instance, which now needs to be set up separately if bootstrapping in order to avoid unwanted includes. Storing the backend description to CAS and corresponding audit checks now take place in main. --- src/buildtool/main/TARGETS | 2 ++ src/buildtool/main/main.cpp | 25 +++++++++++++++++++++++++ src/buildtool/storage/TARGETS | 1 - src/buildtool/storage/target_cache.hpp | 20 +++----------------- 4 files changed, 30 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index c4c7f122..c7b88655 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -33,8 +33,10 @@ , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/serve_service", "serve_server_implementation"] , ["src/buildtool/storage", "file_chunker"] + , ["src/buildtool/storage", "backend_description"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/execution_api/common", "api_bundle"] + , ["src/utils/cpp", "gsl"] , "common" , "cli" , "version" diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 716545a5..a82a6db8 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -80,7 +80,9 @@ #include "src/buildtool/progress_reporting/progress_reporter.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/serve_service/serve_server_implementation.hpp" +#include "src/buildtool/storage/backend_description.hpp" #include "src/buildtool/storage/garbage_collector.hpp" +#include "src/utils/cpp/gsl.hpp" #endif // BOOTSTRAP_BUILD_TOOL namespace { @@ -291,6 +293,23 @@ void SetupFileChunker() { FileChunker::Initialize(); } +/// \brief Write backend description (which determines the target cache shard) +/// to CAS. +void StoreTargetCacheShard(StorageConfig const& storage_config, + Storage const& storage) noexcept { + auto backend_description = + DescribeBackend(RemoteExecutionConfig::RemoteAddress(), + RemoteExecutionConfig::PlatformProperties(), + RemoteExecutionConfig::DispatchList()); + if (not backend_description) { + Logger::Log(LogLevel::Error, backend_description.error()); + std::exit(kExitFailure); + } + [[maybe_unused]] auto id = storage.CAS().StoreBlob(*backend_description); + EnsuresAudit(id and ArtifactDigest{*id}.hash() == + storage_config.backend_description_id); +} + #endif // BOOTSTRAP_BUILD_TOOL // returns path relative to `root`. @@ -847,6 +866,7 @@ auto main(int argc, char* argv[]) -> int { return kExitFailure; } auto const storage = Storage::Create(&*storage_config); + StoreTargetCacheShard(*storage_config, storage); SetupExecutionServiceConfig(arguments.service); ApiBundle const exec_apis{&*storage_config, @@ -879,6 +899,7 @@ auto main(int argc, char* argv[]) -> int { return kExitFailure; } auto const storage = Storage::Create(&*storage_config); + StoreTargetCacheShard(*storage_config, storage); ApiBundle const serve_apis{ &*storage_config, @@ -937,6 +958,10 @@ auto main(int argc, char* argv[]) -> int { } auto const storage = Storage::Create(&*storage_config); +#ifndef BOOTSTRAP_BUILD_TOOL + StoreTargetCacheShard(*storage_config, storage); +#endif // BOOTSTRAP_BUILD_TOOL + auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs : arguments.common.jobs; diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index 53982e2b..71e0de6a 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -68,7 +68,6 @@ , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/compatibility", "compatibility"] - , ["src/buildtool/execution_api/remote", "config"] ] , "stage": ["src", "buildtool", "storage"] , "private-deps": diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp index e5a64737..2f52e100 100644 --- a/src/buildtool/storage/target_cache.hpp +++ b/src/buildtool/storage/target_cache.hpp @@ -27,7 +27,6 @@ #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" @@ -36,7 +35,6 @@ #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/buildtool/storage/uplinker.hpp" -#include "src/utils/cpp/gsl.hpp" /// \brief The high-level target cache for storing export target's data. /// Supports global uplinking across all generations. The uplink is @@ -58,16 +56,10 @@ class TargetCache { GenerationConfig const& config, gsl::not_null const*> const& uplinker) : cas_{*cas}, - file_store_{config.target_cache / ComputeShard()}, + file_store_{config.target_cache / + config.storage_config->backend_description_id}, uplinker_{*uplinker}, - explicit_shard_{std::nullopt} { - if constexpr (kDoGlobalUplink) { - // write backend description (shard) to CAS - [[maybe_unused]] auto id = - cas_.StoreBlob(RemoteExecutionConfig::DescribeBackend()); - EnsuresAudit(id and ArtifactDigest{*id}.hash() == ComputeShard()); - } - } + explicit_shard_{std::nullopt} {} /// \brief Returns a new TargetCache backed by the same CAS, but the /// FileStorage uses the given \p shard. This is particularly useful for the @@ -152,12 +144,6 @@ class TargetCache { LocalGenerationTC const& latest, std::string const& key_digest) const noexcept -> bool; - [[nodiscard]] static auto ComputeShard() noexcept -> std::string { - return ArtifactDigest::Create( - RemoteExecutionConfig::DescribeBackend()) - .hash(); - } - [[nodiscard]] auto DownloadKnownArtifacts( TargetCacheEntry const& value, ArtifactDownloader const& downloader) const noexcept -> bool; -- cgit v1.2.3