diff options
| -rw-r--r-- | src/buildtool/main/build_utils.cpp | 9 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/target.cpp | 13 | ||||
| -rw-r--r-- | src/buildtool/storage/garbage_collector.cpp | 12 | ||||
| -rw-r--r-- | src/buildtool/storage/garbage_collector.hpp | 5 | ||||
| -rw-r--r-- | src/buildtool/storage/target_cache.hpp | 23 | ||||
| -rw-r--r-- | src/buildtool/storage/target_cache.tpp | 3 |
6 files changed, 34 insertions, 31 deletions
diff --git a/src/buildtool/main/build_utils.cpp b/src/buildtool/main/build_utils.cpp index 492e80d6..5b8d142d 100644 --- a/src/buildtool/main/build_utils.cpp +++ b/src/buildtool/main/build_utils.cpp @@ -83,15 +83,6 @@ auto CreateTargetCacheWriterMap( TargetCacheKey tc_key{key}; // check if entry actually needs storing if (not cache_targets.contains(tc_key)) { - // sanity check: if not in the map, then it must be in cache - if (not tc.Read(tc_key)) { - (*logger)( - fmt::format("Target-cache key {} is neither stored " - "nor marked for storing", - key.ToString()), - /*fatal=*/true); - return; - } // entry already in target-cache, so nothing to be done (*setter)(nullptr); return; diff --git a/src/buildtool/serve_api/serve_service/target.cpp b/src/buildtool/serve_api/serve_service/target.cpp index 67102c49..c5b9aa60 100644 --- a/src/buildtool/serve_api/serve_service/target.cpp +++ b/src/buildtool/serve_api/serve_service/target.cpp @@ -180,13 +180,14 @@ auto TargetService::ServeTarget( } // get a target cache instance with the correct computed shard - auto const& tc = Storage::Instance().TargetCache().WithShard( - untagged_execution_backend.hash()); + auto shard = address ? std::make_optional(untagged_execution_backend.hash()) + : std::nullopt; + auto const& tc = Storage::Instance().TargetCache().WithShard(shard); auto const& tc_key = TargetCacheKey{{target_cache_key_digest, ObjectType::File}}; // check if target-level cache entry has already been computed - if (auto target_entry = tc->Read(tc_key); target_entry) { + if (auto target_entry = tc.Read(tc_key); target_entry) { // make sure all artifacts referenced in the target cache value are in // the remote cas @@ -413,7 +414,7 @@ auto TargetService::ServeTarget( auto result = AnalyseTarget(configured_target, &result_map, &repository_config, - *tc, + tc, &stats, RemoteServeConfig::Jobs(), std::nullopt /*request_action_input*/, @@ -528,7 +529,7 @@ auto TargetService::ServeTarget( traverser.GetLocalApi(), traverser.GetRemoteApi(), RemoteServeConfig::TCStrategy(), - *tc, + tc, &logger, true /*strict_logging*/); @@ -565,7 +566,7 @@ auto TargetService::ServeTarget( // now that the target cache key is in, make sure remote CAS has all // required entries - if (auto target_entry = tc->Read(tc_key); target_entry) { + if (auto target_entry = tc.Read(tc_key); target_entry) { // make sure all artifacts referenced in the target cache value are in // the remote cas std::vector<Artifact::ObjectInfo> tc_artifacts; diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp index 82dbc8d2..7ed8d572 100644 --- a/src/buildtool/storage/garbage_collector.cpp +++ b/src/buildtool/storage/garbage_collector.cpp @@ -97,12 +97,16 @@ auto GarbageCollector::GlobalUplinkActionCacheEntry( } auto GarbageCollector::GlobalUplinkTargetCacheEntry( - TargetCacheKey const& key) noexcept -> bool { + TargetCacheKey const& key, + std::optional<std::string> const& shard) noexcept -> bool { // Try to find target-cache entry in all generations. - auto const& latest_tc = Storage::Generation(0).TargetCache(); + auto const& latest_tc = + Storage::Generation(0).TargetCache().WithShard(shard); for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { - if (Storage::Generation(i).TargetCache().LocalUplinkEntry(latest_tc, - key)) { + if (Storage::Generation(i) + .TargetCache() + .WithShard(shard) + .LocalUplinkEntry(latest_tc, key)) { return true; } } diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp index c10eebc2..c4ceaf8a 100644 --- a/src/buildtool/storage/garbage_collector.hpp +++ b/src/buildtool/storage/garbage_collector.hpp @@ -60,9 +60,12 @@ class GarbageCollector { /// \brief Uplink entry from target cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. /// \param key Target cache key to uplink entry for. + /// \param shard Optional explicit shard, if the default is not intended. /// \returns true if cache entry was found and successfully uplinked. [[nodiscard]] auto static GlobalUplinkTargetCacheEntry( - TargetCacheKey const& key) noexcept -> bool; + TargetCacheKey const& key, + std::optional<std::string> const& shard = std::nullopt) noexcept + -> bool; /// \brief Trigger garbage collection; unless no_rotation is given, this /// will include rotation of generations and deleting the oldest generation. diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp index 8c92a0a0..e54c2e42 100644 --- a/src/buildtool/storage/target_cache.hpp +++ b/src/buildtool/storage/target_cache.hpp @@ -50,11 +50,12 @@ class TargetCache { TargetCache(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas, std::filesystem::path const& store_path, - bool compute_shard = true) + std::optional<std::string> const& explicit_shard = std::nullopt) : cas_{std::move(cas)}, - file_store_{compute_shard ? store_path / ComputeShard() - : store_path} { - if (kDoGlobalUplink && compute_shard) { + file_store_{explicit_shard ? store_path / *explicit_shard + : store_path / ComputeShard()}, + explicit_shard_{explicit_shard} { + if (kDoGlobalUplink && not explicit_shard) { // write backend description (shard) to CAS [[maybe_unused]] auto id = cas_->StoreBlob(StorageConfig::ExecutionBackendDescription()); @@ -68,12 +69,12 @@ class TargetCache { /// according to the client's request and not following the server /// configuration. It is caller's responsibility to check that \p shard is a /// valid path. - [[nodiscard]] auto WithShard(const std::string& shard) const - -> std::unique_ptr<TargetCache> { - return std::make_unique<TargetCache<kDoGlobalUplink>>( - cas_, - file_store_.StorageRoot().parent_path() / shard, - /*compute_shard=*/false); + [[nodiscard]] auto WithShard(const std::optional<std::string>& shard) const + -> TargetCache { + return shard + ? TargetCache<kDoGlobalUplink>( + cas_, file_store_.StorageRoot().parent_path(), *shard) + : *this; } TargetCache(TargetCache const&) = default; @@ -94,6 +95,7 @@ class TargetCache { /// \brief Read existing entry and object info from the target cache. /// \param key The target-cache key to read the entry from. + /// \param shard Optional explicit shard, if the default is not intended. /// \returns Pair of cache entry and its object info on success or nullopt. [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; @@ -122,6 +124,7 @@ class TargetCache { kStoreMode, /*kSetEpochTime=*/false> file_store_; + std::optional<std::string> explicit_shard_{std::nullopt}; template <bool kIsLocalGeneration = not kDoGlobalUplink> requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( diff --git a/src/buildtool/storage/target_cache.tpp b/src/buildtool/storage/target_cache.tpp index ed1991e8..7be62c35 100644 --- a/src/buildtool/storage/target_cache.tpp +++ b/src/buildtool/storage/target_cache.tpp @@ -49,7 +49,8 @@ auto TargetCache<kDoGlobalUplink>::Read( if constexpr (kDoGlobalUplink) { // Uplink any existing target cache entry in storage generations [[maybe_unused]] auto found = - GarbageCollector::GlobalUplinkTargetCacheEntry(key); + GarbageCollector::GlobalUplinkTargetCacheEntry(key, + explicit_shard_); } auto const entry = |