diff options
| -rw-r--r-- | src/buildtool/storage/garbage_collector.cpp | 40 | ||||
| -rw-r--r-- | src/buildtool/storage/garbage_collector.hpp | 14 | ||||
| -rw-r--r-- | test/buildtool/storage/large_object_cas.test.cpp | 24 |
3 files changed, 70 insertions, 8 deletions
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp index 1ac1bf93..5014c998 100644 --- a/src/buildtool/storage/garbage_collector.cpp +++ b/src/buildtool/storage/garbage_collector.cpp @@ -70,7 +70,14 @@ auto GarbageCollector::LockFilePath( auto GarbageCollector::TriggerGarbageCollection( StorageConfig const& storage_config, - bool no_rotation) noexcept -> bool { + bool no_rotation, + bool gc_all) noexcept -> bool { + if (no_rotation and gc_all) { + Logger::Log(LogLevel::Error, + "no_rotation and gc_all cannot be set together"); + return false; + } + std::string const remove_me = "remove-me"; auto pid = CreateProcessUniqueId(); @@ -165,15 +172,38 @@ auto GarbageCollector::TriggerGarbageCollection( // Compactification must take place before rotating generations. // Otherwise, an interruption of the process during compactification // would lead to an invalid old generation. - if (not GarbageCollector::Compactify(storage_config, - MessageLimits::kMaxGrpcLength)) { + if (not gc_all and not GarbageCollector::Compactify( + storage_config, MessageLimits::kMaxGrpcLength)) { Logger::Log(LogLevel::Error, "Failed to compactify the youngest generation."); return false; } + if (gc_all) { + // Rename all cache generations starting from the oldest generation. + // If the process gets interrupted, the youngest cache stays + // available. + for (int i = static_cast<int>(storage_config.num_generations) - 1; + i >= 0; + --i) { + auto remove_me_dir = + storage_config.CacheRoot() / + fmt::format("{}{}", remove_me_prefix, remove_me_counter++); + to_remove.emplace_back(remove_me_dir); + + auto cache_root = storage_config.GenerationCacheRoot(i); + if (FileSystemManager::IsDirectory(cache_root) and + not FileSystemManager::Rename(cache_root, remove_me_dir)) { + Logger::Log(LogLevel::Error, + "Failed to rename {} to {}", + cache_root.string(), + remove_me_dir.string()); + return false; + } + } + } // Rotate generations unless told not to do so - if (not no_rotation) { + else if (not no_rotation) { auto remove_me_dir = storage_config.CacheRoot() / fmt::format("{}{}", remove_me_prefix, remove_me_counter++); @@ -216,6 +246,8 @@ auto GarbageCollector::TriggerGarbageCollection( auto GarbageCollector::Compactify(StorageConfig const& storage_config, size_t threshold) noexcept -> bool { + Logger::Log(LogLevel::Performance, "Compactification has been started"); + // Compactification must be done for both native and compatible storages. static constexpr std::array kHashes = {HashFunction::Type::GitSHA1, HashFunction::Type::PlainSHA256}; diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp index f4966c28..e6bc8739 100644 --- a/src/buildtool/storage/garbage_collector.hpp +++ b/src/buildtool/storage/garbage_collector.hpp @@ -26,12 +26,18 @@ /// Responsible for deleting oldest generation. class GarbageCollector { public: - /// \brief Trigger garbage collection; unless no_rotation is given, this - /// will include rotation of generations and deleting the oldest generation. - /// \returns true on success. + /// \brief Trigger garbage collection + /// \param storage_config Storage to collect garbage in + /// \param no_rotation Skip rotation of generation and perform only steps + /// that can be done without losing existing cache. Incompatible with + /// gc_all. + /// \param gc_all Remove all cache generations at once. Incompatible with + /// no_rotation. + /// \return true on success. [[nodiscard]] auto static TriggerGarbageCollection( StorageConfig const& storage_config, - bool no_rotation = false) noexcept -> bool; + bool no_rotation = false, + bool gc_all = false) noexcept -> bool; /// \brief Acquire shared lock to prevent garbage collection from running. /// \param storage_config Storage to be locked. diff --git a/test/buildtool/storage/large_object_cas.test.cpp b/test/buildtool/storage/large_object_cas.test.cpp index 9de22e9b..ce6fe7e5 100644 --- a/test/buildtool/storage/large_object_cas.test.cpp +++ b/test/buildtool/storage/large_object_cas.test.cpp @@ -526,6 +526,30 @@ TEST_CASE("LocalCAS: Split-Splice", "[storage]") { } } +TEST_CASE("Skip compactification", "[storage]") { + auto const config = TestStorageConfig::Create(); + auto const storage = Storage::Create(&config.Get()); + + auto const& cas = storage.CAS(); + + // Create a large object in CAS: + using LargeTestUtils::File; + auto object = File::Create(cas, File::kLargeId, File::kLargeSize); + REQUIRE(object.has_value()); + auto& [digest, path] = *object; + REQUIRE(cas.BlobPath(digest, /*is_executable=*/false).has_value()); + + // Trigger garbage collection with --all + REQUIRE(GarbageCollector::TriggerGarbageCollection( + config.Get(), /*no_rotation=*/false, /*gc_all=*/true)); + + // Check no generation remains: + for (std::size_t i = 0; i < config.Get().num_generations; ++i) { + CHECK_FALSE( + FileSystemManager::Exists(config.Get().GenerationCacheRoot(i))); + } +} + // Test uplinking of nested large objects: // A large tree depends on a number of nested objects: // |