From b88adc43bb6ffe914ca9303e3001624e36fd64fa Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Tue, 2 Apr 2024 18:37:16 +0200 Subject: Compactification: Split large entries. During garbage collection split and remove from the storage every entry that is larger than a threshold. --- src/buildtool/storage/garbage_collector.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/buildtool/storage/garbage_collector.cpp') diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp index 1b7dc09b..3d077daf 100644 --- a/src/buildtool/storage/garbage_collector.cpp +++ b/src/buildtool/storage/garbage_collector.cpp @@ -26,6 +26,7 @@ #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" @@ -242,7 +243,7 @@ auto GarbageCollector::TriggerGarbageCollection(bool no_rotation) noexcept // Compactification must take place before rotating generations. // Otherwise, an interruption of the process during compactification // would lead to an invalid old generation. - if (not GarbageCollector::Compactify()) { + if (not GarbageCollector::Compactify(kMaxBatchTransferSize)) { Logger::Log(LogLevel::Error, "Failed to compactify the youngest generation."); return false; @@ -290,7 +291,7 @@ auto GarbageCollector::TriggerGarbageCollection(bool no_rotation) noexcept return success; } -auto GarbageCollector::Compactify() noexcept -> bool { +auto GarbageCollector::Compactify(size_t threshold) noexcept -> bool { const bool mode = Compatibility::IsCompatible(); // Return to the initial compatibility mode once done: @@ -299,12 +300,13 @@ auto GarbageCollector::Compactify() noexcept -> bool { }); // Compactification must be done for both native and compatible storages. - auto compactify = [](bool compatible) -> bool { + auto compactify = [threshold](bool compatible) -> bool { auto const storage = ::Generation(StorageConfig::GenerationCacheDir(0, compatible)); Compatibility::SetCompatible(compatible); - return Compactifier::RemoveSpliced(storage.CAS()); + return Compactifier::RemoveSpliced(storage.CAS()) and + Compactifier::SplitLarge(storage.CAS(), threshold); }; return compactify(mode) and compactify(not mode); } -- cgit v1.2.3