summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/storage/garbage_collector.cpp161
-rw-r--r--src/buildtool/storage/garbage_collector.hpp6
2 files changed, 136 insertions, 31 deletions
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp
index bd156f0c..01d803dc 100644
--- a/src/buildtool/storage/garbage_collector.cpp
+++ b/src/buildtool/storage/garbage_collector.cpp
@@ -16,6 +16,7 @@
#include "src/buildtool/storage/garbage_collector.hpp"
+#include <filesystem>
#include <vector>
#include "nlohmann/json.hpp"
@@ -34,6 +35,27 @@
#include "src/buildtool/storage/target_cache_entry.hpp"
#include "src/utils/cpp/hex_string.hpp"
+namespace {
+
+auto RemoveDirs(const std::vector<std::filesystem::path>& directories) -> bool {
+ bool success = true;
+ for (auto const& d : directories) {
+ if (FileSystemManager::IsDirectory(d)) {
+ if (not FileSystemManager::RemoveDirectory(d,
+ /*recursively=*/true)) {
+ Logger::Log(LogLevel::Warning,
+ "Failed to remove directory {}",
+ d.string());
+ success = false; // We failed to remove all, but still try the
+ // others to clean up as much as possible
+ }
+ }
+ }
+ return success;
+}
+
+} // namespace
+
auto GarbageCollector::GlobalUplinkBlob(bazel_re::Digest const& digest,
bool is_executable) noexcept -> bool {
// Try to find blob in all generations.
@@ -99,56 +121,137 @@ auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path {
return StorageConfig::CacheRoot() / "gc.lock";
}
-auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool {
+auto GarbageCollector::TriggerGarbageCollection(bool no_rotation) noexcept
+ -> bool {
+ auto const kRemoveMe = std::string{"remove-me"};
+
auto pid = CreateProcessUniqueId();
if (not pid) {
return false;
}
- auto remove_me = std::string{"remove-me-"} + *pid;
- auto remove_me_dir = StorageConfig::CacheRoot() / remove_me;
- if (FileSystemManager::IsDirectory(remove_me_dir)) {
- if (not FileSystemManager::RemoveDirectory(remove_me_dir,
- /*recursively=*/true)) {
+ auto remove_me_prefix = kRemoveMe + *pid + std::string{"-"};
+ std::vector<std::filesystem::path> to_remove{};
+
+ // With a shared lock, we can remove all directories with the given prefix,
+ // as we own the process id.
+ {
+ auto lock = SharedLock();
+ if (not lock) {
+ Logger::Log(LogLevel::Error,
+ "Failed to get a shared lock the local build root");
+ return false;
+ }
+
+ for (auto const& entry :
+ std::filesystem::directory_iterator(StorageConfig::CacheRoot())) {
+ if (entry.path().filename().string().find(remove_me_prefix) == 0) {
+ to_remove.emplace_back(entry.path());
+ }
+ }
+ if (not RemoveDirs(to_remove)) {
Logger::Log(LogLevel::Error,
- "Failed to remove directory {}",
- remove_me_dir.string());
+ "Failed to clean up left-over directories under my "
+ "pid. Will not continue");
return false;
}
}
- { // Create scope for critical renaming section protected by advisory lock.
+
+ to_remove.clear();
+ int remove_me_counter{};
+
+ // after releasing the shared lock, wait to get an exclusive lock for doing
+ // the critical renamings
+ {
auto lock = ExclusiveLock();
if (not lock) {
Logger::Log(LogLevel::Error,
"Failed to exclusively lock the local build root");
return false;
}
- for (std::size_t i = StorageConfig::NumGenerations(); i > 0; --i) {
- auto cache_root = StorageConfig::GenerationCacheRoot(i - 1);
- if (FileSystemManager::IsDirectory(cache_root)) {
- auto new_cache_root =
- (i == StorageConfig::NumGenerations())
- ? remove_me_dir
- : StorageConfig::GenerationCacheRoot(i);
- if (not FileSystemManager::Rename(cache_root, new_cache_root)) {
- Logger::Log(LogLevel::Error,
- "Failed to rename {} to {}.",
- cache_root.string(),
- new_cache_root.string());
- return false;
+
+ // Frirst, while he have not yet created any to-remove directories, grab
+ // all existing remove-me directories; they're left overs, as the clean
+ // up of owned directories is done with a shared lock.
+ std::vector<std::filesystem::path> left_over{};
+ for (auto const& entry :
+ std::filesystem::directory_iterator(StorageConfig::CacheRoot())) {
+ if (entry.path().filename().string().find(kRemoveMe) == 0) {
+ left_over.emplace_back(entry.path());
+ }
+ }
+ for (auto const& d : left_over) {
+ auto new_name =
+ StorageConfig::CacheRoot() /
+ fmt::format("{}{}", remove_me_prefix, remove_me_counter++);
+ if (FileSystemManager::Rename(d, new_name)) {
+ to_remove.emplace_back(new_name);
+ }
+ else {
+ Logger::Log(LogLevel::Warning,
+ "Failed to rename {} to {}",
+ d.string(),
+ new_name.string());
+ }
+ }
+
+ // Now that the have to exclusive lock, try to move out ephemeral data;
+ // as it is still under the generation regime, it is not a huge problem
+ // if that fails.
+ auto ephemeral = StorageConfig::EphemeralRoot();
+ if (FileSystemManager::IsDirectory(ephemeral)) {
+ auto remove_me_dir =
+ StorageConfig::CacheRoot() /
+ fmt::format("{}{}", remove_me_prefix, remove_me_counter++);
+ if (FileSystemManager::Rename(ephemeral, remove_me_dir)) {
+ to_remove.emplace_back(remove_me_dir);
+ }
+ else {
+ Logger::Log(LogLevel::Warning,
+ "Failed to rename {} to {}.",
+ ephemeral.string(),
+ remove_me_dir.string());
+ }
+ }
+ // Rotate generations unless told not to do so
+ if (not no_rotation) {
+ auto remove_me_dir =
+ StorageConfig::CacheRoot() /
+ fmt::format("{}{}", remove_me_prefix, remove_me_counter++);
+ to_remove.emplace_back(remove_me_dir);
+ for (std::size_t i = StorageConfig::NumGenerations(); i > 0; --i) {
+ auto cache_root = StorageConfig::GenerationCacheRoot(i - 1);
+ if (FileSystemManager::IsDirectory(cache_root)) {
+ auto new_cache_root =
+ (i == StorageConfig::NumGenerations())
+ ? remove_me_dir
+ : StorageConfig::GenerationCacheRoot(i);
+ if (not FileSystemManager::Rename(cache_root,
+ new_cache_root)) {
+ Logger::Log(LogLevel::Error,
+ "Failed to rename {} to {}.",
+ cache_root.string(),
+ new_cache_root.string());
+ return false;
+ }
}
}
}
}
- if (FileSystemManager::IsDirectory(remove_me_dir)) {
- if (not FileSystemManager::RemoveDirectory(remove_me_dir,
- /*recursively=*/true)) {
- Logger::Log(LogLevel::Warning,
- "Failed to remove directory {}",
- remove_me_dir.string());
+
+ // After releasing the exlusive lock, get a shared lock and remove what we
+ // have to remove
+ bool success{};
+ {
+ auto lock = SharedLock();
+ if (not lock) {
+ Logger::Log(LogLevel::Error,
+ "Failed to get a shared lock the local build root");
return false;
}
+ success = RemoveDirs(to_remove);
}
- return true;
+
+ return success;
}
#endif // BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp
index 76bbc054..b2f3a940 100644
--- a/src/buildtool/storage/garbage_collector.hpp
+++ b/src/buildtool/storage/garbage_collector.hpp
@@ -64,9 +64,11 @@ class GarbageCollector {
[[nodiscard]] auto static GlobalUplinkTargetCacheEntry(
TargetCacheKey const& key) noexcept -> bool;
- /// \brief Trigger deletion of oldest generation.
+ /// \brief Trigger gargabe collection; unless no_rotation is given, this
+ /// will include rotation of generations and deleting the oldest generation.
/// \returns true on success.
- [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool;
+ [[nodiscard]] auto static TriggerGarbageCollection(
+ bool no_rotation = false) noexcept -> bool;
/// \brief Acquire shared lock to prevent garbage collection from running.
/// \returns The acquired lock file on success or nullopt otherwise.
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 03:11:11 +0000