diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/buildtool/storage/TARGETS | 13 | ||||
-rw-r--r-- | src/buildtool/storage/repository_garbage_collector.cpp | 119 | ||||
-rw-r--r-- | src/buildtool/storage/repository_garbage_collector.hpp | 49 |
3 files changed, 181 insertions, 0 deletions
diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index 71e0de6a..1cd5fed9 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -18,6 +18,19 @@ ] , "stage": ["src", "buildtool", "storage"] } +, "repository_garbage_collector": + { "type": ["@", "rules", "CC", "library"] + , "name": ["repository_garbage_collector"] + , "hdrs": ["repository_garbage_collector.hpp"] + , "srcs": ["repository_garbage_collector.cpp"] + , "deps": ["config", ["src/utils/cpp", "file_locking"]] + , "private-deps": + [ ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/logging", "logging"] + ] + , "stage": ["src", "buildtool", "storage"] + } , "storage": { "type": ["@", "rules", "CC", "library"] , "name": ["storage"] diff --git a/src/buildtool/storage/repository_garbage_collector.cpp b/src/buildtool/storage/repository_garbage_collector.cpp new file mode 100644 index 00000000..f1025795 --- /dev/null +++ b/src/buildtool/storage/repository_garbage_collector.cpp @@ -0,0 +1,119 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/storage/repository_garbage_collector.hpp" + +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto RepositoryGarbageCollector::SharedLock( + StorageConfig const& storage_config) noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/true); +} + +auto RepositoryGarbageCollector::ExclusiveLock( + StorageConfig const& storage_config) noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(storage_config), /*is_shared=*/false); +} + +auto RepositoryGarbageCollector::LockFilePath( + StorageConfig const& storage_config) noexcept -> std::filesystem::path { + return storage_config.RepositoryRoot() / "gc.lock"; +} + +auto RepositoryGarbageCollector::TriggerGarbageCollection( + StorageConfig const& storage_config) noexcept -> bool { + auto const kRemoveMe = std::string{"remove-me"}; + + auto pid = CreateProcessUniqueId(); + if (not pid) { + return false; + } + auto remove_me = storage_config.RepositoryRoot() / (kRemoveMe + *pid); + + // With a shared lock, we can remove that directory, if it exists, + // as we own the process id. + { + auto lock = SharedLock(storage_config); + if (not lock) { + Logger::Log(LogLevel::Error, + "Failed to get a shared lock the for repository root"); + return false; + } + if (FileSystemManager::IsDirectory(remove_me)) { + if (not FileSystemManager::RemoveDirectory(remove_me)) { + Logger::Log(LogLevel::Error, + "Failed to remove directory {}", + remove_me.string()); + return false; + } + } + else { + if (not FileSystemManager::RemoveFile(remove_me)) { + Logger::Log( + LogLevel::Error, "Failed to remove {}", remove_me.string()); + return false; + } + } + } + + // after releasing the shared lock, wait to get an exclusive lock for doing + // the critical renaming + { + auto lock = ExclusiveLock(storage_config); + if (not lock) { + Logger::Log(LogLevel::Error, + "Failed to exclusively lock the local repository root"); + return false; + } + + for (std::size_t i = storage_config.num_generations; i > 0; i--) { + auto from = storage_config.RepositoryGenerationRoot(i - 1); + auto to = i < storage_config.num_generations + ? storage_config.RepositoryGenerationRoot(i) + : remove_me; + if (FileSystemManager::IsDirectory(from)) { + if (not FileSystemManager::Rename(from, to)) { + Logger::Log(LogLevel::Error, + "Failed to rename {} to {}", + from.string(), + to.string()); + return false; + } + } + } + } + + // Finally, with a shared lock, clean up the directory to be removed + { + auto lock = SharedLock(storage_config); + if (not lock) { + Logger::Log(LogLevel::Error, + "Failed to get a shared lock for the repository root"); + return false; + } + if (FileSystemManager::IsDirectory(remove_me)) { + if (not FileSystemManager::RemoveDirectory(remove_me)) { + Logger::Log(LogLevel::Error, + "Failed to remove directory {}", + remove_me.string()); + return false; + } + } + } + + return true; +} diff --git a/src/buildtool/storage/repository_garbage_collector.hpp b/src/buildtool/storage/repository_garbage_collector.hpp new file mode 100644 index 00000000..1bb3867a --- /dev/null +++ b/src/buildtool/storage/repository_garbage_collector.hpp @@ -0,0 +1,49 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_REPOSITORY_GARBAGE_COLLECTOR_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_REPOSITORY_GARBAGE_COLLECTOR_HPP + +#include <filesystem> +#include <optional> + +#include "src/buildtool/storage/config.hpp" +#include "src/utils/cpp/file_locking.hpp" + +/// \brief Global garbage collector implementation. +/// Responsible for deleting oldest generation. +class RepositoryGarbageCollector { + public: + /// \brief Trigger garbage collection, i.e., rotate the generations and + /// delete the oldest. \returns true on success. + [[nodiscard]] auto static TriggerGarbageCollection( + StorageConfig const& storage_config) noexcept -> bool; + + /// \brief Acquire shared lock to prevent garbage collection from running. + /// \param storage_config Storage to be locked. + /// \returns The acquired lock file on success or nullopt otherwise. + [[nodiscard]] auto static SharedLock( + StorageConfig const& storage_config) noexcept + -> std::optional<LockFile>; + + private: + [[nodiscard]] auto static ExclusiveLock( + StorageConfig const& storage_config) noexcept + -> std::optional<LockFile>; + + [[nodiscard]] auto static LockFilePath( + StorageConfig const& storage_config) noexcept -> std::filesystem::path; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP |