summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/build_engine/target_map/TARGETS1
-rw-r--r--src/buildtool/build_engine/target_map/target_cache.cpp20
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_entry.cpp5
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_entry.hpp4
-rw-r--r--src/buildtool/execution_api/local/TARGETS26
-rw-r--r--src/buildtool/execution_api/local/garbage_collector.cpp567
-rw-r--r--src/buildtool/execution_api/local/garbage_collector.hpp82
-rw-r--r--src/buildtool/execution_api/local/local_ac.hpp19
-rw-r--r--src/buildtool/execution_api/local/local_cas.hpp51
-rw-r--r--src/buildtool/file_system/file_storage.hpp34
-rw-r--r--src/buildtool/main/main.cpp6
11 files changed, 777 insertions, 38 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
index abf20ccc..a599b6ee 100644
--- a/src/buildtool/build_engine/target_map/TARGETS
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -101,6 +101,7 @@
, "private-deps":
[ ["@", "fmt", "", "fmt"]
, ["src/buildtool/execution_api/local", "config"]
+ , ["src/buildtool/execution_api/local", "garbage_collector"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/logging", "log_level"]
]
diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp
index 493b7505..48e761fd 100644
--- a/src/buildtool/build_engine/target_map/target_cache.cpp
+++ b/src/buildtool/build_engine/target_map/target_cache.cpp
@@ -23,6 +23,9 @@
#include "src/buildtool/execution_api/local/config.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/log_level.hpp"
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
+#endif
auto TargetCache::Store(TargetCacheKey const& key,
TargetCacheEntry const& value) const noexcept -> bool {
@@ -46,16 +49,23 @@ auto TargetCache::Store(TargetCacheKey const& key,
}
auto TargetCache::Read(TargetCacheKey const& key) const noexcept
- -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> {
- auto entry_path = file_store_.GetPath(key.Id().digest.hash());
- auto const entry =
- FileSystemManager::ReadFile(entry_path, ObjectType::File);
- if (not entry.has_value()) {
+ -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo> > {
+ auto id = key.Id().digest.hash();
+ auto entry_path = file_store_.GetPath(id);
+#ifndef BOOTSTRAP_BUILD_TOOL
+ // Try to find target-cache entry in CAS generations and uplink if required.
+ auto found = GarbageCollector::FindAndUplinkTargetCacheEntry(id);
+#else
+ auto found = FileSystemManager::IsFile(entry_path);
+#endif
+ if (not found) {
logger_.Emit(LogLevel::Debug,
"Cache miss, entry not found {}",
entry_path.string());
return std::nullopt;
}
+ auto const entry =
+ FileSystemManager::ReadFile(entry_path, ObjectType::File);
if (auto info = Artifact::ObjectInfo::FromString(*entry)) {
if (auto path = CAS().BlobPath(info->digest)) {
if (auto value = FileSystemManager::ReadFile(*path)) {
diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.cpp b/src/buildtool/build_engine/target_map/target_cache_entry.cpp
index 45815056..139c706b 100644
--- a/src/buildtool/build_engine/target_map/target_cache_entry.cpp
+++ b/src/buildtool/build_engine/target_map/target_cache_entry.cpp
@@ -33,6 +33,11 @@ auto TargetCacheEntry::FromTarget(
return std::nullopt;
}
+auto TargetCacheEntry::FromJson(nlohmann::json desc) noexcept
+ -> TargetCacheEntry {
+ return TargetCacheEntry(std::move(desc));
+}
+
auto TargetCacheEntry::ToResult() const noexcept
-> std::optional<TargetResult> {
return TargetResult::FromJson(desc_);
diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.hpp b/src/buildtool/build_engine/target_map/target_cache_entry.hpp
index b6d1351c..1e54a003 100644
--- a/src/buildtool/build_engine/target_map/target_cache_entry.hpp
+++ b/src/buildtool/build_engine/target_map/target_cache_entry.hpp
@@ -40,6 +40,10 @@ class TargetCacheEntry {
std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
replacements) noexcept -> std::optional<TargetCacheEntry>;
+ // Create a target-cache entry from a json description.
+ [[nodiscard]] static auto FromJson(nlohmann::json desc) noexcept
+ -> TargetCacheEntry;
+
// Obtain TargetResult from cache entry.
[[nodiscard]] auto ToResult() const noexcept -> std::optional<TargetResult>;
diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS
index ecf6b9ca..31f2c655 100644
--- a/src/buildtool/execution_api/local/TARGETS
+++ b/src/buildtool/execution_api/local/TARGETS
@@ -30,6 +30,7 @@
, "srcs": ["local_action.cpp", "local_storage.cpp"]
, "deps":
[ "config"
+ , "garbage_collector"
, ["@", "fmt", "", "fmt"]
, ["@", "gsl-lite", "", "gsl-lite"]
, ["src/buildtool/execution_api/common", "common"]
@@ -38,7 +39,6 @@
, ["src/buildtool/file_system", "file_storage"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/compatibility", "compatibility"]
- , ["src/buildtool/logging", "logging"]
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "bazel_types"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
@@ -50,4 +50,28 @@
, ["src/buildtool/system", "system_command"]
]
}
+, "garbage_collector":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["garbage_collector"]
+ , "hdrs": ["garbage_collector.hpp"]
+ , "srcs": ["garbage_collector.cpp"]
+ , "deps": [["src/utils/cpp", "file_locking"]]
+ , "stage": ["src", "buildtool", "execution_api", "local"]
+ , "private-deps":
+ [ "config"
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "bazel_types"]
+ , ["src/buildtool/compatibility", "compatibility"]
+ , ["src/buildtool/build_engine/target_map", "target_cache_entry"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/file_system", "file_storage"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/file_system", "git_repo"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/logging", "log_level"]
+ , ["src/utils/cpp", "hex_string"]
+ ]
+ }
}
diff --git a/src/buildtool/execution_api/local/garbage_collector.cpp b/src/buildtool/execution_api/local/garbage_collector.cpp
new file mode 100644
index 00000000..31af07e5
--- /dev/null
+++ b/src/buildtool/execution_api/local/garbage_collector.cpp
@@ -0,0 +1,567 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
+
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
+#include "src/buildtool/compatibility/native_support.hpp"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/file_system/file_storage.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/git_repo.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+auto GarbageCollector::FindAndUplinkBlob(std::string const& id,
+ bool is_executable) noexcept -> bool {
+ // Try to find blob in all generations.
+ for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) {
+ if (UplinkBlob(i, id, is_executable)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+auto GarbageCollector::FindAndUplinkTree(std::string const& id) noexcept
+ -> bool {
+ // Try to find tree in all generations.
+ for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) {
+ if (UplinkTree(i, id)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+auto GarbageCollector::FindAndUplinkActionCacheEntry(
+ std::string const& id) noexcept -> bool {
+ // Try to find action-cache entry in all generations.
+ for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) {
+ if (UplinkActionCacheEntry(i, id)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+auto GarbageCollector::FindAndUplinkTargetCacheEntry(
+ std::string const& id) noexcept -> bool {
+ // Try to find target-cache entry in all generations.
+ for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) {
+ if (UplinkTargetCacheEntry(i, id)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+auto GarbageCollector::SharedLock() noexcept -> std::optional<LockFile> {
+ return LockFile::Acquire(LockFilePath(), /*is_shared=*/true);
+}
+
+auto GarbageCollector::ExclusiveLock() noexcept -> std::optional<LockFile> {
+ return LockFile::Acquire(LockFilePath(), /*is_shared=*/false);
+}
+
+auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path {
+ return LocalExecutionConfig::CacheRoot() / "gc.lock";
+}
+
+auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool {
+ auto pid = CreateProcessUniqueId();
+ if (not pid) {
+ return false;
+ }
+ auto remove_me = std::string{"remove-me-"} + *pid;
+ auto remove_me_dir = LocalExecutionConfig::CacheRoot() / remove_me;
+ if (FileSystemManager::IsDirectory(remove_me_dir)) {
+ if (not FileSystemManager::RemoveDirectory(remove_me_dir,
+ /*recursively=*/true)) {
+ Logger::Log(LogLevel::Error,
+ "Failed to remove directory {}",
+ remove_me_dir.string());
+ return false;
+ }
+ }
+ { // Create scope for critical renaming section protected by advisory lock.
+ auto lock = ExclusiveLock();
+ if (not lock) {
+ Logger::Log(LogLevel::Error,
+ "Failed to exclusively lock the local build root");
+ return false;
+ }
+ for (int i = LocalExecutionConfig::NumGenerations() - 1; i >= 0; i--) {
+ auto cache_root = LocalExecutionConfig::CacheRoot(i);
+ if (FileSystemManager::IsDirectory(cache_root)) {
+ auto new_cache_root =
+ (i == LocalExecutionConfig::NumGenerations() - 1)
+ ? remove_me_dir
+ : LocalExecutionConfig::CacheRoot(i + 1);
+ if (not FileSystemManager::Rename(cache_root, new_cache_root)) {
+ Logger::Log(LogLevel::Error,
+ "Failed to rename {} to {}.",
+ cache_root.string(),
+ new_cache_root.string());
+ return false;
+ }
+ }
+ }
+ }
+ if (FileSystemManager::IsDirectory(remove_me_dir)) {
+ if (not FileSystemManager::RemoveDirectory(remove_me_dir,
+ /*recursively=*/true)) {
+ Logger::Log(LogLevel::Warning,
+ "Failed to remove directory {}",
+ remove_me_dir.string());
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkBlob(int index,
+ std::string const& id,
+ bool is_executable) noexcept -> bool {
+ // Determine blob path of given generation.
+ auto root =
+ is_executable
+ ? LocalExecutionConfig::CASDir<ObjectType::Executable>(index)
+ : LocalExecutionConfig::CASDir<ObjectType::File>(index);
+ auto blob_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(blob_path)) {
+ return false;
+ }
+
+ // Determine blob path in latest generation.
+ auto root_latest =
+ is_executable ? LocalExecutionConfig::CASDir<ObjectType::Executable>(0)
+ : LocalExecutionConfig::CASDir<ObjectType::File>(0);
+ auto blob_path_latest = GetStoragePath(root_latest, id);
+ if (not FileSystemManager::IsFile(blob_path_latest)) {
+ // Uplink blob from older generation to the latest generation.
+ if (not FileSystemManager::CreateDirectory(
+ blob_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ blob_path,
+ blob_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(blob_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// NOLINTNEXTLINE(misc-no-recursion)
+auto GarbageCollector::UplinkTree(int index, std::string const& id) noexcept
+ -> bool {
+ // Determine tree path of given generation.
+ auto root = LocalExecutionConfig::CASDir<ObjectType::Tree>(index);
+ auto tree_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(tree_path)) {
+ return false;
+ }
+
+ // Determine tree path in latest generation.
+ auto root_latest = LocalExecutionConfig::CASDir<ObjectType::Tree>(0);
+ auto tree_path_latest = GetStoragePath(root_latest, id);
+ if (not FileSystemManager::IsFile(tree_path_latest)) {
+ // Determine tree entries.
+ auto content = FileSystemManager::ReadFile(tree_path);
+ auto tree_entries = GitRepo::ReadTreeData(*content,
+ id,
+ /*is_hex_id=*/true);
+ if (not tree_entries) {
+ return false;
+ }
+
+ // Uplink tree entries.
+ for (auto const& [raw_id, entry_vector] : *tree_entries) {
+ // Process only first entry from 'entry_vector' since all entries
+ // represent the same blob, just with different names.
+ auto entry = entry_vector.front();
+ auto hash = ToHexString(raw_id);
+ if (entry.type == ObjectType::Tree) {
+ if (not UplinkTree(index, hash)) {
+ return false;
+ }
+ }
+ else {
+ if (not UplinkBlob(
+ index, hash, entry.type == ObjectType::Executable)) {
+ return false;
+ }
+ }
+ }
+
+ // Uplink tree from older generation to the latest generation.
+ if (not FileSystemManager::CreateDirectory(
+ tree_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ tree_path,
+ tree_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(tree_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkBazelTree(int index,
+ std::string const& id) noexcept -> bool {
+ // Determine bazel tree path of given generation.
+ auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index);
+ auto tree_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(tree_path)) {
+ return false;
+ }
+
+ // Determine bazel tree entries.
+ auto content = FileSystemManager::ReadFile(tree_path);
+ bazel_re::Tree tree{};
+ if (not tree.ParseFromString(*content)) {
+ return false;
+ }
+
+ // Uplink bazel tree entries.
+ auto dir = tree.root();
+ for (auto const& file : dir.files()) {
+ if (not UplinkBlob(index,
+ NativeSupport::Unprefix(file.digest().hash()),
+ file.is_executable())) {
+ return false;
+ }
+ }
+ for (auto const& directory : dir.directories()) {
+ if (not UplinkBazelDirectory(
+ index, NativeSupport::Unprefix(directory.digest().hash()))) {
+ return false;
+ }
+ }
+
+ // Determine bazel tree path in latest generation.
+ auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0);
+ auto tree_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink bazel tree from older generation to the latest generation.
+ if (not FileSystemManager::IsFile(tree_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ tree_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ tree_path,
+ tree_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(tree_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// NOLINTNEXTLINE(misc-no-recursion)
+auto GarbageCollector::UplinkBazelDirectory(int index,
+ std::string const& id) noexcept
+ -> bool {
+ // Determine bazel directory path of given generation.
+ auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index);
+ auto dir_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(dir_path)) {
+ return false;
+ }
+
+ // Determine bazel directory entries.
+ auto content = FileSystemManager::ReadFile(dir_path);
+ bazel_re::Directory dir{};
+ if (not dir.ParseFromString(*content)) {
+ return false;
+ }
+
+ // Uplink bazel directory entries.
+ for (auto const& file : dir.files()) {
+ if (not UplinkBlob(index,
+ NativeSupport::Unprefix(file.digest().hash()),
+ file.is_executable())) {
+ return false;
+ }
+ }
+ for (auto const& directory : dir.directories()) {
+ if (not UplinkBazelDirectory(
+ index, NativeSupport::Unprefix(directory.digest().hash()))) {
+ return false;
+ }
+ }
+
+ // Determine bazel directory path in latest generation.
+ auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0);
+ auto dir_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink bazel directory from older generation to the latest generation.
+ if (not FileSystemManager::IsFile(dir_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ dir_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ dir_path,
+ dir_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(dir_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkActionCacheEntry(int index,
+ std::string const& id) noexcept
+ -> bool {
+ // Determine action-cache entry path of given generation.
+ auto root = LocalExecutionConfig::ActionCacheDir(index);
+ auto entry_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(entry_path)) {
+ return false;
+ }
+
+ // Determine action-cache entry location.
+ auto content = FileSystemManager::ReadFile(entry_path, ObjectType::File);
+ bazel_re::Digest digest{};
+ if (not digest.ParseFromString(*content)) {
+ return false;
+ }
+
+ // Uplink action-cache entry blob.
+ if (not UplinkActionCacheEntryBlob(
+ index, NativeSupport::Unprefix(digest.hash()))) {
+ return false;
+ }
+
+ // Determine action-cache entry path in latest generation.
+ auto root_latest = LocalExecutionConfig::ActionCacheDir(0);
+ auto entry_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink action-cache entry from older generation to the latest
+ // generation.
+ if (not FileSystemManager::IsFile(entry_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ entry_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ entry_path,
+ entry_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(entry_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkActionCacheEntryBlob(
+ int index,
+ std::string const& id) noexcept -> bool {
+
+ // Determine action-cache entry blob path of given generation.
+ auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index);
+ auto entry_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(entry_path)) {
+ return false;
+ }
+
+ // Determine artifacts referenced by action-cache entry.
+ auto content = FileSystemManager::ReadFile(entry_path);
+ bazel_re::ActionResult result{};
+ if (not result.ParseFromString(*content)) {
+ return false;
+ }
+
+ // Uplink referenced artifacts.
+ for (auto const& file : result.output_files()) {
+ if (not UplinkBlob(index,
+ NativeSupport::Unprefix(file.digest().hash()),
+ file.is_executable())) {
+ return false;
+ }
+ }
+ for (auto const& directory : result.output_directories()) {
+ if (Compatibility::IsCompatible()) {
+ if (not UplinkBazelTree(
+ index,
+ NativeSupport::Unprefix(directory.tree_digest().hash()))) {
+ return false;
+ }
+ }
+ else {
+ if (not UplinkTree(
+ index,
+ NativeSupport::Unprefix(directory.tree_digest().hash()))) {
+ return false;
+ }
+ }
+ }
+
+ // Determine action-cache entry blob path in latest generation.
+ auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0);
+ auto entry_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink action-cache entry blob from older generation to the latest
+ // generation.
+ if (not FileSystemManager::IsFile(entry_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ entry_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ entry_path,
+ entry_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(entry_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkTargetCacheEntry(int index,
+ std::string const& id) noexcept
+ -> bool {
+
+ // Determine target-cache entry path of given generation.
+ auto root = LocalExecutionConfig::TargetCacheDir(index);
+ auto entry_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(entry_path)) {
+ return false;
+ }
+
+ // Determine target-cache entry location.
+ auto content = FileSystemManager::ReadFile(entry_path);
+ auto info = Artifact::ObjectInfo::FromString(*content);
+ if (not info) {
+ return false;
+ }
+
+ // Uplink target-cache entry blob.
+ if (not UplinkTargetCacheEntryBlob(index, info->digest.hash())) {
+ return false;
+ }
+
+ // Determine target-cache entry path in latest generation.
+ auto root_latest = LocalExecutionConfig::TargetCacheDir(0);
+ auto entry_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink target-cache entry from older generation to the latest
+ // generation.
+ if (not FileSystemManager::IsFile(entry_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ entry_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ entry_path,
+ entry_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(entry_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto GarbageCollector::UplinkTargetCacheEntryBlob(
+ int index,
+ std::string const& id) noexcept -> bool {
+
+ // Determine target-cache entry blob path of given generation.
+ auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index);
+ auto entry_path = GetStoragePath(root, id);
+ if (not FileSystemManager::IsFile(entry_path)) {
+ return false;
+ }
+
+ // Determine artifacts referenced by target-cache entry.
+ auto content = FileSystemManager::ReadFile(entry_path);
+ nlohmann::json json_desc{};
+ try {
+ json_desc = nlohmann::json::parse(*content);
+ } catch (std::exception const& ex) {
+ return false;
+ }
+ auto entry = TargetCacheEntry::FromJson(json_desc);
+ std::vector<Artifact::ObjectInfo> artifacts_info;
+ if (not entry.ToArtifacts(&artifacts_info)) {
+ return false;
+ }
+
+ // Uplink referenced artifacts.
+ for (auto const& info : artifacts_info) {
+ auto hash = info.digest.hash();
+ if (info.type == ObjectType::Tree) {
+ if (Compatibility::IsCompatible()) {
+ if (not UplinkBazelDirectory(index, hash)) {
+ return false;
+ }
+ }
+ else {
+ if (not UplinkTree(index, hash)) {
+ return false;
+ }
+ }
+ }
+ else {
+ if (not UplinkBlob(
+ index, hash, info.type == ObjectType::Executable)) {
+ return false;
+ }
+ }
+ }
+
+ // Determine target-cache entry blob path in latest generation.
+ auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0);
+ auto entry_path_latest = GetStoragePath(root_latest, id);
+
+ // Uplink target-cache entry blob from older generation to the latest
+ // generation.
+ if (not FileSystemManager::IsFile(entry_path_latest)) {
+ if (not FileSystemManager::CreateDirectory(
+ entry_path_latest.parent_path())) {
+ return false;
+ }
+ if (not FileSystemManager::CreateFileHardlink(
+ entry_path,
+ entry_path_latest,
+ /*log_failure_at=*/LogLevel::Debug) and
+ not FileSystemManager::IsFile(entry_path_latest)) {
+ return false;
+ }
+ }
+ return true;
+}
diff --git a/src/buildtool/execution_api/local/garbage_collector.hpp b/src/buildtool/execution_api/local/garbage_collector.hpp
new file mode 100644
index 00000000..405c8900
--- /dev/null
+++ b/src/buildtool/execution_api/local/garbage_collector.hpp
@@ -0,0 +1,82 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP
+
+#include <optional>
+#include <string>
+
+#include "src/utils/cpp/file_locking.hpp"
+
+/// \brief
+class GarbageCollector {
+ public:
+ [[nodiscard]] auto static FindAndUplinkBlob(std::string const& id,
+ bool is_executable) noexcept
+ -> bool;
+
+ [[nodiscard]] auto static FindAndUplinkTree(std::string const& id) noexcept
+ -> bool;
+
+ [[nodiscard]] auto static FindAndUplinkActionCacheEntry(
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static FindAndUplinkTargetCacheEntry(
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool;
+
+ [[nodiscard]] auto static SharedLock() noexcept -> std::optional<LockFile>;
+
+ [[nodiscard]] auto static ExclusiveLock() noexcept
+ -> std::optional<LockFile>;
+
+ private:
+ [[nodiscard]] auto static LockFilePath() noexcept -> std::filesystem::path;
+
+ [[nodiscard]] auto static UplinkBlob(int index,
+ std::string const& id,
+ bool is_executable) noexcept -> bool;
+
+ [[nodiscard]] auto static UplinkTree(int index,
+ std::string const& id) noexcept
+ -> bool;
+
+ [[nodiscard]] auto static UplinkBazelTree(int index,
+ std::string const& id) noexcept
+ -> bool;
+
+ [[nodiscard]] auto static UplinkBazelDirectory(
+ int index,
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static UplinkActionCacheEntry(
+ int index,
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static UplinkActionCacheEntryBlob(
+ int index,
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static UplinkTargetCacheEntry(
+ int index,
+ std::string const& id) noexcept -> bool;
+
+ [[nodiscard]] auto static UplinkTargetCacheEntryBlob(
+ int index,
+ std::string const& id) noexcept -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP
diff --git a/src/buildtool/execution_api/local/local_ac.hpp b/src/buildtool/execution_api/local/local_ac.hpp
index 03945d16..423282be 100644
--- a/src/buildtool/execution_api/local/local_ac.hpp
+++ b/src/buildtool/execution_api/local/local_ac.hpp
@@ -19,6 +19,7 @@
#include "src/buildtool/common/bazel_types.hpp"
#include "src/buildtool/execution_api/common/execution_common.hpp"
#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
@@ -47,21 +48,23 @@ class LocalAC {
[[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id)
const noexcept -> std::optional<bazel_re::ActionResult> {
- auto entry_path =
- file_store_.GetPath(NativeSupport::Unprefix(action_id.hash()));
- bazel_re::Digest digest{};
- auto const entry =
- FileSystemManager::ReadFile(entry_path, ObjectType::File);
- if (not entry.has_value()) {
+ auto id = NativeSupport::Unprefix(action_id.hash());
+ auto entry_path = file_store_.GetPath(id);
+ // Try to find action-cache entry in CAS generations and uplink if
+ // required.
+ if (not GarbageCollector::FindAndUplinkActionCacheEntry(id)) {
logger_.Emit(LogLevel::Debug,
"Cache miss, entry not found {}",
entry_path.string());
return std::nullopt;
}
+ auto const entry =
+ FileSystemManager::ReadFile(entry_path, ObjectType::File);
+ bazel_re::Digest digest{};
if (not digest.ParseFromString(*entry)) {
logger_.Emit(LogLevel::Warning,
"Parsing cache entry failed for action {}",
- NativeSupport::Unprefix(action_id.hash()));
+ id);
return std::nullopt;
}
auto src_path = cas_->BlobPath(digest);
@@ -74,7 +77,7 @@ class LocalAC {
}
logger_.Emit(LogLevel::Warning,
"Parsing action result failed for action {}",
- NativeSupport::Unprefix(action_id.hash()));
+ id);
return std::nullopt;
}
diff --git a/src/buildtool/execution_api/local/local_cas.hpp b/src/buildtool/execution_api/local/local_cas.hpp
index 2f181ca8..afda71ed 100644
--- a/src/buildtool/execution_api/local/local_cas.hpp
+++ b/src/buildtool/execution_api/local/local_cas.hpp
@@ -25,6 +25,9 @@
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
+#endif
template <ObjectType kType = ObjectType::File>
class LocalCAS {
@@ -42,6 +45,13 @@ class LocalCAS {
return instance;
}
+ auto Reset() noexcept -> void {
+ file_store_ = FileStorage<kStorageType,
+ StoreMode::FirstWins,
+ /*kSetEpochTime=*/true>{
+ LocalExecutionConfig::CASDir<kType>(0)};
+ }
+
[[nodiscard]] auto StoreBlobFromBytes(std::string const& bytes)
const noexcept -> std::optional<bazel_re::Digest> {
return StoreBlob(bytes, /*is_owner=*/true);
@@ -57,11 +67,17 @@ class LocalCAS {
-> std::optional<std::filesystem::path> {
auto id = NativeSupport::Unprefix(digest.hash());
auto blob_path = file_store_.GetPath(id);
- if (FileSystemManager::IsFile(blob_path)) {
- return blob_path;
+#ifndef BOOTSTRAP_BUILD_TOOL
+ // Try to find blob in CAS generations and uplink if required.
+ auto found = FindAndUplinkBlob(id);
+#else
+ auto found = FileSystemManager::IsFile(blob_path);
+#endif
+ if (not found) {
+ logger_.Emit(LogLevel::Debug, "Blob not found {}", id);
+ return std::nullopt;
}
- logger_.Emit(LogLevel::Debug, "Blob not found {}", id);
- return std::nullopt;
+ return blob_path;
}
private:
@@ -109,16 +125,35 @@ class LocalCAS {
-> std::optional<bazel_re::Digest> {
auto digest = CreateDigest(data);
if (digest) {
- if (StoreBlobData(
- NativeSupport::Unprefix(digest->hash()), data, is_owner)) {
+ auto id = NativeSupport::Unprefix(digest->hash());
+#ifndef BOOTSTRAP_BUILD_TOOL
+ if (FindAndUplinkBlob(id)) {
+ return digest;
+ }
+#endif
+ if (StoreBlobData(id, data, is_owner)) {
return digest;
}
- logger_.Emit(
- LogLevel::Debug, "Failed to store blob {}.", digest->hash());
+ logger_.Emit(LogLevel::Debug, "Failed to store blob {}.", id);
}
logger_.Emit(LogLevel::Debug, "Failed to create digest.");
return std::nullopt;
}
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+ [[nodiscard]] auto FindAndUplinkBlob(std::string const& id) const noexcept
+ -> bool {
+ if (Compatibility::IsCompatible()) {
+ return GarbageCollector::FindAndUplinkBlob(
+ id, kType == ObjectType::Executable);
+ }
+ if (kType == ObjectType::Tree) {
+ return GarbageCollector::FindAndUplinkTree(id);
+ }
+ return GarbageCollector::FindAndUplinkBlob(
+ id, kType == ObjectType::Executable);
+ }
+#endif
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP
diff --git a/src/buildtool/file_system/file_storage.hpp b/src/buildtool/file_system/file_storage.hpp
index f320375d..c8b30005 100644
--- a/src/buildtool/file_system/file_storage.hpp
+++ b/src/buildtool/file_system/file_storage.hpp
@@ -21,6 +21,20 @@
#include "src/buildtool/execution_api/common/execution_common.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
+/// \brief Determines the storage path of a blob identified by a hash value. The
+/// same sharding technique as used in git is applied, meaning, the hash value
+/// is separated into a directory part and file part. Two characters are used
+/// for the directory part, the rest for the file, which results in 256 possible
+/// directories.
+/// \param root The root path of the storage.
+/// \param id The hash value of the blob.
+/// \returns The sharded file path.
+[[nodiscard]] static inline auto GetStoragePath(
+ std::filesystem::path const& root,
+ std::string const& id) noexcept -> std::filesystem::path {
+ return root / id.substr(0, 2) / id.substr(2, id.size() - 2);
+}
+
enum class StoreMode {
// First thread to write conflicting file wins.
FirstWins,
@@ -56,12 +70,12 @@ class FileStorage {
[[nodiscard]] auto GetPath(std::string const& id) const noexcept
-> std::filesystem::path {
- return GetShardedPath(id);
+ return GetStoragePath(storage_root_, id);
}
private:
static constexpr bool kFdLess{kType == ObjectType::Executable};
- std::filesystem::path const storage_root_{};
+ std::filesystem::path storage_root_{};
/// \brief Add file to storage from file path via link or copy and rename.
/// If a race-condition occurs, the winning thread will be the one
@@ -72,7 +86,7 @@ class FileStorage {
[[nodiscard]] auto AtomicAddFromFile(std::string const& id,
std::filesystem::path const& path,
bool is_owner) const noexcept -> bool {
- auto file_path = GetShardedPath(id);
+ auto file_path = GetPath(id);
if ((kMode == StoreMode::LastWins or
not FileSystemManager::Exists(file_path)) and
FileSystemManager::CreateDirectory(file_path.parent_path())) {
@@ -115,7 +129,7 @@ class FileStorage {
[[nodiscard]] auto AtomicAddFromBytes(
std::string const& id,
std::string const& bytes) const noexcept -> bool {
- auto file_path = GetShardedPath(id);
+ auto file_path = GetPath(id);
if (kMode == StoreMode::LastWins or
not FileSystemManager::Exists(file_path)) {
auto unique_path = CreateUniquePath(file_path);
@@ -131,18 +145,6 @@ class FileStorage {
return FileSystemManager::IsFile(file_path);
}
- /// \brief Determines the storage path of a blob identified by a hash value.
- /// The same sharding technique as git is used, meaning, the hash value is
- /// separated into a directory part and file part. Two characters are used
- /// for the directory part, the rest for the file, which results in 256
- /// possible directories.
- /// \param id The hash value.
- /// \returns The sharded file path.
- [[nodiscard]] auto GetShardedPath(std::string const& id) const noexcept
- -> std::filesystem::path {
- return storage_root_ / id.substr(0, 2) / id.substr(2, id.size() - 2);
- }
-
/// \brief Create file from file path.
[[nodiscard]] static auto CreateFileFromPath(
std::filesystem::path const& file_path,
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 5bb707f5..5fab02f6 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -39,6 +39,7 @@
#include "src/buildtool/main/install_cas.hpp"
#ifndef BOOTSTRAP_BUILD_TOOL
#include "src/buildtool/auth/authentication.hpp"
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
#include "src/buildtool/graph_traverser/graph_traverser.hpp"
#include "src/buildtool/progress_reporting/base_progress_reporter.hpp"
#endif
@@ -1176,6 +1177,11 @@ auto main(int argc, char* argv[]) -> int {
DetermineRoots(arguments.common, arguments.analysis);
#ifndef BOOTSTRAP_BUILD_TOOL
+ auto lock = GarbageCollector::SharedLock();
+ if (not lock) {
+ return kExitFailure;
+ }
+
if (arguments.cmd == SubCommand::kTraverse) {
if (arguments.graph.git_cas) {
if (Compatibility::IsCompatible()) {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-05 18:25:01 +0000