diff options
Diffstat (limited to 'src')
56 files changed, 2013 insertions, 1783 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS index 82d1eca8..54eb2e77 100644 --- a/src/buildtool/build_engine/target_map/TARGETS +++ b/src/buildtool/build_engine/target_map/TARGETS @@ -15,8 +15,8 @@ , "name": ["result_map"] , "hdrs": ["result_map.hpp"] , "deps": - [ "target_cache" - , ["src/buildtool/common", "tree"] + [ ["src/buildtool/common", "tree"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/expression", "expression"] @@ -28,7 +28,6 @@ , ["src/buildtool/logging", "logging"] , ["src/buildtool/common", "common"] , ["src/buildtool/build_engine/base_maps", "entity_name"] - , "target_cache_key" ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } @@ -52,9 +51,8 @@ ] , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": - [ "target_cache" - , "target_cache_key" - , ["src/buildtool/common", "common"] + [ ["src/buildtool/common", "common"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "field_reader"] , ["src/buildtool/build_engine/expression", "expression"] @@ -81,66 +79,4 @@ ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } -, "target_cache": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache"] - , "hdrs": ["target_cache.hpp"] - , "srcs": ["target_cache.cpp"] - , "deps": - [ "target_cache_key" - , "target_cache_entry" - , ["@", "gsl-lite", "", "gsl-lite"] - , ["@", "json", "", "json"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "config"] - , ["src/buildtool/execution_api/local", "garbage_collector"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["src/buildtool/logging", "log_level"] - ] - } -, "target_cache_key": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache_key"] - , "hdrs": ["target_cache_key.hpp"] - , "srcs": ["target_cache_key.cpp"] - , "deps": - [ ["src/buildtool/build_engine/base_maps", "entity_name_data"] - , ["src/buildtool/build_engine/expression", "expression"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["src/buildtool/common", "config"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - ] - } -, "target_cache_entry": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache_entry"] - , "hdrs": ["target_cache_entry.hpp"] - , "srcs": ["target_cache_entry.cpp"] - , "deps": - [ ["@", "gsl-lite", "", "gsl-lite"] - , ["@", "json", "", "json"] - , ["src/buildtool/build_engine/analysed_target", "target"] - , ["src/buildtool/build_engine/expression", "expression"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/common", "artifact_description"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - ] - } } diff --git a/src/buildtool/build_engine/target_map/export.cpp b/src/buildtool/build_engine/target_map/export.cpp index 9005f037..187a192d 100644 --- a/src/buildtool/build_engine/target_map/export.cpp +++ b/src/buildtool/build_engine/target_map/export.cpp @@ -18,10 +18,8 @@ #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/buildtool/storage/storage.hpp" namespace { auto const kExpectedFields = std::unordered_set<std::string>{"config_doc", @@ -145,7 +143,7 @@ void ExportRule( ComputeTargetCacheKey(*exported_target, target_config); if (target_cache_key) { if (auto target_cache_value = - TargetCache::Instance().Read(*target_cache_key)) { + Storage::Instance().TargetCache().Read(*target_cache_key)) { auto const& [entry, info] = *target_cache_value; if (auto result = entry.ToResult()) { auto deps_info = TargetGraphInformation{ diff --git a/src/buildtool/build_engine/target_map/result_map.hpp b/src/buildtool/build_engine/target_map/result_map.hpp index edb1a472..235b9cc3 100644 --- a/src/buildtool/build_engine/target_map/result_map.hpp +++ b/src/buildtool/build_engine/target_map/result_map.hpp @@ -30,14 +30,14 @@ #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/target_cache.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/hash_combine.hpp" namespace BuildMaps::Target { diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp deleted file mode 100644 index 3e149ddd..00000000 --- a/src/buildtool/build_engine/target_map/target_cache.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/build_engine/target_map/target_cache.hpp" - -#include <exception> -#include <vector> - -#include <fmt/core.h> - -#include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/log_level.hpp" -#ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#endif - -auto TargetCache::Store(TargetCacheKey const& key, - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept - -> bool { - // Before a target-cache entry is stored in local CAS, make sure any created - // artifact for this target is downloaded from the remote CAS to the local - // CAS. - if (not DownloadKnownArtifacts(value, downloader)) { - return false; - } - if (auto digest = CAS().StoreBlobFromBytes(value.ToJson().dump(2))) { - auto data = - Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File} - .ToString(); - logger_.Emit(LogLevel::Debug, - "Adding entry for key {} as {}", - key.Id().ToString(), - data); - return file_store_.AddFromBytes(key.Id().digest.hash(), data); - } - return false; -} - -auto TargetCache::Read(TargetCacheKey const& key) const noexcept - -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo> > { - auto id = key.Id().digest.hash(); - auto entry_path = file_store_.GetPath(id); -#ifndef BOOTSTRAP_BUILD_TOOL - // Try to find target-cache entry in CAS generations and uplink if required. - auto found = GarbageCollector::FindAndUplinkTargetCacheEntry(id); -#else - auto found = FileSystemManager::IsFile(entry_path); -#endif - if (not found) { - logger_.Emit(LogLevel::Debug, - "Cache miss, entry not found {}", - entry_path.string()); - return std::nullopt; - } - auto const entry = - FileSystemManager::ReadFile(entry_path, ObjectType::File); - if (auto info = Artifact::ObjectInfo::FromString(*entry)) { - if (auto path = CAS().BlobPath(info->digest)) { - if (auto value = FileSystemManager::ReadFile(*path)) { - try { - return std::make_pair( - TargetCacheEntry{nlohmann::json::parse(*value)}, - std::move(*info)); - } catch (std::exception const& ex) { - logger_.Emit(LogLevel::Warning, - "Parsing entry for key {} failed with:\n{}", - key.Id().ToString(), - ex.what()); - } - } - } - } - logger_.Emit(LogLevel::Warning, - "Reading entry for key {} failed", - key.Id().ToString()); - return std::nullopt; -} - -auto TargetCache::DownloadKnownArtifacts( - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool { - std::vector<Artifact::ObjectInfo> artifacts_info; - return downloader and value.ToArtifacts(&artifacts_info) and - downloader(artifacts_info); -} - -auto TargetCache::ComputeCacheDir(int index) -> std::filesystem::path { - [[maybe_unused]] auto id = CAS().StoreBlobFromBytes( - LocalExecutionConfig::ExecutionBackendDescription()); - return LocalExecutionConfig::TargetCacheDir(index); -} diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp deleted file mode 100644 index 8bce88c0..00000000 --- a/src/buildtool/build_engine/target_map/target_cache.hpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP - -#include <filesystem> -#include <functional> -#include <optional> -#include <utility> - -#include <gsl-lite/gsl-lite.hpp> -#include <nlohmann/json.hpp> - -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/logging/logger.hpp" - -class TargetCache { - public: - using ArtifactDownloader = - std::function<bool(std::vector<Artifact::ObjectInfo> const&)>; - - TargetCache() = default; - TargetCache(TargetCache const&) = delete; - TargetCache(TargetCache&&) = delete; - auto operator=(TargetCache const&) -> TargetCache& = delete; - auto operator=(TargetCache&&) -> TargetCache& = delete; - ~TargetCache() noexcept = default; - - [[nodiscard]] static auto Instance() -> TargetCache& { - static TargetCache instance; - return instance; - } - - // Store new key entry pair in the target cache. - [[nodiscard]] auto Store( - TargetCacheKey const& key, - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool; - - // Read existing entry and object info for given key from the target cache. - [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept - -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; - - private: - Logger logger_{"TargetCache"}; - FileStorage<ObjectType::File, - StoreMode::LastWins, - /*kSetEpochTime=*/false> - file_store_{ComputeCacheDir(0)}; - - [[nodiscard]] auto DownloadKnownArtifacts( - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool; - [[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& { - return LocalCAS<ObjectType::File>::Instance(); - } - [[nodiscard]] static auto ComputeCacheDir(int index) - -> std::filesystem::path; -}; - -namespace std { -template <> -struct hash<TargetCacheKey> { - [[nodiscard]] auto operator()(TargetCacheKey const& k) const { - return std::hash<Artifact::ObjectInfo>{}(k.Id()); - } -}; -} // namespace std - -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index 39352c31..c8d9f878 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -105,6 +105,7 @@ , "deps": [ ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_cas"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/multithreading", "atomic_value"] ] , "stage": ["src", "buildtool", "common"] diff --git a/src/buildtool/common/bazel_types.hpp b/src/buildtool/common/bazel_types.hpp index d797b554..85e06764 100644 --- a/src/buildtool/common/bazel_types.hpp +++ b/src/buildtool/common/bazel_types.hpp @@ -21,6 +21,9 @@ #ifdef BOOTSTRAP_BUILD_TOOL +#include <cstdint> +#include <string> + namespace build::bazel::remote::execution::v2 { struct Digest { std::string hash_; diff --git a/src/buildtool/common/repository_config.cpp b/src/buildtool/common/repository_config.cpp index 73546a08..a3b81de1 100644 --- a/src/buildtool/common/repository_config.cpp +++ b/src/buildtool/common/repository_config.cpp @@ -14,7 +14,7 @@ #include "src/buildtool/common/repository_config.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/utils/automata/dfa_minimizer.hpp" auto RepositoryConfig::RepositoryInfo::BaseContentDescription() const @@ -43,8 +43,8 @@ auto RepositoryConfig::RepositoryKey(std::string const& repo) const noexcept return data->key.SetOnceAndGet( [this, &unique]() -> std::optional<std::string> { if (auto graph = BuildGraphForRepository(unique)) { - auto& cas = LocalCAS<ObjectType::File>::Instance(); - if (auto digest = cas.StoreBlobFromBytes(graph->dump(2))) { + auto const& cas = Storage::Instance().CAS(); + if (auto digest = cas.StoreBlob(graph->dump(2))) { return ArtifactDigest{*digest}.hash(); } } diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS index 739e32e2..42de3e7b 100644 --- a/src/buildtool/execution_api/execution_service/TARGETS +++ b/src/buildtool/execution_api/execution_service/TARGETS @@ -13,7 +13,7 @@ , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl-lite", "", "gsl-lite"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/file_system", "file_system_manager"] , "operation_cache" ] @@ -31,8 +31,7 @@ , ["src/buildtool/logging", "logging"] , ["src/buildtool/common", "bazel_types"] ] - , "private-deps": - [["src/buildtool/execution_api/local", "garbage_collector"]] + , "private-deps": [["src/buildtool/storage", "storage"]] } , "cas_server": { "type": ["@", "rules", "CC", "library"] @@ -49,7 +48,7 @@ , "private-deps": [ ["src/buildtool/compatibility", "compatibility"] , ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] ] } , "server_implementation": @@ -89,7 +88,7 @@ , ["src/buildtool/execution_api/common", "bytestream-common"] , ["src/utils/cpp", "tmp_dir"] , ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] ] } , "capabilities_server": diff --git a/src/buildtool/execution_api/execution_service/ac_server.cpp b/src/buildtool/execution_api/execution_service/ac_server.cpp index 8746a0cd..f8c5be4f 100644 --- a/src/buildtool/execution_api/execution_service/ac_server.cpp +++ b/src/buildtool/execution_api/execution_service/ac_server.cpp @@ -15,7 +15,7 @@ #include "src/buildtool/execution_api/execution_service/ac_server.hpp" #include "fmt/format.h" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" auto ActionCacheServiceImpl::GetActionResult( ::grpc::ServerContext* /*context*/, @@ -30,7 +30,7 @@ auto ActionCacheServiceImpl::GetActionResult( logger_.Emit(LogLevel::Error, str); return grpc::Status{grpc::StatusCode::INTERNAL, str}; } - auto x = ac_.CachedResult(request->action_digest()); + auto x = storage_->ActionCache().CachedResult(request->action_digest()); if (!x) { return grpc::Status{ grpc::StatusCode::NOT_FOUND, diff --git a/src/buildtool/execution_api/execution_service/ac_server.hpp b/src/buildtool/execution_api/execution_service/ac_server.hpp index bea61841..ce31e0ce 100644 --- a/src/buildtool/execution_api/execution_service/ac_server.hpp +++ b/src/buildtool/execution_api/execution_service/ac_server.hpp @@ -17,8 +17,8 @@ #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/local/local_ac.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class ActionCacheServiceImpl final : public bazel_re::ActionCache::Service { public: @@ -60,8 +60,7 @@ class ActionCacheServiceImpl final : public bazel_re::ActionCache::Service { ::bazel_re::ActionResult* response) -> ::grpc::Status override; private: - LocalCAS<ObjectType::File> cas_{}; - LocalAC ac_{&cas_}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service"}; }; diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index 37bfebc6..234c4adc 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -21,7 +21,7 @@ #include "fmt/format.h" #include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/common/bytestream_common.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { @@ -64,11 +64,12 @@ auto BytestreamServiceImpl::Read( if (NativeSupport::IsTree(*hash)) { ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, true}; - path = storage_.TreePath(static_cast<bazel_re::Digest>(dgst)); + path = storage_->CAS().TreePath(static_cast<bazel_re::Digest>(dgst)); } else { ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, false}; - path = storage_.BlobPath(static_cast<bazel_re::Digest>(dgst), false); + path = storage_->CAS().BlobPath(static_cast<bazel_re::Digest>(dgst), + false); } if (!path) { auto str = fmt::format("could not find {}", *hash); @@ -135,14 +136,14 @@ auto BytestreamServiceImpl::Write( return grpc::Status{grpc::StatusCode::INTERNAL, str}; } if (NativeSupport::IsTree(*hash)) { - if (not storage_.StoreTree(tmp)) { + if (not storage_->CAS().StoreTree(tmp)) { auto str = fmt::format("could not store tree {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } } else { - if (not storage_.StoreBlob(tmp)) { + if (not storage_->CAS().StoreBlob(tmp, /*is_executable=*/false)) { auto str = fmt::format("could not store blob {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.hpp b/src/buildtool/execution_api/execution_service/bytestream_server.hpp index e2627328..b69b5344 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.hpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.hpp @@ -16,8 +16,8 @@ #define BYTESTREAM_SERVER_HPP #include "google/bytestream/bytestream.grpc.pb.h" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class BytestreamServiceImpl : public ::google::bytestream::ByteStream::Service { public: @@ -76,7 +76,7 @@ class BytestreamServiceImpl : public ::google::bytestream::ByteStream::Service { -> ::grpc::Status override; private: - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service:bytestream"}; }; diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp index 07b9f3bf..7815f557 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.cpp +++ b/src/buildtool/execution_api/execution_service/cas_server.cpp @@ -16,7 +16,7 @@ #include "fmt/format.h" #include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" static constexpr std::size_t kJustHashLength = 42; static constexpr std::size_t kSHA256Length = 64; @@ -51,12 +51,12 @@ auto CASServiceImpl::FindMissingBlobs( } logger_.Emit(LogLevel::Trace, "FindMissingBlobs: {}", hash); if (NativeSupport::IsTree(hash)) { - if (!storage_.TreePath(x)) { + if (!storage_->CAS().TreePath(x)) { auto* d = response->add_missing_blob_digests(); d->CopyFrom(x); } } - else if (!storage_.BlobPath(x, false)) { + else if (!storage_->CAS().BlobPath(x, false)) { auto* d = response->add_missing_blob_digests(); d->CopyFrom(x); } @@ -99,7 +99,7 @@ auto CASServiceImpl::BatchUpdateBlobs( auto* r = response->add_responses(); r->mutable_digest()->CopyFrom(x.digest()); if (NativeSupport::IsTree(hash)) { - auto const& dgst = storage_.StoreTree(x.data()); + auto const& dgst = storage_->CAS().StoreTree(x.data()); if (!dgst) { auto const& str = fmt::format( "BatchUpdateBlobs: could not upload tree {}", hash); @@ -111,7 +111,7 @@ auto CASServiceImpl::BatchUpdateBlobs( } } else { - auto const& dgst = storage_.StoreBlob(x.data(), false); + auto const& dgst = storage_->CAS().StoreBlob(x.data(), false); if (!dgst) { auto const& str = fmt::format( "BatchUpdateBlobs: could not upload blob {}", hash); @@ -141,10 +141,10 @@ auto CASServiceImpl::BatchReadBlobs( r->mutable_digest()->CopyFrom(digest); std::optional<std::filesystem::path> path; if (NativeSupport::IsTree(digest.hash())) { - path = storage_.TreePath(digest); + path = storage_->CAS().TreePath(digest); } else { - path = storage_.BlobPath(digest, false); + path = storage_->CAS().BlobPath(digest, false); } if (!path) { google::rpc::Status status; diff --git a/src/buildtool/execution_api/execution_service/cas_server.hpp b/src/buildtool/execution_api/execution_service/cas_server.hpp index b3c11706..520263b6 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.hpp +++ b/src/buildtool/execution_api/execution_service/cas_server.hpp @@ -16,8 +16,8 @@ #define CAS_SERVER_HPP #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class CASServiceImpl final : public bazel_re::ContentAddressableStorage::Service { @@ -119,7 +119,7 @@ class CASServiceImpl final std::string const& computed) const noexcept -> std::optional<std::string>; - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service"}; }; #endif diff --git a/src/buildtool/execution_api/execution_service/execution_server.cpp b/src/buildtool/execution_api/execution_service/execution_server.cpp index 5f21f5ac..ed22e023 100644 --- a/src/buildtool/execution_api/execution_service/execution_server.cpp +++ b/src/buildtool/execution_api/execution_service/execution_server.cpp @@ -24,8 +24,8 @@ #include "fmt/format.h" #include "gsl-lite/gsl-lite.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" static void UpdateTimeStamp(::google::longrunning::Operation* op) { ::google::protobuf::Timestamp t; @@ -40,7 +40,7 @@ auto ExecutionServiceImpl::GetAction(::bazel_re::ExecuteRequest const* request) const noexcept -> std::pair<std::optional<::bazel_re::Action>, std::optional<std::string>> { // get action description - auto path = storage_.BlobPath(request->action_digest(), false); + auto path = storage_->CAS().BlobPath(request->action_digest(), false); if (!path) { auto str = fmt::format("could not retrieve blob {} from cas", request->action_digest().hash()); @@ -59,8 +59,8 @@ auto ExecutionServiceImpl::GetAction(::bazel_re::ExecuteRequest const* request) } path = Compatibility::IsCompatible() - ? storage_.BlobPath(action.input_root_digest(), false) - : storage_.TreePath(action.input_root_digest()); + ? storage_->CAS().BlobPath(action.input_root_digest(), false) + : storage_->CAS().TreePath(action.input_root_digest()); if (!path) { auto str = fmt::format("could not retrieve input root {} from cas", @@ -75,7 +75,7 @@ auto ExecutionServiceImpl::GetCommand(::bazel_re::Action const& action) const noexcept -> std::pair<std::optional<::bazel_re::Command>, std::optional<std::string>> { - auto path = storage_.BlobPath(action.command_digest(), false); + auto path = storage_->CAS().BlobPath(action.command_digest(), false); if (!path) { auto str = fmt::format("could not retrieve blob {} from cas", action.command_digest().hash()); @@ -141,10 +141,10 @@ auto ExecutionServiceImpl::GetIExecutionAction( } static auto GetDirectoryFromDigest(::bazel_re::Digest const& digest, - LocalStorage const& storage) noexcept + Storage const& storage) noexcept -> std::optional<::bazel_re::Directory> { // determine directory path from digest - auto const& path = storage.BlobPath(digest, /*is_executable=*/false); + auto const& path = storage.CAS().BlobPath(digest, /*is_executable=*/false); if (not path) { return std::nullopt; } @@ -166,7 +166,7 @@ static auto GetDirectoryFromDigest(::bazel_re::Digest const& digest, // NOLINTNEXTLINE(misc-no-recursion) static auto CollectChildDirectoriesRecursively( ::bazel_re::Directory const& root, - LocalStorage const& storage, + Storage const& storage, gsl::not_null<std::unordered_map<::bazel_re::Digest, ::bazel_re::Directory>*> map) noexcept -> bool { @@ -196,7 +196,7 @@ static auto CollectChildDirectoriesRecursively( } static auto GetChildrenFromDirectory(::bazel_re::Directory const& root, - LocalStorage const& storage) noexcept + Storage const& storage) noexcept -> std::optional<std::vector<::bazel_re::Directory>> { // determine child directories std::unordered_map<::bazel_re::Digest, ::bazel_re::Directory> map{}; @@ -232,7 +232,7 @@ static auto GetChildrenFromDirectory(::bazel_re::Directory const& root, static auto CreateTreeDigestFromDirectoryDigest( ::bazel_re::Digest const& dir_digest, - LocalStorage const& storage) noexcept -> std::optional<::bazel_re::Digest> { + Storage const& storage) noexcept -> std::optional<::bazel_re::Digest> { // determine root directory message auto root = GetDirectoryFromDigest(dir_digest, storage); if (not root) { @@ -256,7 +256,8 @@ static auto CreateTreeDigestFromDirectoryDigest( // serialize and store tree message auto content = tree.SerializeAsString(); - auto tree_digest = storage.StoreBlob(content, /*is_executable=*/false); + auto tree_digest = + storage.CAS().StoreBlob(content, /*is_executable=*/false); if (not tree_digest) { return std::nullopt; } @@ -266,7 +267,7 @@ static auto CreateTreeDigestFromDirectoryDigest( static auto AddOutputPaths(::bazel_re::ExecuteResponse* response, IExecutionResponse::Ptr const& execution, - LocalStorage const& storage) noexcept -> bool { + Storage const& storage) noexcept -> bool { auto const& size = static_cast<int>(execution->Artifacts().size()); response->mutable_result()->mutable_output_files()->Reserve(size); response->mutable_result()->mutable_output_directories()->Reserve(size); @@ -311,7 +312,7 @@ auto ExecutionServiceImpl::AddResult( IExecutionResponse::Ptr const& i_execution_response, std::string const& action_hash) const noexcept -> std::optional<std::string> { - if (not AddOutputPaths(response, i_execution_response, storage_)) { + if (not AddOutputPaths(response, i_execution_response, *storage_)) { auto str = fmt::format("Error in creating output paths of action {}", action_hash); logger_.Emit(LogLevel::Error, str); @@ -320,8 +321,8 @@ auto ExecutionServiceImpl::AddResult( auto* result = response->mutable_result(); result->set_exit_code(i_execution_response->ExitCode()); if (i_execution_response->HasStdErr()) { - auto dgst = storage_.StoreBlob(i_execution_response->StdErr(), - /*is_executable=*/false); + auto dgst = storage_->CAS().StoreBlob(i_execution_response->StdErr(), + /*is_executable=*/false); if (!dgst) { auto str = fmt::format("Could not store stderr of action {}", action_hash); @@ -331,8 +332,8 @@ auto ExecutionServiceImpl::AddResult( result->mutable_stderr_digest()->CopyFrom(*dgst); } if (i_execution_response->HasStdOut()) { - auto dgst = storage_.StoreBlob(i_execution_response->StdOut(), - /*is_executable=*/false); + auto dgst = storage_->CAS().StoreBlob(i_execution_response->StdOut(), + /*is_executable=*/false); if (!dgst) { auto str = fmt::format("Could not store stdout of action {}", action_hash); @@ -375,8 +376,8 @@ auto ExecutionServiceImpl::StoreActionResult( ::bazel_re::Action const& action) const noexcept -> std::optional<std::string> { if (i_execution_response->ExitCode() == 0 && !action.do_not_cache() && - !storage_.StoreActionResult(request->action_digest(), - execute_response.result())) { + !storage_->ActionCache().StoreResult(request->action_digest(), + execute_response.result())) { auto str = fmt::format("Could not store action result for action {}", request->action_digest().hash()); logger_.Emit(LogLevel::Error, str); diff --git a/src/buildtool/execution_api/execution_service/execution_server.hpp b/src/buildtool/execution_api/execution_service/execution_server.hpp index 65ace208..6d99fb6a 100644 --- a/src/buildtool/execution_api/execution_service/execution_server.hpp +++ b/src/buildtool/execution_api/execution_service/execution_server.hpp @@ -18,8 +18,8 @@ #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class ExecutionServiceImpl final : public bazel_re::Execution::Service { public: @@ -107,7 +107,7 @@ class ExecutionServiceImpl final : public bazel_re::Execution::Service { -> ::grpc::Status override; private: - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); IExecutionApi::Ptr api_{new LocalApi()}; Logger logger_{"execution-service"}; diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index 31f2c655..28377cc2 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -19,24 +19,17 @@ , "local": { "type": ["@", "rules", "CC", "library"] , "name": ["local"] - , "hdrs": - [ "local_api.hpp" - , "local_action.hpp" - , "local_response.hpp" - , "local_storage.hpp" - , "local_cas.hpp" - , "local_ac.hpp" - ] - , "srcs": ["local_action.cpp", "local_storage.cpp"] + , "hdrs": ["local_api.hpp", "local_action.hpp", "local_response.hpp"] + , "srcs": ["local_action.cpp"] , "deps": [ "config" - , "garbage_collector" , ["@", "fmt", "", "fmt"] , ["@", "gsl-lite", "", "gsl-lite"] + , ["src/buildtool/storage", "storage"] + , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] - , ["src/buildtool/file_system", "file_storage"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/common", "common"] @@ -50,28 +43,4 @@ , ["src/buildtool/system", "system_command"] ] } -, "garbage_collector": - { "type": ["@", "rules", "CC", "library"] - , "name": ["garbage_collector"] - , "hdrs": ["garbage_collector.hpp"] - , "srcs": ["garbage_collector.cpp"] - , "deps": [["src/utils/cpp", "file_locking"]] - , "stage": ["src", "buildtool", "execution_api", "local"] - , "private-deps": - [ "config" - , ["@", "json", "", "json"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/common", "bazel_types"] - , ["src/buildtool/compatibility", "compatibility"] - , ["src/buildtool/build_engine/target_map", "target_cache_entry"] - , ["src/buildtool/execution_api/common", "common"] - , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["src/buildtool/file_system", "git_repo"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - , ["src/utils/cpp", "hex_string"] - ] - } } diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp index 3faeb303..a1882776 100644 --- a/src/buildtool/execution_api/local/config.hpp +++ b/src/buildtool/execution_api/local/config.hpp @@ -15,26 +15,15 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP -#ifdef __unix__ -#include <pwd.h> -#include <sys/types.h> -#include <unistd.h> -#else -#error "Non-unix is not supported yet" -#endif - #include <filesystem> #include <functional> #include <string> #include <vector> -#include <fmt/core.h> #include <gsl-lite/gsl-lite.hpp> #include <nlohmann/json.hpp> #include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -43,59 +32,12 @@ /// \brief Store global build system configuration. class LocalExecutionConfig { struct ConfigData { - - // Build root directory. All the cache dirs are subdirs of build_root. - // By default, build_root is set to $HOME/.cache/just. - // If the user uses --local-build-root PATH, - // then build_root will be set to PATH. - std::filesystem::path build_root{}; - - // cache_root points to the root of the cache dirs. - std::filesystem::path cache_root{}; - - // cache_root_generations holds root directories for all cache - // generations (default: two generations). The latest generation points - // to one the following directories: - // build_root/protocol-dependent/generation-0/{git-sha1,compatible-sha256} - // git-sha1 is the current default. If the user passes the flag - // --compatible, then the subfolder compatible_sha256 is used - std::vector<std::filesystem::path> cache_root_generations{"", ""}; - // Launcher to be prepended to action's command before executed. // Default: ["env", "--"] std::vector<std::string> launcher{"env", "--"}; }; - // different folder for different caching protocol - [[nodiscard]] static auto UpdatePathForCompatibility( - std::filesystem::path const& dir) -> std::filesystem::path { - return dir / (Compatibility::IsCompatible() ? "compatible-sha256" - : "git-sha1"); - } - public: - [[nodiscard]] static auto SetBuildRoot( - std::filesystem::path const& dir) noexcept -> bool { - if (FileSystemManager::IsRelativePath(dir)) { - Logger::Log(LogLevel::Error, - "Build root must be absolute path but got '{}'.", - dir.string()); - return false; - } - Data().build_root = dir; - // In case we re-set build_root, we are sure that the cache roots are - // recomputed as well. - Data().cache_root = std::filesystem::path{}; - Data().cache_root_generations = - std::vector(NumGenerations(), std::filesystem::path{}); - // Pre-initialize cache roots to avoid race condition during lazy - // initialization by multiple threads. - for (int i = 0; i < NumGenerations(); ++i) { - [[maybe_unused]] auto root = CacheRoot(i); - } - return true; - } - [[nodiscard]] static auto SetLauncher( std::vector<std::string> const& launcher) noexcept -> bool { try { @@ -109,140 +51,11 @@ class LocalExecutionConfig { return true; } - /// \brief Specifies the number of cache generations. - static auto SetNumGenerations(int num_generations) noexcept -> void { - gsl_ExpectsAudit(num_generations > 0); - Data().cache_root_generations = - std::vector(num_generations, std::filesystem::path{}); - } - - [[nodiscard]] static auto NumGenerations() noexcept -> int { - return Data().cache_root_generations.size(); - } - - /// \brief User directory. - [[nodiscard]] static auto GetUserDir() noexcept -> std::filesystem::path { - return GetUserHome() / ".cache" / "just"; - } - - /// \brief Build directory, defaults to user directory if not set - [[nodiscard]] static auto BuildRoot() noexcept -> std::filesystem::path { - auto& build_root = Data().build_root; - if (build_root.empty()) { - build_root = GetUserDir(); - } - return build_root; - } - - [[nodiscard]] static auto CacheRoot() noexcept -> std::filesystem::path { - auto& cache_root = Data().cache_root; - if (cache_root.empty()) { - cache_root = BuildRoot() / "protocol-dependent"; - } - return cache_root; - } - - [[nodiscard]] static auto CacheRoot(int index) noexcept - -> std::filesystem::path { - gsl_ExpectsAudit(index >= 0 and - index < Data().cache_root_generations.size()); - auto& cache_root = Data().cache_root_generations[index]; - if (cache_root.empty()) { - auto generation = - std::string{"generation-"} + std::to_string(index); - cache_root = CacheRoot() / generation; - } - return cache_root; - } - - [[nodiscard]] static auto CacheRootDir(int index) noexcept - -> std::filesystem::path { - return UpdatePathForCompatibility(CacheRoot(index)); - } - - // CAS directory based on the type of the file. - template <ObjectType kType> - [[nodiscard]] static inline auto CASDir(int index) noexcept - -> std::filesystem::path { - char t = ToChar(kType); - if constexpr (kType == ObjectType::Tree) { - if (Compatibility::IsCompatible()) { - t = ToChar(ObjectType::File); - } - } - static auto const kSuffix = std::string{"cas"} + t; - return CacheRootDir(index) / kSuffix; - } - - /// \brief Action cache directory - [[nodiscard]] static auto ActionCacheDir(int index) noexcept - -> std::filesystem::path { - return CacheRootDir(index) / "ac"; - } - - /// \brief Target cache root directory - [[nodiscard]] static auto TargetCacheRoot(int index) noexcept - -> std::filesystem::path { - return CacheRootDir(index) / "tc"; - } - - /// \brief Target cache directory for the used execution backend. - [[nodiscard]] static auto TargetCacheDir(int index) noexcept - -> std::filesystem::path { - return TargetCacheRoot(index) / - ArtifactDigest::Create<ObjectType::File>( - ExecutionBackendDescription()) - .hash(); - } - - /// \brief String representation of the used execution backend. - [[nodiscard]] static auto ExecutionBackendDescription() noexcept - -> std::string { - auto address = RemoteExecutionConfig::RemoteAddress(); - auto properties = RemoteExecutionConfig::PlatformProperties(); - try { - // json::dump with json::error_handler_t::replace will not throw an - // exception if invalid UTF-8 sequences are detected in the input. - // Instead, it will replace them with the UTF-8 replacement - // character, but still it needs to be inside a try-catch clause to - // ensure the noexcept modifier of the enclosing function. - return nlohmann::json{ - {"remote_address", - address ? nlohmann::json{fmt::format( - "{}:{}", address->host, address->port)} - : nlohmann::json{}}, - {"platform_properties", properties}} - .dump(2, ' ', false, nlohmann::json::error_handler_t::replace); - } catch (...) { - return ""; - } - } - [[nodiscard]] static auto GetLauncher() noexcept -> std::vector<std::string> { return Data().launcher; } - /// \brief Determine user root directory - [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { - char const* root{nullptr}; - -#ifdef __unix__ - root = std::getenv("HOME"); - if (root == nullptr) { - root = getpwuid(getuid())->pw_dir; - } -#endif - - if (root == nullptr) { - Logger::Log(LogLevel::Error, - "Cannot determine user home directory."); - std::exit(EXIT_FAILURE); - } - - return root; - } - private: [[nodiscard]] static auto Data() noexcept -> ConfigData& { static ConfigData instance{}; diff --git a/src/buildtool/execution_api/local/garbage_collector.cpp b/src/buildtool/execution_api/local/garbage_collector.cpp deleted file mode 100644 index a0b68811..00000000 --- a/src/buildtool/execution_api/local/garbage_collector.cpp +++ /dev/null @@ -1,514 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/local/garbage_collector.hpp" - -#include <vector> - -#include <nlohmann/json.hpp> - -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/logging/log_level.hpp" -#include "src/buildtool/logging/logger.hpp" -#include "src/utils/cpp/hex_string.hpp" - -auto GarbageCollector::FindAndUplinkBlob(std::string const& id, - bool is_executable) noexcept -> bool { - // Try to find blob in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkBlob(i, id, is_executable)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkTree(std::string const& id) noexcept - -> bool { - // Try to find tree in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkTree(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkActionCacheEntry( - std::string const& id) noexcept -> bool { - // Try to find action-cache entry in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkActionCacheEntry(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkTargetCacheEntry( - std::string const& id) noexcept -> bool { - // Try to find target-cache entry in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkTargetCacheEntry(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::SharedLock() noexcept -> std::optional<LockFile> { - return LockFile::Acquire(LockFilePath(), /*is_shared=*/true); -} - -auto GarbageCollector::ExclusiveLock() noexcept -> std::optional<LockFile> { - return LockFile::Acquire(LockFilePath(), /*is_shared=*/false); -} - -auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path { - return LocalExecutionConfig::CacheRoot() / "gc.lock"; -} - -auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool { - auto pid = CreateProcessUniqueId(); - if (not pid) { - return false; - } - auto remove_me = std::string{"remove-me-"} + *pid; - auto remove_me_dir = LocalExecutionConfig::CacheRoot() / remove_me; - if (FileSystemManager::IsDirectory(remove_me_dir)) { - if (not FileSystemManager::RemoveDirectory(remove_me_dir, - /*recursively=*/true)) { - Logger::Log(LogLevel::Error, - "Failed to remove directory {}", - remove_me_dir.string()); - return false; - } - } - { // Create scope for critical renaming section protected by advisory lock. - auto lock = ExclusiveLock(); - if (not lock) { - Logger::Log(LogLevel::Error, - "Failed to exclusively lock the local build root"); - return false; - } - for (int i = LocalExecutionConfig::NumGenerations() - 1; i >= 0; i--) { - auto cache_root = LocalExecutionConfig::CacheRoot(i); - if (FileSystemManager::IsDirectory(cache_root)) { - auto new_cache_root = - (i == LocalExecutionConfig::NumGenerations() - 1) - ? remove_me_dir - : LocalExecutionConfig::CacheRoot(i + 1); - if (not FileSystemManager::Rename(cache_root, new_cache_root)) { - Logger::Log(LogLevel::Error, - "Failed to rename {} to {}.", - cache_root.string(), - new_cache_root.string()); - return false; - } - } - } - } - if (FileSystemManager::IsDirectory(remove_me_dir)) { - if (not FileSystemManager::RemoveDirectory(remove_me_dir, - /*recursively=*/true)) { - Logger::Log(LogLevel::Warning, - "Failed to remove directory {}", - remove_me_dir.string()); - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkBlob(int index, - std::string const& id, - bool is_executable) noexcept -> bool { - // Determine blob path of given generation. - auto root = - is_executable - ? LocalExecutionConfig::CASDir<ObjectType::Executable>(index) - : LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto blob_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(blob_path)) { - return false; - } - - // Determine blob path in latest generation. - auto root_latest = - is_executable ? LocalExecutionConfig::CASDir<ObjectType::Executable>(0) - : LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto blob_path_latest = GetStoragePath(root_latest, id); - if (not FileSystemManager::IsFile(blob_path_latest)) { - // Uplink blob from older generation to the latest generation. - if (not FileSystemManager::CreateDirectory( - blob_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - blob_path, - blob_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(blob_path_latest)) { - return false; - } - } - return true; -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto GarbageCollector::UplinkTree(int index, std::string const& id) noexcept - -> bool { - // Determine tree path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::Tree>(index); - auto tree_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(tree_path)) { - return false; - } - - // Determine tree path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::Tree>(0); - auto tree_path_latest = GetStoragePath(root_latest, id); - if (not FileSystemManager::IsFile(tree_path_latest)) { - // Determine tree entries. - auto content = FileSystemManager::ReadFile(tree_path); - auto tree_entries = GitRepo::ReadTreeData(*content, - id, - /*is_hex_id=*/true); - if (not tree_entries) { - return false; - } - - // Uplink tree entries. - for (auto const& [raw_id, entry_vector] : *tree_entries) { - // Process only first entry from 'entry_vector' since all entries - // represent the same blob, just with different names. - auto entry = entry_vector.front(); - auto hash = ToHexString(raw_id); - if (entry.type == ObjectType::Tree) { - if (not UplinkTree(index, hash)) { - return false; - } - } - else { - if (not UplinkBlob( - index, hash, entry.type == ObjectType::Executable)) { - return false; - } - } - } - - // Uplink tree from older generation to the latest generation. - if (not FileSystemManager::CreateDirectory( - tree_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - tree_path, - tree_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(tree_path_latest)) { - return false; - } - } - return true; -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto GarbageCollector::UplinkBazelDirectory(int index, - std::string const& id) noexcept - -> bool { - // Determine bazel directory path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto dir_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(dir_path)) { - return false; - } - - // Determine bazel directory entries. - auto content = FileSystemManager::ReadFile(dir_path); - bazel_re::Directory dir{}; - if (not dir.ParseFromString(*content)) { - return false; - } - - // Uplink bazel directory entries. - for (auto const& file : dir.files()) { - if (not UplinkBlob(index, - NativeSupport::Unprefix(file.digest().hash()), - file.is_executable())) { - return false; - } - } - for (auto const& directory : dir.directories()) { - if (not UplinkBazelDirectory( - index, NativeSupport::Unprefix(directory.digest().hash()))) { - return false; - } - } - - // Determine bazel directory path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto dir_path_latest = GetStoragePath(root_latest, id); - - // Uplink bazel directory from older generation to the latest generation. - if (not FileSystemManager::IsFile(dir_path_latest)) { - if (not FileSystemManager::CreateDirectory( - dir_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - dir_path, - dir_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(dir_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkActionCacheEntry(int index, - std::string const& id) noexcept - -> bool { - // Determine action-cache entry path of given generation. - auto root = LocalExecutionConfig::ActionCacheDir(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine action-cache entry location. - auto content = FileSystemManager::ReadFile(entry_path, ObjectType::File); - bazel_re::Digest digest{}; - if (not digest.ParseFromString(*content)) { - return false; - } - - // Uplink action-cache entry blob. - if (not UplinkActionCacheEntryBlob( - index, NativeSupport::Unprefix(digest.hash()))) { - return false; - } - - // Determine action-cache entry path in latest generation. - auto root_latest = LocalExecutionConfig::ActionCacheDir(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink action-cache entry from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkActionCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool { - - // Determine action-cache entry blob path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine artifacts referenced by action-cache entry. - auto content = FileSystemManager::ReadFile(entry_path); - bazel_re::ActionResult result{}; - if (not result.ParseFromString(*content)) { - return false; - } - - // Uplink referenced artifacts. - for (auto const& file : result.output_files()) { - if (not UplinkBlob(index, - NativeSupport::Unprefix(file.digest().hash()), - file.is_executable())) { - return false; - } - } - for (auto const& directory : result.output_directories()) { - if (Compatibility::IsCompatible()) { - if (not UplinkBazelDirectory( - index, - NativeSupport::Unprefix(directory.tree_digest().hash()))) { - return false; - } - } - else { - if (not UplinkTree( - index, - NativeSupport::Unprefix(directory.tree_digest().hash()))) { - return false; - } - } - } - - // Determine action-cache entry blob path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink action-cache entry blob from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkTargetCacheEntry(int index, - std::string const& id) noexcept - -> bool { - - // Determine target-cache entry path of given generation. - auto root = LocalExecutionConfig::TargetCacheDir(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine target-cache entry location. - auto content = FileSystemManager::ReadFile(entry_path); - auto info = Artifact::ObjectInfo::FromString(*content); - if (not info) { - return false; - } - - // Uplink target-cache entry blob. - if (not UplinkTargetCacheEntryBlob(index, info->digest.hash())) { - return false; - } - - // Determine target-cache entry path in latest generation. - auto root_latest = LocalExecutionConfig::TargetCacheDir(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink target-cache entry from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkTargetCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool { - - // Determine target-cache entry blob path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine artifacts referenced by target-cache entry. - auto content = FileSystemManager::ReadFile(entry_path); - nlohmann::json json_desc{}; - try { - json_desc = nlohmann::json::parse(*content); - } catch (std::exception const& ex) { - return false; - } - auto entry = TargetCacheEntry::FromJson(json_desc); - std::vector<Artifact::ObjectInfo> artifacts_info; - if (not entry.ToArtifacts(&artifacts_info)) { - return false; - } - - // Uplink referenced artifacts. - for (auto const& info : artifacts_info) { - auto hash = info.digest.hash(); - if (info.type == ObjectType::Tree) { - if (Compatibility::IsCompatible()) { - if (not UplinkBazelDirectory(index, hash)) { - return false; - } - } - else { - if (not UplinkTree(index, hash)) { - return false; - } - } - } - else { - if (not UplinkBlob( - index, hash, info.type == ObjectType::Executable)) { - return false; - } - } - } - - // Determine target-cache entry blob path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink target-cache entry blob from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} diff --git a/src/buildtool/execution_api/local/garbage_collector.hpp b/src/buildtool/execution_api/local/garbage_collector.hpp deleted file mode 100644 index 6b89c841..00000000 --- a/src/buildtool/execution_api/local/garbage_collector.hpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP - -#include <optional> -#include <string> - -#include "src/utils/cpp/file_locking.hpp" - -/// \brief -class GarbageCollector { - public: - [[nodiscard]] auto static FindAndUplinkBlob(std::string const& id, - bool is_executable) noexcept - -> bool; - - [[nodiscard]] auto static FindAndUplinkTree(std::string const& id) noexcept - -> bool; - - [[nodiscard]] auto static FindAndUplinkActionCacheEntry( - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static FindAndUplinkTargetCacheEntry( - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool; - - [[nodiscard]] auto static SharedLock() noexcept -> std::optional<LockFile>; - - [[nodiscard]] auto static ExclusiveLock() noexcept - -> std::optional<LockFile>; - - private: - [[nodiscard]] auto static LockFilePath() noexcept -> std::filesystem::path; - - [[nodiscard]] auto static UplinkBlob(int index, - std::string const& id, - bool is_executable) noexcept -> bool; - - [[nodiscard]] auto static UplinkTree(int index, - std::string const& id) noexcept - -> bool; - - [[nodiscard]] auto static UplinkBazelDirectory( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkActionCacheEntry( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkActionCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkTargetCacheEntry( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkTargetCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP diff --git a/src/buildtool/execution_api/local/local_ac.hpp b/src/buildtool/execution_api/local/local_ac.hpp deleted file mode 100644 index 423282be..00000000 --- a/src/buildtool/execution_api/local/local_ac.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP - -#include "gsl-lite/gsl-lite.hpp" -#include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/logger.hpp" - -class LocalAC { - public: - explicit LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas) noexcept - : cas_{std::move(cas)} {}; - - LocalAC(LocalAC const&) = delete; - LocalAC(LocalAC&&) = delete; - auto operator=(LocalAC const&) -> LocalAC& = delete; - auto operator=(LocalAC&&) -> LocalAC& = delete; - ~LocalAC() noexcept = default; - - [[nodiscard]] auto StoreResult( - bazel_re::Digest const& action_id, - bazel_re::ActionResult const& result) const noexcept -> bool { - auto bytes = result.SerializeAsString(); - auto digest = cas_->StoreBlobFromBytes(bytes); - return (digest and file_store_.AddFromBytes( - NativeSupport::Unprefix(action_id.hash()), - digest->SerializeAsString())); - } - - [[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id) - const noexcept -> std::optional<bazel_re::ActionResult> { - auto id = NativeSupport::Unprefix(action_id.hash()); - auto entry_path = file_store_.GetPath(id); - // Try to find action-cache entry in CAS generations and uplink if - // required. - if (not GarbageCollector::FindAndUplinkActionCacheEntry(id)) { - logger_.Emit(LogLevel::Debug, - "Cache miss, entry not found {}", - entry_path.string()); - return std::nullopt; - } - auto const entry = - FileSystemManager::ReadFile(entry_path, ObjectType::File); - bazel_re::Digest digest{}; - if (not digest.ParseFromString(*entry)) { - logger_.Emit(LogLevel::Warning, - "Parsing cache entry failed for action {}", - id); - return std::nullopt; - } - auto src_path = cas_->BlobPath(digest); - bazel_re::ActionResult result{}; - if (src_path) { - auto const bytes = FileSystemManager::ReadFile(*src_path); - if (bytes.has_value() and result.ParseFromString(*bytes)) { - return result; - } - } - logger_.Emit(LogLevel::Warning, - "Parsing action result failed for action {}", - id); - return std::nullopt; - } - - private: - // The action cache stores the results of failed actions. For those to be - // overwritable by subsequent runs we need to choose the store mode "last - // wins" for the underlying file storage. - static constexpr auto kStoreMode = StoreMode::LastWins; - - Logger logger_{"LocalAC"}; - gsl::not_null<LocalCAS<ObjectType::File>*> cas_; - FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false> - file_store_{LocalExecutionConfig::ActionCacheDir(0)}; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index 414773c2..209c6a5d 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -20,9 +20,11 @@ #include "gsl-lite/gsl-lite.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/system/system_command.hpp" namespace { @@ -48,6 +50,24 @@ class BuildCleanupAnchor { std::filesystem::path const build_path{}; }; +[[nodiscard]] auto CreateDigestFromLocalOwnedTree( + gsl::not_null<Storage const*> const& storage, + std::filesystem::path const& dir_path) -> std::optional<bazel_re::Digest> { + auto const& cas = storage->CAS(); + auto store_blob = [&cas](auto path, auto is_exec) { + return cas.StoreBlob</*kOwner=*/true>(path, is_exec); + }; + auto store_tree = [&cas](auto bytes, + auto /*dir*/) -> std::optional<bazel_re::Digest> { + return cas.StoreTree(bytes); + }; + return Compatibility::IsCompatible() + ? BazelMsgFactory::CreateDirectoryDigestFromLocalTree( + dir_path, store_blob, store_tree) + : BazelMsgFactory::CreateGitTreeDigestFromLocalTree( + dir_path, store_blob, store_tree); +} + } // namespace auto LocalAction::Execute(Logger const* logger) noexcept @@ -65,7 +85,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept } if (do_cache) { - if (auto result = storage_->CachedActionResult(action)) { + if (auto result = storage_->ActionCache().CachedResult(action)) { if (result->exit_code() == 0) { return IExecutionResponse::Ptr{ new LocalResponse{action.hash(), @@ -95,7 +115,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept -> std::optional<Output> { auto exec_path = - CreateUniquePath(LocalExecutionConfig::CacheRootDir(0) / "exec_root" / + CreateUniquePath(StorageConfig::BuildRoot() / "exec_root" / NativeSupport::Unprefix(action_id.hash())); if (not exec_path) { @@ -135,7 +155,8 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept if (CollectAndStoreOutputs(&result.action, build_root)) { if (cache_flag_ == CacheFlag::CacheOutput) { - if (not storage_->StoreActionResult(action_id, result.action)) { + if (not storage_->ActionCache().StoreResult(action_id, + result.action)) { logger_.Emit(LogLevel::Warning, "failed to store action results"); } @@ -152,7 +173,7 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept auto LocalAction::StageFile(std::filesystem::path const& target_path, Artifact::ObjectInfo const& info) const -> bool { auto blob_path = - storage_->BlobPath(info.digest, IsExecutableObject(info.type)); + storage_->CAS().BlobPath(info.digest, IsExecutableObject(info.type)); if (not blob_path) { logger_.Emit(LogLevel::Error, @@ -171,7 +192,8 @@ auto LocalAction::StageInputFiles( return false; } - auto infos = storage_->RecursivelyReadTreeLeafs(root_digest_, exec_path); + auto infos = + storage_->CAS().RecursivelyReadTreeLeafs(root_digest_, exec_path); if (not infos) { return false; } @@ -236,7 +258,7 @@ auto LocalAction::CollectOutputFile(std::filesystem::path const& exec_path, } bool is_executable = IsExecutableObject(*type); auto digest = - storage_->StoreBlob</*kOwner=*/true>(file_path, is_executable); + storage_->CAS().StoreBlob</*kOwner=*/true>(file_path, is_executable); if (digest) { auto out_file = bazel_re::OutputFile{}; out_file.set_path(local_path); @@ -257,30 +279,8 @@ auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path, Logger::Log(LogLevel::Error, "expected directory at {}", local_path); return std::nullopt; } - std::optional<bazel_re::Digest> digest{std::nullopt}; - if (Compatibility::IsCompatible()) { - digest = BazelMsgFactory::CreateDirectoryDigestFromLocalTree( - dir_path, - [this](auto path, auto is_exec) { - return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); - }, - [this](auto bytes, - auto /*dir*/) -> std::optional<bazel_re::Digest> { - return storage_->StoreBlob(bytes); - }); - } - else { - digest = BazelMsgFactory::CreateGitTreeDigestFromLocalTree( - dir_path, - [this](auto path, auto is_exec) { - return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); - }, - [this](auto bytes, - auto /*entries*/) -> std::optional<bazel_re::Digest> { - return storage_->StoreTree(bytes); - }); - } - if (digest) { + + if (auto digest = CreateDigestFromLocalOwnedTree(storage_, dir_path)) { auto out_dir = bazel_re::OutputDirectory{}; out_dir.set_path(local_path); out_dir.set_allocated_tree_digest( @@ -322,7 +322,8 @@ auto LocalAction::CollectAndStoreOutputs( auto LocalAction::DigestFromOwnedFile(std::filesystem::path const& file_path) const noexcept -> gsl::owner<bazel_re::Digest*> { - if (auto digest = storage_->StoreBlob</*kOwner=*/true>(file_path)) { + if (auto digest = storage_->CAS().StoreBlob</*kOwner=*/true>( + file_path, /*is_executable=*/false)) { return new bazel_re::Digest{std::move(*digest)}; } return nullptr; diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp index fe908360..4fba44ba 100644 --- a/src/buildtool/execution_api/local/local_action.hpp +++ b/src/buildtool/execution_api/local/local_action.hpp @@ -24,8 +24,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" +#include "src/buildtool/storage/storage.hpp" class LocalApi; @@ -50,7 +49,7 @@ class LocalAction final : public IExecutionAction { private: Logger logger_{"LocalExecution"}; - std::shared_ptr<LocalStorage> storage_; + gsl::not_null<Storage const*> storage_; ArtifactDigest root_digest_{}; std::vector<std::string> cmdline_{}; std::vector<std::string> output_files_{}; @@ -60,7 +59,7 @@ class LocalAction final : public IExecutionAction { std::chrono::milliseconds timeout_{kDefaultTimeout}; CacheFlag cache_flag_{CacheFlag::CacheOutput}; - LocalAction(std::shared_ptr<LocalStorage> storage, + LocalAction(gsl::not_null<Storage const*> storage, ArtifactDigest root_digest, std::vector<std::string> command, std::vector<std::string> output_files, diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 1528ff23..dbfb3504 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -30,8 +30,8 @@ #include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief API for local execution. class LocalApi final : public IExecutionApi { @@ -68,7 +68,7 @@ class LocalApi final : public IExecutionApi { auto const& info = artifacts_info[i]; if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively - auto const infos = storage_->RecursivelyReadTreeLeafs( + auto const infos = storage_->CAS().RecursivelyReadTreeLeafs( info.digest, output_paths[i]); if (not infos or not RetrieveToPaths(infos->second, infos->first)) { @@ -76,7 +76,7 @@ class LocalApi final : public IExecutionApi { } } else { - auto const blob_path = storage_->BlobPath( + auto const blob_path = storage_->CAS().BlobPath( info.digest, IsExecutableObject(info.type)); if (not blob_path or not FileSystemManager::CreateDirectory( @@ -107,7 +107,7 @@ class LocalApi final : public IExecutionApi { if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT auto const success = - storage_->DumpToStream(info, out, raw_tree); + storage_->CAS().DumpToStream(info, out, raw_tree); std::fclose(out); if (not success) { Logger::Log(LogLevel::Error, @@ -159,7 +159,7 @@ class LocalApi final : public IExecutionApi { for (auto const& info : missing_artifacts_info) { // Recursively process trees. if (IsTreeObject(info.type)) { - auto const& infos = storage_->ReadDirectTreeEntries( + auto const& infos = storage_->CAS().ReadDirectTreeEntries( info.digest, std::filesystem::path{}); if (not infos or not RetrieveToCas(infos->second, api)) { return false; @@ -169,9 +169,9 @@ class LocalApi final : public IExecutionApi { // Determine artifact path. auto const& path = IsTreeObject(info.type) - ? storage_->TreePath(info.digest) - : storage_->BlobPath(info.digest, - IsExecutableObject(info.type)); + ? storage_->CAS().TreePath(info.digest) + : storage_->CAS().BlobPath(info.digest, + IsExecutableObject(info.type)); if (not path) { return false; } @@ -213,8 +213,8 @@ class LocalApi final : public IExecutionApi { for (auto const& blob : blobs) { auto const is_tree = NativeSupport::IsTree(blob.digest.hash()); auto cas_digest = - is_tree ? storage_->StoreTree(blob.data) - : storage_->StoreBlob(blob.data, blob.is_exec); + is_tree ? storage_->CAS().StoreTree(blob.data) + : storage_->CAS().StoreBlob(blob.data, blob.is_exec); if (not cas_digest or not std::equal_to<bazel_re::Digest>{}( *cas_digest, blob.digest)) { return false; @@ -331,8 +331,8 @@ class LocalApi final : public IExecutionApi { -> bool final { return static_cast<bool>( NativeSupport::IsTree(static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->TreePath(digest) - : storage_->BlobPath(digest, false)); + ? storage_->CAS().TreePath(digest) + : storage_->CAS().BlobPath(digest, false)); } [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests) @@ -341,8 +341,8 @@ class LocalApi final : public IExecutionApi { for (auto const& digest : digests) { auto const& path = NativeSupport::IsTree( static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->TreePath(digest) - : storage_->BlobPath(digest, false); + ? storage_->CAS().TreePath(digest) + : storage_->CAS().BlobPath(digest, false); if (not path) { result.push_back(digest); } @@ -351,7 +351,7 @@ class LocalApi final : public IExecutionApi { } private: - std::shared_ptr<LocalStorage> storage_{std::make_shared<LocalStorage>()}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP diff --git a/src/buildtool/execution_api/local/local_response.hpp b/src/buildtool/execution_api/local/local_response.hpp index e8ffc234..93b12009 100644 --- a/src/buildtool/execution_api/local/local_response.hpp +++ b/src/buildtool/execution_api/local/local_response.hpp @@ -17,8 +17,8 @@ #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief Response of a LocalAction. class LocalResponse final : public IExecutionResponse { @@ -35,8 +35,8 @@ class LocalResponse final : public IExecutionResponse { return (output_.action.stdout_digest().size_bytes() != 0); } auto StdErr() noexcept -> std::string final { - if (auto path = storage_->BlobPath(output_.action.stderr_digest(), - /*is_executable=*/false)) { + if (auto path = storage_->CAS().BlobPath(output_.action.stderr_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -45,8 +45,8 @@ class LocalResponse final : public IExecutionResponse { return {}; } auto StdOut() noexcept -> std::string final { - if (auto path = storage_->BlobPath(output_.action.stdout_digest(), - /*is_executable=*/false)) { + if (auto path = storage_->CAS().BlobPath(output_.action.stdout_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -103,12 +103,11 @@ class LocalResponse final : public IExecutionResponse { private: std::string action_id_{}; LocalAction::Output output_{}; - gsl::not_null<std::shared_ptr<LocalStorage>> storage_; + gsl::not_null<Storage const*> storage_; - explicit LocalResponse( - std::string action_id, - LocalAction::Output output, - gsl::not_null<std::shared_ptr<LocalStorage>> storage) noexcept + explicit LocalResponse(std::string action_id, + LocalAction::Output output, + gsl::not_null<Storage const*> storage) noexcept : action_id_{std::move(action_id)}, output_{std::move(output)}, storage_{std::move(storage)} {} diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp deleted file mode 100644 index eaeaa617..00000000 --- a/src/buildtool/execution_api/local/local_storage.cpp +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/local/local_api.hpp" - -namespace { - -[[nodiscard]] auto ReadDirectory( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& digest) noexcept - -> std::optional<bazel_re::Directory> { - if (auto const path = storage->BlobPath(digest, /*is_executable=*/false)) { - if (auto const content = FileSystemManager::ReadFile(*path)) { - return BazelMsgFactory::MessageFromString<bazel_re::Directory>( - *content); - } - } - Logger::Log(LogLevel::Error, - "Directory {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto ReadGitTree( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& digest) noexcept - -> std::optional<GitRepo::tree_entries_t> { - if (auto const path = storage->TreePath(digest)) { - if (auto const content = FileSystemManager::ReadFile(*path)) { - return GitRepo::ReadTreeData( - *content, - HashFunction::ComputeTreeHash(*content).Bytes(), - /*is_hex_id=*/false); - } - } - Logger::Log(LogLevel::Error, - "Tree {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto DumpToStream(gsl::not_null<FILE*> const& stream, - std::optional<std::string> const& data) noexcept - -> bool { - if (data) { - std::fwrite(data->data(), 1, data->size(), stream); - return true; - } - return false; -} - -[[nodiscard]] auto TreeToStream( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& tree_digest, - gsl::not_null<FILE*> const& stream) noexcept -> bool { - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(storage, tree_digest)) { - return DumpToStream(stream, - BazelMsgFactory::DirectoryToString(*dir)); - } - } - else { - if (auto entries = ReadGitTree(storage, tree_digest)) { - return DumpToStream(stream, - BazelMsgFactory::GitTreeToString(*entries)); - } - } - return false; -} - -[[nodiscard]] auto BlobToStream( - gsl::not_null<LocalStorage const*> const& storage, - Artifact::ObjectInfo const& blob_info, - gsl::not_null<FILE*> const& stream) noexcept -> bool { - constexpr std::size_t kChunkSize{512}; - auto path = - storage->BlobPath(blob_info.digest, IsExecutableObject(blob_info.type)); - if (not path and not Compatibility::IsCompatible()) { - // in native mode, lookup object in tree cas to dump tree as blob - path = storage->TreePath(blob_info.digest); - } - if (path) { - std::string data(kChunkSize, '\0'); - if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) { - while (auto size = std::fread(data.data(), 1, kChunkSize, in)) { - std::fwrite(data.data(), 1, size, stream); - } - std::fclose(in); - return true; - } - } - return false; -} - -} // namespace - -auto LocalStorage::RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (ReadObjectInfosRecursively(store_info, parent, tree_digest)) { - return std::make_pair(std::move(paths), std::move(infos)); - } - return std::nullopt; -} - -auto LocalStorage::ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - else { - if (auto entries = ReadGitTree(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - return std::make_pair(std::move(paths), std::move(infos)); -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto LocalStorage::ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool { - // read from CAS - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, [this, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, parent / path, info.digest) - : store_info(parent / path, info); - }); - } - } - else { - if (auto entries = ReadGitTree(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, [this, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, parent / path, info.digest) - : store_info(parent / path, info); - }); - } - } - return false; -} - -auto LocalStorage::DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool { - return IsTreeObject(info.type) and not raw_tree - ? TreeToStream(this, info.digest, stream) - : BlobToStream(this, info, stream); -} diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp deleted file mode 100644 index dc9316f8..00000000 --- a/src/buildtool/execution_api/local/local_storage.hpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP - -#include <optional> - -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/local_ac.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" - -class LocalStorage { - public: - /// \brief Store blob from file path with x-bit determined from file system. - template <bool kOwner = false> - [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path) - const noexcept -> std::optional<bazel_re::Digest> { - return StoreBlob<kOwner>(file_path, - FileSystemManager::IsExecutable(file_path)); - } - - /// \brief Store blob from file path with x-bit. - template <bool kOwner = false> - [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path, - bool is_executable) const noexcept - -> std::optional<bazel_re::Digest> { - if (is_executable) { - return cas_exec_.StoreBlobFromFile(file_path, kOwner); - } - return cas_file_.StoreBlobFromFile(file_path, kOwner); - } - - /// \brief Store blob from bytes with x-bit (default: non-executable). - [[nodiscard]] auto StoreBlob(std::string const& bytes, - bool is_executable = false) const noexcept - -> std::optional<bazel_re::Digest> { - return is_executable ? cas_exec_.StoreBlobFromBytes(bytes) - : cas_file_.StoreBlobFromBytes(bytes); - } - - [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept - -> std::optional<bazel_re::Digest> { - return cas_tree_.StoreBlobFromBytes(bytes); - } - - [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) - const noexcept -> std::optional<bazel_re::Digest> { - return cas_tree_.StoreBlobFromFile(file_path); - } - - /// \brief Obtain blob path from digest with x-bit. - /// NOLINTNEXTLINE(misc-no-recursion) - [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest, - bool is_executable) const noexcept - -> std::optional<std::filesystem::path> { - auto const path = is_executable ? cas_exec_.BlobPath(digest) - : cas_file_.BlobPath(digest); - return path ? path : TrySyncBlob(digest, is_executable); - } - - [[nodiscard]] auto TreePath(bazel_re::Digest const& digest) const noexcept - -> std::optional<std::filesystem::path> { - return cas_tree_.BlobPath(digest); - } - - [[nodiscard]] auto StoreActionResult( - bazel_re::Digest const& action_id, - bazel_re::ActionResult const& result) const noexcept -> bool { - return ac_.StoreResult(action_id, result); - } - - [[nodiscard]] auto CachedActionResult(bazel_re::Digest const& action_id) - const noexcept -> std::optional<bazel_re::ActionResult> { - return ac_.CachedResult(action_id); - } - - /// \brief Traverses a tree recursively and retrieves object infos of all - /// found blobs (leafs). Tree objects are not added to the result list, but - /// converted to a path name. - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - - /// \brief Reads the flat content of a tree and returns object infos of all - /// its direct entries (trees and blobs). - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - - [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool; - - private: - LocalCAS<ObjectType::File> cas_file_{}; - LocalCAS<ObjectType::Executable> cas_exec_{}; - LocalCAS<ObjectType::Tree> cas_tree_{}; - LocalAC ac_{&cas_file_}; - - /// \brief Try to sync blob between file CAS and executable CAS. - /// \param digest Blob digest. - /// \param to_executable Sync direction. - /// \returns Path to blob in target CAS. - /// NOLINTNEXTLINE(misc-no-recursion) - [[nodiscard]] auto TrySyncBlob(bazel_re::Digest const& digest, - bool to_executable) const noexcept - -> std::optional<std::filesystem::path> { - std::optional<std::filesystem::path> const src_blob{ - to_executable ? cas_file_.BlobPath(digest) - : cas_exec_.BlobPath(digest)}; - if (src_blob and StoreBlob(*src_blob, to_executable)) { - return BlobPath(digest, to_executable); - } - return std::nullopt; - } - - [[nodiscard]] auto ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP diff --git a/src/buildtool/file_system/TARGETS b/src/buildtool/file_system/TARGETS index 323f6fdc..38df9c2b 100644 --- a/src/buildtool/file_system/TARGETS +++ b/src/buildtool/file_system/TARGETS @@ -15,6 +15,17 @@ ] , "stage": ["src", "buildtool", "file_system"] } +, "object_cas": + { "type": ["@", "rules", "CC", "library"] + , "name": ["object_cas"] + , "hdrs": ["object_cas.hpp"] + , "deps": + [ "file_storage" + , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/file_system", "file_system_manager"] + ] + , "stage": ["src", "buildtool", "file_system"] + } , "file_system_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["file_system_manager"] diff --git a/src/buildtool/file_system/file_storage.hpp b/src/buildtool/file_system/file_storage.hpp index c8b30005..922cbc54 100644 --- a/src/buildtool/file_system/file_storage.hpp +++ b/src/buildtool/file_system/file_storage.hpp @@ -21,20 +21,6 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" -/// \brief Determines the storage path of a blob identified by a hash value. The -/// same sharding technique as used in git is applied, meaning, the hash value -/// is separated into a directory part and file part. Two characters are used -/// for the directory part, the rest for the file, which results in 256 possible -/// directories. -/// \param root The root path of the storage. -/// \param id The hash value of the blob. -/// \returns The sharded file path. -[[nodiscard]] static inline auto GetStoragePath( - std::filesystem::path const& root, - std::string const& id) noexcept -> std::filesystem::path { - return root / id.substr(0, 2) / id.substr(2, id.size() - 2); -} - enum class StoreMode { // First thread to write conflicting file wins. FirstWins, @@ -45,7 +31,11 @@ enum class StoreMode { LastWins }; -template <ObjectType kType, StoreMode kMode, bool kSetEpochTime> +template <ObjectType kType, + StoreMode kMode, + bool kSetEpochTime, + class = std::enable_if_t<kType == ObjectType::File or + kType == ObjectType::Executable>> class FileStorage { public: explicit FileStorage(std::filesystem::path storage_root) noexcept @@ -68,9 +58,16 @@ class FileStorage { return AtomicAddFromBytes(id, bytes); } + /// \brief Determines the storage path of a blob identified by a hash value. + /// The same sharding technique as used in git is applied, meaning, the hash + /// value is separated into a directory part and file part. Two characters + /// are used for the directory part, the rest for the file, which results in + /// 256 possible directories. + /// \param id The hash value of the blob. + /// \returns The sharded file path. [[nodiscard]] auto GetPath(std::string const& id) const noexcept -> std::filesystem::path { - return GetStoragePath(storage_root_, id); + return storage_root_ / id.substr(0, 2) / id.substr(2, id.size() - 2); } private: diff --git a/src/buildtool/execution_api/local/local_cas.hpp b/src/buildtool/file_system/object_cas.hpp index afda71ed..2af59bb9 100644 --- a/src/buildtool/execution_api/local/local_cas.hpp +++ b/src/buildtool/file_system/object_cas.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,68 +12,82 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP +#include <memory> #include <sstream> #include <thread> #include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/logger.hpp" -#ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#endif -template <ObjectType kType = ObjectType::File> -class LocalCAS { +/// \brief CAS for storing objects as plain blobs. +/// Automatically computes the digest for storing a blob from file path/bytes. +/// The actual object type is given as a template parameter. Depending on the +/// object type, files written to the file system may have different properties +/// (e.g., the x-bit set) or the digest may be computed differently (e.g., tree +/// digests in non-compatible mode). Supports custom "exists callback", which +/// is used to check blob existence before every read and write operation. +/// \tparam kType The object type to store as blob. +template <ObjectType kType> +class ObjectCAS { public: - LocalCAS() noexcept = default; - - LocalCAS(LocalCAS const&) = delete; - LocalCAS(LocalCAS&&) = delete; - auto operator=(LocalCAS const&) -> LocalCAS& = delete; - auto operator=(LocalCAS&&) -> LocalCAS& = delete; - ~LocalCAS() noexcept = default; - - [[nodiscard]] static auto Instance() noexcept -> LocalCAS<kType>& { - static auto instance = LocalCAS<kType>{}; - return instance; - } - - auto Reset() noexcept -> void { - file_store_ = FileStorage<kStorageType, - StoreMode::FirstWins, - /*kSetEpochTime=*/true>{ - LocalExecutionConfig::CASDir<kType>(0)}; - } - + /// \brief Callback type for checking blob existence. + /// \returns true if a blob for the given digest exists at the given path. + using ExistsFunc = std::function<bool(bazel_re::Digest const&, + std::filesystem::path const&)>; + + /// Default callback for checking blob existence. + static inline ExistsFunc const kDefaultExists = [](auto /*digest*/, + auto path) { + return FileSystemManager::IsFile(path); + }; + + /// \brief Create new object CAS in store_path directory. + /// The optional "exists callback" is used to check blob existence before + /// every read and write operation. It promises that a blob for the given + /// digest exists at the given path if true was returned. + /// \param store_path The path to use for storing blobs. + /// \param exists (optional) Function for checking blob existence. + explicit ObjectCAS(std::filesystem::path const& store_path, + ExistsFunc exists = kDefaultExists) + : file_store_{store_path}, exists_{std::move(exists)} {} + + ObjectCAS(ObjectCAS const&) = delete; + ObjectCAS(ObjectCAS&&) = delete; + auto operator=(ObjectCAS const&) -> ObjectCAS& = delete; + auto operator=(ObjectCAS&&) -> ObjectCAS& = delete; + ~ObjectCAS() noexcept = default; + + /// \brief Store blob from bytes. + /// \param bytes The bytes do create the blob from. + /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromBytes(std::string const& bytes) const noexcept -> std::optional<bazel_re::Digest> { return StoreBlob(bytes, /*is_owner=*/true); } + /// \brief Store blob from file path. + /// \param file_path The path of the file to store as blob. + /// \param is_owner Indicates ownership for optimization (hardlink). + /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromFile(std::filesystem::path const& file_path, bool is_owner = false) const noexcept -> std::optional<bazel_re::Digest> { return StoreBlob(file_path, is_owner); } + /// \brief Get path to blob. + /// \param digest Digest of the blob to lookup. + /// \returns Path to blob if found or nullopt otherwise. [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest) const noexcept -> std::optional<std::filesystem::path> { auto id = NativeSupport::Unprefix(digest.hash()); auto blob_path = file_store_.GetPath(id); -#ifndef BOOTSTRAP_BUILD_TOOL - // Try to find blob in CAS generations and uplink if required. - auto found = FindAndUplinkBlob(id); -#else - auto found = FileSystemManager::IsFile(blob_path); -#endif - if (not found) { + if (not IsAvailable(digest, blob_path)) { logger_.Emit(LogLevel::Debug, "Blob not found {}", id); return std::nullopt; } @@ -85,10 +99,11 @@ class LocalCAS { static constexpr auto kStorageType = kType == ObjectType::Tree ? ObjectType::File : kType; - Logger logger_{std::string{"LocalCAS"} + ToChar(kType)}; + Logger logger_{std::string{"ObjectCAS"} + ToChar(kType)}; FileStorage<kStorageType, StoreMode::FirstWins, /*kSetEpochTime=*/true> - file_store_{LocalExecutionConfig::CASDir<kType>(0)}; + file_store_; + ExistsFunc exists_; [[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept -> std::optional<bazel_re::Digest> { @@ -105,6 +120,16 @@ class LocalCAS { return std::nullopt; } + [[nodiscard]] auto IsAvailable( + bazel_re::Digest const& digest, + std::filesystem::path const& path) const noexcept -> bool { + try { + return exists_ and exists_(digest, path); + } catch (...) { + return false; + } + } + /// \brief Store blob from bytes to storage. [[nodiscard]] auto StoreBlobData(std::string const& blob_id, std::string const& bytes, @@ -123,14 +148,11 @@ class LocalCAS { template <class T> [[nodiscard]] auto StoreBlob(T const& data, bool is_owner) const noexcept -> std::optional<bazel_re::Digest> { - auto digest = CreateDigest(data); - if (digest) { + if (auto digest = CreateDigest(data)) { auto id = NativeSupport::Unprefix(digest->hash()); -#ifndef BOOTSTRAP_BUILD_TOOL - if (FindAndUplinkBlob(id)) { + if (IsAvailable(*digest, file_store_.GetPath(id))) { return digest; } -#endif if (StoreBlobData(id, data, is_owner)) { return digest; } @@ -139,21 +161,6 @@ class LocalCAS { logger_.Emit(LogLevel::Debug, "Failed to create digest."); return std::nullopt; } - -#ifndef BOOTSTRAP_BUILD_TOOL - [[nodiscard]] auto FindAndUplinkBlob(std::string const& id) const noexcept - -> bool { - if (Compatibility::IsCompatible()) { - return GarbageCollector::FindAndUplinkBlob( - id, kType == ObjectType::Executable); - } - if (kType == ObjectType::Tree) { - return GarbageCollector::FindAndUplinkTree(id); - } - return GarbageCollector::FindAndUplinkBlob( - id, kType == ObjectType::Executable); - } -#endif }; -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 6bfa5512..d729f499 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -6,15 +6,12 @@ , "private-deps": [ ["src/buildtool/common", "cli"] , ["src/buildtool/common", "config"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "progress_reporter"] - , ["src/buildtool/execution_api/local", "garbage_collector"] , ["src/buildtool/build_engine/target_map", "result_map"] - , ["src/buildtool/build_engine/target_map", "target_cache"] - , ["src/buildtool/build_engine/target_map", "target_cache_key"] - , ["src/buildtool/build_engine/target_map", "target_cache_entry"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "concepts"] diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 5b72b640..6801f2b2 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -24,27 +24,29 @@ #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/main/analyse.hpp" #include "src/buildtool/main/constants.hpp" #include "src/buildtool/main/describe.hpp" #include "src/buildtool/main/exit_codes.hpp" #include "src/buildtool/main/install_cas.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/target_cache.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/execution_api/execution_service/server_implementation.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" #endif #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" @@ -300,10 +302,11 @@ void SetupLogging(LogArguments const& clargs) { void SetupExecutionConfig(EndpointArguments const& eargs, BuildArguments const& bargs, RebuildArguments const& rargs) { + using StorageConfig = StorageConfig; using LocalConfig = LocalExecutionConfig; using RemoteConfig = RemoteExecutionConfig; if (not(not eargs.local_root or - (LocalConfig::SetBuildRoot(*eargs.local_root))) or + (StorageConfig::SetBuildRoot(*eargs.local_root))) or not(not bargs.local_launcher or LocalConfig::SetLauncher(*bargs.local_launcher))) { Logger::Log(LogLevel::Error, "failed to configure local execution."); @@ -1217,7 +1220,7 @@ void WriteTargetCacheEntries( [&key = key, &target = target, &extra_infos, &downloader]() { if (auto entry = TargetCacheEntry::FromTarget(target, extra_infos)) { - if (not TargetCache::Instance().Store( + if (not Storage::Instance().TargetCache().Store( key, *entry, downloader)) { Logger::Log(LogLevel::Warning, "Failed writing target cache entry for {}", diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS new file mode 100644 index 00000000..59d93660 --- /dev/null +++ b/src/buildtool/storage/TARGETS @@ -0,0 +1,47 @@ +{ "config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config"] + , "hdrs": ["config.hpp"] + , "deps": + [ ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/execution_api/remote", "config"] + , ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + ] + , "stage": ["src", "buildtool", "storage"] + } +, "storage": + { "type": ["@", "rules", "CC", "library"] + , "name": ["storage"] + , "hdrs": + [ "storage.hpp" + , "local_cas.hpp" + , "local_cas.tpp" + , "local_ac.hpp" + , "local_ac.tpp" + , "target_cache.hpp" + , "target_cache.tpp" + , "target_cache_key.hpp" + , "target_cache_entry.hpp" + , "garbage_collector.hpp" + ] + , "srcs": + ["target_cache_key.cpp", "target_cache_entry.cpp", "garbage_collector.cpp"] + , "deps": + [ "config" + , ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/file_system", "object_cas"] + , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/execution_api/remote", "config"] + , ["src/buildtool/build_engine/analysed_target", "target"] + , ["src/buildtool/build_engine/base_maps", "entity_name_data"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/utils/cpp", "file_locking"] + , ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + ] + , "stage": ["src", "buildtool", "storage"] + } +} diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp new file mode 100644 index 00000000..e2964bb2 --- /dev/null +++ b/src/buildtool/storage/config.hpp @@ -0,0 +1,165 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP + +#ifdef __unix__ +#include <pwd.h> +#include <sys/types.h> +#include <unistd.h> +#else +#error "Non-unix is not supported yet" +#endif + +#include <filesystem> +#include <functional> +#include <string> +#include <vector> + +#include <fmt/core.h> +#include <gsl-lite/gsl-lite.hpp> +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/execution_api/remote/config.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +/// \brief Global storage configuration. +class StorageConfig { + struct ConfigData { + // Build root directory. All the storage dirs are subdirs of build_root. + // By default, build_root is set to $HOME/.cache/just. + // If the user uses --local-build-root PATH, + // then build_root will be set to PATH. + std::filesystem::path build_root{kDefaultBuildRoot}; + + // Number of total storage generations (default: two generations). + std::size_t num_generations{2}; + }; + + public: + /// \brief Determine user home directory + [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { + char const* root{nullptr}; + +#ifdef __unix__ + root = std::getenv("HOME"); + if (root == nullptr) { + root = getpwuid(getuid())->pw_dir; + } +#endif + + if (root == nullptr) { + Logger::Log(LogLevel::Error, + "Cannot determine user home directory."); + std::exit(EXIT_FAILURE); + } + + return root; + } + + static inline auto const kDefaultBuildRoot = + GetUserHome() / ".cache" / "just"; + + [[nodiscard]] static auto SetBuildRoot( + std::filesystem::path const& dir) noexcept -> bool { + if (FileSystemManager::IsRelativePath(dir)) { + Logger::Log(LogLevel::Error, + "Build root must be absolute path but got '{}'.", + dir.string()); + return false; + } + Data().build_root = dir; + return true; + } + + /// \brief Specifies the number of storage generations. + static auto SetNumGenerations(std::size_t num_generations) noexcept + -> void { + Data().num_generations = num_generations; + } + + /// \brief Number of storage generations. + [[nodiscard]] static auto NumGenerations() noexcept -> std::size_t { + return Data().num_generations; + } + + /// \brief Build directory, defaults to user directory if not set + [[nodiscard]] static auto BuildRoot() noexcept -> std::filesystem::path { + return Data().build_root; + } + + /// \brief Root directory of all storage generations. + [[nodiscard]] static auto CacheRoot() noexcept -> std::filesystem::path { + return BuildRoot() / "protocol-dependent"; + } + + /// \brief Root directory of specific storage generation for compatible and + /// non-compatible protocol types. + [[nodiscard]] static auto GenerationCacheRoot(std::size_t index) noexcept + -> std::filesystem::path { + gsl_ExpectsAudit(index < Data().num_generations); + auto generation = std::string{"generation-"} + std::to_string(index); + return CacheRoot() / generation; + } + + /// \brief Storage directory of specific generation and protocol type. + [[nodiscard]] static auto GenerationCacheDir(std::size_t index) noexcept + -> std::filesystem::path { + return UpdatePathForCompatibility(GenerationCacheRoot(index)); + } + + /// \brief String representation of the used execution backend. + [[nodiscard]] static auto ExecutionBackendDescription() noexcept + -> std::string { + auto address = RemoteExecutionConfig::RemoteAddress(); + auto properties = RemoteExecutionConfig::PlatformProperties(); + try { + // json::dump with json::error_handler_t::replace will not throw an + // exception if invalid UTF-8 sequences are detected in the input. + // Instead, it will replace them with the UTF-8 replacement + // character, but still it needs to be inside a try-catch clause to + // ensure the noexcept modifier of the enclosing function. + return nlohmann::json{ + {"remote_address", + address ? nlohmann::json{fmt::format( + "{}:{}", address->host, address->port)} + : nlohmann::json{}}, + {"platform_properties", properties}} + .dump(2, ' ', false, nlohmann::json::error_handler_t::replace); + } catch (...) { + return ""; + } + } + + private: + [[nodiscard]] static auto Data() noexcept -> ConfigData& { + static ConfigData instance{}; + return instance; + } + + // different folder for different caching protocol + [[nodiscard]] static auto UpdatePathForCompatibility( + std::filesystem::path const& dir) -> std::filesystem::path { + return dir / (Compatibility::IsCompatible() ? "compatible-sha256" + : "git-sha1"); + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp new file mode 100644 index 00000000..3e486672 --- /dev/null +++ b/src/buildtool/storage/garbage_collector.cpp @@ -0,0 +1,155 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BOOTSTRAP_BUILD_TOOL + +#include "src/buildtool/storage/garbage_collector.hpp" + +#include <vector> + +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/utils/cpp/hex_string.hpp" + +auto GarbageCollector::GlobalUplinkBlob(bazel_re::Digest const& digest, + bool is_executable) noexcept -> bool { + // Try to find blob in all generations. + auto const& latest_cas = Storage::Generation(0).CAS(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + // Note that we uplink with _skip_sync_ as we want to prefer hard links + // from older generations over copies from the companion file/exec CAS. + if (Storage::Generation(i).CAS().LocalUplinkBlob( + latest_cas, digest, is_executable, /*skip_sync=*/true)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkTree(bazel_re::Digest const& digest) noexcept + -> bool { + // Try to find tree in all generations. + auto const& latest_cas = Storage::Generation(0).CAS(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).CAS().LocalUplinkTree(latest_cas, digest)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkActionCacheEntry( + bazel_re::Digest const& action_id) noexcept -> bool { + // Try to find action-cache entry in all generations. + auto const& latest_ac = Storage::Generation(0).ActionCache(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).ActionCache().LocalUplinkEntry(latest_ac, + action_id)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkTargetCacheEntry( + TargetCacheKey const& key) noexcept -> bool { + // Try to find target-cache entry in all generations. + auto const& latest_tc = Storage::Generation(0).TargetCache(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).TargetCache().LocalUplinkEntry(latest_tc, + key)) { + return true; + } + } + return false; +} + +auto GarbageCollector::SharedLock() noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(), /*is_shared=*/true); +} + +auto GarbageCollector::ExclusiveLock() noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(), /*is_shared=*/false); +} + +auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path { + return StorageConfig::CacheRoot() / "gc.lock"; +} + +auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool { + auto pid = CreateProcessUniqueId(); + if (not pid) { + return false; + } + auto remove_me = std::string{"remove-me-"} + *pid; + auto remove_me_dir = StorageConfig::CacheRoot() / remove_me; + if (FileSystemManager::IsDirectory(remove_me_dir)) { + if (not FileSystemManager::RemoveDirectory(remove_me_dir, + /*recursively=*/true)) { + Logger::Log(LogLevel::Error, + "Failed to remove directory {}", + remove_me_dir.string()); + return false; + } + } + { // Create scope for critical renaming section protected by advisory lock. + auto lock = ExclusiveLock(); + if (not lock) { + Logger::Log(LogLevel::Error, + "Failed to exclusively lock the local build root"); + return false; + } + for (std::size_t i = StorageConfig::NumGenerations(); i > 0; --i) { + auto cache_root = StorageConfig::GenerationCacheRoot(i - 1); + if (FileSystemManager::IsDirectory(cache_root)) { + auto new_cache_root = + (i == StorageConfig::NumGenerations()) + ? remove_me_dir + : StorageConfig::GenerationCacheRoot(i); + if (not FileSystemManager::Rename(cache_root, new_cache_root)) { + Logger::Log(LogLevel::Error, + "Failed to rename {} to {}.", + cache_root.string(), + new_cache_root.string()); + return false; + } + } + } + } + if (FileSystemManager::IsDirectory(remove_me_dir)) { + if (not FileSystemManager::RemoveDirectory(remove_me_dir, + /*recursively=*/true)) { + Logger::Log(LogLevel::Warning, + "Failed to remove directory {}", + remove_me_dir.string()); + return false; + } + } + return true; +} + +#endif // BOOTSTRAP_BUILD_TOOL diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp new file mode 100644 index 00000000..76bbc054 --- /dev/null +++ b/src/buildtool/storage/garbage_collector.hpp @@ -0,0 +1,82 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP + +#include <functional> +#include <optional> +#include <string> + +#include "src/utils/cpp/file_locking.hpp" + +// forward declarations +namespace build::bazel::remote::execution::v2 { +class Digest; +} +namespace bazel_re = build::bazel::remote::execution::v2; +class TargetCacheKey; + +/// \brief Global garbage collector implementation. +/// Responsible for deleting oldest generation and uplinking across all +/// generations to latest generation. +class GarbageCollector { + public: + /// \brief Uplink blob across LocalCASes from all generations to latest. + /// Note that blobs will NOT be synced between file/executable CAS. + /// \param digest Digest of the blob to uplink. + /// \param is_executable Indicate that blob is an executable. + /// \returns true if blob was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkBlob(bazel_re::Digest const& digest, + bool is_executable) noexcept + -> bool; + + /// \brief Uplink tree across LocalCASes from all generations to latest. + /// Note that the tree will be deeply uplinked, i.e., all entries referenced + /// by this tree will be uplinked before (including sub-trees). + /// \param digest Digest of the tree to uplink. + /// \returns true if tree was found and successfully uplinked (deep). + [[nodiscard]] auto static GlobalUplinkTree( + bazel_re::Digest const& digest) noexcept -> bool; + + /// \brief Uplink entry from action cache across all generations to latest. + /// Note that the entry will be uplinked including all referenced items. + /// \param action_id Id of the action to uplink entry for. + /// \returns true if cache entry was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkActionCacheEntry( + bazel_re::Digest const& action_id) noexcept -> bool; + + /// \brief Uplink entry from target cache across all generations to latest. + /// Note that the entry will be uplinked including all referenced items. + /// \param key Target cache key to uplink entry for. + /// \returns true if cache entry was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkTargetCacheEntry( + TargetCacheKey const& key) noexcept -> bool; + + /// \brief Trigger deletion of oldest generation. + /// \returns true on success. + [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool; + + /// \brief Acquire shared lock to prevent garbage collection from running. + /// \returns The acquired lock file on success or nullopt otherwise. + [[nodiscard]] auto static SharedLock() noexcept -> std::optional<LockFile>; + + private: + [[nodiscard]] auto static ExclusiveLock() noexcept + -> std::optional<LockFile>; + + [[nodiscard]] auto static LockFilePath() noexcept -> std::filesystem::path; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp new file mode 100644 index 00000000..313abd7f --- /dev/null +++ b/src/buildtool/storage/local_ac.hpp @@ -0,0 +1,101 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP + +#include "gsl-lite/gsl-lite.hpp" +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/local_cas.hpp" + +// forward declarations +namespace build::bazel::remote::execution::v2 { +class Digest; +class ActionResult; +} // namespace build::bazel::remote::execution::v2 +namespace bazel_re = build::bazel::remote::execution::v2; + +/// \brief The action cache for storing action results. +/// Supports global uplinking across all generations using the garbage +/// collector. The uplink is automatically performed for every entry that is +/// read and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalAC { + public: + /// Local AC generation used by GC without global uplink. + using LocalGenerationAC = LocalAC</*kDoGlobalUplink=*/false>; + + LocalAC(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas, + std::filesystem::path const& store_path) noexcept + : cas_{std::move(cas)}, file_store_{store_path} {}; + + LocalAC(LocalAC const&) = default; + LocalAC(LocalAC&&) noexcept = default; + auto operator=(LocalAC const&) -> LocalAC& = default; + auto operator=(LocalAC&&) noexcept -> LocalAC& = default; + ~LocalAC() noexcept = default; + + /// \brief Store action result. + /// \param action_id The id of the action that produced the result. + /// \param result The action result to store. + /// \returns true on success. + [[nodiscard]] auto StoreResult( + bazel_re::Digest const& action_id, + bazel_re::ActionResult const& result) const noexcept -> bool; + + /// \brief Read cached action result. + /// \param action_id The id of the action the result was produced by. + /// \returns The action result if found or nullopt otherwise. + [[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id) + const noexcept -> std::optional<bazel_re::ActionResult>; + + /// \brief Uplink entry from this generation to latest LocalAC generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalAC generation. + /// \param action_id The id of the action used as entry key. + /// \returns True if entry was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( + LocalGenerationAC const& latest, + bazel_re::Digest const& action_id) const noexcept -> bool; + + private: + // The action cache stores the results of failed actions. For those to be + // overwritable by subsequent runs we need to choose the store mode "last + // wins" for the underlying file storage. Unless this is a generation + // (disabled global uplink), then we never want to overwrite any entries. + static constexpr auto kStoreMode = + kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; + + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("LocalAC")}; + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false> + file_store_; + + [[nodiscard]] auto ReadResult(bazel_re::Digest const& digest) const noexcept + -> std::optional<bazel_re::ActionResult>; +}; + +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/storage/local_ac.tpp" +#endif + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp new file mode 100644 index 00000000..9c6eca77 --- /dev/null +++ b/src/buildtool/storage/local_ac.tpp @@ -0,0 +1,133 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP + +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/storage/local_ac.hpp" + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::StoreResult( + bazel_re::Digest const& action_id, + bazel_re::ActionResult const& result) const noexcept -> bool { + auto bytes = result.SerializeAsString(); + auto digest = cas_->StoreBlob(bytes); + return (digest and + file_store_.AddFromBytes(NativeSupport::Unprefix(action_id.hash()), + digest->SerializeAsString())); +} + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::CachedResult(bazel_re::Digest const& action_id) + const noexcept -> std::optional<bazel_re::ActionResult> { + auto id = NativeSupport::Unprefix(action_id.hash()); + auto entry_path = file_store_.GetPath(id); + + if constexpr (kDoGlobalUplink) { + // Uplink any existing action-cache entries in storage generations + [[maybe_unused]] bool found = + GarbageCollector::GlobalUplinkActionCacheEntry(action_id); + } + + auto const entry = + FileSystemManager::ReadFile(entry_path, ObjectType::File); + if (not entry) { + logger_->Emit(LogLevel::Debug, + "Cache miss, entry not found {}", + entry_path.string()); + return std::nullopt; + } + bazel_re::Digest digest{}; + if (not digest.ParseFromString(*entry)) { + logger_->Emit( + LogLevel::Warning, "Parsing cache entry failed for action {}", id); + return std::nullopt; + } + auto result = ReadResult(digest); + if (not result) { + logger_->Emit(LogLevel::Warning, + "Parsing action result failed for action {}", + id); + } + return result; +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( + LocalGenerationAC const& latest, + bazel_re::Digest const& action_id) const noexcept -> bool { + // Determine action cache key path in latest generation. + auto key_digest = NativeSupport::Unprefix(action_id.hash()); + if (FileSystemManager::IsFile(latest.file_store_.GetPath(key_digest))) { + return true; + } + + // Read cache key + auto cache_key = file_store_.GetPath(key_digest); + auto content = FileSystemManager::ReadFile(cache_key); + if (not content) { + return false; + } + + // Read result (cache value) + bazel_re::Digest result_digest{}; + if (not result_digest.ParseFromString(*content)) { + return false; + } + auto result = ReadResult(result_digest); + if (not result) { + return false; + } + + // Uplink result content + for (auto const& file : result->output_files()) { + if (not cas_->LocalUplinkBlob( + *latest.cas_, file.digest(), file.is_executable())) { + return false; + } + } + for (auto const& directory : result->output_directories()) { + if (not cas_->LocalUplinkTree(*latest.cas_, directory.tree_digest())) { + return false; + } + } + + // Uplink result (cache value) + if (not cas_->LocalUplinkBlob(*latest.cas_, + result_digest, + /*is_executable=*/false)) { + return false; + } + + // Uplink cache key + return latest.file_store_.AddFromFile( + key_digest, cache_key, /*is_owner=*/true); +} + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::ReadResult(bazel_re::Digest const& digest) + const noexcept -> std::optional<bazel_re::ActionResult> { + bazel_re::ActionResult result{}; + if (auto src_path = cas_->BlobPath(digest, /*is_executable=*/false)) { + auto const bytes = FileSystemManager::ReadFile(*src_path); + if (bytes.has_value() and result.ParseFromString(*bytes)) { + return result; + } + } + return std::nullopt; +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp new file mode 100644 index 00000000..af613b1d --- /dev/null +++ b/src/buildtool/storage/local_cas.hpp @@ -0,0 +1,242 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP + +#include <optional> +#include <unordered_set> + +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_cas.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" + +/// \brief The local (logical) CAS for storing blobs and trees. +/// Blobs can be stored/queried as executable or non-executable. Trees might be +/// treated differently depending on the compatibility mode. Supports global +/// uplinking across all generations using the garbage collector. The uplink +/// is automatically performed for every entry that is read and every entry that +/// is stored and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalCAS { + public: + /// Local CAS generation used by GC without global uplink. + using LocalGenerationCAS = LocalCAS</*kDoGlobalUplink=*/false>; + + /// \brief Create local CAS with base path. + /// Note that the base path is concatenated by a single character + /// 'f'/'x'/'t' for each internally used physical CAS. + /// \param base The base path for the CAS. + explicit LocalCAS(std::filesystem::path const& base) + : cas_file_{base.string() + 'f', Uplinker<ObjectType::File>()}, + cas_exec_{base.string() + 'x', Uplinker<ObjectType::Executable>()}, + cas_tree_{base.string() + (Compatibility::IsCompatible() ? 'f' : 't'), + Uplinker<ObjectType::Tree>()} {} + + /// \brief Store blob from file path with x-bit. + /// \tparam kOwner Indicates ownership for optimization (hardlink). + /// \param file_path The path of the file to store as blob. + /// \param is_executable Store blob with executable permissions. + /// \returns Digest of the stored blob or nullopt otherwise. + template <bool kOwner = false> + [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path, + bool is_executable) const noexcept + -> std::optional<bazel_re::Digest> { + return is_executable ? cas_exec_.StoreBlobFromFile(file_path, kOwner) + : cas_file_.StoreBlobFromFile(file_path, kOwner); + } + + /// \brief Store blob from bytes with x-bit (default: non-executable). + /// \param bytes The bytes to create the blob from. + /// \param is_executable Store blob with executable permissions. + /// \returns Digest of the stored blob or nullopt otherwise. + [[nodiscard]] auto StoreBlob(std::string const& bytes, + bool is_executable = false) const noexcept + -> std::optional<bazel_re::Digest> { + return is_executable ? cas_exec_.StoreBlobFromBytes(bytes) + : cas_file_.StoreBlobFromBytes(bytes); + } + + /// \brief Store tree from file path. + /// \param file_path The path of the file to store as tree. + /// \returns Digest of the stored tree or nullopt otherwise. + [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) + const noexcept -> std::optional<bazel_re::Digest> { + return cas_tree_.StoreBlobFromFile(file_path); + } + + /// \brief Store tree from bytes. + /// \param bytes The bytes to create the tree from. + /// \returns Digest of the stored tree or nullopt otherwise. + [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept + -> std::optional<bazel_re::Digest> { + return cas_tree_.StoreBlobFromBytes(bytes); + } + + /// \brief Obtain blob path from digest with x-bit. + /// Performs a synchronization if blob is only available with inverse x-bit. + /// \param digest Digest of the blob to lookup. + /// \param is_executable Lookup blob with executable permissions. + /// \returns Path to the blob if found or nullopt otherwise. + [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest, + bool is_executable) const noexcept + -> std::optional<std::filesystem::path> { + auto const path = BlobPathNoSync(digest, is_executable); + return path ? path : TrySyncBlob(digest, is_executable); + } + + /// \brief Obtain tree path from digest. + /// \param digest Digest of the tree to lookup. + /// \returns Path to the tree if found or nullopt otherwise. + [[nodiscard]] auto TreePath(bazel_re::Digest const& digest) const noexcept + -> std::optional<std::filesystem::path> { + return cas_tree_.BlobPath(digest); + } + + /// \brief Traverses a tree recursively and retrieves object infos of all + /// found blobs (leafs). Tree objects are not added to the result list, but + /// converted to a path name. + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \returns Pair of vectors, first containing filesystem paths, second + /// containing object infos. + [[nodiscard]] auto RecursivelyReadTreeLeafs( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>>; + + /// \brief Reads the flat content of a tree and returns object infos of all + /// its direct entries (trees and blobs). + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \returns Pair of vectors, first containing filesystem paths, second + /// containing object infos. + [[nodiscard]] auto ReadDirectTreeEntries( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>>; + + /// \brief Dump artifact to file stream. + /// Tree artifacts are pretty-printed (i.e., contents are listed) unless + /// raw_tree is set, then the raw tree will be written to the file stream. + /// \param info The object info of the artifact to dump. + /// \param stream The file stream to dump to. + /// \param raw_tree Dump tree as raw blob. + /// \returns true on success. + [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept -> bool; + + /// \brief Uplink blob from this generation to latest LocalCAS generation. + /// Performs a synchronization if requested and if blob is only available + /// with inverse x-bit. This function is only available for instances that + /// are used as local GC generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalCAS generation. + /// \param digest The digest of the blob to uplink. + /// \param is_executable Uplink blob with executable permissions. + /// \param skip_sync Do not sync between file/executable CAS. + /// \returns True if blob was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBlob( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + bool is_executable, + bool skip_sync = false) const noexcept -> bool; + + /// \brief Uplink tree from this generation to latest LocalCAS generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). Trees are uplinked deep, + /// including all referenced entries. Note that in compatible mode we do not + /// have the notion of "tree" and instead trees are stored as blobs. + /// Therefore, in compatible mode this function is only used by instances + /// that are aware of trees, such as output directories in action results or + /// tree artifacts from target cache. + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalCAS generation. + /// \param digest The digest of the tree to uplink. + /// \returns True if tree was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool; + + private: + ObjectCAS<ObjectType::File> cas_file_; + ObjectCAS<ObjectType::Executable> cas_exec_; + ObjectCAS<ObjectType::Tree> cas_tree_; + + /// \brief Provides uplink via "exists callback" for physical object CAS. + template <ObjectType kType> + [[nodiscard]] static auto Uplinker() -> + typename ObjectCAS<kType>::ExistsFunc { + if constexpr (kDoGlobalUplink) { + return [](auto digest, auto /*path*/) { + if (not Compatibility::IsCompatible()) { + // in non-compatible mode, do explicit deep tree uplink + if constexpr (IsTreeObject(kType)) { + return GarbageCollector::GlobalUplinkTree(digest); + } + } + // in compatible mode, treat all trees as blobs + return GarbageCollector::GlobalUplinkBlob( + digest, IsExecutableObject(kType)); + }; + } + return ObjectCAS<kType>::kDefaultExists; + } + + /// \brief Get blob path without sync between file/executable CAS. + [[nodiscard]] auto BlobPathNoSync(bazel_re::Digest const& digest, + bool is_executable) const noexcept + -> std::optional<std::filesystem::path> { + return is_executable ? cas_exec_.BlobPath(digest) + : cas_file_.BlobPath(digest); + } + + /// \brief Try to sync blob between file CAS and executable CAS. + /// \param digest Blob digest. + /// \param to_executable Sync direction. + /// \returns Path to blob in target CAS. + [[nodiscard]] auto TrySyncBlob(bazel_re::Digest const& digest, + bool to_executable) const noexcept + -> std::optional<std::filesystem::path> { + auto const src_blob = BlobPathNoSync(digest, not to_executable); + if (src_blob and StoreBlob(*src_blob, to_executable)) { + return BlobPathNoSync(digest, to_executable); + } + return std::nullopt; + } + + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkGitTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool; + + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBazelDirectory( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen) + const noexcept -> bool; +}; + +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/storage/local_cas.tpp" +#endif + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp new file mode 100644 index 00000000..3664d929 --- /dev/null +++ b/src/buildtool/storage/local_cas.tpp @@ -0,0 +1,367 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP + +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/storage/local_cas.hpp" + +namespace detail { + +template <class T_CAS> +[[nodiscard]] auto ReadDirectory(T_CAS const& cas, + bazel_re::Digest const& digest) noexcept + -> std::optional<bazel_re::Directory> { + if (auto const path = cas.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return BazelMsgFactory::MessageFromString<bazel_re::Directory>( + *content); + } + } + Logger::Log(LogLevel::Error, + "Directory {} not found in CAS", + NativeSupport::Unprefix(digest.hash())); + return std::nullopt; +} + +template <class T_CAS> +[[nodiscard]] auto ReadGitTree(T_CAS const& cas, + bazel_re::Digest const& digest) noexcept + -> std::optional<GitRepo::tree_entries_t> { + if (auto const path = cas.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return GitRepo::ReadTreeData( + *content, + HashFunction::ComputeTreeHash(*content).Bytes(), + /*is_hex_id=*/false); + } + } + Logger::Log(LogLevel::Error, + "Tree {} not found in CAS", + NativeSupport::Unprefix(digest.hash())); + return std::nullopt; +} + +[[nodiscard]] inline auto DumpToStream( + gsl::not_null<FILE*> const& stream, + std::optional<std::string> const& data) noexcept -> bool { + if (data) { + std::fwrite(data->data(), 1, data->size(), stream); + return true; + } + return false; +} + +template <class T_CAS> +[[nodiscard]] auto TreeToStream(T_CAS const& cas, + bazel_re::Digest const& tree_digest, + gsl::not_null<FILE*> const& stream) noexcept + -> bool { + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(cas, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::DirectoryToString(*dir)); + } + } + else { + if (auto entries = ReadGitTree(cas, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::GitTreeToString(*entries)); + } + } + return false; +} + +template <class T_CAS> +[[nodiscard]] auto BlobToStream(T_CAS const& cas, + Artifact::ObjectInfo const& blob_info, + gsl::not_null<FILE*> const& stream) noexcept + -> bool { + constexpr std::size_t kChunkSize{512}; + auto path = + cas.BlobPath(blob_info.digest, IsExecutableObject(blob_info.type)); + if (not path and not Compatibility::IsCompatible()) { + // in native mode, lookup object in tree cas to dump tree as blob + path = cas.TreePath(blob_info.digest); + } + if (path) { + std::string data(kChunkSize, '\0'); + if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) { + while (auto size = std::fread(data.data(), 1, kChunkSize, in)) { + std::fwrite(data.data(), 1, size, stream); + } + std::fclose(in); + return true; + } + } + return false; +} + +template <class T_CAS> +auto ReadObjectInfosRecursively( + T_CAS const& cas, + BazelMsgFactory::InfoStoreFunc const& store_info, + std::filesystem::path const& parent, + bazel_re::Digest const& digest) noexcept -> bool { + // read from CAS + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(cas, digest)) { + return BazelMsgFactory::ReadObjectInfosFromDirectory( + *dir, [&cas, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(cas, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); + } + } + else { + if (auto entries = ReadGitTree(cas, digest)) { + return BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, [&cas, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(cas, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); + } + } + return false; +} + +} // namespace detail + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::RecursivelyReadTreeLeafs( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>> { + std::vector<std::filesystem::path> paths{}; + std::vector<Artifact::ObjectInfo> infos{}; + + auto store_info = [&paths, &infos](auto path, auto info) { + paths.emplace_back(path); + infos.emplace_back(info); + return true; + }; + + if (detail::ReadObjectInfosRecursively( + *this, store_info, parent, tree_digest)) { + return std::make_pair(std::move(paths), std::move(infos)); + } + return std::nullopt; +} + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::ReadDirectTreeEntries( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>> { + std::vector<std::filesystem::path> paths{}; + std::vector<Artifact::ObjectInfo> infos{}; + + auto store_info = [&paths, &infos](auto path, auto info) { + paths.emplace_back(path); + infos.emplace_back(info); + return true; + }; + + if (Compatibility::IsCompatible()) { + if (auto dir = detail::ReadDirectory(*this, tree_digest)) { + if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + *dir, [&store_info, &parent](auto path, auto info) { + return store_info(parent / path, info); + })) { + return std::nullopt; + } + } + } + else { + if (auto entries = detail::ReadGitTree(*this, tree_digest)) { + if (not BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, [&store_info, &parent](auto path, auto info) { + return store_info(parent / path, info); + })) { + return std::nullopt; + } + } + } + return std::make_pair(std::move(paths), std::move(infos)); +} + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept + -> bool { + return IsTreeObject(info.type) and not raw_tree + ? detail::TreeToStream(*this, info.digest, stream) + : detail::BlobToStream(*this, info, stream); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBlob( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + bool is_executable, + bool skip_sync) const noexcept -> bool { + // Determine blob path in latest generation. + auto blob_path_latest = latest.BlobPathNoSync(digest, is_executable); + if (blob_path_latest) { + return true; + } + + // Determine blob path of given generation. + auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable) + : BlobPath(digest, is_executable); + if (not blob_path) { + return false; + } + + // Uplink blob from older generation to the latest generation. + return blob_path_latest.has_value() or + latest.StoreBlob</*kOwner=*/true>(*blob_path, is_executable); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool { + if (Compatibility::IsCompatible()) { + std::unordered_set<bazel_re::Digest> seen{}; + return LocalUplinkBazelDirectory(latest, digest, &seen); + } + return LocalUplinkGitTree(latest, digest); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool { + // Determine tree path in latest generation. + auto tree_path_latest = latest.cas_tree_.BlobPath(digest); + if (tree_path_latest) { + return true; + } + + // Determine tree path of given generation. + auto tree_path = cas_tree_.BlobPath(digest); + if (not tree_path) { + return false; + } + + // Determine tree entries. + auto content = FileSystemManager::ReadFile(*tree_path); + auto id = NativeSupport::Unprefix(digest.hash()); + auto tree_entries = GitRepo::ReadTreeData(*content, + id, + /*is_hex_id=*/true); + if (not tree_entries) { + return false; + } + + // Uplink tree entries. + for (auto const& [raw_id, entry_vector] : *tree_entries) { + // Process only first entry from 'entry_vector' since all + // entries represent the same blob, just with different + // names. + auto entry = entry_vector.front(); + auto hash = ToHexString(raw_id); + auto digest = ArtifactDigest{hash, 0, IsTreeObject(entry.type)}; + if (entry.type == ObjectType::Tree) { + if (not LocalUplinkGitTree(latest, digest)) { + return false; + } + } + else { + if (not LocalUplinkBlob( + latest, digest, IsExecutableObject(entry.type))) { + return false; + } + } + } + + // Uplink tree from older generation to the latest generation. + return latest.cas_tree_ + .StoreBlobFromFile(*tree_path, + /*is_owner=*/true) + .has_value(); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>:: + LocalUplinkBazelDirectory( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen) + const noexcept -> bool { + // Skip already uplinked directories + if (seen->contains(digest)) { + return true; + } + + // Determine bazel directory path of given generation. + auto dir_path = cas_tree_.BlobPath(digest); + if (not dir_path) { + return false; + } + + // Determine bazel directory entries. + auto content = FileSystemManager::ReadFile(*dir_path); + bazel_re::Directory dir{}; + if (not dir.ParseFromString(*content)) { + return false; + } + + // Uplink bazel directory entries. + for (auto const& file : dir.files()) { + if (not LocalUplinkBlob(latest, file.digest(), file.is_executable())) { + return false; + } + } + for (auto const& directory : dir.directories()) { + if (not LocalUplinkBazelDirectory(latest, directory.digest(), seen)) { + return false; + } + } + + // Determine bazel directory path in latest generation. + auto dir_path_latest = latest.cas_tree_.BlobPath(digest); + + // Uplink bazel directory from older generation to the latest + // generation. + if (dir_path_latest.has_value() or + latest.cas_tree_.StoreBlobFromFile(*dir_path, + /*is_owner=*/true)) { + try { + seen->emplace(digest); + return true; + } catch (...) { + } + } + return false; +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP diff --git a/src/buildtool/storage/storage.hpp b/src/buildtool/storage/storage.hpp new file mode 100644 index 00000000..fe6c1e1f --- /dev/null +++ b/src/buildtool/storage/storage.hpp @@ -0,0 +1,133 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP + +#include <optional> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/local_ac.hpp" +#include "src/buildtool/storage/local_cas.hpp" +#include "src/buildtool/storage/target_cache.hpp" + +/// \brief The local storage for accessing CAS and caches. +/// Maintains an instance of LocalCAS, LocalAC, TargetCache. Supports global +/// uplinking across all generations using the garbage collector. The uplink is +/// automatically performed by the affected storage instance (CAS, action cache, +/// target cache). +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalStorage { + public: + explicit LocalStorage(std::filesystem::path const& storage_path) + : cas_{std::make_shared<LocalCAS<kDoGlobalUplink>>(storage_path / + "cas")}, + ac_{cas_, storage_path / "ac"}, + tc_{cas_, storage_path / "tc"} {} + + /// \brief Get the CAS instance. + [[nodiscard]] auto CAS() const noexcept + -> LocalCAS<kDoGlobalUplink> const& { + return *cas_; + } + + /// \brief Get the action cache instance. + [[nodiscard]] auto ActionCache() const noexcept + -> LocalAC<kDoGlobalUplink> const& { + return ac_; + } + + /// \brief Get the target cache instance. + [[nodiscard]] auto TargetCache() const noexcept + -> TargetCache<kDoGlobalUplink> const& { + return tc_; + } + + private: + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + LocalAC<kDoGlobalUplink> ac_; + ::TargetCache<kDoGlobalUplink> tc_; +}; + +#ifdef BOOTSTRAP_BUILD_TOOL +// disable global uplinking (via garbage collector) for global storage singleton +constexpr auto kDefaultDoGlobalUplink = false; +#else +constexpr auto kDefaultDoGlobalUplink = true; +#endif + +/// \brief Generation type, local storage without global uplinking. +using Generation = LocalStorage</*kDoGlobalUplink=*/false>; + +/// \brief Global storage singleton class, valid throughout the entire tool. +/// Configured via \ref StorageConfig. +class Storage : public LocalStorage<kDefaultDoGlobalUplink> { + public: + /// \brief Get the global storage instance. + /// Build root is read from \ref StorageConfig::BuildRoot(). + /// \returns The global storage singleton instance. + [[nodiscard]] static auto Instance() noexcept -> Storage const& { + return GetStorage(); + } + + /// \brief Get specific storage generation. + /// Number of generations is read from \ref StorageConfig::NumGenerations(). + /// \param index the generation index (0 is latest). + /// \returns The specific storage generation. + [[nodiscard]] static auto Generation(std::size_t index) noexcept + -> ::Generation const& { + return GetGenerations()[index]; + } + + /// \brief Reinitialize storage instance and generations. + /// Use if global \ref StorageConfig was changed. Not thread-safe! + static void Reinitialize() noexcept { + GetStorage() = CreateStorage(); + GetGenerations() = CreateGenerations(); + } + + private: + using LocalStorage<kDefaultDoGlobalUplink>::LocalStorage; + + [[nodiscard]] static auto CreateStorage() noexcept -> Storage { + return Storage{StorageConfig::GenerationCacheDir(0)}; + } + + [[nodiscard]] static auto CreateGenerations() noexcept + -> std::vector<::Generation> { + auto count = StorageConfig::NumGenerations(); + std::vector<::Generation> generations{}; + generations.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + generations.emplace_back(StorageConfig::GenerationCacheDir(i)); + } + return generations; + } + + [[nodiscard]] static auto GetStorage() noexcept -> Storage& { + static auto instance = CreateStorage(); + return instance; + } + + [[nodiscard]] static auto GetGenerations() noexcept + -> std::vector<::Generation>& { + static auto generations = CreateGenerations(); + return generations; + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp new file mode 100644 index 00000000..c9f0fbe6 --- /dev/null +++ b/src/buildtool/storage/target_cache.hpp @@ -0,0 +1,132 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP + +#include <filesystem> +#include <functional> +#include <optional> +#include <utility> + +#include <gsl-lite/gsl-lite.hpp> +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/local_cas.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" + +/// \brief The high-level target cache for storing export target's data. +/// Supports global uplinking across all generations using the garbage +/// collector. The uplink is automatically performed for every entry that is +/// read and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class TargetCache { + public: + /// Local target cache generation used by GC without global uplink. + using LocalGenerationTC = TargetCache</*kDoGlobalUplink=*/false>; + + /// Callback type for downloading known artifacts to local CAS. + using ArtifactDownloader = + std::function<bool(std::vector<Artifact::ObjectInfo> const&)>; + + TargetCache(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas, + std::filesystem::path const& store_path) + : cas_{std::move(cas)}, file_store_{store_path / ComputeShard()} { + if constexpr (kDoGlobalUplink) { + // write backend description (shard) to CAS + [[maybe_unused]] auto id = + cas_->StoreBlob(StorageConfig::ExecutionBackendDescription()); + gsl_EnsuresAudit(id and + ArtifactDigest{*id}.hash() == ComputeShard()); + } + } + + TargetCache(TargetCache const&) = default; + TargetCache(TargetCache&&) noexcept = default; + auto operator=(TargetCache const&) -> TargetCache& = default; + auto operator=(TargetCache&&) noexcept -> TargetCache& = default; + ~TargetCache() noexcept = default; + + /// \brief Store new key-entry pair in the target cache. + /// \param key The target-cache key. + /// \param value The target-cache value to store. + /// \param downloader Callback for obtaining known artifacts to local CAS. + /// \returns true on success. + [[nodiscard]] auto Store( + TargetCacheKey const& key, + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool; + + /// \brief Read existing entry and object info from the target cache. + /// \param key The target-cache key to read the entry from. + /// \returns Pair of cache entry and its object info on success or nullopt. + [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; + + /// \brief Uplink entry from this to latest target cache generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest target cache generation. + /// \param key The target-cache key for the entry to uplink. + /// \returns True if entry was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( + LocalGenerationTC const& latest, + TargetCacheKey const& key) const noexcept -> bool; + + private: + // By default, overwrite existing entries. Unless this is a generation + // (disabled global uplink), then we never want to overwrite any entries. + static constexpr auto kStoreMode = + kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; + + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("TargetCache")}; + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + FileStorage<ObjectType::File, + kStoreMode, + /*kSetEpochTime=*/false> + file_store_; + + [[nodiscard]] static auto ComputeShard() noexcept -> std::string { + return ArtifactDigest::Create<ObjectType::File>( + StorageConfig::ExecutionBackendDescription()) + .hash(); + } + + [[nodiscard]] auto DownloadKnownArtifacts( + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool; +}; + +#include "src/buildtool/storage/target_cache.tpp" + +namespace std { +template <> +struct hash<TargetCacheKey> { + [[nodiscard]] auto operator()(TargetCacheKey const& k) const { + return std::hash<Artifact::ObjectInfo>{}(k.Id()); + } +}; +} // namespace std + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP diff --git a/src/buildtool/storage/target_cache.tpp b/src/buildtool/storage/target_cache.tpp new file mode 100644 index 00000000..d1c9d01c --- /dev/null +++ b/src/buildtool/storage/target_cache.tpp @@ -0,0 +1,168 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP + +#include "src/buildtool/storage/target_cache.hpp" + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::Store( + TargetCacheKey const& key, + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool { + if (not DownloadKnownArtifacts(value, downloader)) { + return false; + } + if (auto digest = cas_->StoreBlob(value.ToJson().dump(2))) { + auto data = + Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File} + .ToString(); + logger_->Emit(LogLevel::Debug, + "Adding entry for key {} as {}", + key.Id().ToString(), + data); + return file_store_.AddFromBytes(key.Id().digest.hash(), data); + } + return false; +} + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::Read( + TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> { + auto id = key.Id().digest.hash(); + auto entry_path = file_store_.GetPath(id); + + if constexpr (kDoGlobalUplink) { + // Uplink any existing target cache entry in storage generations + [[maybe_unused]] auto found = + GarbageCollector::GlobalUplinkTargetCacheEntry(key); + } + + auto const entry = + FileSystemManager::ReadFile(entry_path, ObjectType::File); + if (not entry) { + logger_->Emit(LogLevel::Debug, + "Cache miss, entry not found {}", + entry_path.string()); + return std::nullopt; + } + if (auto info = Artifact::ObjectInfo::FromString(*entry)) { + if (auto path = cas_->BlobPath(info->digest, /*is_executable=*/false)) { + if (auto value = FileSystemManager::ReadFile(*path)) { + try { + return std::make_pair( + TargetCacheEntry{nlohmann::json::parse(*value)}, + std::move(*info)); + } catch (std::exception const& ex) { + logger_->Emit(LogLevel::Warning, + "Parsing entry for key {} failed with:\n{}", + key.Id().ToString(), + ex.what()); + } + } + } + } + logger_->Emit(LogLevel::Warning, + "Reading entry for key {} failed", + key.Id().ToString()); + return std::nullopt; +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto TargetCache< + kDoGlobalUplink>::LocalUplinkEntry(LocalGenerationTC const& latest, + TargetCacheKey const& key) const noexcept + -> bool { + // Determine target cache key path of given generation. + auto key_digest = key.Id().digest.hash(); + if (FileSystemManager::IsFile(latest.file_store_.GetPath(key_digest))) { + return true; + } + + // Determine target cache entry location. + auto cache_key = file_store_.GetPath(key_digest); + auto raw_key = FileSystemManager::ReadFile(cache_key); + if (not raw_key) { + return false; + } + + // Determine target cache entry location. + auto entry_info = Artifact::ObjectInfo::FromString(*raw_key); + if (not entry_info) { + return false; + } + + // Determine target cache entry blob path of given generation. + auto cache_entry = + cas_->BlobPath(entry_info->digest, /*is_executable=*/false); + if (not cache_entry) { + return false; + } + + // Determine artifacts referenced by target cache entry. + auto raw_entry = FileSystemManager::ReadFile(*cache_entry); + if (not raw_entry) { + return false; + } + nlohmann::json json_desc{}; + try { + json_desc = nlohmann::json::parse(*raw_entry); + } catch (std::exception const& ex) { + return false; + } + auto entry = TargetCacheEntry::FromJson(json_desc); + std::vector<Artifact::ObjectInfo> artifacts_info; + if (not entry.ToArtifacts(&artifacts_info)) { + return false; + } + + // Uplink referenced artifacts. + for (auto const& info : artifacts_info) { + if (info.type == ObjectType::Tree) { + if (not cas_->LocalUplinkTree(*latest.cas_, info.digest)) { + return false; + } + } + else if (not cas_->LocalUplinkBlob(*latest.cas_, + info.digest, + IsExecutableObject(info.type))) { + return false; + } + } + + // Uplink target cache entry blob. + if (not cas_->LocalUplinkBlob(*latest.cas_, + entry_info->digest, + /*is_executable=*/false)) { + return false; + } + + // Uplink target cache key + return latest.file_store_.AddFromFile( + key_digest, cache_key, /*is_owner=*/true); +} + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::DownloadKnownArtifacts( + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool { + std::vector<Artifact::ObjectInfo> artifacts_info; + return downloader and value.ToArtifacts(&artifacts_info) and + downloader(artifacts_info); +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.cpp b/src/buildtool/storage/target_cache_entry.cpp index 139c706b..683b2f02 100644 --- a/src/buildtool/build_engine/target_map/target_cache_entry.cpp +++ b/src/buildtool/storage/target_cache_entry.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" #include <algorithm> #include <exception> diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.hpp b/src/buildtool/storage/target_cache_entry.hpp index 1e54a003..b2002f80 100644 --- a/src/buildtool/build_engine/target_map/target_cache_entry.hpp +++ b/src/buildtool/storage/target_cache_entry.hpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP #include <optional> #include <unordered_map> @@ -30,9 +30,9 @@ // Entry for target cache. Created from target, contains TargetResult. class TargetCacheEntry { - friend class TargetCache; - public: + explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} + // Create the entry from target with replacement artifacts/infos. // Replacement artifacts must replace all non-known artifacts by known. [[nodiscard]] static auto FromTarget( @@ -53,16 +53,15 @@ class TargetCacheEntry { gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) const noexcept -> bool; - private: - nlohmann::json desc_; - - explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} [[nodiscard]] auto ToJson() const& -> nlohmann::json const& { return desc_; } [[nodiscard]] auto ToJson() && -> nlohmann::json { return std::move(desc_); } + + private: + nlohmann::json desc_; }; -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP diff --git a/src/buildtool/build_engine/target_map/target_cache_key.cpp b/src/buildtool/storage/target_cache_key.cpp index 6408c2a6..11ea5c84 100644 --- a/src/buildtool/build_engine/target_map/target_cache_key.cpp +++ b/src/buildtool/storage/target_cache_key.cpp @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #include <exception> #include <nlohmann/json.hpp> #include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" auto TargetCacheKey::Create(std::string const& repo_key, BuildMaps::Base::NamedTarget const& target_name, @@ -35,8 +35,8 @@ auto TargetCacheKey::Create(std::string const& repo_key, {"target_name", nlohmann::json{target_name.module, target_name.name}.dump()}, {"effective_config", effective_config.ToString()}}}; - static auto const& cas = LocalCAS<ObjectType::File>::Instance(); - if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) { + if (auto target_key = Storage::Instance().CAS().StoreBlob( + target_desc.dump(2), /*is_executable=*/false)) { return TargetCacheKey{ {ArtifactDigest{*target_key}, ObjectType::File}}; } diff --git a/src/buildtool/build_engine/target_map/target_cache_key.hpp b/src/buildtool/storage/target_cache_key.hpp index 116a6a07..70d06464 100644 --- a/src/buildtool/build_engine/target_map/target_cache_key.hpp +++ b/src/buildtool/storage/target_cache_key.hpp @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP +#include <functional> #include <optional> #include <utility> @@ -46,4 +47,4 @@ class TargetCacheKey { Artifact::ObjectInfo id_; }; -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 04276225..b2bf6cd2 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -33,7 +33,7 @@ , "srcs": ["utils.cpp"] , "deps": [ ["src/utils/cpp", "tmp_dir"] - , ["src/buildtool/execution_api/local", "config"] + , ["src/buildtool/storage", "config"] , ["src/buildtool/main", "constants"] , ["src/buildtool/build_engine/expression", "expression"] ] diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index 02eaf8a5..34c1953d 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -223,7 +223,7 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { root_path = *ws_root; } if (root == "home") { - root_path = LocalExecutionConfig::GetUserHome(); + root_path = StorageConfig::GetUserHome(); } if (root == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); @@ -262,7 +262,7 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { root_path = *ws_root; } if (root_str == "home") { - root_path = LocalExecutionConfig::GetUserHome(); + root_path = StorageConfig::GetUserHome(); } if (root_str == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); @@ -1416,9 +1416,9 @@ auto main(int argc, char* argv[]) -> int { arguments.common.explicit_distdirs.begin(), arguments.common.explicit_distdirs.end()); - // Setup LocalExecutionConfig to store the local_build_root properly + // Setup LocalStorageConfig to store the local_build_root properly // and make the cas and git cache roots available - if (not LocalExecutionConfig::SetBuildRoot( + if (not StorageConfig::SetBuildRoot( *arguments.common.just_mr_paths->root)) { Logger::Log(LogLevel::Error, "Failed to configure local build root."); diff --git a/src/other_tools/just_mr/utils.cpp b/src/other_tools/just_mr/utils.cpp index 24511742..ca891875 100644 --- a/src/other_tools/just_mr/utils.cpp +++ b/src/other_tools/just_mr/utils.cpp @@ -14,14 +14,14 @@ #include "src/other_tools/just_mr/utils.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/path.hpp" namespace JustMR::Utils { auto GetGitCacheRoot() noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "git"; + return StorageConfig::BuildRoot() / "git"; } auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, @@ -42,20 +42,19 @@ auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { // try to create parent dir - auto parent_path = - LocalExecutionConfig::BuildRoot() / "tmp-workspaces" / type; + auto parent_path = StorageConfig::BuildRoot() / "tmp-workspaces" / type; return TmpDir::Create(parent_path); } auto GetArchiveTreeIDFile(std::string const& repo_type, std::string const& content) noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "tree-map" / repo_type / content; + return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; } auto GetDistdirTreeIDFile(std::string const& content) noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "distfiles-tree-map" / content; + return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; } auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, @@ -79,23 +78,24 @@ auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, auto AddToCAS(std::string const& data) noexcept -> std::optional<std::filesystem::path> { // get file CAS instance - auto const& casf = LocalCAS<ObjectType::File>::Instance(); - // write to casf - auto digest = casf.StoreBlobFromBytes(data); + auto const& cas = Storage::Instance().CAS(); + // write to cas + auto digest = cas.StoreBlob(data); if (digest) { - return casf.BlobPath(*digest); + return cas.BlobPath(*digest, /*is_executable=*/false); } return std::nullopt; } void AddDistfileToCAS(std::filesystem::path const& distfile, JustMR::PathsPtr const& just_mr_paths) noexcept { - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); for (auto const& dirpath : just_mr_paths->distdirs) { auto candidate = dirpath / distfile; if (FileSystemManager::Exists(candidate)) { // try to add to CAS - [[maybe_unused]] auto digest = casf.StoreBlobFromFile(candidate); + [[maybe_unused]] auto digest = + cas.StoreBlob(candidate, /*is_executable=*/false); } } } diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 5cb82eb3..dddd4b92 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -19,8 +19,8 @@ #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/main/constants.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/tmp_dir.hpp" /* Paths and constants required by just-mr */ @@ -29,12 +29,11 @@ std::unordered_set<std::string> const kLocationTypes{"workspace", "home", "system"}; auto const kDefaultJustPath = "just"; -auto const kDefaultRCPath = LocalExecutionConfig::GetUserHome() / ".just-mrrc"; -auto const kDefaultBuildRoot = LocalExecutionConfig::GetUserDir(); +auto const kDefaultRCPath = StorageConfig::GetUserHome() / ".just-mrrc"; +auto const kDefaultBuildRoot = StorageConfig::kDefaultBuildRoot; auto const kDefaultCheckoutLocationsFile = - LocalExecutionConfig::GetUserHome() / ".just-local.json"; -auto const kDefaultDistdirs = - LocalExecutionConfig::GetUserHome() / ".distfiles"; + StorageConfig::GetUserHome() / ".just-local.json"; +auto const kDefaultDistdirs = StorageConfig::GetUserHome() / ".distfiles"; std::vector<std::string> const kAltDirs = {"target_root", "rule_root", diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 5b692026..1220d50f 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -15,8 +15,8 @@ #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/buildtool/crypto/hasher.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/utils/curl_easy_handle.hpp" @@ -61,9 +61,9 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, JustMRStatistics::Instance().IncrementQueuedCounter(); } // check if content already in CAS - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto digest = ArtifactDigest(key.content, 0, false); - if (casf.BlobPath(digest)) { + if (cas.BlobPath(digest, /*is_executable=*/false)) { (*setter)(true); return; } @@ -74,7 +74,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, : std::filesystem::path(key.fetch_url).filename().string()); JustMR::Utils::AddDistfileToCAS(repo_distfile, just_mr_paths); // check if content is in CAS now - if (casf.BlobPath(digest)) { + if (cas.BlobPath(digest, /*is_executable=*/false)) { (*setter)(true); return; } diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp index 39474b12..1c903fe3 100644 --- a/src/other_tools/ops_maps/repo_fetch_map.cpp +++ b/src/other_tools/ops_maps/repo_fetch_map.cpp @@ -14,8 +14,8 @@ #include "src/other_tools/ops_maps/repo_fetch_map.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/utils.hpp" @@ -54,9 +54,10 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, logger]([[maybe_unused]] auto const& values) { // content is now in CAS // copy content from CAS into fetch_dir - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto content_path = - casf.BlobPath(ArtifactDigest(content, 0, false)); + cas.BlobPath(ArtifactDigest(content, 0, false), + /*is_executable=*/false); if (content_path) { auto target_name = fetch_dir / distfile; if (FileSystemManager::Exists(target_name)) { @@ -100,9 +101,10 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, } else { // copy content from CAS into fetch_dir - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto content_path = - casf.BlobPath(ArtifactDigest(key.archive.content, 0, false)); + cas.BlobPath(ArtifactDigest(key.archive.content, 0, false), + /*is_executable=*/false); if (content_path) { auto target_name = fetch_dir / distfile; if (FileSystemManager::Exists(target_name)) { diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 584fabc7..f36f3142 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -14,8 +14,8 @@ #include "src/other_tools/root_maps/content_git_map.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/utils/archive_ops.hpp" @@ -159,10 +159,11 @@ auto CreateContentGitMap( /*fatal=*/true); return; } - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); // content is in CAS if here, so no need to check nullopt auto content_cas_path = - casf.BlobPath(ArtifactDigest(content_id, 0, false)) + cas.BlobPath(ArtifactDigest(content_id, 0, false), + /*is_executable=*/false) .value(); auto res = ExtractArchive( content_cas_path, repo_type, tmp_dir->GetPath()); diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 3289430b..ef21b7cc 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -17,9 +17,8 @@ #include <algorithm> #include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" @@ -33,12 +32,13 @@ namespace { std::shared_ptr<std::unordered_map<std::string, std::string>> const& content_list, std::filesystem::path const& tmp_dir) noexcept -> bool { - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); return std::all_of(content_list->begin(), content_list->end(), - [&casf, tmp_dir](auto const& kv) { - auto content_path = casf.BlobPath( - ArtifactDigest(kv.second, 0, false)); + [&cas, tmp_dir](auto const& kv) { + auto content_path = + cas.BlobPath(ArtifactDigest(kv.second, 0, false), + /*is_executable=*/false); if (content_path) { return FileSystemManager::CreateFileHardlink( *content_path, // from: cas_path/content_id |