diff options
author | Oliver Reiche <oliver.reiche@huawei.com> | 2023-02-24 15:49:14 +0100 |
---|---|---|
committer | Oliver Reiche <oliver.reiche@huawei.com> | 2023-03-13 17:28:59 +0100 |
commit | 42b452e72f536b63bac080880b8daa481099793c (patch) | |
tree | 0af1a60e5e0fefb5572df38e092def3d8ac74c05 /src | |
parent | 546bf5beebf2eb75e7b325f8f18969b4dd34a169 (diff) | |
download | justbuild-42b452e72f536b63bac080880b8daa481099793c.tar.gz |
Storage: Reworked storage and garbage collection
The improved GC implementation uses refactored storage
classes instead of directly accessing "unknown" file paths.
The required storage class refactoring is quite substantial
and outlined in the following paragraphs.
The module `buildtool/file_system` was extended by:
- `ObjectCAS`: a plain CAS implementation for
reading/writing blobs and computing digests for a given
`ObjectType`. Depending on that type, files written to the
file system may have different properties (e.g., the x-bit
set) or the digest may be computed differently (e.g., tree
digests in non-compatible mode).
A new module `buildtool/storage` was introduced containing:
- `LocalCAS`: provides a common interface for the "logical
CAS", which internally combines three `ObjectCAS`s, one
for each `ObjectType` (file, executable, tree).
- `LocalAC`: implements the action cache, which needs the
`LocalCAS` for storing cache values.
- `TargetCache`: implements the high-level target cache,
which also needs the `LocalCAS` for storing cache values.
- `LocalStorage`: combines the storage classes `LocalCAS`,
`LocalAC`, and `TargetCache`. Those are initialized with
settings from `StorageConfig`, such as the build root base
path or number of generations for the garbage collector.
`LocalStorage` is templated with a Boolean parameter
`kDoGlobalUplink`, which indicates that, on every
read/write access, the garbage collector should be used
for uplinking across all generations (global).
- `GarbageCollector`: responsible for garbage collection and
the global uplinking across all generations. To do so, it
employs instances of `LocalStorage` with `kDoGlobalUplink`
set to false, in order to avoid endless recursion. The
actual (local) uplinking within two single generations is
performed by the corresponding storage class (e.g.,
`TargetCache` implements uplinking of target cache entries
between two target cache generations etc.). Thereby, the
actual knowledge how data should be uplinked is
implemented by the instance that is responsible for
creating the data in the first place.
Diffstat (limited to 'src')
56 files changed, 2013 insertions, 1783 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS index 82d1eca8..54eb2e77 100644 --- a/src/buildtool/build_engine/target_map/TARGETS +++ b/src/buildtool/build_engine/target_map/TARGETS @@ -15,8 +15,8 @@ , "name": ["result_map"] , "hdrs": ["result_map.hpp"] , "deps": - [ "target_cache" - , ["src/buildtool/common", "tree"] + [ ["src/buildtool/common", "tree"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/build_engine/analysed_target", "target"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/build_engine/expression", "expression"] @@ -28,7 +28,6 @@ , ["src/buildtool/logging", "logging"] , ["src/buildtool/common", "common"] , ["src/buildtool/build_engine/base_maps", "entity_name"] - , "target_cache_key" ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } @@ -52,9 +51,8 @@ ] , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": - [ "target_cache" - , "target_cache_key" - , ["src/buildtool/common", "common"] + [ ["src/buildtool/common", "common"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "field_reader"] , ["src/buildtool/build_engine/expression", "expression"] @@ -81,66 +79,4 @@ ] , "stage": ["src", "buildtool", "build_engine", "target_map"] } -, "target_cache": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache"] - , "hdrs": ["target_cache.hpp"] - , "srcs": ["target_cache.cpp"] - , "deps": - [ "target_cache_key" - , "target_cache_entry" - , ["@", "gsl-lite", "", "gsl-lite"] - , ["@", "json", "", "json"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "config"] - , ["src/buildtool/execution_api/local", "garbage_collector"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["src/buildtool/logging", "log_level"] - ] - } -, "target_cache_key": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache_key"] - , "hdrs": ["target_cache_key.hpp"] - , "srcs": ["target_cache_key.cpp"] - , "deps": - [ ["src/buildtool/build_engine/base_maps", "entity_name_data"] - , ["src/buildtool/build_engine/expression", "expression"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["src/buildtool/common", "config"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - ] - } -, "target_cache_entry": - { "type": ["@", "rules", "CC", "library"] - , "name": ["target_cache_entry"] - , "hdrs": ["target_cache_entry.hpp"] - , "srcs": ["target_cache_entry.cpp"] - , "deps": - [ ["@", "gsl-lite", "", "gsl-lite"] - , ["@", "json", "", "json"] - , ["src/buildtool/build_engine/analysed_target", "target"] - , ["src/buildtool/build_engine/expression", "expression"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/common", "artifact_description"] - ] - , "stage": ["src", "buildtool", "build_engine", "target_map"] - , "private-deps": - [ ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - ] - } } diff --git a/src/buildtool/build_engine/target_map/export.cpp b/src/buildtool/build_engine/target_map/export.cpp index 9005f037..187a192d 100644 --- a/src/buildtool/build_engine/target_map/export.cpp +++ b/src/buildtool/build_engine/target_map/export.cpp @@ -18,10 +18,8 @@ #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/buildtool/storage/storage.hpp" namespace { auto const kExpectedFields = std::unordered_set<std::string>{"config_doc", @@ -145,7 +143,7 @@ void ExportRule( ComputeTargetCacheKey(*exported_target, target_config); if (target_cache_key) { if (auto target_cache_value = - TargetCache::Instance().Read(*target_cache_key)) { + Storage::Instance().TargetCache().Read(*target_cache_key)) { auto const& [entry, info] = *target_cache_value; if (auto result = entry.ToResult()) { auto deps_info = TargetGraphInformation{ diff --git a/src/buildtool/build_engine/target_map/result_map.hpp b/src/buildtool/build_engine/target_map/result_map.hpp index edb1a472..235b9cc3 100644 --- a/src/buildtool/build_engine/target_map/result_map.hpp +++ b/src/buildtool/build_engine/target_map/result_map.hpp @@ -30,14 +30,14 @@ #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/target_cache.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/hash_combine.hpp" namespace BuildMaps::Target { diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp deleted file mode 100644 index 3e149ddd..00000000 --- a/src/buildtool/build_engine/target_map/target_cache.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/build_engine/target_map/target_cache.hpp" - -#include <exception> -#include <vector> - -#include <fmt/core.h> - -#include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/log_level.hpp" -#ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#endif - -auto TargetCache::Store(TargetCacheKey const& key, - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept - -> bool { - // Before a target-cache entry is stored in local CAS, make sure any created - // artifact for this target is downloaded from the remote CAS to the local - // CAS. - if (not DownloadKnownArtifacts(value, downloader)) { - return false; - } - if (auto digest = CAS().StoreBlobFromBytes(value.ToJson().dump(2))) { - auto data = - Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File} - .ToString(); - logger_.Emit(LogLevel::Debug, - "Adding entry for key {} as {}", - key.Id().ToString(), - data); - return file_store_.AddFromBytes(key.Id().digest.hash(), data); - } - return false; -} - -auto TargetCache::Read(TargetCacheKey const& key) const noexcept - -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo> > { - auto id = key.Id().digest.hash(); - auto entry_path = file_store_.GetPath(id); -#ifndef BOOTSTRAP_BUILD_TOOL - // Try to find target-cache entry in CAS generations and uplink if required. - auto found = GarbageCollector::FindAndUplinkTargetCacheEntry(id); -#else - auto found = FileSystemManager::IsFile(entry_path); -#endif - if (not found) { - logger_.Emit(LogLevel::Debug, - "Cache miss, entry not found {}", - entry_path.string()); - return std::nullopt; - } - auto const entry = - FileSystemManager::ReadFile(entry_path, ObjectType::File); - if (auto info = Artifact::ObjectInfo::FromString(*entry)) { - if (auto path = CAS().BlobPath(info->digest)) { - if (auto value = FileSystemManager::ReadFile(*path)) { - try { - return std::make_pair( - TargetCacheEntry{nlohmann::json::parse(*value)}, - std::move(*info)); - } catch (std::exception const& ex) { - logger_.Emit(LogLevel::Warning, - "Parsing entry for key {} failed with:\n{}", - key.Id().ToString(), - ex.what()); - } - } - } - } - logger_.Emit(LogLevel::Warning, - "Reading entry for key {} failed", - key.Id().ToString()); - return std::nullopt; -} - -auto TargetCache::DownloadKnownArtifacts( - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool { - std::vector<Artifact::ObjectInfo> artifacts_info; - return downloader and value.ToArtifacts(&artifacts_info) and - downloader(artifacts_info); -} - -auto TargetCache::ComputeCacheDir(int index) -> std::filesystem::path { - [[maybe_unused]] auto id = CAS().StoreBlobFromBytes( - LocalExecutionConfig::ExecutionBackendDescription()); - return LocalExecutionConfig::TargetCacheDir(index); -} diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp deleted file mode 100644 index 8bce88c0..00000000 --- a/src/buildtool/build_engine/target_map/target_cache.hpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP - -#include <filesystem> -#include <functional> -#include <optional> -#include <utility> - -#include <gsl-lite/gsl-lite.hpp> -#include <nlohmann/json.hpp> - -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/logging/logger.hpp" - -class TargetCache { - public: - using ArtifactDownloader = - std::function<bool(std::vector<Artifact::ObjectInfo> const&)>; - - TargetCache() = default; - TargetCache(TargetCache const&) = delete; - TargetCache(TargetCache&&) = delete; - auto operator=(TargetCache const&) -> TargetCache& = delete; - auto operator=(TargetCache&&) -> TargetCache& = delete; - ~TargetCache() noexcept = default; - - [[nodiscard]] static auto Instance() -> TargetCache& { - static TargetCache instance; - return instance; - } - - // Store new key entry pair in the target cache. - [[nodiscard]] auto Store( - TargetCacheKey const& key, - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool; - - // Read existing entry and object info for given key from the target cache. - [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept - -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; - - private: - Logger logger_{"TargetCache"}; - FileStorage<ObjectType::File, - StoreMode::LastWins, - /*kSetEpochTime=*/false> - file_store_{ComputeCacheDir(0)}; - - [[nodiscard]] auto DownloadKnownArtifacts( - TargetCacheEntry const& value, - ArtifactDownloader const& downloader) const noexcept -> bool; - [[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& { - return LocalCAS<ObjectType::File>::Instance(); - } - [[nodiscard]] static auto ComputeCacheDir(int index) - -> std::filesystem::path; -}; - -namespace std { -template <> -struct hash<TargetCacheKey> { - [[nodiscard]] auto operator()(TargetCacheKey const& k) const { - return std::hash<Artifact::ObjectInfo>{}(k.Id()); - } -}; -} // namespace std - -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index 39352c31..c8d9f878 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -105,6 +105,7 @@ , "deps": [ ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_cas"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/multithreading", "atomic_value"] ] , "stage": ["src", "buildtool", "common"] diff --git a/src/buildtool/common/bazel_types.hpp b/src/buildtool/common/bazel_types.hpp index d797b554..85e06764 100644 --- a/src/buildtool/common/bazel_types.hpp +++ b/src/buildtool/common/bazel_types.hpp @@ -21,6 +21,9 @@ #ifdef BOOTSTRAP_BUILD_TOOL +#include <cstdint> +#include <string> + namespace build::bazel::remote::execution::v2 { struct Digest { std::string hash_; diff --git a/src/buildtool/common/repository_config.cpp b/src/buildtool/common/repository_config.cpp index 73546a08..a3b81de1 100644 --- a/src/buildtool/common/repository_config.cpp +++ b/src/buildtool/common/repository_config.cpp @@ -14,7 +14,7 @@ #include "src/buildtool/common/repository_config.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/utils/automata/dfa_minimizer.hpp" auto RepositoryConfig::RepositoryInfo::BaseContentDescription() const @@ -43,8 +43,8 @@ auto RepositoryConfig::RepositoryKey(std::string const& repo) const noexcept return data->key.SetOnceAndGet( [this, &unique]() -> std::optional<std::string> { if (auto graph = BuildGraphForRepository(unique)) { - auto& cas = LocalCAS<ObjectType::File>::Instance(); - if (auto digest = cas.StoreBlobFromBytes(graph->dump(2))) { + auto const& cas = Storage::Instance().CAS(); + if (auto digest = cas.StoreBlob(graph->dump(2))) { return ArtifactDigest{*digest}.hash(); } } diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS index 739e32e2..42de3e7b 100644 --- a/src/buildtool/execution_api/execution_service/TARGETS +++ b/src/buildtool/execution_api/execution_service/TARGETS @@ -13,7 +13,7 @@ , "private-deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl-lite", "", "gsl-lite"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/file_system", "file_system_manager"] , "operation_cache" ] @@ -31,8 +31,7 @@ , ["src/buildtool/logging", "logging"] , ["src/buildtool/common", "bazel_types"] ] - , "private-deps": - [["src/buildtool/execution_api/local", "garbage_collector"]] + , "private-deps": [["src/buildtool/storage", "storage"]] } , "cas_server": { "type": ["@", "rules", "CC", "library"] @@ -49,7 +48,7 @@ , "private-deps": [ ["src/buildtool/compatibility", "compatibility"] , ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] ] } , "server_implementation": @@ -89,7 +88,7 @@ , ["src/buildtool/execution_api/common", "bytestream-common"] , ["src/utils/cpp", "tmp_dir"] , ["@", "fmt", "", "fmt"] - , ["src/buildtool/execution_api/local", "garbage_collector"] + , ["src/buildtool/storage", "storage"] ] } , "capabilities_server": diff --git a/src/buildtool/execution_api/execution_service/ac_server.cpp b/src/buildtool/execution_api/execution_service/ac_server.cpp index 8746a0cd..f8c5be4f 100644 --- a/src/buildtool/execution_api/execution_service/ac_server.cpp +++ b/src/buildtool/execution_api/execution_service/ac_server.cpp @@ -15,7 +15,7 @@ #include "src/buildtool/execution_api/execution_service/ac_server.hpp" #include "fmt/format.h" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" auto ActionCacheServiceImpl::GetActionResult( ::grpc::ServerContext* /*context*/, @@ -30,7 +30,7 @@ auto ActionCacheServiceImpl::GetActionResult( logger_.Emit(LogLevel::Error, str); return grpc::Status{grpc::StatusCode::INTERNAL, str}; } - auto x = ac_.CachedResult(request->action_digest()); + auto x = storage_->ActionCache().CachedResult(request->action_digest()); if (!x) { return grpc::Status{ grpc::StatusCode::NOT_FOUND, diff --git a/src/buildtool/execution_api/execution_service/ac_server.hpp b/src/buildtool/execution_api/execution_service/ac_server.hpp index bea61841..ce31e0ce 100644 --- a/src/buildtool/execution_api/execution_service/ac_server.hpp +++ b/src/buildtool/execution_api/execution_service/ac_server.hpp @@ -17,8 +17,8 @@ #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/local/local_ac.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class ActionCacheServiceImpl final : public bazel_re::ActionCache::Service { public: @@ -60,8 +60,7 @@ class ActionCacheServiceImpl final : public bazel_re::ActionCache::Service { ::bazel_re::ActionResult* response) -> ::grpc::Status override; private: - LocalCAS<ObjectType::File> cas_{}; - LocalAC ac_{&cas_}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service"}; }; diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index 37bfebc6..234c4adc 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -21,7 +21,7 @@ #include "fmt/format.h" #include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/common/bytestream_common.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { @@ -64,11 +64,12 @@ auto BytestreamServiceImpl::Read( if (NativeSupport::IsTree(*hash)) { ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, true}; - path = storage_.TreePath(static_cast<bazel_re::Digest>(dgst)); + path = storage_->CAS().TreePath(static_cast<bazel_re::Digest>(dgst)); } else { ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, false}; - path = storage_.BlobPath(static_cast<bazel_re::Digest>(dgst), false); + path = storage_->CAS().BlobPath(static_cast<bazel_re::Digest>(dgst), + false); } if (!path) { auto str = fmt::format("could not find {}", *hash); @@ -135,14 +136,14 @@ auto BytestreamServiceImpl::Write( return grpc::Status{grpc::StatusCode::INTERNAL, str}; } if (NativeSupport::IsTree(*hash)) { - if (not storage_.StoreTree(tmp)) { + if (not storage_->CAS().StoreTree(tmp)) { auto str = fmt::format("could not store tree {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } } else { - if (not storage_.StoreBlob(tmp)) { + if (not storage_->CAS().StoreBlob(tmp, /*is_executable=*/false)) { auto str = fmt::format("could not store blob {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.hpp b/src/buildtool/execution_api/execution_service/bytestream_server.hpp index e2627328..b69b5344 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.hpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.hpp @@ -16,8 +16,8 @@ #define BYTESTREAM_SERVER_HPP #include "google/bytestream/bytestream.grpc.pb.h" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class BytestreamServiceImpl : public ::google::bytestream::ByteStream::Service { public: @@ -76,7 +76,7 @@ class BytestreamServiceImpl : public ::google::bytestream::ByteStream::Service { -> ::grpc::Status override; private: - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service:bytestream"}; }; diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp index 07b9f3bf..7815f557 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.cpp +++ b/src/buildtool/execution_api/execution_service/cas_server.cpp @@ -16,7 +16,7 @@ #include "fmt/format.h" #include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" static constexpr std::size_t kJustHashLength = 42; static constexpr std::size_t kSHA256Length = 64; @@ -51,12 +51,12 @@ auto CASServiceImpl::FindMissingBlobs( } logger_.Emit(LogLevel::Trace, "FindMissingBlobs: {}", hash); if (NativeSupport::IsTree(hash)) { - if (!storage_.TreePath(x)) { + if (!storage_->CAS().TreePath(x)) { auto* d = response->add_missing_blob_digests(); d->CopyFrom(x); } } - else if (!storage_.BlobPath(x, false)) { + else if (!storage_->CAS().BlobPath(x, false)) { auto* d = response->add_missing_blob_digests(); d->CopyFrom(x); } @@ -99,7 +99,7 @@ auto CASServiceImpl::BatchUpdateBlobs( auto* r = response->add_responses(); r->mutable_digest()->CopyFrom(x.digest()); if (NativeSupport::IsTree(hash)) { - auto const& dgst = storage_.StoreTree(x.data()); + auto const& dgst = storage_->CAS().StoreTree(x.data()); if (!dgst) { auto const& str = fmt::format( "BatchUpdateBlobs: could not upload tree {}", hash); @@ -111,7 +111,7 @@ auto CASServiceImpl::BatchUpdateBlobs( } } else { - auto const& dgst = storage_.StoreBlob(x.data(), false); + auto const& dgst = storage_->CAS().StoreBlob(x.data(), false); if (!dgst) { auto const& str = fmt::format( "BatchUpdateBlobs: could not upload blob {}", hash); @@ -141,10 +141,10 @@ auto CASServiceImpl::BatchReadBlobs( r->mutable_digest()->CopyFrom(digest); std::optional<std::filesystem::path> path; if (NativeSupport::IsTree(digest.hash())) { - path = storage_.TreePath(digest); + path = storage_->CAS().TreePath(digest); } else { - path = storage_.BlobPath(digest, false); + path = storage_->CAS().BlobPath(digest, false); } if (!path) { google::rpc::Status status; diff --git a/src/buildtool/execution_api/execution_service/cas_server.hpp b/src/buildtool/execution_api/execution_service/cas_server.hpp index b3c11706..520263b6 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.hpp +++ b/src/buildtool/execution_api/execution_service/cas_server.hpp @@ -16,8 +16,8 @@ #define CAS_SERVER_HPP #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class CASServiceImpl final : public bazel_re::ContentAddressableStorage::Service { @@ -119,7 +119,7 @@ class CASServiceImpl final std::string const& computed) const noexcept -> std::optional<std::string>; - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service"}; }; #endif diff --git a/src/buildtool/execution_api/execution_service/execution_server.cpp b/src/buildtool/execution_api/execution_service/execution_server.cpp index 5f21f5ac..ed22e023 100644 --- a/src/buildtool/execution_api/execution_service/execution_server.cpp +++ b/src/buildtool/execution_api/execution_service/execution_server.cpp @@ -24,8 +24,8 @@ #include "fmt/format.h" #include "gsl-lite/gsl-lite.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" static void UpdateTimeStamp(::google::longrunning::Operation* op) { ::google::protobuf::Timestamp t; @@ -40,7 +40,7 @@ auto ExecutionServiceImpl::GetAction(::bazel_re::ExecuteRequest const* request) const noexcept -> std::pair<std::optional<::bazel_re::Action>, std::optional<std::string>> { // get action description - auto path = storage_.BlobPath(request->action_digest(), false); + auto path = storage_->CAS().BlobPath(request->action_digest(), false); if (!path) { auto str = fmt::format("could not retrieve blob {} from cas", request->action_digest().hash()); @@ -59,8 +59,8 @@ auto ExecutionServiceImpl::GetAction(::bazel_re::ExecuteRequest const* request) } path = Compatibility::IsCompatible() - ? storage_.BlobPath(action.input_root_digest(), false) - : storage_.TreePath(action.input_root_digest()); + ? storage_->CAS().BlobPath(action.input_root_digest(), false) + : storage_->CAS().TreePath(action.input_root_digest()); if (!path) { auto str = fmt::format("could not retrieve input root {} from cas", @@ -75,7 +75,7 @@ auto ExecutionServiceImpl::GetCommand(::bazel_re::Action const& action) const noexcept -> std::pair<std::optional<::bazel_re::Command>, std::optional<std::string>> { - auto path = storage_.BlobPath(action.command_digest(), false); + auto path = storage_->CAS().BlobPath(action.command_digest(), false); if (!path) { auto str = fmt::format("could not retrieve blob {} from cas", action.command_digest().hash()); @@ -141,10 +141,10 @@ auto ExecutionServiceImpl::GetIExecutionAction( } static auto GetDirectoryFromDigest(::bazel_re::Digest const& digest, - LocalStorage const& storage) noexcept + Storage const& storage) noexcept -> std::optional<::bazel_re::Directory> { // determine directory path from digest - auto const& path = storage.BlobPath(digest, /*is_executable=*/false); + auto const& path = storage.CAS().BlobPath(digest, /*is_executable=*/false); if (not path) { return std::nullopt; } @@ -166,7 +166,7 @@ static auto GetDirectoryFromDigest(::bazel_re::Digest const& digest, // NOLINTNEXTLINE(misc-no-recursion) static auto CollectChildDirectoriesRecursively( ::bazel_re::Directory const& root, - LocalStorage const& storage, + Storage const& storage, gsl::not_null<std::unordered_map<::bazel_re::Digest, ::bazel_re::Directory>*> map) noexcept -> bool { @@ -196,7 +196,7 @@ static auto CollectChildDirectoriesRecursively( } static auto GetChildrenFromDirectory(::bazel_re::Directory const& root, - LocalStorage const& storage) noexcept + Storage const& storage) noexcept -> std::optional<std::vector<::bazel_re::Directory>> { // determine child directories std::unordered_map<::bazel_re::Digest, ::bazel_re::Directory> map{}; @@ -232,7 +232,7 @@ static auto GetChildrenFromDirectory(::bazel_re::Directory const& root, static auto CreateTreeDigestFromDirectoryDigest( ::bazel_re::Digest const& dir_digest, - LocalStorage const& storage) noexcept -> std::optional<::bazel_re::Digest> { + Storage const& storage) noexcept -> std::optional<::bazel_re::Digest> { // determine root directory message auto root = GetDirectoryFromDigest(dir_digest, storage); if (not root) { @@ -256,7 +256,8 @@ static auto CreateTreeDigestFromDirectoryDigest( // serialize and store tree message auto content = tree.SerializeAsString(); - auto tree_digest = storage.StoreBlob(content, /*is_executable=*/false); + auto tree_digest = + storage.CAS().StoreBlob(content, /*is_executable=*/false); if (not tree_digest) { return std::nullopt; } @@ -266,7 +267,7 @@ static auto CreateTreeDigestFromDirectoryDigest( static auto AddOutputPaths(::bazel_re::ExecuteResponse* response, IExecutionResponse::Ptr const& execution, - LocalStorage const& storage) noexcept -> bool { + Storage const& storage) noexcept -> bool { auto const& size = static_cast<int>(execution->Artifacts().size()); response->mutable_result()->mutable_output_files()->Reserve(size); response->mutable_result()->mutable_output_directories()->Reserve(size); @@ -311,7 +312,7 @@ auto ExecutionServiceImpl::AddResult( IExecutionResponse::Ptr const& i_execution_response, std::string const& action_hash) const noexcept -> std::optional<std::string> { - if (not AddOutputPaths(response, i_execution_response, storage_)) { + if (not AddOutputPaths(response, i_execution_response, *storage_)) { auto str = fmt::format("Error in creating output paths of action {}", action_hash); logger_.Emit(LogLevel::Error, str); @@ -320,8 +321,8 @@ auto ExecutionServiceImpl::AddResult( auto* result = response->mutable_result(); result->set_exit_code(i_execution_response->ExitCode()); if (i_execution_response->HasStdErr()) { - auto dgst = storage_.StoreBlob(i_execution_response->StdErr(), - /*is_executable=*/false); + auto dgst = storage_->CAS().StoreBlob(i_execution_response->StdErr(), + /*is_executable=*/false); if (!dgst) { auto str = fmt::format("Could not store stderr of action {}", action_hash); @@ -331,8 +332,8 @@ auto ExecutionServiceImpl::AddResult( result->mutable_stderr_digest()->CopyFrom(*dgst); } if (i_execution_response->HasStdOut()) { - auto dgst = storage_.StoreBlob(i_execution_response->StdOut(), - /*is_executable=*/false); + auto dgst = storage_->CAS().StoreBlob(i_execution_response->StdOut(), + /*is_executable=*/false); if (!dgst) { auto str = fmt::format("Could not store stdout of action {}", action_hash); @@ -375,8 +376,8 @@ auto ExecutionServiceImpl::StoreActionResult( ::bazel_re::Action const& action) const noexcept -> std::optional<std::string> { if (i_execution_response->ExitCode() == 0 && !action.do_not_cache() && - !storage_.StoreActionResult(request->action_digest(), - execute_response.result())) { + !storage_->ActionCache().StoreResult(request->action_digest(), + execute_response.result())) { auto str = fmt::format("Could not store action result for action {}", request->action_digest().hash()); logger_.Emit(LogLevel::Error, str); diff --git a/src/buildtool/execution_api/execution_service/execution_server.hpp b/src/buildtool/execution_api/execution_service/execution_server.hpp index 65ace208..6d99fb6a 100644 --- a/src/buildtool/execution_api/execution_service/execution_server.hpp +++ b/src/buildtool/execution_api/execution_service/execution_server.hpp @@ -18,8 +18,8 @@ #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" class ExecutionServiceImpl final : public bazel_re::Execution::Service { public: @@ -107,7 +107,7 @@ class ExecutionServiceImpl final : public bazel_re::Execution::Service { -> ::grpc::Status override; private: - LocalStorage storage_{}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); IExecutionApi::Ptr api_{new LocalApi()}; Logger logger_{"execution-service"}; diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index 31f2c655..28377cc2 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -19,24 +19,17 @@ , "local": { "type": ["@", "rules", "CC", "library"] , "name": ["local"] - , "hdrs": - [ "local_api.hpp" - , "local_action.hpp" - , "local_response.hpp" - , "local_storage.hpp" - , "local_cas.hpp" - , "local_ac.hpp" - ] - , "srcs": ["local_action.cpp", "local_storage.cpp"] + , "hdrs": ["local_api.hpp", "local_action.hpp", "local_response.hpp"] + , "srcs": ["local_action.cpp"] , "deps": [ "config" - , "garbage_collector" , ["@", "fmt", "", "fmt"] , ["@", "gsl-lite", "", "gsl-lite"] + , ["src/buildtool/storage", "storage"] + , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] - , ["src/buildtool/file_system", "file_storage"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/common", "common"] @@ -50,28 +43,4 @@ , ["src/buildtool/system", "system_command"] ] } -, "garbage_collector": - { "type": ["@", "rules", "CC", "library"] - , "name": ["garbage_collector"] - , "hdrs": ["garbage_collector.hpp"] - , "srcs": ["garbage_collector.cpp"] - , "deps": [["src/utils/cpp", "file_locking"]] - , "stage": ["src", "buildtool", "execution_api", "local"] - , "private-deps": - [ "config" - , ["@", "json", "", "json"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/common", "bazel_types"] - , ["src/buildtool/compatibility", "compatibility"] - , ["src/buildtool/build_engine/target_map", "target_cache_entry"] - , ["src/buildtool/execution_api/common", "common"] - , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["src/buildtool/file_system", "git_repo"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "logging"] - , ["src/buildtool/logging", "log_level"] - , ["src/utils/cpp", "hex_string"] - ] - } } diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp index 3faeb303..a1882776 100644 --- a/src/buildtool/execution_api/local/config.hpp +++ b/src/buildtool/execution_api/local/config.hpp @@ -15,26 +15,15 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP -#ifdef __unix__ -#include <pwd.h> -#include <sys/types.h> -#include <unistd.h> -#else -#error "Non-unix is not supported yet" -#endif - #include <filesystem> #include <functional> #include <string> #include <vector> -#include <fmt/core.h> #include <gsl-lite/gsl-lite.hpp> #include <nlohmann/json.hpp> #include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -43,59 +32,12 @@ /// \brief Store global build system configuration. class LocalExecutionConfig { struct ConfigData { - - // Build root directory. All the cache dirs are subdirs of build_root. - // By default, build_root is set to $HOME/.cache/just. - // If the user uses --local-build-root PATH, - // then build_root will be set to PATH. - std::filesystem::path build_root{}; - - // cache_root points to the root of the cache dirs. - std::filesystem::path cache_root{}; - - // cache_root_generations holds root directories for all cache - // generations (default: two generations). The latest generation points - // to one the following directories: - // build_root/protocol-dependent/generation-0/{git-sha1,compatible-sha256} - // git-sha1 is the current default. If the user passes the flag - // --compatible, then the subfolder compatible_sha256 is used - std::vector<std::filesystem::path> cache_root_generations{"", ""}; - // Launcher to be prepended to action's command before executed. // Default: ["env", "--"] std::vector<std::string> launcher{"env", "--"}; }; - // different folder for different caching protocol - [[nodiscard]] static auto UpdatePathForCompatibility( - std::filesystem::path const& dir) -> std::filesystem::path { - return dir / (Compatibility::IsCompatible() ? "compatible-sha256" - : "git-sha1"); - } - public: - [[nodiscard]] static auto SetBuildRoot( - std::filesystem::path const& dir) noexcept -> bool { - if (FileSystemManager::IsRelativePath(dir)) { - Logger::Log(LogLevel::Error, - "Build root must be absolute path but got '{}'.", - dir.string()); - return false; - } - Data().build_root = dir; - // In case we re-set build_root, we are sure that the cache roots are - // recomputed as well. - Data().cache_root = std::filesystem::path{}; - Data().cache_root_generations = - std::vector(NumGenerations(), std::filesystem::path{}); - // Pre-initialize cache roots to avoid race condition during lazy - // initialization by multiple threads. - for (int i = 0; i < NumGenerations(); ++i) { - [[maybe_unused]] auto root = CacheRoot(i); - } - return true; - } - [[nodiscard]] static auto SetLauncher( std::vector<std::string> const& launcher) noexcept -> bool { try { @@ -109,140 +51,11 @@ class LocalExecutionConfig { return true; } - /// \brief Specifies the number of cache generations. - static auto SetNumGenerations(int num_generations) noexcept -> void { - gsl_ExpectsAudit(num_generations > 0); - Data().cache_root_generations = - std::vector(num_generations, std::filesystem::path{}); - } - - [[nodiscard]] static auto NumGenerations() noexcept -> int { - return Data().cache_root_generations.size(); - } - - /// \brief User directory. - [[nodiscard]] static auto GetUserDir() noexcept -> std::filesystem::path { - return GetUserHome() / ".cache" / "just"; - } - - /// \brief Build directory, defaults to user directory if not set - [[nodiscard]] static auto BuildRoot() noexcept -> std::filesystem::path { - auto& build_root = Data().build_root; - if (build_root.empty()) { - build_root = GetUserDir(); - } - return build_root; - } - - [[nodiscard]] static auto CacheRoot() noexcept -> std::filesystem::path { - auto& cache_root = Data().cache_root; - if (cache_root.empty()) { - cache_root = BuildRoot() / "protocol-dependent"; - } - return cache_root; - } - - [[nodiscard]] static auto CacheRoot(int index) noexcept - -> std::filesystem::path { - gsl_ExpectsAudit(index >= 0 and - index < Data().cache_root_generations.size()); - auto& cache_root = Data().cache_root_generations[index]; - if (cache_root.empty()) { - auto generation = - std::string{"generation-"} + std::to_string(index); - cache_root = CacheRoot() / generation; - } - return cache_root; - } - - [[nodiscard]] static auto CacheRootDir(int index) noexcept - -> std::filesystem::path { - return UpdatePathForCompatibility(CacheRoot(index)); - } - - // CAS directory based on the type of the file. - template <ObjectType kType> - [[nodiscard]] static inline auto CASDir(int index) noexcept - -> std::filesystem::path { - char t = ToChar(kType); - if constexpr (kType == ObjectType::Tree) { - if (Compatibility::IsCompatible()) { - t = ToChar(ObjectType::File); - } - } - static auto const kSuffix = std::string{"cas"} + t; - return CacheRootDir(index) / kSuffix; - } - - /// \brief Action cache directory - [[nodiscard]] static auto ActionCacheDir(int index) noexcept - -> std::filesystem::path { - return CacheRootDir(index) / "ac"; - } - - /// \brief Target cache root directory - [[nodiscard]] static auto TargetCacheRoot(int index) noexcept - -> std::filesystem::path { - return CacheRootDir(index) / "tc"; - } - - /// \brief Target cache directory for the used execution backend. - [[nodiscard]] static auto TargetCacheDir(int index) noexcept - -> std::filesystem::path { - return TargetCacheRoot(index) / - ArtifactDigest::Create<ObjectType::File>( - ExecutionBackendDescription()) - .hash(); - } - - /// \brief String representation of the used execution backend. - [[nodiscard]] static auto ExecutionBackendDescription() noexcept - -> std::string { - auto address = RemoteExecutionConfig::RemoteAddress(); - auto properties = RemoteExecutionConfig::PlatformProperties(); - try { - // json::dump with json::error_handler_t::replace will not throw an - // exception if invalid UTF-8 sequences are detected in the input. - // Instead, it will replace them with the UTF-8 replacement - // character, but still it needs to be inside a try-catch clause to - // ensure the noexcept modifier of the enclosing function. - return nlohmann::json{ - {"remote_address", - address ? nlohmann::json{fmt::format( - "{}:{}", address->host, address->port)} - : nlohmann::json{}}, - {"platform_properties", properties}} - .dump(2, ' ', false, nlohmann::json::error_handler_t::replace); - } catch (...) { - return ""; - } - } - [[nodiscard]] static auto GetLauncher() noexcept -> std::vector<std::string> { return Data().launcher; } - /// \brief Determine user root directory - [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { - char const* root{nullptr}; - -#ifdef __unix__ - root = std::getenv("HOME"); - if (root == nullptr) { - root = getpwuid(getuid())->pw_dir; - } -#endif - - if (root == nullptr) { - Logger::Log(LogLevel::Error, - "Cannot determine user home directory."); - std::exit(EXIT_FAILURE); - } - - return root; - } - private: [[nodiscard]] static auto Data() noexcept -> ConfigData& { static ConfigData instance{}; diff --git a/src/buildtool/execution_api/local/garbage_collector.cpp b/src/buildtool/execution_api/local/garbage_collector.cpp deleted file mode 100644 index a0b68811..00000000 --- a/src/buildtool/execution_api/local/garbage_collector.cpp +++ /dev/null @@ -1,514 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/local/garbage_collector.hpp" - -#include <vector> - -#include <nlohmann/json.hpp> - -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/logging/log_level.hpp" -#include "src/buildtool/logging/logger.hpp" -#include "src/utils/cpp/hex_string.hpp" - -auto GarbageCollector::FindAndUplinkBlob(std::string const& id, - bool is_executable) noexcept -> bool { - // Try to find blob in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkBlob(i, id, is_executable)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkTree(std::string const& id) noexcept - -> bool { - // Try to find tree in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkTree(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkActionCacheEntry( - std::string const& id) noexcept -> bool { - // Try to find action-cache entry in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkActionCacheEntry(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::FindAndUplinkTargetCacheEntry( - std::string const& id) noexcept -> bool { - // Try to find target-cache entry in all generations. - for (int i = 0; i < LocalExecutionConfig::NumGenerations(); i++) { - if (UplinkTargetCacheEntry(i, id)) { - return true; - } - } - return false; -} - -auto GarbageCollector::SharedLock() noexcept -> std::optional<LockFile> { - return LockFile::Acquire(LockFilePath(), /*is_shared=*/true); -} - -auto GarbageCollector::ExclusiveLock() noexcept -> std::optional<LockFile> { - return LockFile::Acquire(LockFilePath(), /*is_shared=*/false); -} - -auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path { - return LocalExecutionConfig::CacheRoot() / "gc.lock"; -} - -auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool { - auto pid = CreateProcessUniqueId(); - if (not pid) { - return false; - } - auto remove_me = std::string{"remove-me-"} + *pid; - auto remove_me_dir = LocalExecutionConfig::CacheRoot() / remove_me; - if (FileSystemManager::IsDirectory(remove_me_dir)) { - if (not FileSystemManager::RemoveDirectory(remove_me_dir, - /*recursively=*/true)) { - Logger::Log(LogLevel::Error, - "Failed to remove directory {}", - remove_me_dir.string()); - return false; - } - } - { // Create scope for critical renaming section protected by advisory lock. - auto lock = ExclusiveLock(); - if (not lock) { - Logger::Log(LogLevel::Error, - "Failed to exclusively lock the local build root"); - return false; - } - for (int i = LocalExecutionConfig::NumGenerations() - 1; i >= 0; i--) { - auto cache_root = LocalExecutionConfig::CacheRoot(i); - if (FileSystemManager::IsDirectory(cache_root)) { - auto new_cache_root = - (i == LocalExecutionConfig::NumGenerations() - 1) - ? remove_me_dir - : LocalExecutionConfig::CacheRoot(i + 1); - if (not FileSystemManager::Rename(cache_root, new_cache_root)) { - Logger::Log(LogLevel::Error, - "Failed to rename {} to {}.", - cache_root.string(), - new_cache_root.string()); - return false; - } - } - } - } - if (FileSystemManager::IsDirectory(remove_me_dir)) { - if (not FileSystemManager::RemoveDirectory(remove_me_dir, - /*recursively=*/true)) { - Logger::Log(LogLevel::Warning, - "Failed to remove directory {}", - remove_me_dir.string()); - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkBlob(int index, - std::string const& id, - bool is_executable) noexcept -> bool { - // Determine blob path of given generation. - auto root = - is_executable - ? LocalExecutionConfig::CASDir<ObjectType::Executable>(index) - : LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto blob_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(blob_path)) { - return false; - } - - // Determine blob path in latest generation. - auto root_latest = - is_executable ? LocalExecutionConfig::CASDir<ObjectType::Executable>(0) - : LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto blob_path_latest = GetStoragePath(root_latest, id); - if (not FileSystemManager::IsFile(blob_path_latest)) { - // Uplink blob from older generation to the latest generation. - if (not FileSystemManager::CreateDirectory( - blob_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - blob_path, - blob_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(blob_path_latest)) { - return false; - } - } - return true; -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto GarbageCollector::UplinkTree(int index, std::string const& id) noexcept - -> bool { - // Determine tree path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::Tree>(index); - auto tree_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(tree_path)) { - return false; - } - - // Determine tree path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::Tree>(0); - auto tree_path_latest = GetStoragePath(root_latest, id); - if (not FileSystemManager::IsFile(tree_path_latest)) { - // Determine tree entries. - auto content = FileSystemManager::ReadFile(tree_path); - auto tree_entries = GitRepo::ReadTreeData(*content, - id, - /*is_hex_id=*/true); - if (not tree_entries) { - return false; - } - - // Uplink tree entries. - for (auto const& [raw_id, entry_vector] : *tree_entries) { - // Process only first entry from 'entry_vector' since all entries - // represent the same blob, just with different names. - auto entry = entry_vector.front(); - auto hash = ToHexString(raw_id); - if (entry.type == ObjectType::Tree) { - if (not UplinkTree(index, hash)) { - return false; - } - } - else { - if (not UplinkBlob( - index, hash, entry.type == ObjectType::Executable)) { - return false; - } - } - } - - // Uplink tree from older generation to the latest generation. - if (not FileSystemManager::CreateDirectory( - tree_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - tree_path, - tree_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(tree_path_latest)) { - return false; - } - } - return true; -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto GarbageCollector::UplinkBazelDirectory(int index, - std::string const& id) noexcept - -> bool { - // Determine bazel directory path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto dir_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(dir_path)) { - return false; - } - - // Determine bazel directory entries. - auto content = FileSystemManager::ReadFile(dir_path); - bazel_re::Directory dir{}; - if (not dir.ParseFromString(*content)) { - return false; - } - - // Uplink bazel directory entries. - for (auto const& file : dir.files()) { - if (not UplinkBlob(index, - NativeSupport::Unprefix(file.digest().hash()), - file.is_executable())) { - return false; - } - } - for (auto const& directory : dir.directories()) { - if (not UplinkBazelDirectory( - index, NativeSupport::Unprefix(directory.digest().hash()))) { - return false; - } - } - - // Determine bazel directory path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto dir_path_latest = GetStoragePath(root_latest, id); - - // Uplink bazel directory from older generation to the latest generation. - if (not FileSystemManager::IsFile(dir_path_latest)) { - if (not FileSystemManager::CreateDirectory( - dir_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - dir_path, - dir_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(dir_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkActionCacheEntry(int index, - std::string const& id) noexcept - -> bool { - // Determine action-cache entry path of given generation. - auto root = LocalExecutionConfig::ActionCacheDir(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine action-cache entry location. - auto content = FileSystemManager::ReadFile(entry_path, ObjectType::File); - bazel_re::Digest digest{}; - if (not digest.ParseFromString(*content)) { - return false; - } - - // Uplink action-cache entry blob. - if (not UplinkActionCacheEntryBlob( - index, NativeSupport::Unprefix(digest.hash()))) { - return false; - } - - // Determine action-cache entry path in latest generation. - auto root_latest = LocalExecutionConfig::ActionCacheDir(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink action-cache entry from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkActionCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool { - - // Determine action-cache entry blob path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine artifacts referenced by action-cache entry. - auto content = FileSystemManager::ReadFile(entry_path); - bazel_re::ActionResult result{}; - if (not result.ParseFromString(*content)) { - return false; - } - - // Uplink referenced artifacts. - for (auto const& file : result.output_files()) { - if (not UplinkBlob(index, - NativeSupport::Unprefix(file.digest().hash()), - file.is_executable())) { - return false; - } - } - for (auto const& directory : result.output_directories()) { - if (Compatibility::IsCompatible()) { - if (not UplinkBazelDirectory( - index, - NativeSupport::Unprefix(directory.tree_digest().hash()))) { - return false; - } - } - else { - if (not UplinkTree( - index, - NativeSupport::Unprefix(directory.tree_digest().hash()))) { - return false; - } - } - } - - // Determine action-cache entry blob path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink action-cache entry blob from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkTargetCacheEntry(int index, - std::string const& id) noexcept - -> bool { - - // Determine target-cache entry path of given generation. - auto root = LocalExecutionConfig::TargetCacheDir(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine target-cache entry location. - auto content = FileSystemManager::ReadFile(entry_path); - auto info = Artifact::ObjectInfo::FromString(*content); - if (not info) { - return false; - } - - // Uplink target-cache entry blob. - if (not UplinkTargetCacheEntryBlob(index, info->digest.hash())) { - return false; - } - - // Determine target-cache entry path in latest generation. - auto root_latest = LocalExecutionConfig::TargetCacheDir(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink target-cache entry from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} - -auto GarbageCollector::UplinkTargetCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool { - - // Determine target-cache entry blob path of given generation. - auto root = LocalExecutionConfig::CASDir<ObjectType::File>(index); - auto entry_path = GetStoragePath(root, id); - if (not FileSystemManager::IsFile(entry_path)) { - return false; - } - - // Determine artifacts referenced by target-cache entry. - auto content = FileSystemManager::ReadFile(entry_path); - nlohmann::json json_desc{}; - try { - json_desc = nlohmann::json::parse(*content); - } catch (std::exception const& ex) { - return false; - } - auto entry = TargetCacheEntry::FromJson(json_desc); - std::vector<Artifact::ObjectInfo> artifacts_info; - if (not entry.ToArtifacts(&artifacts_info)) { - return false; - } - - // Uplink referenced artifacts. - for (auto const& info : artifacts_info) { - auto hash = info.digest.hash(); - if (info.type == ObjectType::Tree) { - if (Compatibility::IsCompatible()) { - if (not UplinkBazelDirectory(index, hash)) { - return false; - } - } - else { - if (not UplinkTree(index, hash)) { - return false; - } - } - } - else { - if (not UplinkBlob( - index, hash, info.type == ObjectType::Executable)) { - return false; - } - } - } - - // Determine target-cache entry blob path in latest generation. - auto root_latest = LocalExecutionConfig::CASDir<ObjectType::File>(0); - auto entry_path_latest = GetStoragePath(root_latest, id); - - // Uplink target-cache entry blob from older generation to the latest - // generation. - if (not FileSystemManager::IsFile(entry_path_latest)) { - if (not FileSystemManager::CreateDirectory( - entry_path_latest.parent_path())) { - return false; - } - if (not FileSystemManager::CreateFileHardlink( - entry_path, - entry_path_latest, - /*log_failure_at=*/LogLevel::Debug) and - not FileSystemManager::IsFile(entry_path_latest)) { - return false; - } - } - return true; -} diff --git a/src/buildtool/execution_api/local/garbage_collector.hpp b/src/buildtool/execution_api/local/garbage_collector.hpp deleted file mode 100644 index 6b89c841..00000000 --- a/src/buildtool/execution_api/local/garbage_collector.hpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP - -#include <optional> -#include <string> - -#include "src/utils/cpp/file_locking.hpp" - -/// \brief -class GarbageCollector { - public: - [[nodiscard]] auto static FindAndUplinkBlob(std::string const& id, - bool is_executable) noexcept - -> bool; - - [[nodiscard]] auto static FindAndUplinkTree(std::string const& id) noexcept - -> bool; - - [[nodiscard]] auto static FindAndUplinkActionCacheEntry( - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static FindAndUplinkTargetCacheEntry( - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool; - - [[nodiscard]] auto static SharedLock() noexcept -> std::optional<LockFile>; - - [[nodiscard]] auto static ExclusiveLock() noexcept - -> std::optional<LockFile>; - - private: - [[nodiscard]] auto static LockFilePath() noexcept -> std::filesystem::path; - - [[nodiscard]] auto static UplinkBlob(int index, - std::string const& id, - bool is_executable) noexcept -> bool; - - [[nodiscard]] auto static UplinkTree(int index, - std::string const& id) noexcept - -> bool; - - [[nodiscard]] auto static UplinkBazelDirectory( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkActionCacheEntry( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkActionCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkTargetCacheEntry( - int index, - std::string const& id) noexcept -> bool; - - [[nodiscard]] auto static UplinkTargetCacheEntryBlob( - int index, - std::string const& id) noexcept -> bool; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_GARBAGE_COLLECTOR_HPP diff --git a/src/buildtool/execution_api/local/local_ac.hpp b/src/buildtool/execution_api/local/local_ac.hpp deleted file mode 100644 index 423282be..00000000 --- a/src/buildtool/execution_api/local/local_ac.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP - -#include "gsl-lite/gsl-lite.hpp" -#include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/logger.hpp" - -class LocalAC { - public: - explicit LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas) noexcept - : cas_{std::move(cas)} {}; - - LocalAC(LocalAC const&) = delete; - LocalAC(LocalAC&&) = delete; - auto operator=(LocalAC const&) -> LocalAC& = delete; - auto operator=(LocalAC&&) -> LocalAC& = delete; - ~LocalAC() noexcept = default; - - [[nodiscard]] auto StoreResult( - bazel_re::Digest const& action_id, - bazel_re::ActionResult const& result) const noexcept -> bool { - auto bytes = result.SerializeAsString(); - auto digest = cas_->StoreBlobFromBytes(bytes); - return (digest and file_store_.AddFromBytes( - NativeSupport::Unprefix(action_id.hash()), - digest->SerializeAsString())); - } - - [[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id) - const noexcept -> std::optional<bazel_re::ActionResult> { - auto id = NativeSupport::Unprefix(action_id.hash()); - auto entry_path = file_store_.GetPath(id); - // Try to find action-cache entry in CAS generations and uplink if - // required. - if (not GarbageCollector::FindAndUplinkActionCacheEntry(id)) { - logger_.Emit(LogLevel::Debug, - "Cache miss, entry not found {}", - entry_path.string()); - return std::nullopt; - } - auto const entry = - FileSystemManager::ReadFile(entry_path, ObjectType::File); - bazel_re::Digest digest{}; - if (not digest.ParseFromString(*entry)) { - logger_.Emit(LogLevel::Warning, - "Parsing cache entry failed for action {}", - id); - return std::nullopt; - } - auto src_path = cas_->BlobPath(digest); - bazel_re::ActionResult result{}; - if (src_path) { - auto const bytes = FileSystemManager::ReadFile(*src_path); - if (bytes.has_value() and result.ParseFromString(*bytes)) { - return result; - } - } - logger_.Emit(LogLevel::Warning, - "Parsing action result failed for action {}", - id); - return std::nullopt; - } - - private: - // The action cache stores the results of failed actions. For those to be - // overwritable by subsequent runs we need to choose the store mode "last - // wins" for the underlying file storage. - static constexpr auto kStoreMode = StoreMode::LastWins; - - Logger logger_{"LocalAC"}; - gsl::not_null<LocalCAS<ObjectType::File>*> cas_; - FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false> - file_store_{LocalExecutionConfig::ActionCacheDir(0)}; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index 414773c2..209c6a5d 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -20,9 +20,11 @@ #include "gsl-lite/gsl-lite.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/system/system_command.hpp" namespace { @@ -48,6 +50,24 @@ class BuildCleanupAnchor { std::filesystem::path const build_path{}; }; +[[nodiscard]] auto CreateDigestFromLocalOwnedTree( + gsl::not_null<Storage const*> const& storage, + std::filesystem::path const& dir_path) -> std::optional<bazel_re::Digest> { + auto const& cas = storage->CAS(); + auto store_blob = [&cas](auto path, auto is_exec) { + return cas.StoreBlob</*kOwner=*/true>(path, is_exec); + }; + auto store_tree = [&cas](auto bytes, + auto /*dir*/) -> std::optional<bazel_re::Digest> { + return cas.StoreTree(bytes); + }; + return Compatibility::IsCompatible() + ? BazelMsgFactory::CreateDirectoryDigestFromLocalTree( + dir_path, store_blob, store_tree) + : BazelMsgFactory::CreateGitTreeDigestFromLocalTree( + dir_path, store_blob, store_tree); +} + } // namespace auto LocalAction::Execute(Logger const* logger) noexcept @@ -65,7 +85,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept } if (do_cache) { - if (auto result = storage_->CachedActionResult(action)) { + if (auto result = storage_->ActionCache().CachedResult(action)) { if (result->exit_code() == 0) { return IExecutionResponse::Ptr{ new LocalResponse{action.hash(), @@ -95,7 +115,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept -> std::optional<Output> { auto exec_path = - CreateUniquePath(LocalExecutionConfig::CacheRootDir(0) / "exec_root" / + CreateUniquePath(StorageConfig::BuildRoot() / "exec_root" / NativeSupport::Unprefix(action_id.hash())); if (not exec_path) { @@ -135,7 +155,8 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept if (CollectAndStoreOutputs(&result.action, build_root)) { if (cache_flag_ == CacheFlag::CacheOutput) { - if (not storage_->StoreActionResult(action_id, result.action)) { + if (not storage_->ActionCache().StoreResult(action_id, + result.action)) { logger_.Emit(LogLevel::Warning, "failed to store action results"); } @@ -152,7 +173,7 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept auto LocalAction::StageFile(std::filesystem::path const& target_path, Artifact::ObjectInfo const& info) const -> bool { auto blob_path = - storage_->BlobPath(info.digest, IsExecutableObject(info.type)); + storage_->CAS().BlobPath(info.digest, IsExecutableObject(info.type)); if (not blob_path) { logger_.Emit(LogLevel::Error, @@ -171,7 +192,8 @@ auto LocalAction::StageInputFiles( return false; } - auto infos = storage_->RecursivelyReadTreeLeafs(root_digest_, exec_path); + auto infos = + storage_->CAS().RecursivelyReadTreeLeafs(root_digest_, exec_path); if (not infos) { return false; } @@ -236,7 +258,7 @@ auto LocalAction::CollectOutputFile(std::filesystem::path const& exec_path, } bool is_executable = IsExecutableObject(*type); auto digest = - storage_->StoreBlob</*kOwner=*/true>(file_path, is_executable); + storage_->CAS().StoreBlob</*kOwner=*/true>(file_path, is_executable); if (digest) { auto out_file = bazel_re::OutputFile{}; out_file.set_path(local_path); @@ -257,30 +279,8 @@ auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path, Logger::Log(LogLevel::Error, "expected directory at {}", local_path); return std::nullopt; } - std::optional<bazel_re::Digest> digest{std::nullopt}; - if (Compatibility::IsCompatible()) { - digest = BazelMsgFactory::CreateDirectoryDigestFromLocalTree( - dir_path, - [this](auto path, auto is_exec) { - return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); - }, - [this](auto bytes, - auto /*dir*/) -> std::optional<bazel_re::Digest> { - return storage_->StoreBlob(bytes); - }); - } - else { - digest = BazelMsgFactory::CreateGitTreeDigestFromLocalTree( - dir_path, - [this](auto path, auto is_exec) { - return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); - }, - [this](auto bytes, - auto /*entries*/) -> std::optional<bazel_re::Digest> { - return storage_->StoreTree(bytes); - }); - } - if (digest) { + + if (auto digest = CreateDigestFromLocalOwnedTree(storage_, dir_path)) { auto out_dir = bazel_re::OutputDirectory{}; out_dir.set_path(local_path); out_dir.set_allocated_tree_digest( @@ -322,7 +322,8 @@ auto LocalAction::CollectAndStoreOutputs( auto LocalAction::DigestFromOwnedFile(std::filesystem::path const& file_path) const noexcept -> gsl::owner<bazel_re::Digest*> { - if (auto digest = storage_->StoreBlob</*kOwner=*/true>(file_path)) { + if (auto digest = storage_->CAS().StoreBlob</*kOwner=*/true>( + file_path, /*is_executable=*/false)) { return new bazel_re::Digest{std::move(*digest)}; } return nullptr; diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp index fe908360..4fba44ba 100644 --- a/src/buildtool/execution_api/local/local_action.hpp +++ b/src/buildtool/execution_api/local/local_action.hpp @@ -24,8 +24,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" +#include "src/buildtool/storage/storage.hpp" class LocalApi; @@ -50,7 +49,7 @@ class LocalAction final : public IExecutionAction { private: Logger logger_{"LocalExecution"}; - std::shared_ptr<LocalStorage> storage_; + gsl::not_null<Storage const*> storage_; ArtifactDigest root_digest_{}; std::vector<std::string> cmdline_{}; std::vector<std::string> output_files_{}; @@ -60,7 +59,7 @@ class LocalAction final : public IExecutionAction { std::chrono::milliseconds timeout_{kDefaultTimeout}; CacheFlag cache_flag_{CacheFlag::CacheOutput}; - LocalAction(std::shared_ptr<LocalStorage> storage, + LocalAction(gsl::not_null<Storage const*> storage, ArtifactDigest root_digest, std::vector<std::string> command, std::vector<std::string> output_files, diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 1528ff23..dbfb3504 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -30,8 +30,8 @@ #include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief API for local execution. class LocalApi final : public IExecutionApi { @@ -68,7 +68,7 @@ class LocalApi final : public IExecutionApi { auto const& info = artifacts_info[i]; if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively - auto const infos = storage_->RecursivelyReadTreeLeafs( + auto const infos = storage_->CAS().RecursivelyReadTreeLeafs( info.digest, output_paths[i]); if (not infos or not RetrieveToPaths(infos->second, infos->first)) { @@ -76,7 +76,7 @@ class LocalApi final : public IExecutionApi { } } else { - auto const blob_path = storage_->BlobPath( + auto const blob_path = storage_->CAS().BlobPath( info.digest, IsExecutableObject(info.type)); if (not blob_path or not FileSystemManager::CreateDirectory( @@ -107,7 +107,7 @@ class LocalApi final : public IExecutionApi { if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT auto const success = - storage_->DumpToStream(info, out, raw_tree); + storage_->CAS().DumpToStream(info, out, raw_tree); std::fclose(out); if (not success) { Logger::Log(LogLevel::Error, @@ -159,7 +159,7 @@ class LocalApi final : public IExecutionApi { for (auto const& info : missing_artifacts_info) { // Recursively process trees. if (IsTreeObject(info.type)) { - auto const& infos = storage_->ReadDirectTreeEntries( + auto const& infos = storage_->CAS().ReadDirectTreeEntries( info.digest, std::filesystem::path{}); if (not infos or not RetrieveToCas(infos->second, api)) { return false; @@ -169,9 +169,9 @@ class LocalApi final : public IExecutionApi { // Determine artifact path. auto const& path = IsTreeObject(info.type) - ? storage_->TreePath(info.digest) - : storage_->BlobPath(info.digest, - IsExecutableObject(info.type)); + ? storage_->CAS().TreePath(info.digest) + : storage_->CAS().BlobPath(info.digest, + IsExecutableObject(info.type)); if (not path) { return false; } @@ -213,8 +213,8 @@ class LocalApi final : public IExecutionApi { for (auto const& blob : blobs) { auto const is_tree = NativeSupport::IsTree(blob.digest.hash()); auto cas_digest = - is_tree ? storage_->StoreTree(blob.data) - : storage_->StoreBlob(blob.data, blob.is_exec); + is_tree ? storage_->CAS().StoreTree(blob.data) + : storage_->CAS().StoreBlob(blob.data, blob.is_exec); if (not cas_digest or not std::equal_to<bazel_re::Digest>{}( *cas_digest, blob.digest)) { return false; @@ -331,8 +331,8 @@ class LocalApi final : public IExecutionApi { -> bool final { return static_cast<bool>( NativeSupport::IsTree(static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->TreePath(digest) - : storage_->BlobPath(digest, false)); + ? storage_->CAS().TreePath(digest) + : storage_->CAS().BlobPath(digest, false)); } [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests) @@ -341,8 +341,8 @@ class LocalApi final : public IExecutionApi { for (auto const& digest : digests) { auto const& path = NativeSupport::IsTree( static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->TreePath(digest) - : storage_->BlobPath(digest, false); + ? storage_->CAS().TreePath(digest) + : storage_->CAS().BlobPath(digest, false); if (not path) { result.push_back(digest); } @@ -351,7 +351,7 @@ class LocalApi final : public IExecutionApi { } private: - std::shared_ptr<LocalStorage> storage_{std::make_shared<LocalStorage>()}; + gsl::not_null<Storage const*> storage_ = &Storage::Instance(); }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP diff --git a/src/buildtool/execution_api/local/local_response.hpp b/src/buildtool/execution_api/local/local_response.hpp index e8ffc234..93b12009 100644 --- a/src/buildtool/execution_api/local/local_response.hpp +++ b/src/buildtool/execution_api/local/local_response.hpp @@ -17,8 +17,8 @@ #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" -#include "src/buildtool/execution_api/local/local_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief Response of a LocalAction. class LocalResponse final : public IExecutionResponse { @@ -35,8 +35,8 @@ class LocalResponse final : public IExecutionResponse { return (output_.action.stdout_digest().size_bytes() != 0); } auto StdErr() noexcept -> std::string final { - if (auto path = storage_->BlobPath(output_.action.stderr_digest(), - /*is_executable=*/false)) { + if (auto path = storage_->CAS().BlobPath(output_.action.stderr_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -45,8 +45,8 @@ class LocalResponse final : public IExecutionResponse { return {}; } auto StdOut() noexcept -> std::string final { - if (auto path = storage_->BlobPath(output_.action.stdout_digest(), - /*is_executable=*/false)) { + if (auto path = storage_->CAS().BlobPath(output_.action.stdout_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -103,12 +103,11 @@ class LocalResponse final : public IExecutionResponse { private: std::string action_id_{}; LocalAction::Output output_{}; - gsl::not_null<std::shared_ptr<LocalStorage>> storage_; + gsl::not_null<Storage const*> storage_; - explicit LocalResponse( - std::string action_id, - LocalAction::Output output, - gsl::not_null<std::shared_ptr<LocalStorage>> storage) noexcept + explicit LocalResponse(std::string action_id, + LocalAction::Output output, + gsl::not_null<Storage const*> storage) noexcept : action_id_{std::move(action_id)}, output_{std::move(output)}, storage_{std::move(storage)} {} diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp deleted file mode 100644 index eaeaa617..00000000 --- a/src/buildtool/execution_api/local/local_storage.cpp +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/local/local_api.hpp" - -namespace { - -[[nodiscard]] auto ReadDirectory( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& digest) noexcept - -> std::optional<bazel_re::Directory> { - if (auto const path = storage->BlobPath(digest, /*is_executable=*/false)) { - if (auto const content = FileSystemManager::ReadFile(*path)) { - return BazelMsgFactory::MessageFromString<bazel_re::Directory>( - *content); - } - } - Logger::Log(LogLevel::Error, - "Directory {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto ReadGitTree( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& digest) noexcept - -> std::optional<GitRepo::tree_entries_t> { - if (auto const path = storage->TreePath(digest)) { - if (auto const content = FileSystemManager::ReadFile(*path)) { - return GitRepo::ReadTreeData( - *content, - HashFunction::ComputeTreeHash(*content).Bytes(), - /*is_hex_id=*/false); - } - } - Logger::Log(LogLevel::Error, - "Tree {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto DumpToStream(gsl::not_null<FILE*> const& stream, - std::optional<std::string> const& data) noexcept - -> bool { - if (data) { - std::fwrite(data->data(), 1, data->size(), stream); - return true; - } - return false; -} - -[[nodiscard]] auto TreeToStream( - gsl::not_null<LocalStorage const*> const& storage, - bazel_re::Digest const& tree_digest, - gsl::not_null<FILE*> const& stream) noexcept -> bool { - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(storage, tree_digest)) { - return DumpToStream(stream, - BazelMsgFactory::DirectoryToString(*dir)); - } - } - else { - if (auto entries = ReadGitTree(storage, tree_digest)) { - return DumpToStream(stream, - BazelMsgFactory::GitTreeToString(*entries)); - } - } - return false; -} - -[[nodiscard]] auto BlobToStream( - gsl::not_null<LocalStorage const*> const& storage, - Artifact::ObjectInfo const& blob_info, - gsl::not_null<FILE*> const& stream) noexcept -> bool { - constexpr std::size_t kChunkSize{512}; - auto path = - storage->BlobPath(blob_info.digest, IsExecutableObject(blob_info.type)); - if (not path and not Compatibility::IsCompatible()) { - // in native mode, lookup object in tree cas to dump tree as blob - path = storage->TreePath(blob_info.digest); - } - if (path) { - std::string data(kChunkSize, '\0'); - if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) { - while (auto size = std::fread(data.data(), 1, kChunkSize, in)) { - std::fwrite(data.data(), 1, size, stream); - } - std::fclose(in); - return true; - } - } - return false; -} - -} // namespace - -auto LocalStorage::RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (ReadObjectInfosRecursively(store_info, parent, tree_digest)) { - return std::make_pair(std::move(paths), std::move(infos)); - } - return std::nullopt; -} - -auto LocalStorage::ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - else { - if (auto entries = ReadGitTree(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - return std::make_pair(std::move(paths), std::move(infos)); -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto LocalStorage::ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool { - // read from CAS - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, [this, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, parent / path, info.digest) - : store_info(parent / path, info); - }); - } - } - else { - if (auto entries = ReadGitTree(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, [this, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, parent / path, info.digest) - : store_info(parent / path, info); - }); - } - } - return false; -} - -auto LocalStorage::DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool { - return IsTreeObject(info.type) and not raw_tree - ? TreeToStream(this, info.digest, stream) - : BlobToStream(this, info, stream); -} diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp deleted file mode 100644 index dc9316f8..00000000 --- a/src/buildtool/execution_api/local/local_storage.hpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP - -#include <optional> - -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/local_ac.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" - -class LocalStorage { - public: - /// \brief Store blob from file path with x-bit determined from file system. - template <bool kOwner = false> - [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path) - const noexcept -> std::optional<bazel_re::Digest> { - return StoreBlob<kOwner>(file_path, - FileSystemManager::IsExecutable(file_path)); - } - - /// \brief Store blob from file path with x-bit. - template <bool kOwner = false> - [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path, - bool is_executable) const noexcept - -> std::optional<bazel_re::Digest> { - if (is_executable) { - return cas_exec_.StoreBlobFromFile(file_path, kOwner); - } - return cas_file_.StoreBlobFromFile(file_path, kOwner); - } - - /// \brief Store blob from bytes with x-bit (default: non-executable). - [[nodiscard]] auto StoreBlob(std::string const& bytes, - bool is_executable = false) const noexcept - -> std::optional<bazel_re::Digest> { - return is_executable ? cas_exec_.StoreBlobFromBytes(bytes) - : cas_file_.StoreBlobFromBytes(bytes); - } - - [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept - -> std::optional<bazel_re::Digest> { - return cas_tree_.StoreBlobFromBytes(bytes); - } - - [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) - const noexcept -> std::optional<bazel_re::Digest> { - return cas_tree_.StoreBlobFromFile(file_path); - } - - /// \brief Obtain blob path from digest with x-bit. - /// NOLINTNEXTLINE(misc-no-recursion) - [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest, - bool is_executable) const noexcept - -> std::optional<std::filesystem::path> { - auto const path = is_executable ? cas_exec_.BlobPath(digest) - : cas_file_.BlobPath(digest); - return path ? path : TrySyncBlob(digest, is_executable); - } - - [[nodiscard]] auto TreePath(bazel_re::Digest const& digest) const noexcept - -> std::optional<std::filesystem::path> { - return cas_tree_.BlobPath(digest); - } - - [[nodiscard]] auto StoreActionResult( - bazel_re::Digest const& action_id, - bazel_re::ActionResult const& result) const noexcept -> bool { - return ac_.StoreResult(action_id, result); - } - - [[nodiscard]] auto CachedActionResult(bazel_re::Digest const& action_id) - const noexcept -> std::optional<bazel_re::ActionResult> { - return ac_.CachedResult(action_id); - } - - /// \brief Traverses a tree recursively and retrieves object infos of all - /// found blobs (leafs). Tree objects are not added to the result list, but - /// converted to a path name. - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - - /// \brief Reads the flat content of a tree and returns object infos of all - /// its direct entries (trees and blobs). - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - - [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool; - - private: - LocalCAS<ObjectType::File> cas_file_{}; - LocalCAS<ObjectType::Executable> cas_exec_{}; - LocalCAS<ObjectType::Tree> cas_tree_{}; - LocalAC ac_{&cas_file_}; - - /// \brief Try to sync blob between file CAS and executable CAS. - /// \param digest Blob digest. - /// \param to_executable Sync direction. - /// \returns Path to blob in target CAS. - /// NOLINTNEXTLINE(misc-no-recursion) - [[nodiscard]] auto TrySyncBlob(bazel_re::Digest const& digest, - bool to_executable) const noexcept - -> std::optional<std::filesystem::path> { - std::optional<std::filesystem::path> const src_blob{ - to_executable ? cas_file_.BlobPath(digest) - : cas_exec_.BlobPath(digest)}; - if (src_blob and StoreBlob(*src_blob, to_executable)) { - return BlobPath(digest, to_executable); - } - return std::nullopt; - } - - [[nodiscard]] auto ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP diff --git a/src/buildtool/file_system/TARGETS b/src/buildtool/file_system/TARGETS index 323f6fdc..38df9c2b 100644 --- a/src/buildtool/file_system/TARGETS +++ b/src/buildtool/file_system/TARGETS @@ -15,6 +15,17 @@ ] , "stage": ["src", "buildtool", "file_system"] } +, "object_cas": + { "type": ["@", "rules", "CC", "library"] + , "name": ["object_cas"] + , "hdrs": ["object_cas.hpp"] + , "deps": + [ "file_storage" + , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/file_system", "file_system_manager"] + ] + , "stage": ["src", "buildtool", "file_system"] + } , "file_system_manager": { "type": ["@", "rules", "CC", "library"] , "name": ["file_system_manager"] diff --git a/src/buildtool/file_system/file_storage.hpp b/src/buildtool/file_system/file_storage.hpp index c8b30005..922cbc54 100644 --- a/src/buildtool/file_system/file_storage.hpp +++ b/src/buildtool/file_system/file_storage.hpp @@ -21,20 +21,6 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" -/// \brief Determines the storage path of a blob identified by a hash value. The -/// same sharding technique as used in git is applied, meaning, the hash value -/// is separated into a directory part and file part. Two characters are used -/// for the directory part, the rest for the file, which results in 256 possible -/// directories. -/// \param root The root path of the storage. -/// \param id The hash value of the blob. -/// \returns The sharded file path. -[[nodiscard]] static inline auto GetStoragePath( - std::filesystem::path const& root, - std::string const& id) noexcept -> std::filesystem::path { - return root / id.substr(0, 2) / id.substr(2, id.size() - 2); -} - enum class StoreMode { // First thread to write conflicting file wins. FirstWins, @@ -45,7 +31,11 @@ enum class StoreMode { LastWins }; -template <ObjectType kType, StoreMode kMode, bool kSetEpochTime> +template <ObjectType kType, + StoreMode kMode, + bool kSetEpochTime, + class = std::enable_if_t<kType == ObjectType::File or + kType == ObjectType::Executable>> class FileStorage { public: explicit FileStorage(std::filesystem::path storage_root) noexcept @@ -68,9 +58,16 @@ class FileStorage { return AtomicAddFromBytes(id, bytes); } + /// \brief Determines the storage path of a blob identified by a hash value. + /// The same sharding technique as used in git is applied, meaning, the hash + /// value is separated into a directory part and file part. Two characters + /// are used for the directory part, the rest for the file, which results in + /// 256 possible directories. + /// \param id The hash value of the blob. + /// \returns The sharded file path. [[nodiscard]] auto GetPath(std::string const& id) const noexcept -> std::filesystem::path { - return GetStoragePath(storage_root_, id); + return storage_root_ / id.substr(0, 2) / id.substr(2, id.size() - 2); } private: diff --git a/src/buildtool/execution_api/local/local_cas.hpp b/src/buildtool/file_system/object_cas.hpp index afda71ed..2af59bb9 100644 --- a/src/buildtool/execution_api/local/local_cas.hpp +++ b/src/buildtool/file_system/object_cas.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,68 +12,82 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP +#include <memory> #include <sstream> #include <thread> #include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" -#include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/logger.hpp" -#ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/local/garbage_collector.hpp" -#endif -template <ObjectType kType = ObjectType::File> -class LocalCAS { +/// \brief CAS for storing objects as plain blobs. +/// Automatically computes the digest for storing a blob from file path/bytes. +/// The actual object type is given as a template parameter. Depending on the +/// object type, files written to the file system may have different properties +/// (e.g., the x-bit set) or the digest may be computed differently (e.g., tree +/// digests in non-compatible mode). Supports custom "exists callback", which +/// is used to check blob existence before every read and write operation. +/// \tparam kType The object type to store as blob. +template <ObjectType kType> +class ObjectCAS { public: - LocalCAS() noexcept = default; - - LocalCAS(LocalCAS const&) = delete; - LocalCAS(LocalCAS&&) = delete; - auto operator=(LocalCAS const&) -> LocalCAS& = delete; - auto operator=(LocalCAS&&) -> LocalCAS& = delete; - ~LocalCAS() noexcept = default; - - [[nodiscard]] static auto Instance() noexcept -> LocalCAS<kType>& { - static auto instance = LocalCAS<kType>{}; - return instance; - } - - auto Reset() noexcept -> void { - file_store_ = FileStorage<kStorageType, - StoreMode::FirstWins, - /*kSetEpochTime=*/true>{ - LocalExecutionConfig::CASDir<kType>(0)}; - } - + /// \brief Callback type for checking blob existence. + /// \returns true if a blob for the given digest exists at the given path. + using ExistsFunc = std::function<bool(bazel_re::Digest const&, + std::filesystem::path const&)>; + + /// Default callback for checking blob existence. + static inline ExistsFunc const kDefaultExists = [](auto /*digest*/, + auto path) { + return FileSystemManager::IsFile(path); + }; + + /// \brief Create new object CAS in store_path directory. + /// The optional "exists callback" is used to check blob existence before + /// every read and write operation. It promises that a blob for the given + /// digest exists at the given path if true was returned. + /// \param store_path The path to use for storing blobs. + /// \param exists (optional) Function for checking blob existence. + explicit ObjectCAS(std::filesystem::path const& store_path, + ExistsFunc exists = kDefaultExists) + : file_store_{store_path}, exists_{std::move(exists)} {} + + ObjectCAS(ObjectCAS const&) = delete; + ObjectCAS(ObjectCAS&&) = delete; + auto operator=(ObjectCAS const&) -> ObjectCAS& = delete; + auto operator=(ObjectCAS&&) -> ObjectCAS& = delete; + ~ObjectCAS() noexcept = default; + + /// \brief Store blob from bytes. + /// \param bytes The bytes do create the blob from. + /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromBytes(std::string const& bytes) const noexcept -> std::optional<bazel_re::Digest> { return StoreBlob(bytes, /*is_owner=*/true); } + /// \brief Store blob from file path. + /// \param file_path The path of the file to store as blob. + /// \param is_owner Indicates ownership for optimization (hardlink). + /// \returns Digest of the stored blob or nullopt in case of error. [[nodiscard]] auto StoreBlobFromFile(std::filesystem::path const& file_path, bool is_owner = false) const noexcept -> std::optional<bazel_re::Digest> { return StoreBlob(file_path, is_owner); } + /// \brief Get path to blob. + /// \param digest Digest of the blob to lookup. + /// \returns Path to blob if found or nullopt otherwise. [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest) const noexcept -> std::optional<std::filesystem::path> { auto id = NativeSupport::Unprefix(digest.hash()); auto blob_path = file_store_.GetPath(id); -#ifndef BOOTSTRAP_BUILD_TOOL - // Try to find blob in CAS generations and uplink if required. - auto found = FindAndUplinkBlob(id); -#else - auto found = FileSystemManager::IsFile(blob_path); -#endif - if (not found) { + if (not IsAvailable(digest, blob_path)) { logger_.Emit(LogLevel::Debug, "Blob not found {}", id); return std::nullopt; } @@ -85,10 +99,11 @@ class LocalCAS { static constexpr auto kStorageType = kType == ObjectType::Tree ? ObjectType::File : kType; - Logger logger_{std::string{"LocalCAS"} + ToChar(kType)}; + Logger logger_{std::string{"ObjectCAS"} + ToChar(kType)}; FileStorage<kStorageType, StoreMode::FirstWins, /*kSetEpochTime=*/true> - file_store_{LocalExecutionConfig::CASDir<kType>(0)}; + file_store_; + ExistsFunc exists_; [[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept -> std::optional<bazel_re::Digest> { @@ -105,6 +120,16 @@ class LocalCAS { return std::nullopt; } + [[nodiscard]] auto IsAvailable( + bazel_re::Digest const& digest, + std::filesystem::path const& path) const noexcept -> bool { + try { + return exists_ and exists_(digest, path); + } catch (...) { + return false; + } + } + /// \brief Store blob from bytes to storage. [[nodiscard]] auto StoreBlobData(std::string const& blob_id, std::string const& bytes, @@ -123,14 +148,11 @@ class LocalCAS { template <class T> [[nodiscard]] auto StoreBlob(T const& data, bool is_owner) const noexcept -> std::optional<bazel_re::Digest> { - auto digest = CreateDigest(data); - if (digest) { + if (auto digest = CreateDigest(data)) { auto id = NativeSupport::Unprefix(digest->hash()); -#ifndef BOOTSTRAP_BUILD_TOOL - if (FindAndUplinkBlob(id)) { + if (IsAvailable(*digest, file_store_.GetPath(id))) { return digest; } -#endif if (StoreBlobData(id, data, is_owner)) { return digest; } @@ -139,21 +161,6 @@ class LocalCAS { logger_.Emit(LogLevel::Debug, "Failed to create digest."); return std::nullopt; } - -#ifndef BOOTSTRAP_BUILD_TOOL - [[nodiscard]] auto FindAndUplinkBlob(std::string const& id) const noexcept - -> bool { - if (Compatibility::IsCompatible()) { - return GarbageCollector::FindAndUplinkBlob( - id, kType == ObjectType::Executable); - } - if (kType == ObjectType::Tree) { - return GarbageCollector::FindAndUplinkTree(id); - } - return GarbageCollector::FindAndUplinkBlob( - id, kType == ObjectType::Executable); - } -#endif }; -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_CAS_HPP diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 6bfa5512..d729f499 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -6,15 +6,12 @@ , "private-deps": [ ["src/buildtool/common", "cli"] , ["src/buildtool/common", "config"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/graph_traverser", "graph_traverser"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/progress_reporting", "progress_reporter"] - , ["src/buildtool/execution_api/local", "garbage_collector"] , ["src/buildtool/build_engine/target_map", "result_map"] - , ["src/buildtool/build_engine/target_map", "target_cache"] - , ["src/buildtool/build_engine/target_map", "target_cache_key"] - , ["src/buildtool/build_engine/target_map", "target_cache_entry"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "concepts"] diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 5b72b640..6801f2b2 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -24,27 +24,29 @@ #include "src/buildtool/build_engine/base_maps/entity_name.hpp" #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" -#include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" +#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/main/analyse.hpp" #include "src/buildtool/main/constants.hpp" #include "src/buildtool/main/describe.hpp" #include "src/buildtool/main/exit_codes.hpp" #include "src/buildtool/main/install_cas.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/target_cache.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/execution_api/execution_service/operation_cache.hpp" #include "src/buildtool/execution_api/execution_service/server_implementation.hpp" -#include "src/buildtool/execution_api/local/garbage_collector.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" #endif #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" @@ -300,10 +302,11 @@ void SetupLogging(LogArguments const& clargs) { void SetupExecutionConfig(EndpointArguments const& eargs, BuildArguments const& bargs, RebuildArguments const& rargs) { + using StorageConfig = StorageConfig; using LocalConfig = LocalExecutionConfig; using RemoteConfig = RemoteExecutionConfig; if (not(not eargs.local_root or - (LocalConfig::SetBuildRoot(*eargs.local_root))) or + (StorageConfig::SetBuildRoot(*eargs.local_root))) or not(not bargs.local_launcher or LocalConfig::SetLauncher(*bargs.local_launcher))) { Logger::Log(LogLevel::Error, "failed to configure local execution."); @@ -1217,7 +1220,7 @@ void WriteTargetCacheEntries( [&key = key, &target = target, &extra_infos, &downloader]() { if (auto entry = TargetCacheEntry::FromTarget(target, extra_infos)) { - if (not TargetCache::Instance().Store( + if (not Storage::Instance().TargetCache().Store( key, *entry, downloader)) { Logger::Log(LogLevel::Warning, "Failed writing target cache entry for {}", diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS new file mode 100644 index 00000000..59d93660 --- /dev/null +++ b/src/buildtool/storage/TARGETS @@ -0,0 +1,47 @@ +{ "config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config"] + , "hdrs": ["config.hpp"] + , "deps": + [ ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/execution_api/remote", "config"] + , ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + ] + , "stage": ["src", "buildtool", "storage"] + } +, "storage": + { "type": ["@", "rules", "CC", "library"] + , "name": ["storage"] + , "hdrs": + [ "storage.hpp" + , "local_cas.hpp" + , "local_cas.tpp" + , "local_ac.hpp" + , "local_ac.tpp" + , "target_cache.hpp" + , "target_cache.tpp" + , "target_cache_key.hpp" + , "target_cache_entry.hpp" + , "garbage_collector.hpp" + ] + , "srcs": + ["target_cache_key.cpp", "target_cache_entry.cpp", "garbage_collector.cpp"] + , "deps": + [ "config" + , ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/file_system", "object_cas"] + , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/execution_api/remote", "config"] + , ["src/buildtool/build_engine/analysed_target", "target"] + , ["src/buildtool/build_engine/base_maps", "entity_name_data"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/utils/cpp", "file_locking"] + , ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + ] + , "stage": ["src", "buildtool", "storage"] + } +} diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp new file mode 100644 index 00000000..e2964bb2 --- /dev/null +++ b/src/buildtool/storage/config.hpp @@ -0,0 +1,165 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP + +#ifdef __unix__ +#include <pwd.h> +#include <sys/types.h> +#include <unistd.h> +#else +#error "Non-unix is not supported yet" +#endif + +#include <filesystem> +#include <functional> +#include <string> +#include <vector> + +#include <fmt/core.h> +#include <gsl-lite/gsl-lite.hpp> +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/execution_api/remote/config.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +/// \brief Global storage configuration. +class StorageConfig { + struct ConfigData { + // Build root directory. All the storage dirs are subdirs of build_root. + // By default, build_root is set to $HOME/.cache/just. + // If the user uses --local-build-root PATH, + // then build_root will be set to PATH. + std::filesystem::path build_root{kDefaultBuildRoot}; + + // Number of total storage generations (default: two generations). + std::size_t num_generations{2}; + }; + + public: + /// \brief Determine user home directory + [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { + char const* root{nullptr}; + +#ifdef __unix__ + root = std::getenv("HOME"); + if (root == nullptr) { + root = getpwuid(getuid())->pw_dir; + } +#endif + + if (root == nullptr) { + Logger::Log(LogLevel::Error, + "Cannot determine user home directory."); + std::exit(EXIT_FAILURE); + } + + return root; + } + + static inline auto const kDefaultBuildRoot = + GetUserHome() / ".cache" / "just"; + + [[nodiscard]] static auto SetBuildRoot( + std::filesystem::path const& dir) noexcept -> bool { + if (FileSystemManager::IsRelativePath(dir)) { + Logger::Log(LogLevel::Error, + "Build root must be absolute path but got '{}'.", + dir.string()); + return false; + } + Data().build_root = dir; + return true; + } + + /// \brief Specifies the number of storage generations. + static auto SetNumGenerations(std::size_t num_generations) noexcept + -> void { + Data().num_generations = num_generations; + } + + /// \brief Number of storage generations. + [[nodiscard]] static auto NumGenerations() noexcept -> std::size_t { + return Data().num_generations; + } + + /// \brief Build directory, defaults to user directory if not set + [[nodiscard]] static auto BuildRoot() noexcept -> std::filesystem::path { + return Data().build_root; + } + + /// \brief Root directory of all storage generations. + [[nodiscard]] static auto CacheRoot() noexcept -> std::filesystem::path { + return BuildRoot() / "protocol-dependent"; + } + + /// \brief Root directory of specific storage generation for compatible and + /// non-compatible protocol types. + [[nodiscard]] static auto GenerationCacheRoot(std::size_t index) noexcept + -> std::filesystem::path { + gsl_ExpectsAudit(index < Data().num_generations); + auto generation = std::string{"generation-"} + std::to_string(index); + return CacheRoot() / generation; + } + + /// \brief Storage directory of specific generation and protocol type. + [[nodiscard]] static auto GenerationCacheDir(std::size_t index) noexcept + -> std::filesystem::path { + return UpdatePathForCompatibility(GenerationCacheRoot(index)); + } + + /// \brief String representation of the used execution backend. + [[nodiscard]] static auto ExecutionBackendDescription() noexcept + -> std::string { + auto address = RemoteExecutionConfig::RemoteAddress(); + auto properties = RemoteExecutionConfig::PlatformProperties(); + try { + // json::dump with json::error_handler_t::replace will not throw an + // exception if invalid UTF-8 sequences are detected in the input. + // Instead, it will replace them with the UTF-8 replacement + // character, but still it needs to be inside a try-catch clause to + // ensure the noexcept modifier of the enclosing function. + return nlohmann::json{ + {"remote_address", + address ? nlohmann::json{fmt::format( + "{}:{}", address->host, address->port)} + : nlohmann::json{}}, + {"platform_properties", properties}} + .dump(2, ' ', false, nlohmann::json::error_handler_t::replace); + } catch (...) { + return ""; + } + } + + private: + [[nodiscard]] static auto Data() noexcept -> ConfigData& { + static ConfigData instance{}; + return instance; + } + + // different folder for different caching protocol + [[nodiscard]] static auto UpdatePathForCompatibility( + std::filesystem::path const& dir) -> std::filesystem::path { + return dir / (Compatibility::IsCompatible() ? "compatible-sha256" + : "git-sha1"); + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp new file mode 100644 index 00000000..3e486672 --- /dev/null +++ b/src/buildtool/storage/garbage_collector.cpp @@ -0,0 +1,155 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BOOTSTRAP_BUILD_TOOL + +#include "src/buildtool/storage/garbage_collector.hpp" + +#include <vector> + +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/utils/cpp/hex_string.hpp" + +auto GarbageCollector::GlobalUplinkBlob(bazel_re::Digest const& digest, + bool is_executable) noexcept -> bool { + // Try to find blob in all generations. + auto const& latest_cas = Storage::Generation(0).CAS(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + // Note that we uplink with _skip_sync_ as we want to prefer hard links + // from older generations over copies from the companion file/exec CAS. + if (Storage::Generation(i).CAS().LocalUplinkBlob( + latest_cas, digest, is_executable, /*skip_sync=*/true)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkTree(bazel_re::Digest const& digest) noexcept + -> bool { + // Try to find tree in all generations. + auto const& latest_cas = Storage::Generation(0).CAS(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).CAS().LocalUplinkTree(latest_cas, digest)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkActionCacheEntry( + bazel_re::Digest const& action_id) noexcept -> bool { + // Try to find action-cache entry in all generations. + auto const& latest_ac = Storage::Generation(0).ActionCache(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).ActionCache().LocalUplinkEntry(latest_ac, + action_id)) { + return true; + } + } + return false; +} + +auto GarbageCollector::GlobalUplinkTargetCacheEntry( + TargetCacheKey const& key) noexcept -> bool { + // Try to find target-cache entry in all generations. + auto const& latest_tc = Storage::Generation(0).TargetCache(); + for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) { + if (Storage::Generation(i).TargetCache().LocalUplinkEntry(latest_tc, + key)) { + return true; + } + } + return false; +} + +auto GarbageCollector::SharedLock() noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(), /*is_shared=*/true); +} + +auto GarbageCollector::ExclusiveLock() noexcept -> std::optional<LockFile> { + return LockFile::Acquire(LockFilePath(), /*is_shared=*/false); +} + +auto GarbageCollector::LockFilePath() noexcept -> std::filesystem::path { + return StorageConfig::CacheRoot() / "gc.lock"; +} + +auto GarbageCollector::TriggerGarbageCollection() noexcept -> bool { + auto pid = CreateProcessUniqueId(); + if (not pid) { + return false; + } + auto remove_me = std::string{"remove-me-"} + *pid; + auto remove_me_dir = StorageConfig::CacheRoot() / remove_me; + if (FileSystemManager::IsDirectory(remove_me_dir)) { + if (not FileSystemManager::RemoveDirectory(remove_me_dir, + /*recursively=*/true)) { + Logger::Log(LogLevel::Error, + "Failed to remove directory {}", + remove_me_dir.string()); + return false; + } + } + { // Create scope for critical renaming section protected by advisory lock. + auto lock = ExclusiveLock(); + if (not lock) { + Logger::Log(LogLevel::Error, + "Failed to exclusively lock the local build root"); + return false; + } + for (std::size_t i = StorageConfig::NumGenerations(); i > 0; --i) { + auto cache_root = StorageConfig::GenerationCacheRoot(i - 1); + if (FileSystemManager::IsDirectory(cache_root)) { + auto new_cache_root = + (i == StorageConfig::NumGenerations()) + ? remove_me_dir + : StorageConfig::GenerationCacheRoot(i); + if (not FileSystemManager::Rename(cache_root, new_cache_root)) { + Logger::Log(LogLevel::Error, + "Failed to rename {} to {}.", + cache_root.string(), + new_cache_root.string()); + return false; + } + } + } + } + if (FileSystemManager::IsDirectory(remove_me_dir)) { + if (not FileSystemManager::RemoveDirectory(remove_me_dir, + /*recursively=*/true)) { + Logger::Log(LogLevel::Warning, + "Failed to remove directory {}", + remove_me_dir.string()); + return false; + } + } + return true; +} + +#endif // BOOTSTRAP_BUILD_TOOL diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp new file mode 100644 index 00000000..76bbc054 --- /dev/null +++ b/src/buildtool/storage/garbage_collector.hpp @@ -0,0 +1,82 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP + +#include <functional> +#include <optional> +#include <string> + +#include "src/utils/cpp/file_locking.hpp" + +// forward declarations +namespace build::bazel::remote::execution::v2 { +class Digest; +} +namespace bazel_re = build::bazel::remote::execution::v2; +class TargetCacheKey; + +/// \brief Global garbage collector implementation. +/// Responsible for deleting oldest generation and uplinking across all +/// generations to latest generation. +class GarbageCollector { + public: + /// \brief Uplink blob across LocalCASes from all generations to latest. + /// Note that blobs will NOT be synced between file/executable CAS. + /// \param digest Digest of the blob to uplink. + /// \param is_executable Indicate that blob is an executable. + /// \returns true if blob was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkBlob(bazel_re::Digest const& digest, + bool is_executable) noexcept + -> bool; + + /// \brief Uplink tree across LocalCASes from all generations to latest. + /// Note that the tree will be deeply uplinked, i.e., all entries referenced + /// by this tree will be uplinked before (including sub-trees). + /// \param digest Digest of the tree to uplink. + /// \returns true if tree was found and successfully uplinked (deep). + [[nodiscard]] auto static GlobalUplinkTree( + bazel_re::Digest const& digest) noexcept -> bool; + + /// \brief Uplink entry from action cache across all generations to latest. + /// Note that the entry will be uplinked including all referenced items. + /// \param action_id Id of the action to uplink entry for. + /// \returns true if cache entry was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkActionCacheEntry( + bazel_re::Digest const& action_id) noexcept -> bool; + + /// \brief Uplink entry from target cache across all generations to latest. + /// Note that the entry will be uplinked including all referenced items. + /// \param key Target cache key to uplink entry for. + /// \returns true if cache entry was found and successfully uplinked. + [[nodiscard]] auto static GlobalUplinkTargetCacheEntry( + TargetCacheKey const& key) noexcept -> bool; + + /// \brief Trigger deletion of oldest generation. + /// \returns true on success. + [[nodiscard]] auto static TriggerGarbageCollection() noexcept -> bool; + + /// \brief Acquire shared lock to prevent garbage collection from running. + /// \returns The acquired lock file on success or nullopt otherwise. + [[nodiscard]] auto static SharedLock() noexcept -> std::optional<LockFile>; + + private: + [[nodiscard]] auto static ExclusiveLock() noexcept + -> std::optional<LockFile>; + + [[nodiscard]] auto static LockFilePath() noexcept -> std::filesystem::path; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_GARBAGE_COLLECTOR_HPP diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp new file mode 100644 index 00000000..313abd7f --- /dev/null +++ b/src/buildtool/storage/local_ac.hpp @@ -0,0 +1,101 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP + +#include "gsl-lite/gsl-lite.hpp" +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/local_cas.hpp" + +// forward declarations +namespace build::bazel::remote::execution::v2 { +class Digest; +class ActionResult; +} // namespace build::bazel::remote::execution::v2 +namespace bazel_re = build::bazel::remote::execution::v2; + +/// \brief The action cache for storing action results. +/// Supports global uplinking across all generations using the garbage +/// collector. The uplink is automatically performed for every entry that is +/// read and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalAC { + public: + /// Local AC generation used by GC without global uplink. + using LocalGenerationAC = LocalAC</*kDoGlobalUplink=*/false>; + + LocalAC(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas, + std::filesystem::path const& store_path) noexcept + : cas_{std::move(cas)}, file_store_{store_path} {}; + + LocalAC(LocalAC const&) = default; + LocalAC(LocalAC&&) noexcept = default; + auto operator=(LocalAC const&) -> LocalAC& = default; + auto operator=(LocalAC&&) noexcept -> LocalAC& = default; + ~LocalAC() noexcept = default; + + /// \brief Store action result. + /// \param action_id The id of the action that produced the result. + /// \param result The action result to store. + /// \returns true on success. + [[nodiscard]] auto StoreResult( + bazel_re::Digest const& action_id, + bazel_re::ActionResult const& result) const noexcept -> bool; + + /// \brief Read cached action result. + /// \param action_id The id of the action the result was produced by. + /// \returns The action result if found or nullopt otherwise. + [[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id) + const noexcept -> std::optional<bazel_re::ActionResult>; + + /// \brief Uplink entry from this generation to latest LocalAC generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalAC generation. + /// \param action_id The id of the action used as entry key. + /// \returns True if entry was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( + LocalGenerationAC const& latest, + bazel_re::Digest const& action_id) const noexcept -> bool; + + private: + // The action cache stores the results of failed actions. For those to be + // overwritable by subsequent runs we need to choose the store mode "last + // wins" for the underlying file storage. Unless this is a generation + // (disabled global uplink), then we never want to overwrite any entries. + static constexpr auto kStoreMode = + kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; + + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("LocalAC")}; + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false> + file_store_; + + [[nodiscard]] auto ReadResult(bazel_re::Digest const& digest) const noexcept + -> std::optional<bazel_re::ActionResult>; +}; + +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/storage/local_ac.tpp" +#endif + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_HPP diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp new file mode 100644 index 00000000..9c6eca77 --- /dev/null +++ b/src/buildtool/storage/local_ac.tpp @@ -0,0 +1,133 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP + +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/storage/local_ac.hpp" + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::StoreResult( + bazel_re::Digest const& action_id, + bazel_re::ActionResult const& result) const noexcept -> bool { + auto bytes = result.SerializeAsString(); + auto digest = cas_->StoreBlob(bytes); + return (digest and + file_store_.AddFromBytes(NativeSupport::Unprefix(action_id.hash()), + digest->SerializeAsString())); +} + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::CachedResult(bazel_re::Digest const& action_id) + const noexcept -> std::optional<bazel_re::ActionResult> { + auto id = NativeSupport::Unprefix(action_id.hash()); + auto entry_path = file_store_.GetPath(id); + + if constexpr (kDoGlobalUplink) { + // Uplink any existing action-cache entries in storage generations + [[maybe_unused]] bool found = + GarbageCollector::GlobalUplinkActionCacheEntry(action_id); + } + + auto const entry = + FileSystemManager::ReadFile(entry_path, ObjectType::File); + if (not entry) { + logger_->Emit(LogLevel::Debug, + "Cache miss, entry not found {}", + entry_path.string()); + return std::nullopt; + } + bazel_re::Digest digest{}; + if (not digest.ParseFromString(*entry)) { + logger_->Emit( + LogLevel::Warning, "Parsing cache entry failed for action {}", id); + return std::nullopt; + } + auto result = ReadResult(digest); + if (not result) { + logger_->Emit(LogLevel::Warning, + "Parsing action result failed for action {}", + id); + } + return result; +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( + LocalGenerationAC const& latest, + bazel_re::Digest const& action_id) const noexcept -> bool { + // Determine action cache key path in latest generation. + auto key_digest = NativeSupport::Unprefix(action_id.hash()); + if (FileSystemManager::IsFile(latest.file_store_.GetPath(key_digest))) { + return true; + } + + // Read cache key + auto cache_key = file_store_.GetPath(key_digest); + auto content = FileSystemManager::ReadFile(cache_key); + if (not content) { + return false; + } + + // Read result (cache value) + bazel_re::Digest result_digest{}; + if (not result_digest.ParseFromString(*content)) { + return false; + } + auto result = ReadResult(result_digest); + if (not result) { + return false; + } + + // Uplink result content + for (auto const& file : result->output_files()) { + if (not cas_->LocalUplinkBlob( + *latest.cas_, file.digest(), file.is_executable())) { + return false; + } + } + for (auto const& directory : result->output_directories()) { + if (not cas_->LocalUplinkTree(*latest.cas_, directory.tree_digest())) { + return false; + } + } + + // Uplink result (cache value) + if (not cas_->LocalUplinkBlob(*latest.cas_, + result_digest, + /*is_executable=*/false)) { + return false; + } + + // Uplink cache key + return latest.file_store_.AddFromFile( + key_digest, cache_key, /*is_owner=*/true); +} + +template <bool kDoGlobalUplink> +auto LocalAC<kDoGlobalUplink>::ReadResult(bazel_re::Digest const& digest) + const noexcept -> std::optional<bazel_re::ActionResult> { + bazel_re::ActionResult result{}; + if (auto src_path = cas_->BlobPath(digest, /*is_executable=*/false)) { + auto const bytes = FileSystemManager::ReadFile(*src_path); + if (bytes.has_value() and result.ParseFromString(*bytes)) { + return result; + } + } + return std::nullopt; +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_AC_TPP diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp new file mode 100644 index 00000000..af613b1d --- /dev/null +++ b/src/buildtool/storage/local_cas.hpp @@ -0,0 +1,242 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP + +#include <optional> +#include <unordered_set> + +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_cas.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" + +/// \brief The local (logical) CAS for storing blobs and trees. +/// Blobs can be stored/queried as executable or non-executable. Trees might be +/// treated differently depending on the compatibility mode. Supports global +/// uplinking across all generations using the garbage collector. The uplink +/// is automatically performed for every entry that is read and every entry that +/// is stored and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalCAS { + public: + /// Local CAS generation used by GC without global uplink. + using LocalGenerationCAS = LocalCAS</*kDoGlobalUplink=*/false>; + + /// \brief Create local CAS with base path. + /// Note that the base path is concatenated by a single character + /// 'f'/'x'/'t' for each internally used physical CAS. + /// \param base The base path for the CAS. + explicit LocalCAS(std::filesystem::path const& base) + : cas_file_{base.string() + 'f', Uplinker<ObjectType::File>()}, + cas_exec_{base.string() + 'x', Uplinker<ObjectType::Executable>()}, + cas_tree_{base.string() + (Compatibility::IsCompatible() ? 'f' : 't'), + Uplinker<ObjectType::Tree>()} {} + + /// \brief Store blob from file path with x-bit. + /// \tparam kOwner Indicates ownership for optimization (hardlink). + /// \param file_path The path of the file to store as blob. + /// \param is_executable Store blob with executable permissions. + /// \returns Digest of the stored blob or nullopt otherwise. + template <bool kOwner = false> + [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path, + bool is_executable) const noexcept + -> std::optional<bazel_re::Digest> { + return is_executable ? cas_exec_.StoreBlobFromFile(file_path, kOwner) + : cas_file_.StoreBlobFromFile(file_path, kOwner); + } + + /// \brief Store blob from bytes with x-bit (default: non-executable). + /// \param bytes The bytes to create the blob from. + /// \param is_executable Store blob with executable permissions. + /// \returns Digest of the stored blob or nullopt otherwise. + [[nodiscard]] auto StoreBlob(std::string const& bytes, + bool is_executable = false) const noexcept + -> std::optional<bazel_re::Digest> { + return is_executable ? cas_exec_.StoreBlobFromBytes(bytes) + : cas_file_.StoreBlobFromBytes(bytes); + } + + /// \brief Store tree from file path. + /// \param file_path The path of the file to store as tree. + /// \returns Digest of the stored tree or nullopt otherwise. + [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) + const noexcept -> std::optional<bazel_re::Digest> { + return cas_tree_.StoreBlobFromFile(file_path); + } + + /// \brief Store tree from bytes. + /// \param bytes The bytes to create the tree from. + /// \returns Digest of the stored tree or nullopt otherwise. + [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept + -> std::optional<bazel_re::Digest> { + return cas_tree_.StoreBlobFromBytes(bytes); + } + + /// \brief Obtain blob path from digest with x-bit. + /// Performs a synchronization if blob is only available with inverse x-bit. + /// \param digest Digest of the blob to lookup. + /// \param is_executable Lookup blob with executable permissions. + /// \returns Path to the blob if found or nullopt otherwise. + [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest, + bool is_executable) const noexcept + -> std::optional<std::filesystem::path> { + auto const path = BlobPathNoSync(digest, is_executable); + return path ? path : TrySyncBlob(digest, is_executable); + } + + /// \brief Obtain tree path from digest. + /// \param digest Digest of the tree to lookup. + /// \returns Path to the tree if found or nullopt otherwise. + [[nodiscard]] auto TreePath(bazel_re::Digest const& digest) const noexcept + -> std::optional<std::filesystem::path> { + return cas_tree_.BlobPath(digest); + } + + /// \brief Traverses a tree recursively and retrieves object infos of all + /// found blobs (leafs). Tree objects are not added to the result list, but + /// converted to a path name. + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \returns Pair of vectors, first containing filesystem paths, second + /// containing object infos. + [[nodiscard]] auto RecursivelyReadTreeLeafs( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>>; + + /// \brief Reads the flat content of a tree and returns object infos of all + /// its direct entries (trees and blobs). + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \returns Pair of vectors, first containing filesystem paths, second + /// containing object infos. + [[nodiscard]] auto ReadDirectTreeEntries( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>>; + + /// \brief Dump artifact to file stream. + /// Tree artifacts are pretty-printed (i.e., contents are listed) unless + /// raw_tree is set, then the raw tree will be written to the file stream. + /// \param info The object info of the artifact to dump. + /// \param stream The file stream to dump to. + /// \param raw_tree Dump tree as raw blob. + /// \returns true on success. + [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept -> bool; + + /// \brief Uplink blob from this generation to latest LocalCAS generation. + /// Performs a synchronization if requested and if blob is only available + /// with inverse x-bit. This function is only available for instances that + /// are used as local GC generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalCAS generation. + /// \param digest The digest of the blob to uplink. + /// \param is_executable Uplink blob with executable permissions. + /// \param skip_sync Do not sync between file/executable CAS. + /// \returns True if blob was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBlob( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + bool is_executable, + bool skip_sync = false) const noexcept -> bool; + + /// \brief Uplink tree from this generation to latest LocalCAS generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). Trees are uplinked deep, + /// including all referenced entries. Note that in compatible mode we do not + /// have the notion of "tree" and instead trees are stored as blobs. + /// Therefore, in compatible mode this function is only used by instances + /// that are aware of trees, such as output directories in action results or + /// tree artifacts from target cache. + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest LocalCAS generation. + /// \param digest The digest of the tree to uplink. + /// \returns True if tree was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool; + + private: + ObjectCAS<ObjectType::File> cas_file_; + ObjectCAS<ObjectType::Executable> cas_exec_; + ObjectCAS<ObjectType::Tree> cas_tree_; + + /// \brief Provides uplink via "exists callback" for physical object CAS. + template <ObjectType kType> + [[nodiscard]] static auto Uplinker() -> + typename ObjectCAS<kType>::ExistsFunc { + if constexpr (kDoGlobalUplink) { + return [](auto digest, auto /*path*/) { + if (not Compatibility::IsCompatible()) { + // in non-compatible mode, do explicit deep tree uplink + if constexpr (IsTreeObject(kType)) { + return GarbageCollector::GlobalUplinkTree(digest); + } + } + // in compatible mode, treat all trees as blobs + return GarbageCollector::GlobalUplinkBlob( + digest, IsExecutableObject(kType)); + }; + } + return ObjectCAS<kType>::kDefaultExists; + } + + /// \brief Get blob path without sync between file/executable CAS. + [[nodiscard]] auto BlobPathNoSync(bazel_re::Digest const& digest, + bool is_executable) const noexcept + -> std::optional<std::filesystem::path> { + return is_executable ? cas_exec_.BlobPath(digest) + : cas_file_.BlobPath(digest); + } + + /// \brief Try to sync blob between file CAS and executable CAS. + /// \param digest Blob digest. + /// \param to_executable Sync direction. + /// \returns Path to blob in target CAS. + [[nodiscard]] auto TrySyncBlob(bazel_re::Digest const& digest, + bool to_executable) const noexcept + -> std::optional<std::filesystem::path> { + auto const src_blob = BlobPathNoSync(digest, not to_executable); + if (src_blob and StoreBlob(*src_blob, to_executable)) { + return BlobPathNoSync(digest, to_executable); + } + return std::nullopt; + } + + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkGitTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool; + + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBazelDirectory( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen) + const noexcept -> bool; +}; + +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/storage/local_cas.tpp" +#endif + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_HPP diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp new file mode 100644 index 00000000..3664d929 --- /dev/null +++ b/src/buildtool/storage/local_cas.tpp @@ -0,0 +1,367 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP + +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/storage/local_cas.hpp" + +namespace detail { + +template <class T_CAS> +[[nodiscard]] auto ReadDirectory(T_CAS const& cas, + bazel_re::Digest const& digest) noexcept + -> std::optional<bazel_re::Directory> { + if (auto const path = cas.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return BazelMsgFactory::MessageFromString<bazel_re::Directory>( + *content); + } + } + Logger::Log(LogLevel::Error, + "Directory {} not found in CAS", + NativeSupport::Unprefix(digest.hash())); + return std::nullopt; +} + +template <class T_CAS> +[[nodiscard]] auto ReadGitTree(T_CAS const& cas, + bazel_re::Digest const& digest) noexcept + -> std::optional<GitRepo::tree_entries_t> { + if (auto const path = cas.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return GitRepo::ReadTreeData( + *content, + HashFunction::ComputeTreeHash(*content).Bytes(), + /*is_hex_id=*/false); + } + } + Logger::Log(LogLevel::Error, + "Tree {} not found in CAS", + NativeSupport::Unprefix(digest.hash())); + return std::nullopt; +} + +[[nodiscard]] inline auto DumpToStream( + gsl::not_null<FILE*> const& stream, + std::optional<std::string> const& data) noexcept -> bool { + if (data) { + std::fwrite(data->data(), 1, data->size(), stream); + return true; + } + return false; +} + +template <class T_CAS> +[[nodiscard]] auto TreeToStream(T_CAS const& cas, + bazel_re::Digest const& tree_digest, + gsl::not_null<FILE*> const& stream) noexcept + -> bool { + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(cas, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::DirectoryToString(*dir)); + } + } + else { + if (auto entries = ReadGitTree(cas, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::GitTreeToString(*entries)); + } + } + return false; +} + +template <class T_CAS> +[[nodiscard]] auto BlobToStream(T_CAS const& cas, + Artifact::ObjectInfo const& blob_info, + gsl::not_null<FILE*> const& stream) noexcept + -> bool { + constexpr std::size_t kChunkSize{512}; + auto path = + cas.BlobPath(blob_info.digest, IsExecutableObject(blob_info.type)); + if (not path and not Compatibility::IsCompatible()) { + // in native mode, lookup object in tree cas to dump tree as blob + path = cas.TreePath(blob_info.digest); + } + if (path) { + std::string data(kChunkSize, '\0'); + if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) { + while (auto size = std::fread(data.data(), 1, kChunkSize, in)) { + std::fwrite(data.data(), 1, size, stream); + } + std::fclose(in); + return true; + } + } + return false; +} + +template <class T_CAS> +auto ReadObjectInfosRecursively( + T_CAS const& cas, + BazelMsgFactory::InfoStoreFunc const& store_info, + std::filesystem::path const& parent, + bazel_re::Digest const& digest) noexcept -> bool { + // read from CAS + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(cas, digest)) { + return BazelMsgFactory::ReadObjectInfosFromDirectory( + *dir, [&cas, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(cas, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); + } + } + else { + if (auto entries = ReadGitTree(cas, digest)) { + return BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, [&cas, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(cas, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); + } + } + return false; +} + +} // namespace detail + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::RecursivelyReadTreeLeafs( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>> { + std::vector<std::filesystem::path> paths{}; + std::vector<Artifact::ObjectInfo> infos{}; + + auto store_info = [&paths, &infos](auto path, auto info) { + paths.emplace_back(path); + infos.emplace_back(info); + return true; + }; + + if (detail::ReadObjectInfosRecursively( + *this, store_info, parent, tree_digest)) { + return std::make_pair(std::move(paths), std::move(infos)); + } + return std::nullopt; +} + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::ReadDirectTreeEntries( + bazel_re::Digest const& tree_digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<std::pair<std::vector<std::filesystem::path>, + std::vector<Artifact::ObjectInfo>>> { + std::vector<std::filesystem::path> paths{}; + std::vector<Artifact::ObjectInfo> infos{}; + + auto store_info = [&paths, &infos](auto path, auto info) { + paths.emplace_back(path); + infos.emplace_back(info); + return true; + }; + + if (Compatibility::IsCompatible()) { + if (auto dir = detail::ReadDirectory(*this, tree_digest)) { + if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + *dir, [&store_info, &parent](auto path, auto info) { + return store_info(parent / path, info); + })) { + return std::nullopt; + } + } + } + else { + if (auto entries = detail::ReadGitTree(*this, tree_digest)) { + if (not BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, [&store_info, &parent](auto path, auto info) { + return store_info(parent / path, info); + })) { + return std::nullopt; + } + } + } + return std::make_pair(std::move(paths), std::move(infos)); +} + +template <bool kDoGlobalUplink> +auto LocalCAS<kDoGlobalUplink>::DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept + -> bool { + return IsTreeObject(info.type) and not raw_tree + ? detail::TreeToStream(*this, info.digest, stream) + : detail::BlobToStream(*this, info, stream); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBlob( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + bool is_executable, + bool skip_sync) const noexcept -> bool { + // Determine blob path in latest generation. + auto blob_path_latest = latest.BlobPathNoSync(digest, is_executable); + if (blob_path_latest) { + return true; + } + + // Determine blob path of given generation. + auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable) + : BlobPath(digest, is_executable); + if (not blob_path) { + return false; + } + + // Uplink blob from older generation to the latest generation. + return blob_path_latest.has_value() or + latest.StoreBlob</*kOwner=*/true>(*blob_path, is_executable); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool { + if (Compatibility::IsCompatible()) { + std::unordered_set<bazel_re::Digest> seen{}; + return LocalUplinkBazelDirectory(latest, digest, &seen); + } + return LocalUplinkGitTree(latest, digest); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest) const noexcept -> bool { + // Determine tree path in latest generation. + auto tree_path_latest = latest.cas_tree_.BlobPath(digest); + if (tree_path_latest) { + return true; + } + + // Determine tree path of given generation. + auto tree_path = cas_tree_.BlobPath(digest); + if (not tree_path) { + return false; + } + + // Determine tree entries. + auto content = FileSystemManager::ReadFile(*tree_path); + auto id = NativeSupport::Unprefix(digest.hash()); + auto tree_entries = GitRepo::ReadTreeData(*content, + id, + /*is_hex_id=*/true); + if (not tree_entries) { + return false; + } + + // Uplink tree entries. + for (auto const& [raw_id, entry_vector] : *tree_entries) { + // Process only first entry from 'entry_vector' since all + // entries represent the same blob, just with different + // names. + auto entry = entry_vector.front(); + auto hash = ToHexString(raw_id); + auto digest = ArtifactDigest{hash, 0, IsTreeObject(entry.type)}; + if (entry.type == ObjectType::Tree) { + if (not LocalUplinkGitTree(latest, digest)) { + return false; + } + } + else { + if (not LocalUplinkBlob( + latest, digest, IsExecutableObject(entry.type))) { + return false; + } + } + } + + // Uplink tree from older generation to the latest generation. + return latest.cas_tree_ + .StoreBlobFromFile(*tree_path, + /*is_owner=*/true) + .has_value(); +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>:: + LocalUplinkBazelDirectory( + LocalGenerationCAS const& latest, + bazel_re::Digest const& digest, + gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen) + const noexcept -> bool { + // Skip already uplinked directories + if (seen->contains(digest)) { + return true; + } + + // Determine bazel directory path of given generation. + auto dir_path = cas_tree_.BlobPath(digest); + if (not dir_path) { + return false; + } + + // Determine bazel directory entries. + auto content = FileSystemManager::ReadFile(*dir_path); + bazel_re::Directory dir{}; + if (not dir.ParseFromString(*content)) { + return false; + } + + // Uplink bazel directory entries. + for (auto const& file : dir.files()) { + if (not LocalUplinkBlob(latest, file.digest(), file.is_executable())) { + return false; + } + } + for (auto const& directory : dir.directories()) { + if (not LocalUplinkBazelDirectory(latest, directory.digest(), seen)) { + return false; + } + } + + // Determine bazel directory path in latest generation. + auto dir_path_latest = latest.cas_tree_.BlobPath(digest); + + // Uplink bazel directory from older generation to the latest + // generation. + if (dir_path_latest.has_value() or + latest.cas_tree_.StoreBlobFromFile(*dir_path, + /*is_owner=*/true)) { + try { + seen->emplace(digest); + return true; + } catch (...) { + } + } + return false; +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP diff --git a/src/buildtool/storage/storage.hpp b/src/buildtool/storage/storage.hpp new file mode 100644 index 00000000..fe6c1e1f --- /dev/null +++ b/src/buildtool/storage/storage.hpp @@ -0,0 +1,133 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP + +#include <optional> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/local_ac.hpp" +#include "src/buildtool/storage/local_cas.hpp" +#include "src/buildtool/storage/target_cache.hpp" + +/// \brief The local storage for accessing CAS and caches. +/// Maintains an instance of LocalCAS, LocalAC, TargetCache. Supports global +/// uplinking across all generations using the garbage collector. The uplink is +/// automatically performed by the affected storage instance (CAS, action cache, +/// target cache). +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class LocalStorage { + public: + explicit LocalStorage(std::filesystem::path const& storage_path) + : cas_{std::make_shared<LocalCAS<kDoGlobalUplink>>(storage_path / + "cas")}, + ac_{cas_, storage_path / "ac"}, + tc_{cas_, storage_path / "tc"} {} + + /// \brief Get the CAS instance. + [[nodiscard]] auto CAS() const noexcept + -> LocalCAS<kDoGlobalUplink> const& { + return *cas_; + } + + /// \brief Get the action cache instance. + [[nodiscard]] auto ActionCache() const noexcept + -> LocalAC<kDoGlobalUplink> const& { + return ac_; + } + + /// \brief Get the target cache instance. + [[nodiscard]] auto TargetCache() const noexcept + -> TargetCache<kDoGlobalUplink> const& { + return tc_; + } + + private: + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + LocalAC<kDoGlobalUplink> ac_; + ::TargetCache<kDoGlobalUplink> tc_; +}; + +#ifdef BOOTSTRAP_BUILD_TOOL +// disable global uplinking (via garbage collector) for global storage singleton +constexpr auto kDefaultDoGlobalUplink = false; +#else +constexpr auto kDefaultDoGlobalUplink = true; +#endif + +/// \brief Generation type, local storage without global uplinking. +using Generation = LocalStorage</*kDoGlobalUplink=*/false>; + +/// \brief Global storage singleton class, valid throughout the entire tool. +/// Configured via \ref StorageConfig. +class Storage : public LocalStorage<kDefaultDoGlobalUplink> { + public: + /// \brief Get the global storage instance. + /// Build root is read from \ref StorageConfig::BuildRoot(). + /// \returns The global storage singleton instance. + [[nodiscard]] static auto Instance() noexcept -> Storage const& { + return GetStorage(); + } + + /// \brief Get specific storage generation. + /// Number of generations is read from \ref StorageConfig::NumGenerations(). + /// \param index the generation index (0 is latest). + /// \returns The specific storage generation. + [[nodiscard]] static auto Generation(std::size_t index) noexcept + -> ::Generation const& { + return GetGenerations()[index]; + } + + /// \brief Reinitialize storage instance and generations. + /// Use if global \ref StorageConfig was changed. Not thread-safe! + static void Reinitialize() noexcept { + GetStorage() = CreateStorage(); + GetGenerations() = CreateGenerations(); + } + + private: + using LocalStorage<kDefaultDoGlobalUplink>::LocalStorage; + + [[nodiscard]] static auto CreateStorage() noexcept -> Storage { + return Storage{StorageConfig::GenerationCacheDir(0)}; + } + + [[nodiscard]] static auto CreateGenerations() noexcept + -> std::vector<::Generation> { + auto count = StorageConfig::NumGenerations(); + std::vector<::Generation> generations{}; + generations.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + generations.emplace_back(StorageConfig::GenerationCacheDir(i)); + } + return generations; + } + + [[nodiscard]] static auto GetStorage() noexcept -> Storage& { + static auto instance = CreateStorage(); + return instance; + } + + [[nodiscard]] static auto GetGenerations() noexcept + -> std::vector<::Generation>& { + static auto generations = CreateGenerations(); + return generations; + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_STORAGE_HPP diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp new file mode 100644 index 00000000..c9f0fbe6 --- /dev/null +++ b/src/buildtool/storage/target_cache.hpp @@ -0,0 +1,132 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP + +#include <filesystem> +#include <functional> +#include <optional> +#include <utility> + +#include <gsl-lite/gsl-lite.hpp> +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/garbage_collector.hpp" +#include "src/buildtool/storage/local_cas.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" + +/// \brief The high-level target cache for storing export target's data. +/// Supports global uplinking across all generations using the garbage +/// collector. The uplink is automatically performed for every entry that is +/// read and already exists in an older generation. +/// \tparam kDoGlobalUplink Enable global uplinking via garbage collector. +template <bool kDoGlobalUplink> +class TargetCache { + public: + /// Local target cache generation used by GC without global uplink. + using LocalGenerationTC = TargetCache</*kDoGlobalUplink=*/false>; + + /// Callback type for downloading known artifacts to local CAS. + using ArtifactDownloader = + std::function<bool(std::vector<Artifact::ObjectInfo> const&)>; + + TargetCache(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas, + std::filesystem::path const& store_path) + : cas_{std::move(cas)}, file_store_{store_path / ComputeShard()} { + if constexpr (kDoGlobalUplink) { + // write backend description (shard) to CAS + [[maybe_unused]] auto id = + cas_->StoreBlob(StorageConfig::ExecutionBackendDescription()); + gsl_EnsuresAudit(id and + ArtifactDigest{*id}.hash() == ComputeShard()); + } + } + + TargetCache(TargetCache const&) = default; + TargetCache(TargetCache&&) noexcept = default; + auto operator=(TargetCache const&) -> TargetCache& = default; + auto operator=(TargetCache&&) noexcept -> TargetCache& = default; + ~TargetCache() noexcept = default; + + /// \brief Store new key-entry pair in the target cache. + /// \param key The target-cache key. + /// \param value The target-cache value to store. + /// \param downloader Callback for obtaining known artifacts to local CAS. + /// \returns true on success. + [[nodiscard]] auto Store( + TargetCacheKey const& key, + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool; + + /// \brief Read existing entry and object info from the target cache. + /// \param key The target-cache key to read the entry from. + /// \returns Pair of cache entry and its object info on success or nullopt. + [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; + + /// \brief Uplink entry from this to latest target cache generation. + /// This function is only available for instances that are used as local GC + /// generations (i.e., disabled global uplink). + /// \tparam kIsLocalGeneration True if this instance is a local generation. + /// \param latest The latest target cache generation. + /// \param key The target-cache key for the entry to uplink. + /// \returns True if entry was successfully uplinked. + template <bool kIsLocalGeneration = not kDoGlobalUplink> + requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( + LocalGenerationTC const& latest, + TargetCacheKey const& key) const noexcept -> bool; + + private: + // By default, overwrite existing entries. Unless this is a generation + // (disabled global uplink), then we never want to overwrite any entries. + static constexpr auto kStoreMode = + kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins; + + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("TargetCache")}; + gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_; + FileStorage<ObjectType::File, + kStoreMode, + /*kSetEpochTime=*/false> + file_store_; + + [[nodiscard]] static auto ComputeShard() noexcept -> std::string { + return ArtifactDigest::Create<ObjectType::File>( + StorageConfig::ExecutionBackendDescription()) + .hash(); + } + + [[nodiscard]] auto DownloadKnownArtifacts( + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool; +}; + +#include "src/buildtool/storage/target_cache.tpp" + +namespace std { +template <> +struct hash<TargetCacheKey> { + [[nodiscard]] auto operator()(TargetCacheKey const& k) const { + return std::hash<Artifact::ObjectInfo>{}(k.Id()); + } +}; +} // namespace std + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_HPP diff --git a/src/buildtool/storage/target_cache.tpp b/src/buildtool/storage/target_cache.tpp new file mode 100644 index 00000000..d1c9d01c --- /dev/null +++ b/src/buildtool/storage/target_cache.tpp @@ -0,0 +1,168 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP + +#include "src/buildtool/storage/target_cache.hpp" + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::Store( + TargetCacheKey const& key, + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool { + if (not DownloadKnownArtifacts(value, downloader)) { + return false; + } + if (auto digest = cas_->StoreBlob(value.ToJson().dump(2))) { + auto data = + Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File} + .ToString(); + logger_->Emit(LogLevel::Debug, + "Adding entry for key {} as {}", + key.Id().ToString(), + data); + return file_store_.AddFromBytes(key.Id().digest.hash(), data); + } + return false; +} + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::Read( + TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> { + auto id = key.Id().digest.hash(); + auto entry_path = file_store_.GetPath(id); + + if constexpr (kDoGlobalUplink) { + // Uplink any existing target cache entry in storage generations + [[maybe_unused]] auto found = + GarbageCollector::GlobalUplinkTargetCacheEntry(key); + } + + auto const entry = + FileSystemManager::ReadFile(entry_path, ObjectType::File); + if (not entry) { + logger_->Emit(LogLevel::Debug, + "Cache miss, entry not found {}", + entry_path.string()); + return std::nullopt; + } + if (auto info = Artifact::ObjectInfo::FromString(*entry)) { + if (auto path = cas_->BlobPath(info->digest, /*is_executable=*/false)) { + if (auto value = FileSystemManager::ReadFile(*path)) { + try { + return std::make_pair( + TargetCacheEntry{nlohmann::json::parse(*value)}, + std::move(*info)); + } catch (std::exception const& ex) { + logger_->Emit(LogLevel::Warning, + "Parsing entry for key {} failed with:\n{}", + key.Id().ToString(), + ex.what()); + } + } + } + } + logger_->Emit(LogLevel::Warning, + "Reading entry for key {} failed", + key.Id().ToString()); + return std::nullopt; +} + +template <bool kDoGlobalUplink> +template <bool kIsLocalGeneration> +requires(kIsLocalGeneration) auto TargetCache< + kDoGlobalUplink>::LocalUplinkEntry(LocalGenerationTC const& latest, + TargetCacheKey const& key) const noexcept + -> bool { + // Determine target cache key path of given generation. + auto key_digest = key.Id().digest.hash(); + if (FileSystemManager::IsFile(latest.file_store_.GetPath(key_digest))) { + return true; + } + + // Determine target cache entry location. + auto cache_key = file_store_.GetPath(key_digest); + auto raw_key = FileSystemManager::ReadFile(cache_key); + if (not raw_key) { + return false; + } + + // Determine target cache entry location. + auto entry_info = Artifact::ObjectInfo::FromString(*raw_key); + if (not entry_info) { + return false; + } + + // Determine target cache entry blob path of given generation. + auto cache_entry = + cas_->BlobPath(entry_info->digest, /*is_executable=*/false); + if (not cache_entry) { + return false; + } + + // Determine artifacts referenced by target cache entry. + auto raw_entry = FileSystemManager::ReadFile(*cache_entry); + if (not raw_entry) { + return false; + } + nlohmann::json json_desc{}; + try { + json_desc = nlohmann::json::parse(*raw_entry); + } catch (std::exception const& ex) { + return false; + } + auto entry = TargetCacheEntry::FromJson(json_desc); + std::vector<Artifact::ObjectInfo> artifacts_info; + if (not entry.ToArtifacts(&artifacts_info)) { + return false; + } + + // Uplink referenced artifacts. + for (auto const& info : artifacts_info) { + if (info.type == ObjectType::Tree) { + if (not cas_->LocalUplinkTree(*latest.cas_, info.digest)) { + return false; + } + } + else if (not cas_->LocalUplinkBlob(*latest.cas_, + info.digest, + IsExecutableObject(info.type))) { + return false; + } + } + + // Uplink target cache entry blob. + if (not cas_->LocalUplinkBlob(*latest.cas_, + entry_info->digest, + /*is_executable=*/false)) { + return false; + } + + // Uplink target cache key + return latest.file_store_.AddFromFile( + key_digest, cache_key, /*is_owner=*/true); +} + +template <bool kDoGlobalUplink> +auto TargetCache<kDoGlobalUplink>::DownloadKnownArtifacts( + TargetCacheEntry const& value, + ArtifactDownloader const& downloader) const noexcept -> bool { + std::vector<Artifact::ObjectInfo> artifacts_info; + return downloader and value.ToArtifacts(&artifacts_info) and + downloader(artifacts_info); +} + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_TPP diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.cpp b/src/buildtool/storage/target_cache_entry.cpp index 139c706b..683b2f02 100644 --- a/src/buildtool/build_engine/target_map/target_cache_entry.cpp +++ b/src/buildtool/storage/target_cache_entry.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" #include <algorithm> #include <exception> diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.hpp b/src/buildtool/storage/target_cache_entry.hpp index 1e54a003..b2002f80 100644 --- a/src/buildtool/build_engine/target_map/target_cache_entry.hpp +++ b/src/buildtool/storage/target_cache_entry.hpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP #include <optional> #include <unordered_map> @@ -30,9 +30,9 @@ // Entry for target cache. Created from target, contains TargetResult. class TargetCacheEntry { - friend class TargetCache; - public: + explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} + // Create the entry from target with replacement artifacts/infos. // Replacement artifacts must replace all non-known artifacts by known. [[nodiscard]] static auto FromTarget( @@ -53,16 +53,15 @@ class TargetCacheEntry { gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) const noexcept -> bool; - private: - nlohmann::json desc_; - - explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} [[nodiscard]] auto ToJson() const& -> nlohmann::json const& { return desc_; } [[nodiscard]] auto ToJson() && -> nlohmann::json { return std::move(desc_); } + + private: + nlohmann::json desc_; }; -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_ENTRY_HPP diff --git a/src/buildtool/build_engine/target_map/target_cache_key.cpp b/src/buildtool/storage/target_cache_key.cpp index 6408c2a6..11ea5c84 100644 --- a/src/buildtool/build_engine/target_map/target_cache_key.cpp +++ b/src/buildtool/storage/target_cache_key.cpp @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" #include <exception> #include <nlohmann/json.hpp> #include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/storage.hpp" auto TargetCacheKey::Create(std::string const& repo_key, BuildMaps::Base::NamedTarget const& target_name, @@ -35,8 +35,8 @@ auto TargetCacheKey::Create(std::string const& repo_key, {"target_name", nlohmann::json{target_name.module, target_name.name}.dump()}, {"effective_config", effective_config.ToString()}}}; - static auto const& cas = LocalCAS<ObjectType::File>::Instance(); - if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) { + if (auto target_key = Storage::Instance().CAS().StoreBlob( + target_desc.dump(2), /*is_executable=*/false)) { return TargetCacheKey{ {ArtifactDigest{*target_key}, ObjectType::File}}; } diff --git a/src/buildtool/build_engine/target_map/target_cache_key.hpp b/src/buildtool/storage/target_cache_key.hpp index 116a6a07..70d06464 100644 --- a/src/buildtool/build_engine/target_map/target_cache_key.hpp +++ b/src/buildtool/storage/target_cache_key.hpp @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP -#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP +#include <functional> #include <optional> #include <utility> @@ -46,4 +47,4 @@ class TargetCacheKey { Artifact::ObjectInfo id_; }; -#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_TARGET_CACHE_KEY_HPP diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 04276225..b2bf6cd2 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -33,7 +33,7 @@ , "srcs": ["utils.cpp"] , "deps": [ ["src/utils/cpp", "tmp_dir"] - , ["src/buildtool/execution_api/local", "config"] + , ["src/buildtool/storage", "config"] , ["src/buildtool/main", "constants"] , ["src/buildtool/build_engine/expression", "expression"] ] diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index 02eaf8a5..34c1953d 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -223,7 +223,7 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { root_path = *ws_root; } if (root == "home") { - root_path = LocalExecutionConfig::GetUserHome(); + root_path = StorageConfig::GetUserHome(); } if (root == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); @@ -262,7 +262,7 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { root_path = *ws_root; } if (root_str == "home") { - root_path = LocalExecutionConfig::GetUserHome(); + root_path = StorageConfig::GetUserHome(); } if (root_str == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); @@ -1416,9 +1416,9 @@ auto main(int argc, char* argv[]) -> int { arguments.common.explicit_distdirs.begin(), arguments.common.explicit_distdirs.end()); - // Setup LocalExecutionConfig to store the local_build_root properly + // Setup LocalStorageConfig to store the local_build_root properly // and make the cas and git cache roots available - if (not LocalExecutionConfig::SetBuildRoot( + if (not StorageConfig::SetBuildRoot( *arguments.common.just_mr_paths->root)) { Logger::Log(LogLevel::Error, "Failed to configure local build root."); diff --git a/src/other_tools/just_mr/utils.cpp b/src/other_tools/just_mr/utils.cpp index 24511742..ca891875 100644 --- a/src/other_tools/just_mr/utils.cpp +++ b/src/other_tools/just_mr/utils.cpp @@ -14,14 +14,14 @@ #include "src/other_tools/just_mr/utils.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/path.hpp" namespace JustMR::Utils { auto GetGitCacheRoot() noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "git"; + return StorageConfig::BuildRoot() / "git"; } auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, @@ -42,20 +42,19 @@ auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { // try to create parent dir - auto parent_path = - LocalExecutionConfig::BuildRoot() / "tmp-workspaces" / type; + auto parent_path = StorageConfig::BuildRoot() / "tmp-workspaces" / type; return TmpDir::Create(parent_path); } auto GetArchiveTreeIDFile(std::string const& repo_type, std::string const& content) noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "tree-map" / repo_type / content; + return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; } auto GetDistdirTreeIDFile(std::string const& content) noexcept -> std::filesystem::path { - return LocalExecutionConfig::BuildRoot() / "distfiles-tree-map" / content; + return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; } auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, @@ -79,23 +78,24 @@ auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, auto AddToCAS(std::string const& data) noexcept -> std::optional<std::filesystem::path> { // get file CAS instance - auto const& casf = LocalCAS<ObjectType::File>::Instance(); - // write to casf - auto digest = casf.StoreBlobFromBytes(data); + auto const& cas = Storage::Instance().CAS(); + // write to cas + auto digest = cas.StoreBlob(data); if (digest) { - return casf.BlobPath(*digest); + return cas.BlobPath(*digest, /*is_executable=*/false); } return std::nullopt; } void AddDistfileToCAS(std::filesystem::path const& distfile, JustMR::PathsPtr const& just_mr_paths) noexcept { - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); for (auto const& dirpath : just_mr_paths->distdirs) { auto candidate = dirpath / distfile; if (FileSystemManager::Exists(candidate)) { // try to add to CAS - [[maybe_unused]] auto digest = casf.StoreBlobFromFile(candidate); + [[maybe_unused]] auto digest = + cas.StoreBlob(candidate, /*is_executable=*/false); } } } diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 5cb82eb3..dddd4b92 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -19,8 +19,8 @@ #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/main/constants.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/utils/cpp/tmp_dir.hpp" /* Paths and constants required by just-mr */ @@ -29,12 +29,11 @@ std::unordered_set<std::string> const kLocationTypes{"workspace", "home", "system"}; auto const kDefaultJustPath = "just"; -auto const kDefaultRCPath = LocalExecutionConfig::GetUserHome() / ".just-mrrc"; -auto const kDefaultBuildRoot = LocalExecutionConfig::GetUserDir(); +auto const kDefaultRCPath = StorageConfig::GetUserHome() / ".just-mrrc"; +auto const kDefaultBuildRoot = StorageConfig::kDefaultBuildRoot; auto const kDefaultCheckoutLocationsFile = - LocalExecutionConfig::GetUserHome() / ".just-local.json"; -auto const kDefaultDistdirs = - LocalExecutionConfig::GetUserHome() / ".distfiles"; + StorageConfig::GetUserHome() / ".just-local.json"; +auto const kDefaultDistdirs = StorageConfig::GetUserHome() / ".distfiles"; std::vector<std::string> const kAltDirs = {"target_root", "rule_root", diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 5b692026..1220d50f 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -15,8 +15,8 @@ #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/buildtool/crypto/hasher.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/utils/curl_easy_handle.hpp" @@ -61,9 +61,9 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, JustMRStatistics::Instance().IncrementQueuedCounter(); } // check if content already in CAS - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto digest = ArtifactDigest(key.content, 0, false); - if (casf.BlobPath(digest)) { + if (cas.BlobPath(digest, /*is_executable=*/false)) { (*setter)(true); return; } @@ -74,7 +74,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, : std::filesystem::path(key.fetch_url).filename().string()); JustMR::Utils::AddDistfileToCAS(repo_distfile, just_mr_paths); // check if content is in CAS now - if (casf.BlobPath(digest)) { + if (cas.BlobPath(digest, /*is_executable=*/false)) { (*setter)(true); return; } diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp index 39474b12..1c903fe3 100644 --- a/src/other_tools/ops_maps/repo_fetch_map.cpp +++ b/src/other_tools/ops_maps/repo_fetch_map.cpp @@ -14,8 +14,8 @@ #include "src/other_tools/ops_maps/repo_fetch_map.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/utils.hpp" @@ -54,9 +54,10 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, logger]([[maybe_unused]] auto const& values) { // content is now in CAS // copy content from CAS into fetch_dir - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto content_path = - casf.BlobPath(ArtifactDigest(content, 0, false)); + cas.BlobPath(ArtifactDigest(content, 0, false), + /*is_executable=*/false); if (content_path) { auto target_name = fetch_dir / distfile; if (FileSystemManager::Exists(target_name)) { @@ -100,9 +101,10 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, } else { // copy content from CAS into fetch_dir - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); auto content_path = - casf.BlobPath(ArtifactDigest(key.archive.content, 0, false)); + cas.BlobPath(ArtifactDigest(key.archive.content, 0, false), + /*is_executable=*/false); if (content_path) { auto target_name = fetch_dir / distfile; if (FileSystemManager::Exists(target_name)) { diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 584fabc7..f36f3142 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -14,8 +14,8 @@ #include "src/other_tools/root_maps/content_git_map.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/utils/archive_ops.hpp" @@ -159,10 +159,11 @@ auto CreateContentGitMap( /*fatal=*/true); return; } - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); // content is in CAS if here, so no need to check nullopt auto content_cas_path = - casf.BlobPath(ArtifactDigest(content_id, 0, false)) + cas.BlobPath(ArtifactDigest(content_id, 0, false), + /*is_executable=*/false) .value(); auto res = ExtractArchive( content_cas_path, repo_type, tmp_dir->GetPath()); diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 3289430b..ef21b7cc 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -17,9 +17,8 @@ #include <algorithm> #include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" @@ -33,12 +32,13 @@ namespace { std::shared_ptr<std::unordered_map<std::string, std::string>> const& content_list, std::filesystem::path const& tmp_dir) noexcept -> bool { - auto const& casf = LocalCAS<ObjectType::File>::Instance(); + auto const& cas = Storage::Instance().CAS(); return std::all_of(content_list->begin(), content_list->end(), - [&casf, tmp_dir](auto const& kv) { - auto content_path = casf.BlobPath( - ArtifactDigest(kv.second, 0, false)); + [&cas, tmp_dir](auto const& kv) { + auto content_path = + cas.BlobPath(ArtifactDigest(kv.second, 0, false), + /*is_executable=*/false); if (content_path) { return FileSystemManager::CreateFileHardlink( *content_path, // from: cas_path/content_id |