diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-07-19 12:44:47 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-07-22 17:01:13 +0200 |
commit | b2dff03c841f159b474f445eb558a4ee153b1741 (patch) | |
tree | 2892207a7f7a359769706a107b12faf809d9f180 /src | |
parent | 57388e7624c71762e6aac30f16809c467f96ca58 (diff) | |
download | justbuild-b2dff03c841f159b474f445eb558a4ee153b1741.tar.gz |
Pass HashFunction from StorageConfig to Storage
Diffstat (limited to 'src')
-rw-r--r-- | src/buildtool/file_system/object_cas.hpp | 20 | ||||
-rw-r--r-- | src/buildtool/storage/garbage_collector.cpp | 33 | ||||
-rw-r--r-- | src/buildtool/storage/local_ac.hpp | 6 | ||||
-rw-r--r-- | src/buildtool/storage/local_ac.tpp | 4 | ||||
-rw-r--r-- | src/buildtool/storage/local_cas.hpp | 20 | ||||
-rw-r--r-- | src/buildtool/storage/local_cas.tpp | 5 | ||||
-rw-r--r-- | src/buildtool/storage/storage.hpp | 4 |
7 files changed, 63 insertions, 29 deletions
diff --git a/src/buildtool/file_system/object_cas.hpp b/src/buildtool/file_system/object_cas.hpp index 122778b5..cf960110 100644 --- a/src/buildtool/file_system/object_cas.hpp +++ b/src/buildtool/file_system/object_cas.hpp @@ -56,9 +56,12 @@ class ObjectCAS { /// digest exists at the given path if true was returned. /// \param store_path The path to use for storing blobs. /// \param exists (optional) Function for checking blob existence. - explicit ObjectCAS(std::filesystem::path const& store_path, + explicit ObjectCAS(HashFunction hash_function, + std::filesystem::path const& store_path, ExistsFunc exists = kDefaultExists) - : file_store_{store_path}, exists_{std::move(exists)} {} + : hash_function_{hash_function}, + file_store_{store_path}, + exists_{std::move(exists)} {} ObjectCAS(ObjectCAS const&) = delete; ObjectCAS(ObjectCAS&&) = delete; @@ -109,22 +112,21 @@ class ObjectCAS { static constexpr auto kStorageType = kType == ObjectType::Tree ? ObjectType::File : kType; + HashFunction const hash_function_; Logger logger_{std::string{"ObjectCAS"} + ToChar(kType)}; FileStorage<kStorageType, StoreMode::FirstWins, /*kSetEpochTime=*/true> file_store_; ExistsFunc exists_; - [[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept + [[nodiscard]] auto CreateDigest(std::string const& bytes) const noexcept -> std::optional<bazel_re::Digest> { - return ArtifactDigest::Create<kType>(HashFunction::Instance(), bytes); + return ArtifactDigest::Create<kType>(hash_function_, bytes); } - [[nodiscard]] static auto CreateDigest( - std::filesystem::path const& file_path) noexcept - -> std::optional<bazel_re::Digest> { - return ArtifactDigest::CreateFromFile<kType>(HashFunction::Instance(), - file_path); + [[nodiscard]] auto CreateDigest(std::filesystem::path const& file_path) + const noexcept -> std::optional<bazel_re::Digest> { + return ArtifactDigest::CreateFromFile<kType>(hash_function_, file_path); } [[nodiscard]] auto IsAvailable( diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp index 25a78ddb..b03364f8 100644 --- a/src/buildtool/storage/garbage_collector.cpp +++ b/src/buildtool/storage/garbage_collector.cpp @@ -16,13 +16,14 @@ #include "src/buildtool/storage/garbage_collector.hpp" +#include <array> #include <filesystem> -#include <memory> #include <vector> -#include "nlohmann/json.hpp" +#include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -213,23 +214,35 @@ auto GarbageCollector::TriggerGarbageCollection( auto GarbageCollector::Compactify(StorageConfig const& storage_config, size_t threshold) noexcept -> bool { - const bool mode = Compatibility::IsCompatible(); - // Return to the initial compatibility mode once done: - auto scope_guard = std::shared_ptr<void>(nullptr, [mode](void* /*unused*/) { + auto const guard = gsl::finally([mode = Compatibility::IsCompatible()] { Compatibility::SetCompatible(mode); }); - // Compactification must be done for both native and compatible storages. - auto compactify = [&storage_config, threshold](bool compatible) -> bool { - Compatibility::SetCompatible(compatible); - auto const storage = ::Generation::Create(&storage_config); + auto compactify = [threshold](StorageConfig const& config) -> bool { + Compatibility::SetCompatible(config.hash_function.GetHashType() == + HashFunction::JustHash::Compatible); + auto const storage = ::Generation::Create(&config); return Compactifier::RemoveInvalid(storage.CAS()) and Compactifier::RemoveSpliced(storage.CAS()) and Compactifier::SplitLarge(storage.CAS(), threshold); }; - return compactify(mode) and compactify(not mode); + + // Compactification must be done for both native and compatible storages. + static constexpr std::array kHashes = {HashFunction::JustHash::Native, + HashFunction::JustHash::Compatible}; + auto builder = StorageConfig::Builder{} + .SetBuildRoot(storage_config.build_root) + .SetNumGenerations(storage_config.num_generations); + + return std::all_of( + kHashes.begin(), + kHashes.end(), + [&builder, &compactify](HashFunction::JustHash hash_type) { + auto const config = builder.SetHashType(hash_type).Build(); + return config.has_value() and compactify(*config); + }); } #endif // BOOTSTRAP_BUILD_TOOL diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp index c3694915..920413cd 100644 --- a/src/buildtool/storage/local_ac.hpp +++ b/src/buildtool/storage/local_ac.hpp @@ -48,7 +48,10 @@ class LocalAC { GenerationConfig const& config, gsl::not_null<Uplinker<kDoGlobalUplink> const*> const& uplinker) noexcept - : cas_{*cas}, file_store_{config.action_cache}, uplinker_{*uplinker} {}; + : cas_{*cas}, + file_store_{config.action_cache}, + uplinker_{*uplinker}, + hash_function_{config.storage_config->hash_function} {}; LocalAC(LocalAC const&) = default; LocalAC(LocalAC&&) noexcept = default; @@ -95,6 +98,7 @@ class LocalAC { FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false> file_store_; Uplinker<kDoGlobalUplink> const& uplinker_; + HashFunction const hash_function_; [[nodiscard]] auto ReadResult(bazel_re::Digest const& digest) const noexcept -> std::optional<bazel_re::ActionResult>; diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp index c60e8751..122549bb 100644 --- a/src/buildtool/storage/local_ac.tpp +++ b/src/buildtool/storage/local_ac.tpp @@ -106,7 +106,7 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( if (not cas_.LocalUplinkBlob( latest.cas_, bazel_re::Digest(ArtifactDigest::Create<ObjectType::File>( - HashFunction::Instance(), link.target())), + hash_function_, link.target())), /*is_executable=*/false)) { return false; } @@ -115,7 +115,7 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( if (not cas_.LocalUplinkBlob( latest.cas_, bazel_re::Digest(ArtifactDigest::Create<ObjectType::File>( - HashFunction::Instance(), link.target())), + hash_function_, link.target())), /*is_executable=*/false)) { return false; } diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index aedf5838..1f6a74ed 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -22,6 +22,7 @@ #include "gsl/gsl" #include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_cas.hpp" #include "src/buildtool/storage/config.hpp" @@ -49,12 +50,22 @@ class LocalCAS { explicit LocalCAS( GenerationConfig const& config, gsl::not_null<Uplinker<kDoGlobalUplink> const*> const& uplinker) - : cas_file_{config.cas_f, MakeUplinker<ObjectType::File>(uplinker)}, - cas_exec_{config.cas_x, + : cas_file_{config.storage_config->hash_function, + config.cas_f, + MakeUplinker<ObjectType::File>(uplinker)}, + cas_exec_{config.storage_config->hash_function, + config.cas_x, MakeUplinker<ObjectType::Executable>(uplinker)}, - cas_tree_{config.cas_t, MakeUplinker<ObjectType::Tree>(uplinker)}, + cas_tree_{config.storage_config->hash_function, + config.cas_t, + MakeUplinker<ObjectType::Tree>(uplinker)}, cas_file_large_{this, config, uplinker}, - cas_tree_large_{this, config, uplinker} {} + cas_tree_large_{this, config, uplinker}, + hash_function_{config.storage_config->hash_function} {} + + [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction { + return hash_function_; + } /// \brief Obtain path to the storage root. /// \param type Type of the storage to be obtained. @@ -261,6 +272,7 @@ class LocalCAS { ObjectCAS<ObjectType::Tree> cas_tree_; LargeObjectCAS<kDoGlobalUplink, ObjectType::File> cas_file_large_; LargeObjectCAS<kDoGlobalUplink, ObjectType::Tree> cas_tree_large_; + HashFunction const hash_function_; /// \brief Provides uplink via "exists callback" for physical object CAS. template <ObjectType kType> diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp index 54807580..eba99a67 100644 --- a/src/buildtool/storage/local_cas.tpp +++ b/src/buildtool/storage/local_cas.tpp @@ -19,7 +19,6 @@ #include <utility> // std::move #include "fmt/core.h" -#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/local_cas.hpp" @@ -375,8 +374,8 @@ auto LocalCAS<kDoGlobalUplink>::Splice( // methods can refer to a file that existed before. The direct hash // calculation is done instead. auto const& file_path = large_object.GetPath(); - auto spliced_digest = ArtifactDigest::CreateFromFile<kType>( - HashFunction::Instance(), file_path); + auto spliced_digest = + ArtifactDigest::CreateFromFile<kType>(hash_function_, file_path); if (not spliced_digest) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, "could not calculate digest"}}; diff --git a/src/buildtool/storage/storage.hpp b/src/buildtool/storage/storage.hpp index d4b070e3..2a848925 100644 --- a/src/buildtool/storage/storage.hpp +++ b/src/buildtool/storage/storage.hpp @@ -53,6 +53,10 @@ class LocalStorage final { return LocalStorage<kDoGlobalUplink>{gen_config}; } + [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction { + return cas_->GetHashFunction(); + } + /// \brief Get the CAS instance. [[nodiscard]] auto CAS() const noexcept -> CAS_t const& { return *cas_; } |