summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorMaksim Denisov <denisov.maksim@huawei.com>2024-07-19 12:44:47 +0200
committerMaksim Denisov <denisov.maksim@huawei.com>2024-07-22 17:01:13 +0200
commitb2dff03c841f159b474f445eb558a4ee153b1741 (patch)
tree2892207a7f7a359769706a107b12faf809d9f180 /src
parent57388e7624c71762e6aac30f16809c467f96ca58 (diff)
downloadjustbuild-b2dff03c841f159b474f445eb558a4ee153b1741.tar.gz
Pass HashFunction from StorageConfig to Storage
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/file_system/object_cas.hpp20
-rw-r--r--src/buildtool/storage/garbage_collector.cpp33
-rw-r--r--src/buildtool/storage/local_ac.hpp6
-rw-r--r--src/buildtool/storage/local_ac.tpp4
-rw-r--r--src/buildtool/storage/local_cas.hpp20
-rw-r--r--src/buildtool/storage/local_cas.tpp5
-rw-r--r--src/buildtool/storage/storage.hpp4
7 files changed, 63 insertions, 29 deletions
diff --git a/src/buildtool/file_system/object_cas.hpp b/src/buildtool/file_system/object_cas.hpp
index 122778b5..cf960110 100644
--- a/src/buildtool/file_system/object_cas.hpp
+++ b/src/buildtool/file_system/object_cas.hpp
@@ -56,9 +56,12 @@ class ObjectCAS {
/// digest exists at the given path if true was returned.
/// \param store_path The path to use for storing blobs.
/// \param exists (optional) Function for checking blob existence.
- explicit ObjectCAS(std::filesystem::path const& store_path,
+ explicit ObjectCAS(HashFunction hash_function,
+ std::filesystem::path const& store_path,
ExistsFunc exists = kDefaultExists)
- : file_store_{store_path}, exists_{std::move(exists)} {}
+ : hash_function_{hash_function},
+ file_store_{store_path},
+ exists_{std::move(exists)} {}
ObjectCAS(ObjectCAS const&) = delete;
ObjectCAS(ObjectCAS&&) = delete;
@@ -109,22 +112,21 @@ class ObjectCAS {
static constexpr auto kStorageType =
kType == ObjectType::Tree ? ObjectType::File : kType;
+ HashFunction const hash_function_;
Logger logger_{std::string{"ObjectCAS"} + ToChar(kType)};
FileStorage<kStorageType, StoreMode::FirstWins, /*kSetEpochTime=*/true>
file_store_;
ExistsFunc exists_;
- [[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept
+ [[nodiscard]] auto CreateDigest(std::string const& bytes) const noexcept
-> std::optional<bazel_re::Digest> {
- return ArtifactDigest::Create<kType>(HashFunction::Instance(), bytes);
+ return ArtifactDigest::Create<kType>(hash_function_, bytes);
}
- [[nodiscard]] static auto CreateDigest(
- std::filesystem::path const& file_path) noexcept
- -> std::optional<bazel_re::Digest> {
- return ArtifactDigest::CreateFromFile<kType>(HashFunction::Instance(),
- file_path);
+ [[nodiscard]] auto CreateDigest(std::filesystem::path const& file_path)
+ const noexcept -> std::optional<bazel_re::Digest> {
+ return ArtifactDigest::CreateFromFile<kType>(hash_function_, file_path);
}
[[nodiscard]] auto IsAvailable(
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp
index 25a78ddb..b03364f8 100644
--- a/src/buildtool/storage/garbage_collector.cpp
+++ b/src/buildtool/storage/garbage_collector.cpp
@@ -16,13 +16,14 @@
#include "src/buildtool/storage/garbage_collector.hpp"
+#include <array>
#include <filesystem>
-#include <memory>
#include <vector>
-#include "nlohmann/json.hpp"
+#include "gsl/gsl"
#include "src/buildtool/common/artifact.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/message_limits.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/log_level.hpp"
@@ -213,23 +214,35 @@ auto GarbageCollector::TriggerGarbageCollection(
auto GarbageCollector::Compactify(StorageConfig const& storage_config,
size_t threshold) noexcept -> bool {
- const bool mode = Compatibility::IsCompatible();
-
// Return to the initial compatibility mode once done:
- auto scope_guard = std::shared_ptr<void>(nullptr, [mode](void* /*unused*/) {
+ auto const guard = gsl::finally([mode = Compatibility::IsCompatible()] {
Compatibility::SetCompatible(mode);
});
- // Compactification must be done for both native and compatible storages.
- auto compactify = [&storage_config, threshold](bool compatible) -> bool {
- Compatibility::SetCompatible(compatible);
- auto const storage = ::Generation::Create(&storage_config);
+ auto compactify = [threshold](StorageConfig const& config) -> bool {
+ Compatibility::SetCompatible(config.hash_function.GetHashType() ==
+ HashFunction::JustHash::Compatible);
+ auto const storage = ::Generation::Create(&config);
return Compactifier::RemoveInvalid(storage.CAS()) and
Compactifier::RemoveSpliced(storage.CAS()) and
Compactifier::SplitLarge(storage.CAS(), threshold);
};
- return compactify(mode) and compactify(not mode);
+
+ // Compactification must be done for both native and compatible storages.
+ static constexpr std::array kHashes = {HashFunction::JustHash::Native,
+ HashFunction::JustHash::Compatible};
+ auto builder = StorageConfig::Builder{}
+ .SetBuildRoot(storage_config.build_root)
+ .SetNumGenerations(storage_config.num_generations);
+
+ return std::all_of(
+ kHashes.begin(),
+ kHashes.end(),
+ [&builder, &compactify](HashFunction::JustHash hash_type) {
+ auto const config = builder.SetHashType(hash_type).Build();
+ return config.has_value() and compactify(*config);
+ });
}
#endif // BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp
index c3694915..920413cd 100644
--- a/src/buildtool/storage/local_ac.hpp
+++ b/src/buildtool/storage/local_ac.hpp
@@ -48,7 +48,10 @@ class LocalAC {
GenerationConfig const& config,
gsl::not_null<Uplinker<kDoGlobalUplink> const*> const&
uplinker) noexcept
- : cas_{*cas}, file_store_{config.action_cache}, uplinker_{*uplinker} {};
+ : cas_{*cas},
+ file_store_{config.action_cache},
+ uplinker_{*uplinker},
+ hash_function_{config.storage_config->hash_function} {};
LocalAC(LocalAC const&) = default;
LocalAC(LocalAC&&) noexcept = default;
@@ -95,6 +98,7 @@ class LocalAC {
FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false>
file_store_;
Uplinker<kDoGlobalUplink> const& uplinker_;
+ HashFunction const hash_function_;
[[nodiscard]] auto ReadResult(bazel_re::Digest const& digest) const noexcept
-> std::optional<bazel_re::ActionResult>;
diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp
index c60e8751..122549bb 100644
--- a/src/buildtool/storage/local_ac.tpp
+++ b/src/buildtool/storage/local_ac.tpp
@@ -106,7 +106,7 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry(
if (not cas_.LocalUplinkBlob(
latest.cas_,
bazel_re::Digest(ArtifactDigest::Create<ObjectType::File>(
- HashFunction::Instance(), link.target())),
+ hash_function_, link.target())),
/*is_executable=*/false)) {
return false;
}
@@ -115,7 +115,7 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry(
if (not cas_.LocalUplinkBlob(
latest.cas_,
bazel_re::Digest(ArtifactDigest::Create<ObjectType::File>(
- HashFunction::Instance(), link.target())),
+ hash_function_, link.target())),
/*is_executable=*/false)) {
return false;
}
diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp
index aedf5838..1f6a74ed 100644
--- a/src/buildtool/storage/local_cas.hpp
+++ b/src/buildtool/storage/local_cas.hpp
@@ -22,6 +22,7 @@
#include "gsl/gsl"
#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/file_system/object_cas.hpp"
#include "src/buildtool/storage/config.hpp"
@@ -49,12 +50,22 @@ class LocalCAS {
explicit LocalCAS(
GenerationConfig const& config,
gsl::not_null<Uplinker<kDoGlobalUplink> const*> const& uplinker)
- : cas_file_{config.cas_f, MakeUplinker<ObjectType::File>(uplinker)},
- cas_exec_{config.cas_x,
+ : cas_file_{config.storage_config->hash_function,
+ config.cas_f,
+ MakeUplinker<ObjectType::File>(uplinker)},
+ cas_exec_{config.storage_config->hash_function,
+ config.cas_x,
MakeUplinker<ObjectType::Executable>(uplinker)},
- cas_tree_{config.cas_t, MakeUplinker<ObjectType::Tree>(uplinker)},
+ cas_tree_{config.storage_config->hash_function,
+ config.cas_t,
+ MakeUplinker<ObjectType::Tree>(uplinker)},
cas_file_large_{this, config, uplinker},
- cas_tree_large_{this, config, uplinker} {}
+ cas_tree_large_{this, config, uplinker},
+ hash_function_{config.storage_config->hash_function} {}
+
+ [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction {
+ return hash_function_;
+ }
/// \brief Obtain path to the storage root.
/// \param type Type of the storage to be obtained.
@@ -261,6 +272,7 @@ class LocalCAS {
ObjectCAS<ObjectType::Tree> cas_tree_;
LargeObjectCAS<kDoGlobalUplink, ObjectType::File> cas_file_large_;
LargeObjectCAS<kDoGlobalUplink, ObjectType::Tree> cas_tree_large_;
+ HashFunction const hash_function_;
/// \brief Provides uplink via "exists callback" for physical object CAS.
template <ObjectType kType>
diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp
index 54807580..eba99a67 100644
--- a/src/buildtool/storage/local_cas.tpp
+++ b/src/buildtool/storage/local_cas.tpp
@@ -19,7 +19,6 @@
#include <utility> // std::move
#include "fmt/core.h"
-#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/storage/local_cas.hpp"
@@ -375,8 +374,8 @@ auto LocalCAS<kDoGlobalUplink>::Splice(
// methods can refer to a file that existed before. The direct hash
// calculation is done instead.
auto const& file_path = large_object.GetPath();
- auto spliced_digest = ArtifactDigest::CreateFromFile<kType>(
- HashFunction::Instance(), file_path);
+ auto spliced_digest =
+ ArtifactDigest::CreateFromFile<kType>(hash_function_, file_path);
if (not spliced_digest) {
return unexpected{LargeObjectError{LargeObjectErrorCode::Internal,
"could not calculate digest"}};
diff --git a/src/buildtool/storage/storage.hpp b/src/buildtool/storage/storage.hpp
index d4b070e3..2a848925 100644
--- a/src/buildtool/storage/storage.hpp
+++ b/src/buildtool/storage/storage.hpp
@@ -53,6 +53,10 @@ class LocalStorage final {
return LocalStorage<kDoGlobalUplink>{gen_config};
}
+ [[nodiscard]] auto GetHashFunction() const noexcept -> HashFunction {
+ return cas_->GetHashFunction();
+ }
+
/// \brief Get the CAS instance.
[[nodiscard]] auto CAS() const noexcept -> CAS_t const& { return *cas_; }