summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorMaksim Denisov <denisov.maksim@huawei.com>2024-12-20 16:57:21 +0100
committerMaksim Denisov <denisov.maksim@huawei.com>2025-01-07 14:18:09 +0100
commit972bc9108bc52ade30505ae32b38d663ab02b220 (patch)
treeb3230e6208523a28e5bf4cdaeff3f4852a19eebe /src
parent71ffe14b0083c9c4975f66db4127de9ee5b9a84d (diff)
downloadjustbuild-972bc9108bc52ade30505ae32b38d663ab02b220.tar.gz
Implement rebuilding of StorageConfig
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/computed_roots/evaluate.cpp33
-rw-r--r--src/buildtool/computed_roots/evaluate.hpp2
-rw-r--r--src/buildtool/main/main.cpp10
-rw-r--r--src/buildtool/serve_api/serve_service/serve_server_implementation.cpp3
-rw-r--r--src/buildtool/storage/config.hpp14
-rw-r--r--src/buildtool/storage/garbage_collector.cpp9
-rw-r--r--src/other_tools/just_mr/fetch.cpp3
-rw-r--r--src/other_tools/just_mr/setup.cpp3
8 files changed, 40 insertions, 37 deletions
diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp
index 5ec1e3e5..6c05aadb 100644
--- a/src/buildtool/computed_roots/evaluate.cpp
+++ b/src/buildtool/computed_roots/evaluate.cpp
@@ -493,10 +493,7 @@ void ComputeTreeStructureAndFill(
// Since tree structure works with git trees, a native storage is required.
std::optional<StorageConfig> substitution_storage_config;
if (not ProtocolTraits::IsNative(storage_config->hash_function.GetType())) {
- // Only build root and the hash_type are important:
- auto config = StorageConfig::Builder{}
- .SetBuildRoot(storage_config->build_root)
- .SetNumGenerations(storage_config->num_generations)
+ auto config = StorageConfig::Builder::Rebuild(*storage_config)
.SetHashType(HashFunction::Type::GitSHA1)
.Build();
if (not config) {
@@ -731,7 +728,6 @@ auto EvaluatePrecomputedRoots(
std::string const& main_repo,
ServeApi const* serve,
StorageConfig const& storage_config,
- std::optional<StorageConfig> const& git_storage_config,
GraphTraverser::CommandLineArguments const& traverser_args,
gsl::not_null<const ExecutionContext*> const& context,
std::size_t jobs) -> bool {
@@ -757,15 +753,24 @@ auto EvaluatePrecomputedRoots(
}
// Prepare rehash-function, if rehashing is required
- const bool needs_rehash =
- git_storage_config and
- (git_storage_config->hash_function.GetType() !=
- storage_config.hash_function.GetType());
- auto rehash =
- needs_rehash
- ? std::make_optional<RehashUtils::Rehasher>(
- storage_config, *git_storage_config, context->apis)
- : std::nullopt;
+ std::optional<RehashUtils::Rehasher> rehash;
+ if (not ProtocolTraits::IsNative(
+ storage_config.hash_function.GetType())) {
+ auto native_storage_config =
+ StorageConfig::Builder::Rebuild(storage_config)
+ .SetHashType(HashFunction::Type::GitSHA1)
+ .Build();
+ if (not native_storage_config) {
+ Logger::Log(
+ LogLevel::Error,
+ "Failed to create native storage config for rehashing:\n{}",
+ std::move(native_storage_config).error());
+ return false;
+ }
+ rehash.emplace(storage_config,
+ *std::move(native_storage_config),
+ context->apis);
+ }
// Our RepositoryConfig is a bit problematic: it cannot be copied, hence
// we have to change in place. Moreover, it is tread-safe for read
diff --git a/src/buildtool/computed_roots/evaluate.hpp b/src/buildtool/computed_roots/evaluate.hpp
index e90db25b..2288bfec 100644
--- a/src/buildtool/computed_roots/evaluate.hpp
+++ b/src/buildtool/computed_roots/evaluate.hpp
@@ -16,7 +16,6 @@
#define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_EVALUTE_HPP
#include <cstddef>
-#include <optional>
#include <string>
#include "gsl/gsl"
@@ -31,7 +30,6 @@ auto EvaluatePrecomputedRoots(
std::string const& main_repo,
ServeApi const* serve,
StorageConfig const& storage_config,
- std::optional<StorageConfig> const& git_storage_config,
GraphTraverser::CommandLineArguments const& traverser_args,
gsl::not_null<const ExecutionContext*> const& context,
std::size_t jobs) -> bool;
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index d125041f..5f06cbad 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -1019,22 +1019,12 @@ auto main(int argc, char* argv[]) -> int {
std::size_t eval_root_jobs =
std::lround(std::ceil(std::sqrt(arguments.common.jobs)));
#ifndef BOOTSTRAP_BUILD_TOOL
- const bool need_rehash =
- arguments.protocol.hash_type != HashFunction::Type::GitSHA1;
- const auto git_storage_config =
- need_rehash ? CreateStorageConfig(arguments.endpoint,
- HashFunction::Type::GitSHA1)
- : std::nullopt;
- if (need_rehash and (not git_storage_config)) {
- return kExitFailure;
- }
std::optional<ServeApi> serve = ServeApi::Create(
*serve_config, &local_context, &remote_context, &main_apis);
if (not EvaluatePrecomputedRoots(&repo_config,
main_repo,
serve ? &*serve : nullptr,
*storage_config,
- git_storage_config,
traverse_args,
&exec_context,
eval_root_jobs)) {
diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp
index ffe78822..fce3813b 100644
--- a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp
+++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp
@@ -148,8 +148,7 @@ auto ServeServerImpl::Run(
auto const is_compat = not ProtocolTraits::IsNative(hash_type);
if (is_compat) {
auto config =
- StorageConfig::Builder{}
- .SetBuildRoot(local_context->storage_config->build_root)
+ StorageConfig::Builder::Rebuild(*local_context->storage_config)
.SetHashType(HashFunction::Type::GitSHA1)
.Build();
if (not config) {
diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp
index a5d96e90..ff7c7710 100644
--- a/src/buildtool/storage/config.hpp
+++ b/src/buildtool/storage/config.hpp
@@ -151,6 +151,20 @@ struct StorageConfig final {
class StorageConfig::Builder final {
public:
+ explicit Builder() = default;
+
+ /// \brief Create a configurable builder from an existing config.
+ /// Useful, for example, to make a copy of an existing config and change
+ /// the hash type.
+ [[nodiscard]] static auto Rebuild(StorageConfig const& config) noexcept
+ -> Builder {
+ return Builder{}
+ .SetBuildRoot(config.build_root)
+ .SetNumGenerations(config.num_generations)
+ .SetHashType(config.hash_function.GetType())
+ .SetBackendDescription(config.backend_description);
+ }
+
auto SetBuildRoot(std::filesystem::path value) noexcept -> Builder& {
build_root_ = std::move(value);
return *this;
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp
index fa448552..0f4dcdae 100644
--- a/src/buildtool/storage/garbage_collector.cpp
+++ b/src/buildtool/storage/garbage_collector.cpp
@@ -220,15 +220,14 @@ auto GarbageCollector::Compactify(StorageConfig const& storage_config,
// Compactification must be done for both native and compatible storages.
static constexpr std::array kHashes = {HashFunction::Type::GitSHA1,
HashFunction::Type::PlainSHA256};
- auto builder = StorageConfig::Builder{}
- .SetBuildRoot(storage_config.build_root)
- .SetNumGenerations(storage_config.num_generations);
return std::all_of(
kHashes.begin(),
kHashes.end(),
- [threshold, &builder](HashFunction::Type hash_type) {
- auto const config = builder.SetHashType(hash_type).Build();
+ [threshold, &storage_config](HashFunction::Type hash_type) {
+ auto const config = StorageConfig::Builder::Rebuild(storage_config)
+ .SetHashType(hash_type)
+ .Build();
if (not config) {
return false;
}
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index ab56db41..307f3aaa 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -358,8 +358,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
std::optional<LockFile> compat_lock = std::nullopt;
IExecutionApi::Ptr compat_local_api = nullptr;
if (common_args.compatible) {
- auto config = StorageConfig::Builder{}
- .SetBuildRoot(native_storage_config.build_root)
+ auto config = StorageConfig::Builder::Rebuild(native_storage_config)
.SetHashType(HashFunction::Type::PlainSHA256)
.Build();
if (not config) {
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 3247b06d..55aa328b 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -166,8 +166,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
std::optional<LockFile> compat_lock = std::nullopt;
IExecutionApi::Ptr compat_local_api = nullptr;
if (common_args.compatible) {
- auto config = StorageConfig::Builder{}
- .SetBuildRoot(native_storage_config.build_root)
+ auto config = StorageConfig::Builder::Rebuild(native_storage_config)
.SetHashType(HashFunction::Type::PlainSHA256)
.Build();
if (not config) {