From 27f227fc2762a9796e331d9636033f2d55679947 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 3 Dec 2024 12:21:16 +0100 Subject: Rehash root lookup, if building in compatible mode To avoid unnecessary staging and git imports of computed roots for build in compatible mode, use the rehashing functionality to allow us to compute the git tree of that root in memory. --- src/buildtool/computed_roots/evaluate.cpp | 40 ++++++++++++++++++++++++------- src/buildtool/computed_roots/evaluate.hpp | 2 ++ src/buildtool/main/main.cpp | 10 ++++++++ 3 files changed, 43 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp index 202fc7d1..fc11c377 100644 --- a/src/buildtool/computed_roots/evaluate.cpp +++ b/src/buildtool/computed_roots/evaluate.cpp @@ -40,6 +40,8 @@ #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/computed_roots/analyse_and_build.hpp" #include "src/buildtool/computed_roots/lookup_cache.hpp" +#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" @@ -266,6 +268,7 @@ void ComputeAndFill( traverser_args, gsl::not_null const& context, gsl::not_null const& storage_config, + gsl::not_null*> const& rehash, gsl::not_null const& config_lock, gsl::not_null const& git_lock, std::size_t jobs, @@ -306,7 +309,8 @@ void ComputeAndFill( expected, std::monostate>(std::nullopt); { std::shared_lock computing{*config_lock}; - cache_lookup = LookupCache(target, repository_config, storage, logger); + cache_lookup = + LookupCache(target, repository_config, storage, logger, *rehash); } if (not cache_lookup) { // prerequisite failure; fatal logger call already handled by @@ -401,15 +405,18 @@ void ComputeAndFill( (*setter)(std::move(*result)); } -auto FillRoots(std::size_t jobs, - gsl::not_null const& repository_config, - gsl::not_null const& - traverser_args, - gsl::not_null const& context, - gsl::not_null const& storage_config, - gsl::not_null const& config_lock, - gsl::not_null const& git_lock) -> RootMap { +auto FillRoots( + std::size_t jobs, + gsl::not_null const& repository_config, + gsl::not_null const& + traverser_args, + gsl::not_null const& context, + gsl::not_null const& storage_config, + gsl::not_null*> const& rehash, + gsl::not_null const& config_lock, + gsl::not_null const& git_lock) -> RootMap { RootMap::ValueCreator fill_roots = [storage_config, + rehash, repository_config, traverser_args, context, @@ -435,6 +442,7 @@ auto FillRoots(std::size_t jobs, context, storage_config, config_lock, + rehash, git_lock, jobs, logger = annotated_logger, @@ -444,6 +452,7 @@ auto FillRoots(std::size_t jobs, traverser_args, context, storage_config, + rehash, config_lock, git_lock, jobs, @@ -461,6 +470,7 @@ auto EvaluateComputedRoots( gsl::not_null const& repository_config, std::string const& main_repo, StorageConfig const& storage_config, + std::optional const& git_storage_config, GraphTraverser::CommandLineArguments const& traverser_args, gsl::not_null const& context, std::size_t jobs) -> bool { @@ -485,6 +495,17 @@ auto EvaluateComputedRoots( return false; } + // Prepare rehash-function, if rehashing is required + const bool needs_rehash = + git_storage_config and + (git_storage_config->hash_function.GetType() != + storage_config.hash_function.GetType()); + auto rehash = + needs_rehash + ? std::make_optional( + storage_config, *git_storage_config, context->apis) + : std::nullopt; + // Our RepositoryConfig is a bit problematic: it cannot be copied, hence // we have to change in place. Moreover, it is tread-safe for read // access, but not for writing, so we have to synchronize access out of @@ -496,6 +517,7 @@ auto EvaluateComputedRoots( &traverser_args, context, &storage_config, + &rehash, &repo_config_access, &git_access); bool failed = false; diff --git a/src/buildtool/computed_roots/evaluate.hpp b/src/buildtool/computed_roots/evaluate.hpp index f4a7e6ca..873e2904 100644 --- a/src/buildtool/computed_roots/evaluate.hpp +++ b/src/buildtool/computed_roots/evaluate.hpp @@ -16,6 +16,7 @@ #define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_EVALUTE_HPP #include +#include #include #include "gsl/gsl" @@ -28,6 +29,7 @@ auto EvaluateComputedRoots( gsl::not_null const& repository_config, std::string const& main_repo, StorageConfig const& storage_config, + std::optional const& git_storage_config, GraphTraverser::CommandLineArguments const& traverser_args, gsl::not_null const& context, std::size_t jobs) -> bool; diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index b1e3dbad..34125fcd 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -1041,9 +1041,19 @@ auto main(int argc, char* argv[]) -> int { std::size_t eval_root_jobs = std::lround(std::ceil(std::sqrt(arguments.common.jobs))); #ifndef BOOTSTRAP_BUILD_TOOL + const bool need_rehash = + arguments.protocol.hash_type != HashFunction::Type::GitSHA1; + const auto git_storage_config = + need_rehash ? CreateStorageConfig(arguments.endpoint, + HashFunction::Type::GitSHA1) + : std::nullopt; + if (need_rehash and (not git_storage_config)) { + return kExitFailure; + } if (not EvaluateComputedRoots(&repo_config, main_repo, *storage_config, + git_storage_config, traverse_args, &exec_context, eval_root_jobs)) { -- cgit v1.2.3