diff options
Diffstat (limited to 'src/other_tools/root_maps/content_git_map.cpp')
| -rw-r--r-- | src/other_tools/root_maps/content_git_map.cpp | 134 |
1 files changed, 76 insertions, 58 deletions
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index c73d39a5..ef345ff0 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -20,9 +20,7 @@ #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/root_maps/root_utils.hpp" @@ -52,13 +50,15 @@ namespace { /// root if it was marked absent. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void EnsureRootAsAbsent(std::string const& tree_id, - ArchiveRepoInfo const& key, - ServeApi const* serve, - IExecutionApi const* remote_api, - bool is_cache_hit, - ContentGitMap::SetterPtr const& ws_setter, - ContentGitMap::LoggerPtr const& logger) { +void EnsureRootAsAbsent( + std::string const& tree_id, + ArchiveRepoInfo const& key, + ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + IExecutionApi const* remote_api, + bool is_cache_hit, + ContentGitMap::SetterPtr const& ws_setter, + ContentGitMap::LoggerPtr const& logger) { // this is an absent root if (serve != nullptr) { // check if the serve endpoint has this root @@ -116,7 +116,7 @@ void EnsureRootAsAbsent(std::string const& tree_id, if (not EnsureAbsentRootOnServe( *serve, tree_id, - StorageConfig::Instance().GitRoot(), + storage_config->GitRoot(), remote_api, logger, /*no_sync_is_fatal=*/true)) { @@ -139,13 +139,12 @@ void EnsureRootAsAbsent(std::string const& tree_id, // the tree is known locally, so we can upload it to remote // CAS for the serve endpoint to retrieve it and set up the // root - if (not EnsureAbsentRootOnServe( - *serve, - tree_id, - StorageConfig::Instance().GitRoot(), - remote_api, - logger, - /*no_sync_is_fatal=*/true)) { + if (not EnsureAbsentRootOnServe(*serve, + tree_id, + storage_config->GitRoot(), + remote_api, + logger, + /*no_sync_is_fatal=*/true)) { return; } } @@ -174,6 +173,7 @@ void ResolveContentTree( bool is_cache_hit, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -183,7 +183,7 @@ void ResolveContentTree( if (key.pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - StorageConfig::Instance(), tree_hash, *key.pragma_special); + *storage_config, tree_hash, *key.pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -199,6 +199,7 @@ void ResolveContentTree( EnsureRootAsAbsent(*resolved_tree_id, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, @@ -206,10 +207,9 @@ void ResolveContentTree( } else { (*ws_setter)(std::pair( - nlohmann::json::array( - {FileRoot::kGitTreeMarker, - *resolved_tree_id, - StorageConfig::Instance().GitRoot().string()}), + nlohmann::json::array({FileRoot::kGitTreeMarker, + *resolved_tree_id, + storage_config->GitRoot().string()}), /*is_cache_hit=*/is_cache_hit)); } } @@ -231,6 +231,7 @@ void ResolveContentTree( key, is_absent, serve, + storage_config, remote_api, ts, ws_setter, @@ -259,8 +260,7 @@ void ResolveContentTree( GitOpKey op_key = { .params = { - StorageConfig::Instance() - .GitRoot(), // target_path + storage_config->GitRoot(), // target_path resolved_tree_id, // git_hash "", // branch "Keep referenced tree alive" // message @@ -274,6 +274,7 @@ void ResolveContentTree( tree_id_file, is_absent, serve, + storage_config, remote_api, is_cache_hit, ws_setter, @@ -300,24 +301,22 @@ void ResolveContentTree( EnsureRootAsAbsent(resolved_tree_id, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - resolved_tree_id, - StorageConfig::Instance() - .GitRoot() - .string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)(std::pair( + nlohmann::json::array( + {FileRoot::kGitTreeMarker, + resolved_tree_id, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } }, - [logger, - target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -343,18 +342,18 @@ void ResolveContentTree( EnsureRootAsAbsent(tree_hash, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - tree_hash, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)(std::pair( + nlohmann::json::array({FileRoot::kGitTreeMarker, + tree_hash, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } } } @@ -369,6 +368,7 @@ void WriteIdFileAndSetWSRoot( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -413,6 +413,7 @@ void WriteIdFileAndSetWSRoot( false, /*is_cache_hit*/ is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -430,6 +431,7 @@ void ExtractAndImportToGit( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -438,7 +440,7 @@ void ExtractAndImportToGit( ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // extract archive - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir(key.repo_type); + auto tmp_dir = storage_config->CreateTypedTmpDir(key.repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for {} target {}", key.repo_type, @@ -466,6 +468,7 @@ void ExtractAndImportToGit( key, is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -487,6 +490,7 @@ void ExtractAndImportToGit( archive_tree_id_file, is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -514,6 +518,8 @@ auto CreateContentGitMap( gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, IExecutionApi const* remote_api, bool fetch_absent, std::size_t jobs) -> ContentGitMap { @@ -525,6 +531,8 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, + storage, + storage_config, remote_api, fetch_absent](auto ts, auto setter, @@ -532,7 +540,7 @@ auto CreateContentGitMap( auto /* unused */, auto const& key) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( - StorageConfig::Instance(), key.repo_type, key.archive.content); + *storage_config, key.repo_type, key.archive.content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file auto archive_tree_id = @@ -548,11 +556,11 @@ auto CreateContentGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -562,6 +570,7 @@ auto CreateContentGitMap( key, fetch_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -606,6 +615,7 @@ auto CreateContentGitMap( /*is_cache_hit = */ true, /*is_absent = */ (key.absent and not fetch_absent), serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -613,7 +623,7 @@ auto CreateContentGitMap( setter, logger); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT for " @@ -661,7 +671,7 @@ auto CreateContentGitMap( // a serve endpoint exists we can upload it the root ourselves; // check if content already in CAS - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); auto digest = ArtifactDigest(key.archive.content, 0, false); if (auto content_cas_path = cas.BlobPath(digest, /*is_executable=*/false)) { @@ -670,6 +680,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent = */ true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -686,11 +697,11 @@ auto CreateContentGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -706,6 +717,8 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, + storage, + storage_config, remote_api, ts, setter, @@ -743,7 +756,7 @@ auto CreateContentGitMap( // blob check failed return; } - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); if (res.second) { // blob found; add it to CAS if (not cas.StoreBlob(*res.second, @@ -761,6 +774,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -789,7 +803,7 @@ auto CreateContentGitMap( .filename() .string()); StorageUtils::AddDistfileToCAS( - Storage::Instance(), repo_distfile, just_mr_paths); + *storage, repo_distfile, just_mr_paths); // check if content is in CAS now if (auto content_cas_path = cas.BlobPath(digest, /*is_executable=*/false)) { @@ -800,6 +814,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -816,7 +831,7 @@ auto CreateContentGitMap( key.archive.content), /*fatal=*/true); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -832,6 +847,8 @@ auto CreateContentGitMap( {key.archive}, [archive_tree_id_file, key, + storage, + storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, @@ -839,7 +856,7 @@ auto CreateContentGitMap( setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); auto content_cas_path = cas.BlobPath(ArtifactDigest( key.archive.content, 0, false), @@ -852,6 +869,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/false, /*serve=*/nullptr, + storage_config, /*remote_api=*/nullptr, critical_git_op_map, import_to_git_map, |