From 6242ef6fff52116398913a40634a71292616c126 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 21 Oct 2024 12:51:08 +0200 Subject: just-mr and SourceTree: Use new Git execution api instance In just-mr: to instantiate the new Git api instance, both storage configs, as well as the compatible storage, need to be passed to the maps. While there, use more explicit naming schemes for the storage and CAS instances used. In serve: also acquire gc locks for the local storages when needed to instantiate the new Git api, which now has access to the CAS. In all these instances we also pass, as needed, the local api, which currently still operates only in native mode. This makes no difference currently, but will ensure less changes needed when the future compatible-aware local api will be used instead. --- src/other_tools/root_maps/distdir_git_map.cpp | 82 +++++++++++++++++---------- 1 file changed, 51 insertions(+), 31 deletions(-) (limited to 'src/other_tools/root_maps/distdir_git_map.cpp') diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index a19889b2..82c0c623 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -74,15 +74,15 @@ namespace { /// the setter on success. void ImportFromCASAndSetRoot( DistdirInfo const& key, - StorageConfig const& storage_config, - Storage const& storage, + StorageConfig const& native_storage_config, + Storage const& native_storage, std::filesystem::path const& distdir_tree_id_file, gsl::not_null const& import_to_git_map, gsl::not_null const& ts, DistdirGitMap::SetterPtr const& setter, DistdirGitMap::LoggerPtr const& logger) { // create the links to CAS - auto tmp_dir = storage_config.CreateTypedTmpDir("distdir"); + auto tmp_dir = native_storage_config.CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", @@ -91,7 +91,7 @@ void ImportFromCASAndSetRoot( return; } // link content from CAS into tmp dir - if (not LinkToCAS(storage, key.content_list, tmp_dir->GetPath())) { + if (not LinkToCAS(native_storage, key.content_list, tmp_dir->GetPath())) { (*logger)(fmt::format("Failed to create links to CAS content!", key.content_id), /*fatal=*/true); @@ -104,7 +104,7 @@ void ImportFromCASAndSetRoot( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, - git_root = storage_config.GitRoot().string(), + git_root = native_storage_config.GitRoot().string(), setter, logger](auto const& values) { // check for errors @@ -145,8 +145,10 @@ auto CreateDistdirGitMap( gsl::not_null const& import_to_git_map, gsl::not_null const& critical_git_op_map, ServeApi const* serve, - gsl::not_null const& storage_config, - gsl::not_null const& storage, + gsl::not_null const& native_storage_config, + StorageConfig const* compat_storage_config, + gsl::not_null const& native_storage, + Storage const* compat_storage, gsl::not_null const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap { @@ -154,16 +156,18 @@ auto CreateDistdirGitMap( import_to_git_map, critical_git_op_map, serve, - storage, - storage_config, + native_storage_config, + compat_storage_config, + native_storage, + compat_storage, local_api, remote_api](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { - auto distdir_tree_id_file = - StorageUtils::GetDistdirTreeIDFile(*storage_config, key.content_id); + auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile( + *native_storage_config, key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = @@ -179,11 +183,11 @@ auto CreateDistdirGitMap( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -193,7 +197,10 @@ auto CreateDistdirGitMap( content_id = key.content_id, key, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, setter, logger](auto const& values) { @@ -268,7 +275,12 @@ auto CreateDistdirGitMap( if (not EnsureAbsentRootOnServe( *serve, distdir_tree_id, - storage_config->GitRoot(), + native_storage_config + ->GitRoot(), /*repo_root*/ + native_storage_config, + compat_storage_config, + compat_storage, + &*local_api, remote_api, logger, true /*no_sync_is_fatal*/)) { @@ -293,15 +305,15 @@ auto CreateDistdirGitMap( } else { // set root as present - (*setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - distdir_tree_id, - storage_config->GitRoot().string()}), - /*is_cache_hit=*/true)); + (*setter)(std::pair( + nlohmann::json::array( + {FileRoot::kGitTreeMarker, + distdir_tree_id, + native_storage_config->GitRoot().string()}), + /*is_cache_hit=*/true)); } }, - [logger, target_path = storage_config->GitRoot()]( + [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -415,6 +427,10 @@ auto CreateDistdirGitMap( *serve, tree_id, /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, logger, /*no_sync_is_fatal=*/true)) { @@ -452,6 +468,10 @@ auto CreateDistdirGitMap( *serve, tree_id, /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, logger, /*no_sync_is_fatal=*/true)) { @@ -487,8 +507,8 @@ auto CreateDistdirGitMap( // first, look in the local CAS if (digest and local_api->IsAvailable({*digest})) { ImportFromCASAndSetRoot(key, - *storage_config, - *storage, + *native_storage_config, + *native_storage, distdir_tree_id_file, import_to_git_map, ts, @@ -541,15 +561,15 @@ auto CreateDistdirGitMap( [distdir_tree_id_file, key, import_to_git_map, + native_storage_config, + native_storage, ts, - storage, - storage_config, setter, logger]([[maybe_unused]] auto const& values) { // archive blobs are in CAS ImportFromCASAndSetRoot(key, - *storage_config, - *storage, + *native_storage_config, + *native_storage, distdir_tree_id_file, import_to_git_map, ts, -- cgit v1.2.3