From 6242ef6fff52116398913a40634a71292616c126 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 21 Oct 2024 12:51:08 +0200 Subject: just-mr and SourceTree: Use new Git execution api instance In just-mr: to instantiate the new Git api instance, both storage configs, as well as the compatible storage, need to be passed to the maps. While there, use more explicit naming schemes for the storage and CAS instances used. In serve: also acquire gc locks for the local storages when needed to instantiate the new Git api, which now has access to the CAS. In all these instances we also pass, as needed, the local api, which currently still operates only in native mode. This makes no difference currently, but will ensure less changes needed when the future compatible-aware local api will be used instead. --- src/other_tools/root_maps/fpath_git_map.cpp | 160 +++++++++++++++++++--------- 1 file changed, 110 insertions(+), 50 deletions(-) (limited to 'src/other_tools/root_maps/fpath_git_map.cpp') diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 023dedf8..ca8bdd52 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -31,13 +31,18 @@ namespace { /// \brief Does the serve endpoint checks and sets the workspace root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void CheckServeAndSetRoot(std::string const& tree_id, - std::string const& repo_root, - bool absent, - ServeApi const* serve, - IExecutionApi const* remote_api, - FilePathGitMap::SetterPtr const& ws_setter, - FilePathGitMap::LoggerPtr const& logger) { +void CheckServeAndSetRoot( + std::string const& tree_id, + std::string const& repo_root, + bool absent, + ServeApi const* serve, + gsl::not_null const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, + IExecutionApi const* remote_api, + FilePathGitMap::SetterPtr const& ws_setter, + FilePathGitMap::LoggerPtr const& logger) { // if serve endpoint is given, try to ensure it has this tree available to // be able to build against it. If root is not absent, do not fail if we // don't have a suitable remote endpoint, but warn user nonetheless. @@ -63,6 +68,10 @@ void CheckServeAndSetRoot(std::string const& tree_id, if (not EnsureAbsentRootOnServe(*serve, tree_id, repo_root, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, logger, /*no_sync_is_fatal=*/absent)) { @@ -99,7 +108,10 @@ void ResolveFilePathTree( gsl::not_null const& critical_git_op_map, gsl::not_null const& resolve_symlinks_map, ServeApi const* serve, - gsl::not_null const& storage_config, + gsl::not_null const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null const& ts, FilePathGitMap::SetterPtr const& ws_setter, @@ -107,7 +119,7 @@ void ResolveFilePathTree( if (pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - *storage_config, tree_hash, *pragma_special); + *native_storage_config, tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -122,9 +134,13 @@ void ResolveFilePathTree( // available to be able to build against it; the tree is resolved, // so it is in our Git cache CheckServeAndSetRoot(*resolved_tree_id, - storage_config->GitRoot().string(), + native_storage_config->GitRoot().string(), absent, serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger); @@ -145,7 +161,10 @@ void ResolveFilePathTree( tree_id_file, absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, ws_setter, @@ -155,7 +174,8 @@ void ResolveFilePathTree( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path + native_storage_config + ->GitRoot(), // target_path resolved_tree_id, // git_hash "Keep referenced tree alive" // message }, @@ -167,7 +187,10 @@ void ResolveFilePathTree( tree_id_file, absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger](auto const& values) { @@ -193,14 +216,19 @@ void ResolveFilePathTree( // it; the resolved tree is in the Git cache CheckServeAndSetRoot( resolved_tree_id, - storage_config->GitRoot().string(), + native_storage_config->GitRoot().string(), absent, serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -223,8 +251,17 @@ void ResolveFilePathTree( // tree needs no further processing; // if serve endpoint is given, try to ensure it has this tree available // to be able to build against it - CheckServeAndSetRoot( - tree_hash, repo_root, absent, serve, remote_api, ws_setter, logger); + CheckServeAndSetRoot(tree_hash, + repo_root, + absent, + serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + ws_setter, + logger); } } @@ -236,7 +273,10 @@ auto CreateFilePathGitMap( gsl::not_null const& import_to_git_map, gsl::not_null const& resolve_symlinks_map, ServeApi const* serve, - gsl::not_null const& storage_config, + gsl::not_null const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, std::size_t jobs, std::string const& multi_repo_tool_name, @@ -246,7 +286,10 @@ auto CreateFilePathGitMap( import_to_git_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, multi_repo_tool_name, build_tool_name](auto ts, @@ -285,7 +328,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -324,16 +370,16 @@ auto CreateFilePathGitMap( // resolve tree and set workspace root; tree gets resolved // from source repo into the Git cache, which we first need // to ensure is initialized - GitOpKey op_key = { - .params = - { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = {.params = + { + native_storage_config + ->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -346,7 +392,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -369,13 +418,17 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, logger); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -408,7 +461,7 @@ auto CreateFilePathGitMap( /*fatal=*/false); } // it's not a git repo, so import it to git cache - auto tmp_dir = storage_config->CreateTypedTmpDir("file"); + auto tmp_dir = native_storage_config->CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); @@ -437,7 +490,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -452,21 +508,25 @@ auto CreateFilePathGitMap( std::string tree = values[0]->first; // resolve tree and set workspace root; // we work on the Git CAS directly - ResolveFilePathTree(storage_config->GitRoot().string(), - fpath.string(), - tree, - pragma_special, - values[0]->second, /*source_cas*/ - values[0]->second, /*target_cas*/ - absent, - critical_git_op_map, - resolve_symlinks_map, - serve, - storage_config, - remote_api, - ts, - setter, - logger); + ResolveFilePathTree( + native_storage_config->GitRoot().string(), + fpath.string(), + tree, + pragma_special, + values[0]->second, /*source_cas*/ + values[0]->second, /*target_cas*/ + absent, + critical_git_op_map, + resolve_symlinks_map, + serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + ts, + setter, + logger); }, [logger, target_path = key.fpath](auto const& msg, bool fatal) { (*logger)( -- cgit v1.2.3