just-mr and SourceTree: Use new Git execution api instance

In just-mr: to instantiate the new Git api instance, both storage
configs, as well as the compatible storage, need to be passed to
the maps. While there, use more explicit naming schemes for the
storage and CAS instances used.

In serve: also acquire gc locks for the local storages when needed
to instantiate the new Git api, which now has access to the CAS.

In all these instances we also pass, as needed, the local api, which
currently still operates only in native mode. This makes no
difference currently, but will ensure less changes needed when the
future compatible-aware local api will be used instead.

1 files changed, 51 insertions, 31 deletions

diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index a19889b2..82c0c623 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -74,15 +74,15 @@ namespace {
 /// the setter on success.
 void ImportFromCASAndSetRoot(
     DistdirInfo const& key,
-    StorageConfig const& storage_config,
-    Storage const& storage,
+    StorageConfig const& native_storage_config,
+    Storage const& native_storage,
     std::filesystem::path const& distdir_tree_id_file,
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     gsl::not_null<TaskSystem*> const& ts,
     DistdirGitMap::SetterPtr const& setter,
     DistdirGitMap::LoggerPtr const& logger) {
     // create the links to CAS
-    auto tmp_dir = storage_config.CreateTypedTmpDir("distdir");
+    auto tmp_dir = native_storage_config.CreateTypedTmpDir("distdir");
     if (not tmp_dir) {
         (*logger)(fmt::format("Failed to create tmp path for "
                               "distdir target {}",
@@ -91,7 +91,7 @@ void ImportFromCASAndSetRoot(
         return;
     }
     // link content from CAS into tmp dir
-    if (not LinkToCAS(storage, key.content_list, tmp_dir->GetPath())) {
+    if (not LinkToCAS(native_storage, key.content_list, tmp_dir->GetPath())) {
         (*logger)(fmt::format("Failed to create links to CAS content!",
                               key.content_id),
                   /*fatal=*/true);
@@ -104,7 +104,7 @@ void ImportFromCASAndSetRoot(
         {std::move(c_info)},
         [tmp_dir,  // keep tmp_dir alive
          distdir_tree_id_file,
-         git_root = storage_config.GitRoot().string(),
+         git_root = native_storage_config.GitRoot().string(),
          setter,
          logger](auto const& values) {
             // check for errors
@@ -145,8 +145,10 @@ auto CreateDistdirGitMap(
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     ServeApi const* serve,
-    gsl::not_null<StorageConfig const*> const& storage_config,
-    gsl::not_null<Storage const*> const& storage,
+    gsl::not_null<StorageConfig const*> const& native_storage_config,
+    StorageConfig const* compat_storage_config,
+    gsl::not_null<Storage const*> const& native_storage,
+    Storage const* compat_storage,
     gsl::not_null<IExecutionApi const*> const& local_api,
     IExecutionApi const* remote_api,
     std::size_t jobs) -> DistdirGitMap {
@@ -154,16 +156,18 @@ auto CreateDistdirGitMap(
                            import_to_git_map,
                            critical_git_op_map,
                            serve,
-                           storage,
-                           storage_config,
+                           native_storage_config,
+                           compat_storage_config,
+                           native_storage,
+                           compat_storage,
                            local_api,
                            remote_api](auto ts,
                                        auto setter,
                                        auto logger,
                                        auto /* unused */,
                                        auto const& key) {
-        auto distdir_tree_id_file =
-            StorageUtils::GetDistdirTreeIDFile(*storage_config, key.content_id);
+        auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile(
+            *native_storage_config, key.content_id);
         if (FileSystemManager::Exists(distdir_tree_id_file)) {
             // read distdir_tree_id from file tree_id_file
             auto distdir_tree_id =
@@ -179,11 +183,11 @@ auto CreateDistdirGitMap(
             GitOpKey op_key = {
                 .params =
                     {
-                        storage_config->GitRoot(),  // target_path
-                        "",                         // git_hash
-                        std::nullopt,               // message
-                        std::nullopt,               // source_path
-                        true                        // init_bare
+                        native_storage_config->GitRoot(),  // target_path
+                        "",                                // git_hash
+                        std::nullopt,                      // message
+                        std::nullopt,                      // source_path
+                        true                               // init_bare
                     },
                 .op_type = GitOpType::ENSURE_INIT};
             critical_git_op_map->ConsumeAfterKeysReady(
@@ -193,7 +197,10 @@ auto CreateDistdirGitMap(
                  content_id = key.content_id,
                  key,
                  serve,
-                 storage_config,
+                 native_storage_config,
+                 compat_storage_config,
+                 compat_storage,
+                 local_api,
                  remote_api,
                  setter,
                  logger](auto const& values) {
@@ -268,7 +275,12 @@ auto CreateDistdirGitMap(
                                     if (not EnsureAbsentRootOnServe(
                                             *serve,
                                             distdir_tree_id,
-                                            storage_config->GitRoot(),
+                                            native_storage_config
+                                                ->GitRoot(), /*repo_root*/
+                                            native_storage_config,
+                                            compat_storage_config,
+                                            compat_storage,
+                                            &*local_api,
                                             remote_api,
                                             logger,
                                             true /*no_sync_is_fatal*/)) {
@@ -293,15 +305,15 @@ auto CreateDistdirGitMap(
                     }
                     else {
                         // set root as present
-                        (*setter)(
-                            std::pair(nlohmann::json::array(
-                                          {FileRoot::kGitTreeMarker,
-                                           distdir_tree_id,
-                                           storage_config->GitRoot().string()}),
-                                      /*is_cache_hit=*/true));
+                        (*setter)(std::pair(
+                            nlohmann::json::array(
+                                {FileRoot::kGitTreeMarker,
+                                 distdir_tree_id,
+                                 native_storage_config->GitRoot().string()}),
+                            /*is_cache_hit=*/true));
                     }
                 },
-                [logger, target_path = storage_config->GitRoot()](
+                [logger, target_path = native_storage_config->GitRoot()](
                     auto const& msg, bool fatal) {
                     (*logger)(fmt::format("While running critical Git op "
                                           "ENSURE_INIT for target {}:\n{}",
@@ -415,6 +427,10 @@ auto CreateDistdirGitMap(
                                 *serve,
                                 tree_id,
                                 /*repo_path=*/"",
+                                native_storage_config,
+                                /*compat_storage_config=*/nullptr,
+                                /*compat_storage=*/nullptr,
+                                /*local_api=*/nullptr,
                                 /*remote_api=*/nullptr,
                                 logger,
                                 /*no_sync_is_fatal=*/true)) {
@@ -452,6 +468,10 @@ auto CreateDistdirGitMap(
                                 *serve,
                                 tree_id,
                                 /*repo_path=*/"",
+                                native_storage_config,
+                                /*compat_storage_config=*/nullptr,
+                                /*compat_storage=*/nullptr,
+                                /*local_api=*/nullptr,
                                 /*remote_api=*/nullptr,
                                 logger,
                                 /*no_sync_is_fatal=*/true)) {
@@ -487,8 +507,8 @@ auto CreateDistdirGitMap(
             // first, look in the local CAS
             if (digest and local_api->IsAvailable({*digest})) {
                 ImportFromCASAndSetRoot(key,
-                                        *storage_config,
-                                        *storage,
+                                        *native_storage_config,
+                                        *native_storage,
                                         distdir_tree_id_file,
                                         import_to_git_map,
                                         ts,
@@ -541,15 +561,15 @@ auto CreateDistdirGitMap(
                 [distdir_tree_id_file,
                  key,
                  import_to_git_map,
+                 native_storage_config,
+                 native_storage,
                  ts,
-                 storage,
-                 storage_config,
                  setter,
                  logger]([[maybe_unused]] auto const& values) {
                     // archive blobs are in CAS
                     ImportFromCASAndSetRoot(key,
-                                            *storage_config,
-                                            *storage,
+                                            *native_storage_config,
+                                            *native_storage,
                                             distdir_tree_id_file,
                                             import_to_git_map,
                                             ts,