just-mr: Instantiate both native and compatible storages...

...to be later passed to maps that might need them.

This is a preparatory commit. Currently only the native storage is
actively being used.

3 files changed, 159 insertions, 33 deletions

diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index ddc92bf3..a1180e56 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -121,10 +121,14 @@
     , ["@", "json", "", "json"]
     , ["src/buildtool/auth", "auth"]
     , ["src/buildtool/common/remote", "retry_config"]
+    , ["src/buildtool/crypto", "hash_function"]
+    , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
     , ["src/buildtool/execution_api/common", "api_bundle"]
     , ["src/buildtool/execution_api/common", "common"]
     , ["src/buildtool/execution_api/local", "config"]
     , ["src/buildtool/execution_api/local", "context"]
+    , ["src/buildtool/execution_api/local", "local"]
+    , ["src/buildtool/execution_api/remote", "bazel"]
     , ["src/buildtool/execution_api/remote", "config"]
     , ["src/buildtool/execution_api/remote", "context"]
     , ["src/buildtool/logging", "logging"]
@@ -143,6 +147,7 @@
     , ["src/other_tools/ops_maps", "import_to_git_map"]
     , ["src/other_tools/utils", "parse_archive"]
     , ["src/other_tools/utils", "parse_git_tree"]
+    , ["src/utils/cpp", "file_locking"]
     ]
   }
 , "update":
@@ -191,10 +196,14 @@
     , ["@", "json", "", "json"]
     , ["src/buildtool/auth", "auth"]
     , ["src/buildtool/common/remote", "retry_config"]
+    , ["src/buildtool/crypto", "hash_function"]
+    , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
     , ["src/buildtool/execution_api/common", "api_bundle"]
     , ["src/buildtool/execution_api/common", "common"]
     , ["src/buildtool/execution_api/local", "config"]
     , ["src/buildtool/execution_api/local", "context"]
+    , ["src/buildtool/execution_api/local", "local"]
+    , ["src/buildtool/execution_api/remote", "bazel"]
     , ["src/buildtool/execution_api/remote", "config"]
     , ["src/buildtool/execution_api/remote", "context"]
     , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"]
@@ -217,6 +226,7 @@
     , ["src/other_tools/root_maps", "distdir_git_map"]
     , ["src/other_tools/root_maps", "fpath_git_map"]
     , ["src/other_tools/root_maps", "tree_id_git_map"]
+    , ["src/utils/cpp", "file_locking"]
     ]
   }
 , "launch":
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 5ad1a1f1..04c6c013 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -22,10 +22,14 @@
 #include "nlohmann/json.hpp"
 #include "src/buildtool/auth/authentication.hpp"
 #include "src/buildtool/common/remote/retry_config.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
 #include "src/buildtool/execution_api/common/api_bundle.hpp"
 #include "src/buildtool/execution_api/common/execution_api.hpp"
 #include "src/buildtool/execution_api/local/config.hpp"
 #include "src/buildtool/execution_api/local/context.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
 #include "src/buildtool/execution_api/remote/config.hpp"
 #include "src/buildtool/execution_api/remote/context.hpp"
 #include "src/buildtool/logging/log_level.hpp"
@@ -35,6 +39,7 @@
 #include "src/buildtool/multithreading/task_system.hpp"
 #include "src/buildtool/serve_api/remote/config.hpp"
 #include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/buildtool/storage/garbage_collector.hpp"
 #include "src/other_tools/just_mr/exit_codes.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp"
@@ -47,6 +52,7 @@
 #include "src/other_tools/ops_maps/import_to_git_map.hpp"
 #include "src/other_tools/utils/parse_archive.hpp"
 #include "src/other_tools/utils/parse_git_tree.hpp"
+#include "src/utils/cpp/file_locking.hpp"
 
 auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
                     MultiRepoCommonArguments const& common_args,
@@ -322,53 +328,105 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
         return kExitConfigError;
     }
 
-    // set up the native local context
+    // pack the native local context and create api
     LocalContext const native_local_context{
         .exec_config = &*local_exec_config,
         .storage_config = &native_storage_config,
         .storage = &native_storage};
+    IExecutionApi::Ptr const native_local_api =
+        std::make_shared<LocalApi>(&native_local_context);
+
+    // pack the compatible local context, if needed
+    std::unique_ptr<StorageConfig> compat_storage_config = nullptr;
+    std::unique_ptr<Storage> compat_storage = nullptr;
+    std::unique_ptr<LocalContext> compat_local_context = nullptr;
+    std::optional<LockFile> compat_lock = std::nullopt;
+    if (common_args.compatible) {
+        auto config = StorageConfig::Builder{}
+                          .SetBuildRoot(native_storage_config.build_root)
+                          .SetHashType(HashFunction::Type::PlainSHA256)
+                          .Build();
+        if (not config) {
+            Logger::Log(LogLevel::Error, config.error());
+            return kExitConfigError;
+        }
+        compat_storage_config =
+            std::make_unique<StorageConfig>(*std::move(config));
+        compat_storage = std::make_unique<Storage>(
+            Storage::Create(compat_storage_config.get()));
+        compat_local_context = std::make_unique<LocalContext>(
+            LocalContext{.exec_config = &*local_exec_config,
+                         .storage_config = compat_storage_config.get(),
+                         .storage = compat_storage.get()});
+        // if a compatible storage is created, one must get a lock for it the
+        // same way as done for the native one
+        compat_lock = GarbageCollector::SharedLock(*compat_storage_config);
+        if (not compat_lock) {
+            Logger::Log(LogLevel::Error,
+                        "Failed to acquire compatible storage gc lock");
+            return kExitConfigError;
+        }
+    }
 
     // setup authentication config
-    auto auth_config = JustMR::Utils::CreateAuthConfig(auth_args);
+    auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args);
     if (not auth_config) {
         return kExitConfigError;
     }
 
     // setup the retry config
-    auto retry_config = CreateRetryConfig(retry_args);
+    auto const retry_config = CreateRetryConfig(retry_args);
     if (not retry_config) {
         return kExitConfigError;
     }
 
     // setup remote execution config
-    auto remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig(
+    auto const remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig(
         common_args.remote_execution_address, common_args.remote_serve_address);
     if (not remote_exec_config) {
         return kExitConfigError;
     }
 
-    // pack the remote context instances to be passed to ApiBundle
+    // create the remote api
+    auto const hash_fct =
+        compat_local_context != nullptr
+            ? compat_local_context->storage_config->hash_function
+            : native_local_context.storage_config->hash_function;
+    IExecutionApi::Ptr remote_api = nullptr;
+    if (auto const address = remote_exec_config->remote_address) {
+        ExecutionConfiguration config;
+        config.skip_cache_lookup = false;
+        remote_api = std::make_shared<BazelApi>("remote-execution",
+                                                address->host,
+                                                address->port,
+                                                &*auth_config,
+                                                &*retry_config,
+                                                config,
+                                                &hash_fct);
+    }
+    bool const has_remote_api =
+        remote_api != nullptr and not common_args.compatible;
+
+    // pack the remote context
     RemoteContext const remote_context{.auth = &*auth_config,
                                        .retry_config = &*retry_config,
                                        .exec_config = &*remote_exec_config};
 
-    // setup the APIs for archive fetches; only happens if in native mode
-    auto const apis = ApiBundle::Create(&native_local_context,
-                                        &remote_context,
-                                        /*repo_config=*/nullptr);
-
-    bool const has_remote_api =
-        apis.local != apis.remote and not common_args.compatible;
-
-    // setup the API for serving roots
+    // setup the api for serving roots
     auto serve_config =
         JustMR::Utils::CreateServeConfig(common_args.remote_serve_address);
     if (not serve_config) {
         return kExitConfigError;
     }
+    auto const apis =
+        ApiBundle{.hash_function = hash_fct,
+                  .local = native_local_api,
+                  .remote = has_remote_api ? remote_api : native_local_api};
+    auto serve = ServeApi::Create(*serve_config,
+                                  &native_local_context, /*unused*/
+                                  &remote_context,
+                                  &apis /*unused*/);
 
-    auto serve = ServeApi::Create(
-        *serve_config, &native_local_context, &remote_context, &apis);
     // check configuration of the serve endpoint provided
     if (serve) {
         // if we have a remote endpoint explicitly given by the user, it must
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 0e11127e..012629c2 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -24,10 +24,14 @@
 #include "nlohmann/json.hpp"
 #include "src/buildtool/auth/authentication.hpp"
 #include "src/buildtool/common/remote/retry_config.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
 #include "src/buildtool/execution_api/common/api_bundle.hpp"
 #include "src/buildtool/execution_api/common/execution_api.hpp"
 #include "src/buildtool/execution_api/local/config.hpp"
 #include "src/buildtool/execution_api/local/context.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
 #include "src/buildtool/execution_api/remote/config.hpp"
 #include "src/buildtool/execution_api/remote/context.hpp"
 #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp"
@@ -39,6 +43,7 @@
 #include "src/buildtool/serve_api/remote/config.hpp"
 #include "src/buildtool/serve_api/remote/serve_api.hpp"
 #include "src/buildtool/storage/fs_utils.hpp"
+#include "src/buildtool/storage/garbage_collector.hpp"
 #include "src/other_tools/just_mr/exit_codes.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp"
@@ -54,6 +59,7 @@
 #include "src/other_tools/root_maps/distdir_git_map.hpp"
 #include "src/other_tools/root_maps/fpath_git_map.hpp"
 #include "src/other_tools/root_maps/tree_id_git_map.hpp"
+#include "src/utils/cpp/file_locking.hpp"
 
 auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
                     MultiRepoCommonArguments const& common_args,
@@ -128,58 +134,110 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
                 setup_repos->to_setup.size());
 
     // setup local execution config
-    auto local_exec_config =
+    auto const local_exec_config =
         JustMR::Utils::CreateLocalExecutionConfig(common_args);
     if (not local_exec_config) {
         return std::nullopt;
     }
 
-    // pack the local context instances to be passed to ApiBundle
+    // pack the native local context and create api
     LocalContext const native_local_context{
         .exec_config = &*local_exec_config,
         .storage_config = &native_storage_config,
         .storage = &native_storage};
+    IExecutionApi::Ptr const native_local_api =
+        std::make_shared<LocalApi>(&native_local_context);
+
+    // pack the compatible local context, if needed
+    std::unique_ptr<StorageConfig> compat_storage_config = nullptr;
+    std::unique_ptr<Storage> compat_storage = nullptr;
+    std::unique_ptr<LocalContext> compat_local_context = nullptr;
+    std::optional<LockFile> compat_lock = std::nullopt;
+    if (common_args.compatible) {
+        auto config = StorageConfig::Builder{}
+                          .SetBuildRoot(native_storage_config.build_root)
+                          .SetHashType(HashFunction::Type::PlainSHA256)
+                          .Build();
+        if (not config) {
+            Logger::Log(LogLevel::Error, config.error());
+            return std::nullopt;
+        }
+        compat_storage_config =
+            std::make_unique<StorageConfig>(*std::move(config));
+        compat_storage = std::make_unique<Storage>(
+            Storage::Create(compat_storage_config.get()));
+        compat_local_context = std::make_unique<LocalContext>(
+            LocalContext{.exec_config = &*local_exec_config,
+                         .storage_config = compat_storage_config.get(),
+                         .storage = compat_storage.get()});
+        // if a compatible storage is created, one must get a lock for it the
+        // same way as done for the native one
+        compat_lock = GarbageCollector::SharedLock(*compat_storage_config);
+        if (not compat_lock) {
+            Logger::Log(LogLevel::Error,
+                        "Failed to acquire compatible storage gc lock");
+            return std::nullopt;
+        }
+    }
 
     // setup authentication config
-    auto auth_config = JustMR::Utils::CreateAuthConfig(auth_args);
+    auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args);
     if (not auth_config) {
         return std::nullopt;
     }
 
     // setup the retry config
-    auto retry_config = CreateRetryConfig(retry_args);
+    auto const retry_config = CreateRetryConfig(retry_args);
     if (not retry_config) {
         return std::nullopt;
     }
 
     // setup remote execution config
-    auto remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig(
+    auto const remote_exec_config = JustMR::Utils::CreateRemoteExecutionConfig(
         common_args.remote_execution_address, common_args.remote_serve_address);
     if (not remote_exec_config) {
         return std::nullopt;
     }
 
-    // pack the remote context instances to be passed to ApiBundle
+    // create the remote api
+    auto const hash_fct =
+        compat_local_context != nullptr
+            ? compat_local_context->storage_config->hash_function
+            : native_local_context.storage_config->hash_function;
+    IExecutionApi::Ptr remote_api = nullptr;
+    if (auto const address = remote_exec_config->remote_address) {
+        ExecutionConfiguration config;
+        config.skip_cache_lookup = false;
+        remote_api = std::make_shared<BazelApi>("remote-execution",
+                                                address->host,
+                                                address->port,
+                                                &*auth_config,
+                                                &*retry_config,
+                                                config,
+                                                &hash_fct);
+    }
+    bool const has_remote_api =
+        remote_api != nullptr and not common_args.compatible;
+
+    // pack the remote context
     RemoteContext const remote_context{.auth = &*auth_config,
                                        .retry_config = &*retry_config,
                                        .exec_config = &*remote_exec_config};
 
-    auto const apis = ApiBundle::Create(&native_local_context,
-                                        &remote_context,
-                                        /*repo_config=*/nullptr);
-
-    bool const has_remote_api =
-        apis.local != apis.remote and not common_args.compatible;
-
-    // setup the API for serving roots
+    // setup the api for serving roots
     auto serve_config =
         JustMR::Utils::CreateServeConfig(common_args.remote_serve_address);
     if (not serve_config) {
         return std::nullopt;
     }
-
-    auto serve = ServeApi::Create(
-        *serve_config, &native_local_context, &remote_context, &apis);
+    auto const apis =
+        ApiBundle{.hash_function = hash_fct,
+                  .local = native_local_api,
+                  .remote = has_remote_api ? remote_api : native_local_api};
+    auto serve = ServeApi::Create(*serve_config,
+                                  &native_local_context, /*unused*/
+                                  &remote_context,
+                                  &apis /*unused*/);
 
     // check configuration of the serve endpoint provided
     if (serve) {