Add RehashGitDigest function

5 files changed, 43 insertions, 135 deletions

diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS
index 14843487..a8237ba7 100644
--- a/src/buildtool/execution_api/serve/TARGETS
+++ b/src/buildtool/execution_api/serve/TARGETS
@@ -15,10 +15,8 @@
     ]
   , "stage": ["src", "buildtool", "execution_api", "serve"]
   , "private-deps":
-    [ ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
-    , ["src/buildtool/execution_api/git", "git"]
+    [ ["src/buildtool/execution_api/git", "git"]
     , ["src/buildtool/execution_api/utils", "rehash_utils"]
-    , ["src/buildtool/file_system", "object_type"]
     , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]
     , ["src/utils/cpp", "expected"]
diff --git a/src/buildtool/execution_api/serve/mr_git_api.cpp b/src/buildtool/execution_api/serve/mr_git_api.cpp
index 6630e891..cd2d7a69 100644
--- a/src/buildtool/execution_api/serve/mr_git_api.cpp
+++ b/src/buildtool/execution_api/serve/mr_git_api.cpp
@@ -15,12 +15,9 @@
 #include "src/buildtool/execution_api/serve/mr_git_api.hpp"
 
 #include <utility>
-#include <variant>
 
-#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
 #include "src/buildtool/execution_api/git/git_api.hpp"
 #include "src/buildtool/execution_api/utils/rehash_utils.hpp"
-#include "src/buildtool/file_system/object_type.hpp"
 #include "src/buildtool/logging/log_level.hpp"
 #include "src/buildtool/logging/logger.hpp"
 #include "src/utils/cpp/expected.hpp"
@@ -51,129 +48,16 @@ auto MRGitApi::RetrieveToCas(
         return git_api.RetrieveToCas(artifacts_info, api);
     }
 
-    // in compatible mode: set up needed callbacks for caching digest mappings
-    auto read_rehashed =
-        [native_sc = native_storage_config_,
-         compat_sc = compat_storage_config_](ArtifactDigest const& digest)
-        -> expected<std::optional<Artifact::ObjectInfo>, std::string> {
-        return RehashUtils::ReadRehashedDigest(
-            digest, *native_sc, *compat_sc, /*from_git=*/true);
-    };
-    auto store_rehashed =
-        [native_sc = native_storage_config_,
-         compat_sc = compat_storage_config_](
-            ArtifactDigest const& source_digest,
-            ArtifactDigest const& target_digest,
-            ObjectType obj_type) -> std::optional<std::string> {
-        return RehashUtils::StoreRehashedDigest(source_digest,
-                                                target_digest,
-                                                obj_type,
-                                                *native_sc,
-                                                *compat_sc,
-                                                /*from_git=*/true);
-    };
-
-    // collect the native blobs and rehash them as compatible to be able to
-    // check what is missing in the other api
-    std::vector<Artifact::ObjectInfo> compat_artifacts;
-    compat_artifacts.reserve(artifacts_info.size());
-    for (auto const& native_obj : artifacts_info) {
-        // check if we know already the compatible digest
-        auto cached_obj = read_rehashed(native_obj.digest);
-        if (not cached_obj) {
-            Logger::Log(
-                LogLevel::Error, "MRGitApi: {}", std::move(cached_obj).error());
-            return false;
-        }
-        if (*cached_obj) {
-            // add object to the vector of compatible artifacts
-            compat_artifacts.emplace_back(std::move(cached_obj)->value());
-        }
-        else {
-            // process object; trees need to be handled appropriately
-            if (IsTreeObject(native_obj.type)) {
-                // set up all the callbacks needed
-                auto read_git = [repo_config = repo_config_](
-                                    ArtifactDigest const& digest,
-                                    ObjectType /*type*/)
-                    -> std::optional<
-                        std::variant<std::filesystem::path, std::string>> {
-                    return repo_config->ReadBlobFromGitCAS(digest.hash());
-                };
-                auto store_file =
-                    [cas = &compat_storage_->CAS()](
-                        std::variant<std::filesystem::path, std::string> const&
-                            data,
-                        bool is_exec) -> std::optional<ArtifactDigest> {
-                    if (not std::holds_alternative<std::string>(data)) {
-                        return std::nullopt;
-                    }
-                    return cas->StoreBlob(std::get<std::string>(data), is_exec);
-                };
-                BazelMsgFactory::TreeStoreFunc store_dir =
-                    [cas = &compat_storage_->CAS()](std::string const& content)
-                    -> std::optional<ArtifactDigest> {
-                    return cas->StoreTree(content);
-                };
-                BazelMsgFactory::SymlinkStoreFunc store_symlink =
-                    [cas = &compat_storage_->CAS()](std::string const& content)
-                    -> std::optional<ArtifactDigest> {
-                    return cas->StoreBlob(content);
-                };
-                // get the directory digest
-                auto tree_digest =
-                    BazelMsgFactory::CreateDirectoryDigestFromGitTree(
-                        native_obj.digest,
-                        read_git,
-                        store_file,
-                        store_dir,
-                        store_symlink,
-                        read_rehashed,
-                        store_rehashed);
-                if (not tree_digest) {
-                    Logger::Log(LogLevel::Error,
-                                "MRGitApi: {}",
-                                std::move(tree_digest).error());
-                    return false;
-                }
-                // add object to the vector of compatible artifacts
-                compat_artifacts.emplace_back(
-                    Artifact::ObjectInfo{.digest = *std::move(tree_digest),
-                                         .type = ObjectType::Tree});
-            }
-            else {
-                // blobs are read from repo and added to compatible CAS
-                auto const blob_content =
-                    repo_config_->ReadBlobFromGitCAS(native_obj.digest.hash());
-                if (not blob_content) {
-                    Logger::Log(LogLevel::Error,
-                                "MRGitApi: failed reading Git entry {}",
-                                native_obj.digest.hash());
-                    return false;
-                }
-                auto blob_digest = compat_storage_->CAS().StoreBlob(
-                    *blob_content, IsExecutableObject(native_obj.type));
-                if (not blob_digest) {
-                    Logger::Log(LogLevel::Error,
-                                "MRGitApi: failed to rehash Git entry {}",
-                                native_obj.digest.hash());
-                    return false;
-                }
-                // cache the digest association
-                if (auto error_msg = store_rehashed(
-                        native_obj.digest, *blob_digest, native_obj.type)) {
-                    Logger::Log(
-                        LogLevel::Error, "MRGitApi: {}", *std::move(error_msg));
-                    return false;
-                }
-                // add object to the vector of compatible artifacts
-                compat_artifacts.emplace_back(
-                    Artifact::ObjectInfo{.digest = *std::move(blob_digest),
-                                         .type = native_obj.type});
-            }
-        }
+    auto compat_artifacts =
+        RehashUtils::RehashGitDigest(artifacts_info,
+                                     *native_storage_config_,
+                                     *compat_storage_config_,
+                                     *repo_config_);
+    if (not compat_artifacts) {
+        Logger::Log(LogLevel::Error,
+                    "MRGitApi: {}",
+                    std::move(compat_artifacts).error());
+        return false;
     }
-    // now that we have gathered all the compatible object infos, simply pass
-    // them to a local api that can interact with the remote
-    return compat_local_api_->RetrieveToCas(compat_artifacts, api);
+    return compat_local_api_->RetrieveToCas(*compat_artifacts, api);
 }
diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS
index e54b63f6..abcb7470 100644
--- a/src/buildtool/execution_api/utils/TARGETS
+++ b/src/buildtool/execution_api/utils/TARGETS
@@ -38,6 +38,7 @@
   , "srcs": ["rehash_utils.cpp"]
   , "deps":
     [ ["src/buildtool/common", "common"]
+    , ["src/buildtool/common", "config"]
     , ["src/buildtool/file_system", "object_type"]
     , ["src/buildtool/storage", "config"]
     , ["src/utils/cpp", "expected"]
diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp
index bbaefaeb..c5ca60c2 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.cpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.cpp
@@ -136,11 +136,11 @@ template <std::invocable<ArtifactDigest const&, ObjectType> TReadCallback>
         [&target_storage](
             std::variant<std::filesystem::path, std::string> const& data,
             bool is_exec) -> std::optional<ArtifactDigest> {
-        if (not std::holds_alternative<std::filesystem::path>(data)) {
-            return std::nullopt;
-        }
-        return target_storage.CAS().StoreBlob(
-            std::get<std::filesystem::path>(data), is_exec);
+        return std::visit(
+            [&target_storage, is_exec](auto const& d) {
+                return target_storage.CAS().StoreBlob(d, is_exec);
+            },
+            data);
     };
     BazelMsgFactory::TreeStoreFunc store_dir =
         [&cas = target_storage.CAS()](std::string const& content) {
@@ -252,4 +252,21 @@ auto RehashDigest(std::vector<Artifact::ObjectInfo> const& digests,
                             /*from_git=*/false);
 }
 
+auto RehashGitDigest(std::vector<Artifact::ObjectInfo> const& digests,
+                     StorageConfig const& source_config,
+                     StorageConfig const& target_config,
+                     RepositoryConfig const& repo_config)
+    -> expected<std::vector<Artifact::ObjectInfo>, std::string> {
+    auto read = [&repo_config](
+                    ArtifactDigest const& digest,
+                    ObjectType /*type*/) -> std::optional<std::string> {
+        return repo_config.ReadBlobFromGitCAS(digest.hash());
+    };
+    return RehashDigestImpl(digests,
+                            source_config,
+                            target_config,
+                            std::move(read),
+                            /*from_git=*/true);
+}
+
 }  // namespace RehashUtils
diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp
index e6b21874..7de1cc54 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.hpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.hpp
@@ -21,6 +21,7 @@
 
 #include "src/buildtool/common/artifact.hpp"
 #include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/repository_config.hpp"
 #include "src/buildtool/file_system/object_type.hpp"
 #include "src/buildtool/storage/config.hpp"
 #include "src/utils/cpp/expected.hpp"
@@ -66,6 +67,13 @@ namespace RehashUtils {
     StorageConfig const& target_config)
     -> expected<std::vector<Artifact::ObjectInfo>, std::string>;
 
+[[nodiscard]] auto RehashGitDigest(
+    std::vector<Artifact::ObjectInfo> const& digests,
+    StorageConfig const& source_config,
+    StorageConfig const& target_config,
+    RepositoryConfig const& repo_config)
+    -> expected<std::vector<Artifact::ObjectInfo>, std::string>;
+
 }  // namespace RehashUtils
 
 #endif  // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP