From d3ce7e1633ec8be6d76e43f7e28b4155c9f06b2a Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Mon, 18 Nov 2024 18:07:24 +0100 Subject: Add RehashGitDigest function --- src/buildtool/execution_api/serve/mr_git_api.cpp | 138 ++--------------------- 1 file changed, 11 insertions(+), 127 deletions(-) (limited to 'src/buildtool/execution_api/serve/mr_git_api.cpp') diff --git a/src/buildtool/execution_api/serve/mr_git_api.cpp b/src/buildtool/execution_api/serve/mr_git_api.cpp index 6630e891..cd2d7a69 100644 --- a/src/buildtool/execution_api/serve/mr_git_api.cpp +++ b/src/buildtool/execution_api/serve/mr_git_api.cpp @@ -15,12 +15,9 @@ #include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include -#include -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" -#include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/expected.hpp" @@ -51,129 +48,16 @@ auto MRGitApi::RetrieveToCas( return git_api.RetrieveToCas(artifacts_info, api); } - // in compatible mode: set up needed callbacks for caching digest mappings - auto read_rehashed = - [native_sc = native_storage_config_, - compat_sc = compat_storage_config_](ArtifactDigest const& digest) - -> expected, std::string> { - return RehashUtils::ReadRehashedDigest( - digest, *native_sc, *compat_sc, /*from_git=*/true); - }; - auto store_rehashed = - [native_sc = native_storage_config_, - compat_sc = compat_storage_config_]( - ArtifactDigest const& source_digest, - ArtifactDigest const& target_digest, - ObjectType obj_type) -> std::optional { - return RehashUtils::StoreRehashedDigest(source_digest, - target_digest, - obj_type, - *native_sc, - *compat_sc, - /*from_git=*/true); - }; - - // collect the native blobs and rehash them as compatible to be able to - // check what is missing in the other api - std::vector compat_artifacts; - compat_artifacts.reserve(artifacts_info.size()); - for (auto const& native_obj : artifacts_info) { - // check if we know already the compatible digest - auto cached_obj = read_rehashed(native_obj.digest); - if (not cached_obj) { - Logger::Log( - LogLevel::Error, "MRGitApi: {}", std::move(cached_obj).error()); - return false; - } - if (*cached_obj) { - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back(std::move(cached_obj)->value()); - } - else { - // process object; trees need to be handled appropriately - if (IsTreeObject(native_obj.type)) { - // set up all the callbacks needed - auto read_git = [repo_config = repo_config_]( - ArtifactDigest const& digest, - ObjectType /*type*/) - -> std::optional< - std::variant> { - return repo_config->ReadBlobFromGitCAS(digest.hash()); - }; - auto store_file = - [cas = &compat_storage_->CAS()]( - std::variant const& - data, - bool is_exec) -> std::optional { - if (not std::holds_alternative(data)) { - return std::nullopt; - } - return cas->StoreBlob(std::get(data), is_exec); - }; - BazelMsgFactory::TreeStoreFunc store_dir = - [cas = &compat_storage_->CAS()](std::string const& content) - -> std::optional { - return cas->StoreTree(content); - }; - BazelMsgFactory::SymlinkStoreFunc store_symlink = - [cas = &compat_storage_->CAS()](std::string const& content) - -> std::optional { - return cas->StoreBlob(content); - }; - // get the directory digest - auto tree_digest = - BazelMsgFactory::CreateDirectoryDigestFromGitTree( - native_obj.digest, - read_git, - store_file, - store_dir, - store_symlink, - read_rehashed, - store_rehashed); - if (not tree_digest) { - Logger::Log(LogLevel::Error, - "MRGitApi: {}", - std::move(tree_digest).error()); - return false; - } - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back( - Artifact::ObjectInfo{.digest = *std::move(tree_digest), - .type = ObjectType::Tree}); - } - else { - // blobs are read from repo and added to compatible CAS - auto const blob_content = - repo_config_->ReadBlobFromGitCAS(native_obj.digest.hash()); - if (not blob_content) { - Logger::Log(LogLevel::Error, - "MRGitApi: failed reading Git entry {}", - native_obj.digest.hash()); - return false; - } - auto blob_digest = compat_storage_->CAS().StoreBlob( - *blob_content, IsExecutableObject(native_obj.type)); - if (not blob_digest) { - Logger::Log(LogLevel::Error, - "MRGitApi: failed to rehash Git entry {}", - native_obj.digest.hash()); - return false; - } - // cache the digest association - if (auto error_msg = store_rehashed( - native_obj.digest, *blob_digest, native_obj.type)) { - Logger::Log( - LogLevel::Error, "MRGitApi: {}", *std::move(error_msg)); - return false; - } - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back( - Artifact::ObjectInfo{.digest = *std::move(blob_digest), - .type = native_obj.type}); - } - } + auto compat_artifacts = + RehashUtils::RehashGitDigest(artifacts_info, + *native_storage_config_, + *compat_storage_config_, + *repo_config_); + if (not compat_artifacts) { + Logger::Log(LogLevel::Error, + "MRGitApi: {}", + std::move(compat_artifacts).error()); + return false; } - // now that we have gathered all the compatible object infos, simply pass - // them to a local api that can interact with the remote - return compat_local_api_->RetrieveToCas(compat_artifacts, api); + return compat_local_api_->RetrieveToCas(*compat_artifacts, api); } -- cgit v1.2.3