summaryrefslogtreecommitdiff
path: root/src/buildtool/execution_api
diff options
context:
space:
mode:
Diffstat (limited to 'src/buildtool/execution_api')
-rw-r--r--src/buildtool/execution_api/serve/TARGETS4
-rw-r--r--src/buildtool/execution_api/serve/mr_git_api.cpp138
-rw-r--r--src/buildtool/execution_api/utils/TARGETS1
-rw-r--r--src/buildtool/execution_api/utils/rehash_utils.cpp27
-rw-r--r--src/buildtool/execution_api/utils/rehash_utils.hpp8
5 files changed, 43 insertions, 135 deletions
diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS
index 14843487..a8237ba7 100644
--- a/src/buildtool/execution_api/serve/TARGETS
+++ b/src/buildtool/execution_api/serve/TARGETS
@@ -15,10 +15,8 @@
]
, "stage": ["src", "buildtool", "execution_api", "serve"]
, "private-deps":
- [ ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
- , ["src/buildtool/execution_api/git", "git"]
+ [ ["src/buildtool/execution_api/git", "git"]
, ["src/buildtool/execution_api/utils", "rehash_utils"]
- , ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/logging", "log_level"]
, ["src/buildtool/logging", "logging"]
, ["src/utils/cpp", "expected"]
diff --git a/src/buildtool/execution_api/serve/mr_git_api.cpp b/src/buildtool/execution_api/serve/mr_git_api.cpp
index 6630e891..cd2d7a69 100644
--- a/src/buildtool/execution_api/serve/mr_git_api.cpp
+++ b/src/buildtool/execution_api/serve/mr_git_api.cpp
@@ -15,12 +15,9 @@
#include "src/buildtool/execution_api/serve/mr_git_api.hpp"
#include <utility>
-#include <variant>
-#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
#include "src/buildtool/execution_api/git/git_api.hpp"
#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
-#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/utils/cpp/expected.hpp"
@@ -51,129 +48,16 @@ auto MRGitApi::RetrieveToCas(
return git_api.RetrieveToCas(artifacts_info, api);
}
- // in compatible mode: set up needed callbacks for caching digest mappings
- auto read_rehashed =
- [native_sc = native_storage_config_,
- compat_sc = compat_storage_config_](ArtifactDigest const& digest)
- -> expected<std::optional<Artifact::ObjectInfo>, std::string> {
- return RehashUtils::ReadRehashedDigest(
- digest, *native_sc, *compat_sc, /*from_git=*/true);
- };
- auto store_rehashed =
- [native_sc = native_storage_config_,
- compat_sc = compat_storage_config_](
- ArtifactDigest const& source_digest,
- ArtifactDigest const& target_digest,
- ObjectType obj_type) -> std::optional<std::string> {
- return RehashUtils::StoreRehashedDigest(source_digest,
- target_digest,
- obj_type,
- *native_sc,
- *compat_sc,
- /*from_git=*/true);
- };
-
- // collect the native blobs and rehash them as compatible to be able to
- // check what is missing in the other api
- std::vector<Artifact::ObjectInfo> compat_artifacts;
- compat_artifacts.reserve(artifacts_info.size());
- for (auto const& native_obj : artifacts_info) {
- // check if we know already the compatible digest
- auto cached_obj = read_rehashed(native_obj.digest);
- if (not cached_obj) {
- Logger::Log(
- LogLevel::Error, "MRGitApi: {}", std::move(cached_obj).error());
- return false;
- }
- if (*cached_obj) {
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(std::move(cached_obj)->value());
- }
- else {
- // process object; trees need to be handled appropriately
- if (IsTreeObject(native_obj.type)) {
- // set up all the callbacks needed
- auto read_git = [repo_config = repo_config_](
- ArtifactDigest const& digest,
- ObjectType /*type*/)
- -> std::optional<
- std::variant<std::filesystem::path, std::string>> {
- return repo_config->ReadBlobFromGitCAS(digest.hash());
- };
- auto store_file =
- [cas = &compat_storage_->CAS()](
- std::variant<std::filesystem::path, std::string> const&
- data,
- bool is_exec) -> std::optional<ArtifactDigest> {
- if (not std::holds_alternative<std::string>(data)) {
- return std::nullopt;
- }
- return cas->StoreBlob(std::get<std::string>(data), is_exec);
- };
- BazelMsgFactory::TreeStoreFunc store_dir =
- [cas = &compat_storage_->CAS()](std::string const& content)
- -> std::optional<ArtifactDigest> {
- return cas->StoreTree(content);
- };
- BazelMsgFactory::SymlinkStoreFunc store_symlink =
- [cas = &compat_storage_->CAS()](std::string const& content)
- -> std::optional<ArtifactDigest> {
- return cas->StoreBlob(content);
- };
- // get the directory digest
- auto tree_digest =
- BazelMsgFactory::CreateDirectoryDigestFromGitTree(
- native_obj.digest,
- read_git,
- store_file,
- store_dir,
- store_symlink,
- read_rehashed,
- store_rehashed);
- if (not tree_digest) {
- Logger::Log(LogLevel::Error,
- "MRGitApi: {}",
- std::move(tree_digest).error());
- return false;
- }
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(
- Artifact::ObjectInfo{.digest = *std::move(tree_digest),
- .type = ObjectType::Tree});
- }
- else {
- // blobs are read from repo and added to compatible CAS
- auto const blob_content =
- repo_config_->ReadBlobFromGitCAS(native_obj.digest.hash());
- if (not blob_content) {
- Logger::Log(LogLevel::Error,
- "MRGitApi: failed reading Git entry {}",
- native_obj.digest.hash());
- return false;
- }
- auto blob_digest = compat_storage_->CAS().StoreBlob(
- *blob_content, IsExecutableObject(native_obj.type));
- if (not blob_digest) {
- Logger::Log(LogLevel::Error,
- "MRGitApi: failed to rehash Git entry {}",
- native_obj.digest.hash());
- return false;
- }
- // cache the digest association
- if (auto error_msg = store_rehashed(
- native_obj.digest, *blob_digest, native_obj.type)) {
- Logger::Log(
- LogLevel::Error, "MRGitApi: {}", *std::move(error_msg));
- return false;
- }
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(
- Artifact::ObjectInfo{.digest = *std::move(blob_digest),
- .type = native_obj.type});
- }
- }
+ auto compat_artifacts =
+ RehashUtils::RehashGitDigest(artifacts_info,
+ *native_storage_config_,
+ *compat_storage_config_,
+ *repo_config_);
+ if (not compat_artifacts) {
+ Logger::Log(LogLevel::Error,
+ "MRGitApi: {}",
+ std::move(compat_artifacts).error());
+ return false;
}
- // now that we have gathered all the compatible object infos, simply pass
- // them to a local api that can interact with the remote
- return compat_local_api_->RetrieveToCas(compat_artifacts, api);
+ return compat_local_api_->RetrieveToCas(*compat_artifacts, api);
}
diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS
index e54b63f6..abcb7470 100644
--- a/src/buildtool/execution_api/utils/TARGETS
+++ b/src/buildtool/execution_api/utils/TARGETS
@@ -38,6 +38,7 @@
, "srcs": ["rehash_utils.cpp"]
, "deps":
[ ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "config"]
, ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/storage", "config"]
, ["src/utils/cpp", "expected"]
diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp
index bbaefaeb..c5ca60c2 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.cpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.cpp
@@ -136,11 +136,11 @@ template <std::invocable<ArtifactDigest const&, ObjectType> TReadCallback>
[&target_storage](
std::variant<std::filesystem::path, std::string> const& data,
bool is_exec) -> std::optional<ArtifactDigest> {
- if (not std::holds_alternative<std::filesystem::path>(data)) {
- return std::nullopt;
- }
- return target_storage.CAS().StoreBlob(
- std::get<std::filesystem::path>(data), is_exec);
+ return std::visit(
+ [&target_storage, is_exec](auto const& d) {
+ return target_storage.CAS().StoreBlob(d, is_exec);
+ },
+ data);
};
BazelMsgFactory::TreeStoreFunc store_dir =
[&cas = target_storage.CAS()](std::string const& content) {
@@ -252,4 +252,21 @@ auto RehashDigest(std::vector<Artifact::ObjectInfo> const& digests,
/*from_git=*/false);
}
+auto RehashGitDigest(std::vector<Artifact::ObjectInfo> const& digests,
+ StorageConfig const& source_config,
+ StorageConfig const& target_config,
+ RepositoryConfig const& repo_config)
+ -> expected<std::vector<Artifact::ObjectInfo>, std::string> {
+ auto read = [&repo_config](
+ ArtifactDigest const& digest,
+ ObjectType /*type*/) -> std::optional<std::string> {
+ return repo_config.ReadBlobFromGitCAS(digest.hash());
+ };
+ return RehashDigestImpl(digests,
+ source_config,
+ target_config,
+ std::move(read),
+ /*from_git=*/true);
+}
+
} // namespace RehashUtils
diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp
index e6b21874..7de1cc54 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.hpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.hpp
@@ -21,6 +21,7 @@
#include "src/buildtool/common/artifact.hpp"
#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/storage/config.hpp"
#include "src/utils/cpp/expected.hpp"
@@ -66,6 +67,13 @@ namespace RehashUtils {
StorageConfig const& target_config)
-> expected<std::vector<Artifact::ObjectInfo>, std::string>;
+[[nodiscard]] auto RehashGitDigest(
+ std::vector<Artifact::ObjectInfo> const& digests,
+ StorageConfig const& source_config,
+ StorageConfig const& target_config,
+ RepositoryConfig const& repo_config)
+ -> expected<std::vector<Artifact::ObjectInfo>, std::string>;
+
} // namespace RehashUtils
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 11:31:46 +0000