summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_api/serve/TARGETS3
-rw-r--r--src/buildtool/execution_api/serve/mr_local_api.cpp149
-rw-r--r--src/buildtool/execution_api/utils/TARGETS2
-rw-r--r--src/buildtool/execution_api/utils/rehash_utils.cpp137
-rw-r--r--src/buildtool/execution_api/utils/rehash_utils.hpp7
5 files changed, 156 insertions, 142 deletions
diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS
index cfc23515..14843487 100644
--- a/src/buildtool/execution_api/serve/TARGETS
+++ b/src/buildtool/execution_api/serve/TARGETS
@@ -40,13 +40,10 @@
, "stage": ["src", "buildtool", "execution_api", "serve"]
, "private-deps":
[ ["src/buildtool/common", "protocol_traits"]
- , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
, ["src/buildtool/execution_api/utils", "rehash_utils"]
- , ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/logging", "log_level"]
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/storage", "config"]
- , ["src/buildtool/storage", "storage"]
, ["src/utils/cpp", "expected"]
]
}
diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp
index 55dddafc..2d80e0ac 100644
--- a/src/buildtool/execution_api/serve/mr_local_api.cpp
+++ b/src/buildtool/execution_api/serve/mr_local_api.cpp
@@ -15,16 +15,12 @@
#include "src/buildtool/execution_api/serve/mr_local_api.hpp"
#include <utility>
-#include <variant>
#include "src/buildtool/common/protocol_traits.hpp"
-#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
-#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/buildtool/storage/config.hpp"
-#include "src/buildtool/storage/storage.hpp"
#include "src/utils/cpp/expected.hpp"
MRLocalApi::MRLocalApi(
@@ -83,142 +79,17 @@ auto MRLocalApi::RetrieveToCas(
return compat_local_api_->RetrieveToCas(artifacts_info, api);
}
- // in compatible mode: if passed native digests, one must rehash them;
- // first, set up needed callbacks for caching digest mappings
- auto read_rehashed = [native_storage = native_context_->storage_config,
- compat_storage = compat_context_->storage_config](
- ArtifactDigest const& digest)
- -> expected<std::optional<Artifact::ObjectInfo>, std::string> {
- return RehashUtils::ReadRehashedDigest(
- digest, *native_storage, *compat_storage);
- };
- auto store_rehashed =
- [native_storage = native_context_->storage_config,
- compat_storage = compat_context_->storage_config](
- ArtifactDigest const& source_digest,
- ArtifactDigest const& target_digest,
- ObjectType obj_type) -> std::optional<std::string> {
- return RehashUtils::StoreRehashedDigest(source_digest,
- target_digest,
- obj_type,
- *native_storage,
- *compat_storage);
- };
-
- // collect the native blobs and rehash them as compatible to be able to
- // check what is missing in the other api
- std::vector<Artifact::ObjectInfo> compat_artifacts;
- compat_artifacts.reserve(artifacts_info.size());
- for (auto const& native_obj : artifacts_info) {
- // check if we know already the compatible digest
- auto cached_obj = read_rehashed(native_obj.digest);
- if (not cached_obj) {
- Logger::Log(LogLevel::Error,
- "MRLocalApi: {}",
- std::move(cached_obj).error());
- return false;
- }
- if (*cached_obj) {
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(std::move(cached_obj)->value());
- }
- else {
- // process object; trees need to be handled appropriately
- if (IsTreeObject(native_obj.type)) {
- // set up all the callbacks needed
- auto read_git = [cas = &native_context_->storage->CAS()](
- ArtifactDigest const& digest,
- ObjectType type)
- -> std::optional<
- std::variant<std::filesystem::path, std::string>> {
- return IsTreeObject(type)
- ? cas->TreePath(digest)
- : cas->BlobPath(digest,
- IsExecutableObject(type));
- };
- auto store_file =
- [cas = &compat_context_->storage->CAS()](
- std::variant<std::filesystem::path, std::string> const&
- data,
- bool is_exec) -> std::optional<ArtifactDigest> {
- if (not std::holds_alternative<std::filesystem::path>(
- data)) {
- return std::nullopt;
- }
- return cas->StoreBlob(std::get<std::filesystem::path>(data),
- is_exec);
- };
- BazelMsgFactory::TreeStoreFunc store_dir =
- [cas = &compat_context_->storage->CAS()](
- std::string const& content)
- -> std::optional<ArtifactDigest> {
- return cas->StoreTree(content);
- };
- BazelMsgFactory::SymlinkStoreFunc store_symlink =
- [cas = &compat_context_->storage->CAS()](
- std::string const& content)
- -> std::optional<ArtifactDigest> {
- return cas->StoreBlob(content);
- };
- // get the directory digest
- auto tree_digest =
- BazelMsgFactory::CreateDirectoryDigestFromGitTree(
- native_obj.digest,
- read_git,
- store_file,
- store_dir,
- store_symlink,
- read_rehashed,
- store_rehashed);
- if (not tree_digest) {
- Logger::Log(LogLevel::Error,
- "MRLocalApi: {}",
- std::move(tree_digest).error());
- return false;
- }
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(
- Artifact::ObjectInfo{.digest = *std::move(tree_digest),
- .type = ObjectType::Tree});
- }
- else {
- // blobs can be directly rehashed
- auto const is_exec = IsExecutableObject(native_obj.type);
- auto path = native_context_->storage->CAS().BlobPath(
- native_obj.digest, is_exec);
- if (not path) {
- Logger::Log(
- LogLevel::Error,
- "MRLocalApi: failed to get path of CAS entry {}",
- native_obj.digest.hash());
- return false;
- }
- auto blob_digest =
- compat_context_->storage->CAS().StoreBlob(*path, is_exec);
- if (not blob_digest) {
- Logger::Log(LogLevel::Error,
- "MRLocalApi: failed to rehash CAS entry {}",
- native_obj.digest.hash());
- return false;
- }
- // cache the digest association
- if (auto error_msg = store_rehashed(
- native_obj.digest, *blob_digest, native_obj.type)) {
- Logger::Log(LogLevel::Error,
- "MRLocalApi: {}",
- *std::move(error_msg));
- return false;
- }
- // add object to the vector of compatible artifacts
- compat_artifacts.emplace_back(
- Artifact::ObjectInfo{.digest = *std::move(blob_digest),
- .type = native_obj.type});
- }
- }
+ auto compat_artifacts =
+ RehashUtils::RehashDigest(artifacts_info,
+ *native_context_->storage_config,
+ *compat_context_->storage_config);
+ if (not compat_artifacts) {
+ Logger::Log(LogLevel::Error,
+ "MRLocalApi: {}",
+ std::move(compat_artifacts).error());
+ return false;
}
- // now that we have gathered all the compatible object infos, simply pass
- // them to the compatible local api
- return compat_local_api_->RetrieveToCas(compat_artifacts, api);
+ return compat_local_api_->RetrieveToCas(*compat_artifacts, api);
}
// NOLINTNEXTLINE(google-default-arguments)
diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS
index afaabeb0..e54b63f6 100644
--- a/src/buildtool/execution_api/utils/TARGETS
+++ b/src/buildtool/execution_api/utils/TARGETS
@@ -46,8 +46,10 @@
, "private-deps":
[ ["@", "fmt", "", "fmt"]
, ["src/buildtool/crypto", "hash_function"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/storage", "fs_utils"]
+ , ["src/buildtool/storage", "storage"]
]
}
}
diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp
index 660d8cec..bbaefaeb 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.cpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.cpp
@@ -14,15 +14,20 @@
#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
+#include <concepts>
#include <cstddef> // std::size_t
#include <filesystem>
+#include <functional>
#include <system_error>
#include <utility> // std::move
+#include <variant>
#include "fmt/core.h"
#include "src/buildtool/crypto/hash_function.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/storage/fs_utils.hpp"
+#include "src/buildtool/storage/storage.hpp"
namespace RehashUtils {
@@ -115,4 +120,136 @@ auto StoreRehashedDigest(ArtifactDigest const& source_digest,
return std::nullopt; // a-ok
}
+namespace {
+
+template <std::invocable<ArtifactDigest const&, ObjectType> TReadCallback>
+[[nodiscard]] auto RehashDigestImpl(
+ std::vector<Artifact::ObjectInfo> const& infos,
+ StorageConfig const& source_config,
+ StorageConfig const& target_config,
+ TReadCallback read_callback,
+ bool from_git) -> expected<std::vector<Artifact::ObjectInfo>, std::string> {
+
+ auto const target_storage = Storage::Create(&target_config);
+
+ BazelMsgFactory::BlobStoreFunc store_file =
+ [&target_storage](
+ std::variant<std::filesystem::path, std::string> const& data,
+ bool is_exec) -> std::optional<ArtifactDigest> {
+ if (not std::holds_alternative<std::filesystem::path>(data)) {
+ return std::nullopt;
+ }
+ return target_storage.CAS().StoreBlob(
+ std::get<std::filesystem::path>(data), is_exec);
+ };
+ BazelMsgFactory::TreeStoreFunc store_dir =
+ [&cas = target_storage.CAS()](std::string const& content) {
+ return cas.StoreTree(content);
+ };
+ BazelMsgFactory::SymlinkStoreFunc store_symlink =
+ [&cas = target_storage.CAS()](std::string const& content) {
+ return cas.StoreBlob(content);
+ };
+ BazelMsgFactory::RehashedDigestReadFunc read_rehashed =
+ [&source_config, &target_config, from_git](ArtifactDigest const& digest)
+ -> expected<std::optional<Artifact::ObjectInfo>, std::string> {
+ return RehashUtils::ReadRehashedDigest(
+ digest, source_config, target_config, from_git);
+ };
+ BazelMsgFactory::RehashedDigestStoreFunc store_rehashed =
+ [&source_config, &target_config, from_git](
+ ArtifactDigest const& source_digest,
+ ArtifactDigest const& target_digest,
+ ObjectType obj_type) -> std::optional<std::string> {
+ return RehashUtils::StoreRehashedDigest(source_digest,
+ target_digest,
+ obj_type,
+ source_config,
+ target_config,
+ from_git);
+ };
+
+ // collect the native blobs and rehash them as compatible to be able to
+ // check what is missing in the other api
+ std::vector<Artifact::ObjectInfo> compat_artifacts;
+ compat_artifacts.reserve(infos.size());
+ for (auto const& source_object : infos) {
+ // check if we know already the compatible digest
+ auto cached_obj = read_rehashed(source_object.digest);
+ if (not cached_obj) {
+ return unexpected(std::move(cached_obj).error());
+ }
+ if (*cached_obj) {
+ // add object to the vector of compatible artifacts
+ compat_artifacts.emplace_back(std::move(cached_obj)->value());
+ continue;
+ }
+
+ // process object; trees need to be handled appropriately
+ if (IsTreeObject(source_object.type)) {
+ // get the directory digest
+ auto target_tree =
+ BazelMsgFactory::CreateDirectoryDigestFromGitTree(
+ source_object.digest,
+ read_callback,
+ store_file,
+ store_dir,
+ store_symlink,
+ read_rehashed,
+ store_rehashed);
+ if (not target_tree) {
+ return unexpected(std::move(target_tree).error());
+ }
+ // add object to the vector of compatible artifacts
+ compat_artifacts.emplace_back(Artifact::ObjectInfo{
+ .digest = *std::move(target_tree), .type = ObjectType::Tree});
+ }
+ else {
+ // blobs can be directly rehashed
+ auto path = read_callback(source_object.digest, source_object.type);
+ if (not path) {
+ return unexpected(
+ fmt::format("failed to get path of entry {}",
+ source_object.digest.hash()));
+ }
+ auto target_blob =
+ store_file(*path, IsExecutableObject(source_object.type));
+ if (not target_blob) {
+ return unexpected(fmt::format("failed to rehash entry {}",
+ source_object.digest.hash()));
+ }
+ // cache the digest association
+ if (auto error_msg = store_rehashed(
+ source_object.digest, *target_blob, source_object.type)) {
+ return unexpected(*std::move(error_msg));
+ }
+ // add object to the vector of compatible artifacts
+ compat_artifacts.emplace_back(Artifact::ObjectInfo{
+ .digest = *std::move(target_blob), .type = source_object.type});
+ }
+ }
+
+ return compat_artifacts;
+}
+} // namespace
+
+auto RehashDigest(std::vector<Artifact::ObjectInfo> const& digests,
+ StorageConfig const& source_config,
+ StorageConfig const& target_config)
+ -> expected<std::vector<Artifact::ObjectInfo>, std::string> {
+ auto const source_storage = Storage::Create(&source_config);
+ auto read = [&source_storage](
+ ArtifactDigest const& digest,
+ ObjectType type) -> std::optional<std::filesystem::path> {
+ return IsTreeObject(type) ? source_storage.CAS().TreePath(digest)
+ : source_storage.CAS().BlobPath(
+ digest, IsExecutableObject(type));
+ };
+ return RehashDigestImpl(digests,
+ source_config,
+ target_config,
+ std::move(read),
+ /*from_git=*/false);
+}
+
} // namespace RehashUtils
diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp
index 7ed7cb11..e6b21874 100644
--- a/src/buildtool/execution_api/utils/rehash_utils.hpp
+++ b/src/buildtool/execution_api/utils/rehash_utils.hpp
@@ -17,6 +17,7 @@
#include <optional>
#include <string>
+#include <vector>
#include "src/buildtool/common/artifact.hpp"
#include "src/buildtool/common/artifact_digest.hpp"
@@ -59,6 +60,12 @@ namespace RehashUtils {
bool from_git = false) noexcept
-> std::optional<std::string>;
+[[nodiscard]] auto RehashDigest(
+ std::vector<Artifact::ObjectInfo> const& digests,
+ StorageConfig const& source_config,
+ StorageConfig const& target_config)
+ -> expected<std::vector<Artifact::ObjectInfo>, std::string>;
+
} // namespace RehashUtils
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 20:40:21 +0000