diff options
| -rw-r--r-- | src/buildtool/execution_api/serve/TARGETS | 3 | ||||
| -rw-r--r-- | src/buildtool/execution_api/serve/mr_local_api.cpp | 149 | ||||
| -rw-r--r-- | src/buildtool/execution_api/utils/TARGETS | 2 | ||||
| -rw-r--r-- | src/buildtool/execution_api/utils/rehash_utils.cpp | 137 | ||||
| -rw-r--r-- | src/buildtool/execution_api/utils/rehash_utils.hpp | 7 |
5 files changed, 156 insertions, 142 deletions
diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS index cfc23515..14843487 100644 --- a/src/buildtool/execution_api/serve/TARGETS +++ b/src/buildtool/execution_api/serve/TARGETS @@ -40,13 +40,10 @@ , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": [ ["src/buildtool/common", "protocol_traits"] - , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/utils", "rehash_utils"] - , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/storage", "config"] - , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] } diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp index 55dddafc..2d80e0ac 100644 --- a/src/buildtool/execution_api/serve/mr_local_api.cpp +++ b/src/buildtool/execution_api/serve/mr_local_api.cpp @@ -15,16 +15,12 @@ #include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include <utility> -#include <variant> #include "src/buildtool/common/protocol_traits.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/utils/rehash_utils.hpp" -#include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" MRLocalApi::MRLocalApi( @@ -83,142 +79,17 @@ auto MRLocalApi::RetrieveToCas( return compat_local_api_->RetrieveToCas(artifacts_info, api); } - // in compatible mode: if passed native digests, one must rehash them; - // first, set up needed callbacks for caching digest mappings - auto read_rehashed = [native_storage = native_context_->storage_config, - compat_storage = compat_context_->storage_config]( - ArtifactDigest const& digest) - -> expected<std::optional<Artifact::ObjectInfo>, std::string> { - return RehashUtils::ReadRehashedDigest( - digest, *native_storage, *compat_storage); - }; - auto store_rehashed = - [native_storage = native_context_->storage_config, - compat_storage = compat_context_->storage_config]( - ArtifactDigest const& source_digest, - ArtifactDigest const& target_digest, - ObjectType obj_type) -> std::optional<std::string> { - return RehashUtils::StoreRehashedDigest(source_digest, - target_digest, - obj_type, - *native_storage, - *compat_storage); - }; - - // collect the native blobs and rehash them as compatible to be able to - // check what is missing in the other api - std::vector<Artifact::ObjectInfo> compat_artifacts; - compat_artifacts.reserve(artifacts_info.size()); - for (auto const& native_obj : artifacts_info) { - // check if we know already the compatible digest - auto cached_obj = read_rehashed(native_obj.digest); - if (not cached_obj) { - Logger::Log(LogLevel::Error, - "MRLocalApi: {}", - std::move(cached_obj).error()); - return false; - } - if (*cached_obj) { - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back(std::move(cached_obj)->value()); - } - else { - // process object; trees need to be handled appropriately - if (IsTreeObject(native_obj.type)) { - // set up all the callbacks needed - auto read_git = [cas = &native_context_->storage->CAS()]( - ArtifactDigest const& digest, - ObjectType type) - -> std::optional< - std::variant<std::filesystem::path, std::string>> { - return IsTreeObject(type) - ? cas->TreePath(digest) - : cas->BlobPath(digest, - IsExecutableObject(type)); - }; - auto store_file = - [cas = &compat_context_->storage->CAS()]( - std::variant<std::filesystem::path, std::string> const& - data, - bool is_exec) -> std::optional<ArtifactDigest> { - if (not std::holds_alternative<std::filesystem::path>( - data)) { - return std::nullopt; - } - return cas->StoreBlob(std::get<std::filesystem::path>(data), - is_exec); - }; - BazelMsgFactory::TreeStoreFunc store_dir = - [cas = &compat_context_->storage->CAS()]( - std::string const& content) - -> std::optional<ArtifactDigest> { - return cas->StoreTree(content); - }; - BazelMsgFactory::SymlinkStoreFunc store_symlink = - [cas = &compat_context_->storage->CAS()]( - std::string const& content) - -> std::optional<ArtifactDigest> { - return cas->StoreBlob(content); - }; - // get the directory digest - auto tree_digest = - BazelMsgFactory::CreateDirectoryDigestFromGitTree( - native_obj.digest, - read_git, - store_file, - store_dir, - store_symlink, - read_rehashed, - store_rehashed); - if (not tree_digest) { - Logger::Log(LogLevel::Error, - "MRLocalApi: {}", - std::move(tree_digest).error()); - return false; - } - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back( - Artifact::ObjectInfo{.digest = *std::move(tree_digest), - .type = ObjectType::Tree}); - } - else { - // blobs can be directly rehashed - auto const is_exec = IsExecutableObject(native_obj.type); - auto path = native_context_->storage->CAS().BlobPath( - native_obj.digest, is_exec); - if (not path) { - Logger::Log( - LogLevel::Error, - "MRLocalApi: failed to get path of CAS entry {}", - native_obj.digest.hash()); - return false; - } - auto blob_digest = - compat_context_->storage->CAS().StoreBlob(*path, is_exec); - if (not blob_digest) { - Logger::Log(LogLevel::Error, - "MRLocalApi: failed to rehash CAS entry {}", - native_obj.digest.hash()); - return false; - } - // cache the digest association - if (auto error_msg = store_rehashed( - native_obj.digest, *blob_digest, native_obj.type)) { - Logger::Log(LogLevel::Error, - "MRLocalApi: {}", - *std::move(error_msg)); - return false; - } - // add object to the vector of compatible artifacts - compat_artifacts.emplace_back( - Artifact::ObjectInfo{.digest = *std::move(blob_digest), - .type = native_obj.type}); - } - } + auto compat_artifacts = + RehashUtils::RehashDigest(artifacts_info, + *native_context_->storage_config, + *compat_context_->storage_config); + if (not compat_artifacts) { + Logger::Log(LogLevel::Error, + "MRLocalApi: {}", + std::move(compat_artifacts).error()); + return false; } - // now that we have gathered all the compatible object infos, simply pass - // them to the compatible local api - return compat_local_api_->RetrieveToCas(compat_artifacts, api); + return compat_local_api_->RetrieveToCas(*compat_artifacts, api); } // NOLINTNEXTLINE(google-default-arguments) diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS index afaabeb0..e54b63f6 100644 --- a/src/buildtool/execution_api/utils/TARGETS +++ b/src/buildtool/execution_api/utils/TARGETS @@ -46,8 +46,10 @@ , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "fs_utils"] + , ["src/buildtool/storage", "storage"] ] } } diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp index 660d8cec..bbaefaeb 100644 --- a/src/buildtool/execution_api/utils/rehash_utils.cpp +++ b/src/buildtool/execution_api/utils/rehash_utils.cpp @@ -14,15 +14,20 @@ #include "src/buildtool/execution_api/utils/rehash_utils.hpp" +#include <concepts> #include <cstddef> // std::size_t #include <filesystem> +#include <functional> #include <system_error> #include <utility> // std::move +#include <variant> #include "fmt/core.h" #include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/fs_utils.hpp" +#include "src/buildtool/storage/storage.hpp" namespace RehashUtils { @@ -115,4 +120,136 @@ auto StoreRehashedDigest(ArtifactDigest const& source_digest, return std::nullopt; // a-ok } +namespace { + +template <std::invocable<ArtifactDigest const&, ObjectType> TReadCallback> +[[nodiscard]] auto RehashDigestImpl( + std::vector<Artifact::ObjectInfo> const& infos, + StorageConfig const& source_config, + StorageConfig const& target_config, + TReadCallback read_callback, + bool from_git) -> expected<std::vector<Artifact::ObjectInfo>, std::string> { + + auto const target_storage = Storage::Create(&target_config); + + BazelMsgFactory::BlobStoreFunc store_file = + [&target_storage]( + std::variant<std::filesystem::path, std::string> const& data, + bool is_exec) -> std::optional<ArtifactDigest> { + if (not std::holds_alternative<std::filesystem::path>(data)) { + return std::nullopt; + } + return target_storage.CAS().StoreBlob( + std::get<std::filesystem::path>(data), is_exec); + }; + BazelMsgFactory::TreeStoreFunc store_dir = + [&cas = target_storage.CAS()](std::string const& content) { + return cas.StoreTree(content); + }; + BazelMsgFactory::SymlinkStoreFunc store_symlink = + [&cas = target_storage.CAS()](std::string const& content) { + return cas.StoreBlob(content); + }; + BazelMsgFactory::RehashedDigestReadFunc read_rehashed = + [&source_config, &target_config, from_git](ArtifactDigest const& digest) + -> expected<std::optional<Artifact::ObjectInfo>, std::string> { + return RehashUtils::ReadRehashedDigest( + digest, source_config, target_config, from_git); + }; + BazelMsgFactory::RehashedDigestStoreFunc store_rehashed = + [&source_config, &target_config, from_git]( + ArtifactDigest const& source_digest, + ArtifactDigest const& target_digest, + ObjectType obj_type) -> std::optional<std::string> { + return RehashUtils::StoreRehashedDigest(source_digest, + target_digest, + obj_type, + source_config, + target_config, + from_git); + }; + + // collect the native blobs and rehash them as compatible to be able to + // check what is missing in the other api + std::vector<Artifact::ObjectInfo> compat_artifacts; + compat_artifacts.reserve(infos.size()); + for (auto const& source_object : infos) { + // check if we know already the compatible digest + auto cached_obj = read_rehashed(source_object.digest); + if (not cached_obj) { + return unexpected(std::move(cached_obj).error()); + } + if (*cached_obj) { + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(std::move(cached_obj)->value()); + continue; + } + + // process object; trees need to be handled appropriately + if (IsTreeObject(source_object.type)) { + // get the directory digest + auto target_tree = + BazelMsgFactory::CreateDirectoryDigestFromGitTree( + source_object.digest, + read_callback, + store_file, + store_dir, + store_symlink, + read_rehashed, + store_rehashed); + if (not target_tree) { + return unexpected(std::move(target_tree).error()); + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(Artifact::ObjectInfo{ + .digest = *std::move(target_tree), .type = ObjectType::Tree}); + } + else { + // blobs can be directly rehashed + auto path = read_callback(source_object.digest, source_object.type); + if (not path) { + return unexpected( + fmt::format("failed to get path of entry {}", + source_object.digest.hash())); + } + auto target_blob = + store_file(*path, IsExecutableObject(source_object.type)); + if (not target_blob) { + return unexpected(fmt::format("failed to rehash entry {}", + source_object.digest.hash())); + } + // cache the digest association + if (auto error_msg = store_rehashed( + source_object.digest, *target_blob, source_object.type)) { + return unexpected(*std::move(error_msg)); + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(Artifact::ObjectInfo{ + .digest = *std::move(target_blob), .type = source_object.type}); + } + } + + return compat_artifacts; +} +} // namespace + +auto RehashDigest(std::vector<Artifact::ObjectInfo> const& digests, + StorageConfig const& source_config, + StorageConfig const& target_config) + -> expected<std::vector<Artifact::ObjectInfo>, std::string> { + auto const source_storage = Storage::Create(&source_config); + auto read = [&source_storage]( + ArtifactDigest const& digest, + ObjectType type) -> std::optional<std::filesystem::path> { + return IsTreeObject(type) ? source_storage.CAS().TreePath(digest) + : source_storage.CAS().BlobPath( + digest, IsExecutableObject(type)); + }; + return RehashDigestImpl(digests, + source_config, + target_config, + std::move(read), + /*from_git=*/false); +} + } // namespace RehashUtils diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp index 7ed7cb11..e6b21874 100644 --- a/src/buildtool/execution_api/utils/rehash_utils.hpp +++ b/src/buildtool/execution_api/utils/rehash_utils.hpp @@ -17,6 +17,7 @@ #include <optional> #include <string> +#include <vector> #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" @@ -59,6 +60,12 @@ namespace RehashUtils { bool from_git = false) noexcept -> std::optional<std::string>; +[[nodiscard]] auto RehashDigest( + std::vector<Artifact::ObjectInfo> const& digests, + StorageConfig const& source_config, + StorageConfig const& target_config) + -> expected<std::vector<Artifact::ObjectInfo>, std::string>; + } // namespace RehashUtils #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP |