diff options
Diffstat (limited to 'src/buildtool/execution_api/utils')
| -rw-r--r-- | src/buildtool/execution_api/utils/TARGETS | 2 | ||||
| -rw-r--r-- | src/buildtool/execution_api/utils/rehash_utils.cpp | 137 | ||||
| -rw-r--r-- | src/buildtool/execution_api/utils/rehash_utils.hpp | 7 |
3 files changed, 146 insertions, 0 deletions
diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS index afaabeb0..e54b63f6 100644 --- a/src/buildtool/execution_api/utils/TARGETS +++ b/src/buildtool/execution_api/utils/TARGETS @@ -46,8 +46,10 @@ , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "fs_utils"] + , ["src/buildtool/storage", "storage"] ] } } diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp index 660d8cec..bbaefaeb 100644 --- a/src/buildtool/execution_api/utils/rehash_utils.cpp +++ b/src/buildtool/execution_api/utils/rehash_utils.cpp @@ -14,15 +14,20 @@ #include "src/buildtool/execution_api/utils/rehash_utils.hpp" +#include <concepts> #include <cstddef> // std::size_t #include <filesystem> +#include <functional> #include <system_error> #include <utility> // std::move +#include <variant> #include "fmt/core.h" #include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/fs_utils.hpp" +#include "src/buildtool/storage/storage.hpp" namespace RehashUtils { @@ -115,4 +120,136 @@ auto StoreRehashedDigest(ArtifactDigest const& source_digest, return std::nullopt; // a-ok } +namespace { + +template <std::invocable<ArtifactDigest const&, ObjectType> TReadCallback> +[[nodiscard]] auto RehashDigestImpl( + std::vector<Artifact::ObjectInfo> const& infos, + StorageConfig const& source_config, + StorageConfig const& target_config, + TReadCallback read_callback, + bool from_git) -> expected<std::vector<Artifact::ObjectInfo>, std::string> { + + auto const target_storage = Storage::Create(&target_config); + + BazelMsgFactory::BlobStoreFunc store_file = + [&target_storage]( + std::variant<std::filesystem::path, std::string> const& data, + bool is_exec) -> std::optional<ArtifactDigest> { + if (not std::holds_alternative<std::filesystem::path>(data)) { + return std::nullopt; + } + return target_storage.CAS().StoreBlob( + std::get<std::filesystem::path>(data), is_exec); + }; + BazelMsgFactory::TreeStoreFunc store_dir = + [&cas = target_storage.CAS()](std::string const& content) { + return cas.StoreTree(content); + }; + BazelMsgFactory::SymlinkStoreFunc store_symlink = + [&cas = target_storage.CAS()](std::string const& content) { + return cas.StoreBlob(content); + }; + BazelMsgFactory::RehashedDigestReadFunc read_rehashed = + [&source_config, &target_config, from_git](ArtifactDigest const& digest) + -> expected<std::optional<Artifact::ObjectInfo>, std::string> { + return RehashUtils::ReadRehashedDigest( + digest, source_config, target_config, from_git); + }; + BazelMsgFactory::RehashedDigestStoreFunc store_rehashed = + [&source_config, &target_config, from_git]( + ArtifactDigest const& source_digest, + ArtifactDigest const& target_digest, + ObjectType obj_type) -> std::optional<std::string> { + return RehashUtils::StoreRehashedDigest(source_digest, + target_digest, + obj_type, + source_config, + target_config, + from_git); + }; + + // collect the native blobs and rehash them as compatible to be able to + // check what is missing in the other api + std::vector<Artifact::ObjectInfo> compat_artifacts; + compat_artifacts.reserve(infos.size()); + for (auto const& source_object : infos) { + // check if we know already the compatible digest + auto cached_obj = read_rehashed(source_object.digest); + if (not cached_obj) { + return unexpected(std::move(cached_obj).error()); + } + if (*cached_obj) { + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(std::move(cached_obj)->value()); + continue; + } + + // process object; trees need to be handled appropriately + if (IsTreeObject(source_object.type)) { + // get the directory digest + auto target_tree = + BazelMsgFactory::CreateDirectoryDigestFromGitTree( + source_object.digest, + read_callback, + store_file, + store_dir, + store_symlink, + read_rehashed, + store_rehashed); + if (not target_tree) { + return unexpected(std::move(target_tree).error()); + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(Artifact::ObjectInfo{ + .digest = *std::move(target_tree), .type = ObjectType::Tree}); + } + else { + // blobs can be directly rehashed + auto path = read_callback(source_object.digest, source_object.type); + if (not path) { + return unexpected( + fmt::format("failed to get path of entry {}", + source_object.digest.hash())); + } + auto target_blob = + store_file(*path, IsExecutableObject(source_object.type)); + if (not target_blob) { + return unexpected(fmt::format("failed to rehash entry {}", + source_object.digest.hash())); + } + // cache the digest association + if (auto error_msg = store_rehashed( + source_object.digest, *target_blob, source_object.type)) { + return unexpected(*std::move(error_msg)); + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(Artifact::ObjectInfo{ + .digest = *std::move(target_blob), .type = source_object.type}); + } + } + + return compat_artifacts; +} +} // namespace + +auto RehashDigest(std::vector<Artifact::ObjectInfo> const& digests, + StorageConfig const& source_config, + StorageConfig const& target_config) + -> expected<std::vector<Artifact::ObjectInfo>, std::string> { + auto const source_storage = Storage::Create(&source_config); + auto read = [&source_storage]( + ArtifactDigest const& digest, + ObjectType type) -> std::optional<std::filesystem::path> { + return IsTreeObject(type) ? source_storage.CAS().TreePath(digest) + : source_storage.CAS().BlobPath( + digest, IsExecutableObject(type)); + }; + return RehashDigestImpl(digests, + source_config, + target_config, + std::move(read), + /*from_git=*/false); +} + } // namespace RehashUtils diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp index 7ed7cb11..e6b21874 100644 --- a/src/buildtool/execution_api/utils/rehash_utils.hpp +++ b/src/buildtool/execution_api/utils/rehash_utils.hpp @@ -17,6 +17,7 @@ #include <optional> #include <string> +#include <vector> #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" @@ -59,6 +60,12 @@ namespace RehashUtils { bool from_git = false) noexcept -> std::optional<std::string>; +[[nodiscard]] auto RehashDigest( + std::vector<Artifact::ObjectInfo> const& digests, + StorageConfig const& source_config, + StorageConfig const& target_config) + -> expected<std::vector<Artifact::ObjectInfo>, std::string>; + } // namespace RehashUtils #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP |