diff options
Diffstat (limited to 'src/buildtool/execution_api')
14 files changed, 364 insertions, 136 deletions
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp index 43e16783..993f6d17 100644 --- a/src/buildtool/execution_api/common/execution_api.hpp +++ b/src/buildtool/execution_api/common/execution_api.hpp @@ -135,6 +135,26 @@ class IExecutionApi { [[nodiscard]] virtual auto IsAvailable( std::vector<ArtifactDigest> const& digests) const noexcept -> std::vector<ArtifactDigest> = 0; + + [[nodiscard]] virtual auto SplitBlob(ArtifactDigest const& /*blob_digest*/) + const noexcept -> std::optional<std::vector<ArtifactDigest>> { + return std::nullopt; + } + + [[nodiscard]] virtual auto BlobSplitSupport() const noexcept -> bool { + return false; + } + + [[nodiscard]] virtual auto SpliceBlob( + ArtifactDigest const& /*blob_digest*/, + std::vector<ArtifactDigest> const& /*chunk_digests*/) const noexcept + -> std::optional<ArtifactDigest> { + return std::nullopt; + } + + [[nodiscard]] virtual auto BlobSpliceSupport() const noexcept -> bool { + return false; + } }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS index 85068f0f..e494711c 100644 --- a/src/buildtool/execution_api/execution_service/TARGETS +++ b/src/buildtool/execution_api/execution_service/TARGETS @@ -58,12 +58,7 @@ , ["@", "fmt", "", "fmt"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "verify_hash"] - , ["src/buildtool/common", "common"] - , ["src/buildtool/file_system", "git_repo"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/logging", "log_level"] - , ["src/utils/cpp", "hex_string"] - , "file_chunker" + , "cas_utils" ] } , "server_implementation": @@ -151,4 +146,27 @@ , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "private-deps": [["@", "gsl", "", "gsl"]] } +, "cas_utils": + { "type": ["@", "rules", "CC", "library"] + , "name": ["cas_utils"] + , "hdrs": ["cas_utils.hpp"] + , "srcs": ["cas_utils.cpp"] + , "stage": ["src", "buildtool", "execution_api", "execution_service"] + , "deps": + [ ["@", "grpc", "", "grpc++"] + , ["src/buildtool/common", "bazel_types"] + , ["src/buildtool/storage", "storage"] + ] + , "private-deps": + [ "file_chunker" + , ["@", "fmt", "", "fmt"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/compatibility", "compatibility"] + , ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/storage", "fs_utils"] + , ["src/utils/cpp", "hex_string"] + ] + } } diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp index 49939c20..4fd78323 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.cpp +++ b/src/buildtool/execution_api/execution_service/cas_server.cpp @@ -21,15 +21,10 @@ #include <vector> #include "fmt/core.h" -#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/execution_service/file_chunker.hpp" -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/storage/garbage_collector.hpp" -#include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/verify_hash.hpp" static constexpr std::size_t kGitSHA1Length = 42; @@ -100,46 +95,6 @@ auto CASServiceImpl::CheckDigestConsistency(bazel_re::Digest const& ref, return std::nullopt; } -auto CASServiceImpl::EnsureTreeInvariant(std::string const& data, - std::string const& hash) const noexcept - -> std::optional<std::string> { - auto entries = GitRepo::ReadTreeData( - data, - NativeSupport::Unprefix(hash), - [](auto const& /*unused*/) { return true; }, - /*is_hex_id=*/true); - if (not entries) { - auto str = fmt::format("Could not read tree data {}", hash); - logger_.Emit(LogLevel::Error, str); - return str; - } - for (auto const& entry : *entries) { - for (auto const& item : entry.second) { - auto digest = static_cast<bazel_re::Digest>( - ArtifactDigest{ToHexString(entry.first), - /*size is unknown*/ 0, - IsTreeObject(item.type)}); - if (not(IsTreeObject(item.type) - ? storage_->CAS().TreePath(digest) - : storage_->CAS().BlobPath(digest, false))) { - auto str = fmt::format( - "Tree invariant violated {}: missing element {}", - hash, - digest.hash()); - logger_.Emit(LogLevel::Error, str); - return str; - } - // The GitRepo::tree_entries_t data structure maps the object id to - // a list of entries of that object in possibly multiple trees. It - // is sufficient to check the existence of only one of these entries - // to be sure that the object is in CAS since they all have the same - // content. - break; - } - } - return std::nullopt; -} - auto CASServiceImpl::BatchUpdateBlobs( ::grpc::ServerContext* /*context*/, const ::bazel_re::BatchUpdateBlobsRequest* request, @@ -166,7 +121,8 @@ auto CASServiceImpl::BatchUpdateBlobs( if (NativeSupport::IsTree(hash)) { // In native mode: for trees, check whether the tree invariant holds // before storing the actual tree object. - if (auto err = EnsureTreeInvariant(x.data(), hash)) { + if (auto err = + CASUtils::EnsureTreeInvariant(x.data(), hash, *storage_)) { return ::grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, *err}; } @@ -273,55 +229,20 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/, logger_.Emit(LogLevel::Info, "SplitBlob({})", blob_digest.hash()); - // Check blob existence. - auto path = std::optional<std::filesystem::path>{}; - if (NativeSupport::IsTree(blob_digest.hash())) { - path = storage_->CAS().TreePath(blob_digest); - } - else { - path = storage_->CAS().BlobPath(blob_digest, true); - if (not path) { - path = storage_->CAS().BlobPath(blob_digest, false); - } - } - if (not path) { - auto str = - fmt::format("SplitBlob: blob not found {}", blob_digest.hash()); - logger_.Emit(LogLevel::Error, str); - return ::grpc::Status{grpc::StatusCode::NOT_FOUND, str}; - } - - // Split blob into chunks, store each chunk in CAS, and collect chunk - // digests. - auto chunker = FileChunker{*path}; - if (not chunker.IsOpen()) { - auto str = fmt::format("SplitBlob: could not open blob for reading"); + // Split blob into chunks. + auto split_result = CASUtils::SplitBlob(blob_digest, *storage_); + if (std::holds_alternative<grpc::Status>(split_result)) { + auto status = std::get<grpc::Status>(split_result); + auto str = fmt::format("SplitBlob: {}", status.error_message()); logger_.Emit(LogLevel::Error, str); - return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; - } - - auto chunk_digests = std::vector<bazel_re::Digest>{}; - while (auto chunk = chunker.NextChunk()) { - auto chunk_digest = storage_->CAS().StoreBlob(*chunk, false); - if (not chunk_digest) { - auto str = - fmt::format("SplitBlob: could not store chunk of blob {}", - blob_digest.hash()); - logger_.Emit(LogLevel::Error, str); - return ::grpc::Status{grpc::StatusCode::RESOURCE_EXHAUSTED, str}; - } - chunk_digests.emplace_back(*chunk_digest); - } - if (not chunker.Finished()) { - auto str = - fmt::format("SplitBlob: could split blob {}", blob_digest.hash()); - logger_.Emit(LogLevel::Error, str); - return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; + return ::grpc::Status{status.error_code(), str}; } + auto chunk_digests = std::get<std::vector<bazel_re::Digest>>(split_result); logger_.Emit(LogLevel::Debug, [&blob_digest, &chunk_digests]() { std::stringstream ss{}; ss << "Split blob " << blob_digest.hash() << ":" - << blob_digest.size_bytes() << " into [ "; + << blob_digest.size_bytes() << " into " << chunk_digests.size() + << " chunks: [ "; for (auto const& chunk_digest : chunk_digests) { ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes() << " "; diff --git a/src/buildtool/execution_api/execution_service/cas_server.hpp b/src/buildtool/execution_api/execution_service/cas_server.hpp index c9f7cff4..306c2833 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.hpp +++ b/src/buildtool/execution_api/execution_service/cas_server.hpp @@ -147,10 +147,6 @@ class CASServiceImpl final bazel_re::Digest const& computed) const noexcept -> std::optional<std::string>; - [[nodiscard]] auto EnsureTreeInvariant( - std::string const& data, - std::string const& hash) const noexcept -> std::optional<std::string>; - gsl::not_null<Storage const*> storage_ = &Storage::Instance(); Logger logger_{"execution-service"}; }; diff --git a/src/buildtool/execution_api/execution_service/cas_utils.cpp b/src/buildtool/execution_api/execution_service/cas_utils.cpp new file mode 100644 index 00000000..3da56e28 --- /dev/null +++ b/src/buildtool/execution_api/execution_service/cas_utils.cpp @@ -0,0 +1,155 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/execution_service/cas_utils.hpp" + +#include <fstream> + +#include "fmt/core.h" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/execution_service/file_chunker.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/fs_utils.hpp" +#include "src/utils/cpp/hex_string.hpp" + +auto CASUtils::EnsureTreeInvariant(std::string const& data, + std::string const& hash, + Storage const& storage) noexcept + -> std::optional<std::string> { + auto entries = GitRepo::ReadTreeData( + data, + NativeSupport::Unprefix(hash), + [](auto const& /*unused*/) { return true; }, + /*is_hex_id=*/true); + if (not entries) { + return fmt::format("could not read tree data {}", hash); + } + for (auto const& entry : *entries) { + for (auto const& item : entry.second) { + auto digest = static_cast<bazel_re::Digest>( + ArtifactDigest{ToHexString(entry.first), + /*size is unknown*/ 0, + IsTreeObject(item.type)}); + if (not(IsTreeObject(item.type) + ? storage.CAS().TreePath(digest) + : storage.CAS().BlobPath(digest, false))) { + return fmt::format( + "tree invariant violated {}: missing element {}", + hash, + digest.hash()); + } + // The GitRepo::tree_entries_t data structure maps the object id to + // a list of entries of that object in possibly multiple trees. It + // is sufficient to check the existence of only one of these entries + // to be sure that the object is in CAS since they all have the same + // content. + break; + } + } + return std::nullopt; +} + +auto CASUtils::SplitBlob(bazel_re::Digest const& blob_digest, + Storage const& storage) noexcept + -> std::variant<std::vector<bazel_re::Digest>, grpc::Status> { + + // Check blob existence. + auto path = NativeSupport::IsTree(blob_digest.hash()) + ? storage.CAS().TreePath(blob_digest) + : storage.CAS().BlobPath(blob_digest, false); + if (not path) { + return grpc::Status{ + grpc::StatusCode::NOT_FOUND, + fmt::format("blob not found {}", blob_digest.hash())}; + } + + // Split blob into chunks, store each chunk in CAS, and collect chunk + // digests. + auto chunker = FileChunker{*path}; + if (not chunker.IsOpen()) { + return grpc::Status{ + grpc::StatusCode::INTERNAL, + fmt::format("could not open blob {}", blob_digest.hash())}; + } + auto chunk_digests = std::vector<bazel_re::Digest>{}; + while (auto chunk = chunker.NextChunk()) { + auto chunk_digest = storage.CAS().StoreBlob(*chunk, false); + if (not chunk_digest) { + return grpc::Status{grpc::StatusCode::INTERNAL, + fmt::format("could not store chunk of blob {}", + blob_digest.hash())}; + } + chunk_digests.emplace_back(*chunk_digest); + } + if (not chunker.Finished()) { + return grpc::Status{ + grpc::StatusCode::INTERNAL, + fmt::format("could not split blob {}", blob_digest.hash())}; + } + + return chunk_digests; +} + +auto CASUtils::SpliceBlob(bazel_re::Digest const& blob_digest, + std::vector<bazel_re::Digest> const& chunk_digests, + Storage const& storage) noexcept + -> std::variant<bazel_re::Digest, grpc::Status> { + + // Assemble blob from chunks. + auto tmp_dir = StorageUtils::CreateTypedTmpDir("splice"); + auto tmp_file = tmp_dir->GetPath() / "blob"; + { + std::ofstream tmp(tmp_file, std::ios::binary); + for (auto const& chunk_digest : chunk_digests) { + // Check chunk existence (only check file CAS). + auto path = storage.CAS().BlobPath(chunk_digest, false); + if (not path) { + return grpc::Status{ + grpc::StatusCode::NOT_FOUND, + fmt::format("chunk not found {}", chunk_digest.hash())}; + } + // Load chunk data. + auto chunk_data = FileSystemManager::ReadFile(*path); + if (not chunk_data) { + return grpc::Status{grpc::StatusCode::INTERNAL, + fmt::format("could read chunk data {}", + chunk_digest.hash())}; + } + tmp << *chunk_data; + } + } + + // Store resulting blob in according CAS. + auto const& hash = blob_digest.hash(); + if (NativeSupport::IsTree(hash)) { + auto const& digest = + storage.CAS().StoreTree</* kOwner= */ true>(tmp_file); + if (not digest) { + return grpc::Status{grpc::StatusCode::INTERNAL, + fmt::format("could not store tree {}", hash)}; + } + return *digest; + } + + auto const& digest = + storage.CAS().StoreBlob</* kOwner= */ true>(tmp_file, false); + if (not digest) { + return grpc::Status{grpc::StatusCode::INTERNAL, + fmt::format("could not store blob {}", hash)}; + } + return *digest; +} diff --git a/src/buildtool/execution_api/execution_service/cas_utils.hpp b/src/buildtool/execution_api/execution_service/cas_utils.hpp new file mode 100644 index 00000000..b22260b8 --- /dev/null +++ b/src/buildtool/execution_api/execution_service/cas_utils.hpp @@ -0,0 +1,45 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP + +#include <optional> +#include <string> +#include <variant> +#include <vector> + +#include "grpcpp/support/status.h" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/storage/storage.hpp" + +class CASUtils { + public: + [[nodiscard]] static auto EnsureTreeInvariant( + std::string const& data, + std::string const& hash, + Storage const& storage) noexcept -> std::optional<std::string>; + + [[nodiscard]] static auto SplitBlob(bazel_re::Digest const& blob_digest, + Storage const& storage) noexcept + -> std::variant<std::vector<bazel_re::Digest>, grpc::Status>; + + [[nodiscard]] static auto SpliceBlob( + bazel_re::Digest const& blob_digest, + std::vector<bazel_re::Digest> const& chunk_digests, + Storage const& storage) noexcept + -> std::variant<bazel_re::Digest, grpc::Status>; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index c8895c20..f4cfde76 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -26,6 +26,7 @@ , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] + , ["@", "grpc", "", "grpc++"] , ["src/buildtool/common", "config"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/execution_api/common", "common"] @@ -36,6 +37,7 @@ , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] , ["src/buildtool/logging", "logging"] + , ["src/buildtool/execution_api/execution_service", "cas_utils"] ] , "stage": ["src", "buildtool", "execution_api", "local"] , "private-deps": diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 1e3de092..5880475e 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -15,14 +15,18 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP +#include <iterator> #include <map> #include <memory> #include <optional> +#include <sstream> #include <string> #include <unordered_map> +#include <variant> #include <vector> #include "fmt/core.h" +#include "grpcpp/support/status.h" #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/compatibility/compatibility.hpp" @@ -30,6 +34,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" #include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" @@ -454,6 +459,38 @@ class LocalApi final : public IExecutionApi { return result; } + [[nodiscard]] auto SpliceBlob( + ArtifactDigest const& blob_digest, + std::vector<ArtifactDigest> const& chunk_digests) const noexcept + -> std::optional<ArtifactDigest> final { + Logger::Log(LogLevel::Debug, + "SpliceBlob({}, {} chunks)", + blob_digest.hash(), + chunk_digests.size()); + auto digests = std::vector<bazel_re::Digest>{}; + digests.reserve(chunk_digests.size()); + std::transform( + chunk_digests.cbegin(), + chunk_digests.cend(), + std::back_inserter(digests), + [](auto const& artifact_digest) { + return static_cast<bazel_re::Digest>(artifact_digest); + }); + auto splice_result = CASUtils::SpliceBlob( + static_cast<bazel_re::Digest>(blob_digest), digests, *storage_); + if (std::holds_alternative<grpc::Status>(splice_result)) { + auto* status = std::get_if<grpc::Status>(&splice_result); + Logger::Log(LogLevel::Error, status->error_message()); + return std::nullopt; + } + auto* digest = std::get_if<bazel_re::Digest>(&splice_result); + return ArtifactDigest{*digest}; + } + + [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final { + return true; + } + private: std::optional<gsl::not_null<RepositoryConfig*>> repo_config_{}; gsl::not_null<Storage const*> storage_ = &Storage::Instance(); diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index 48cf3133..157016f7 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -17,7 +17,8 @@ #include <algorithm> #include <atomic> #include <cstdint> -#include <fstream> +#include <iterator> +#include <sstream> #include <unordered_map> #include <unordered_set> @@ -120,7 +121,7 @@ namespace { // Fetch unknown chunks. auto digest_set = std::unordered_set<bazel_re::Digest>{ - (*chunk_digests).begin(), (*chunk_digests).end()}; + chunk_digests->begin(), chunk_digests->end()}; auto unique_digests = std::vector<bazel_re::Digest>{digest_set.begin(), digest_set.end()}; auto missing_digests = ::IsAvailable(unique_digests, api); @@ -129,30 +130,24 @@ namespace { } // Assemble blob from chunks. - auto tmp_dir = StorageUtils::CreateTypedTmpDir("splice"); - auto tmp_file = tmp_dir->GetPath() / "blob"; - std::size_t total_size{}; - { - std::ofstream tmp(tmp_file, std::ios::binary); - for (auto const& chunk_digest : *chunk_digests) { - auto info = - Artifact::ObjectInfo{.digest = ArtifactDigest{chunk_digest}, - .type = ObjectType::File}; - auto chunk_data = api->RetrieveToMemory(info); - if (not chunk_data) { - Logger::Log(LogLevel::Error, - "could not load blob chunk in memory: ", - chunk_digest.hash()); - return false; - } - tmp << *chunk_data; - total_size += chunk_data->size(); - } + auto artifact_digests = std::vector<ArtifactDigest>{}; + artifact_digests.reserve(chunk_digests->size()); + std::transform(chunk_digests->cbegin(), + chunk_digests->cend(), + std::back_inserter(artifact_digests), + [](auto const& digest) { return ArtifactDigest{digest}; }); + auto digest = api->SpliceBlob(artifact_info.digest, artifact_digests); + if (not digest) { + // If blob splicing failed, fall back to regular fetching. + return ::RetrieveToCas({artifact_info.digest}, api, network, info_map); } Logger::Log( LogLevel::Debug, - [&artifact_info, &unique_digests, &missing_digests, &total_size]() { + [&artifact_info, + &unique_digests, + &missing_digests, + total_size = digest->size()]() { auto missing_digest_set = std::unordered_set<bazel_re::Digest>{ missing_digests.begin(), missing_digests.end()}; std::uint64_t transmitted_bytes{0}; @@ -172,7 +167,7 @@ namespace { artifact_info.ToString()); }); - return api->UploadFile(tmp_file, artifact_info.type); + return true; } } // namespace @@ -584,3 +579,23 @@ auto BazelApi::CreateAction( } return result; } + +[[nodiscard]] auto BazelApi::SplitBlob(ArtifactDigest const& blob_digest) + const noexcept -> std::optional<std::vector<ArtifactDigest>> { + auto chunk_digests = + network_->SplitBlob(static_cast<bazel_re::Digest>(blob_digest)); + if (not chunk_digests) { + return std::nullopt; + } + auto artifact_digests = std::vector<ArtifactDigest>{}; + artifact_digests.reserve(chunk_digests->size()); + std::transform(chunk_digests->cbegin(), + chunk_digests->cend(), + std::back_inserter(artifact_digests), + [](auto const& digest) { return ArtifactDigest{digest}; }); + return artifact_digests; +} + +[[nodiscard]] auto BazelApi::BlobSplitSupport() const noexcept -> bool { + return network_->BlobSplitSupport(); +} diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index aae90907..2d514ecd 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -96,6 +96,11 @@ class BazelApi final : public IExecutionApi { Artifact::ObjectInfo const& artifact_info) noexcept -> std::optional<std::string> final; + [[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest) + const noexcept -> std::optional<std::vector<ArtifactDigest>> final; + + [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final; + private: std::shared_ptr<BazelNetwork> network_; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index f193bf8b..473dd5ba 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -93,7 +93,7 @@ namespace { return blob_split_support_map[instance_name]; } } - auto supported = BlobSplitSupport(instance_name, stub); + auto supported = ::BlobSplitSupport(instance_name, stub); logger->Emit(LogLevel::Debug, "Blob split support for \"{}\": {}", instance_name, @@ -312,14 +312,14 @@ auto BazelCasClient::ReadSingleBlob(std::string const& instance_name, } auto BazelCasClient::SplitBlob(std::string const& instance_name, - bazel_re::Digest const& digest) noexcept - -> std::optional<std::vector<bazel_re::Digest>> { + bazel_re::Digest const& blob_digest) + const noexcept -> std::optional<std::vector<bazel_re::Digest>> { if (not BlobSplitSupportCached(instance_name, stub_, &logger_)) { return std::nullopt; } bazel_re::SplitBlobRequest request{}; request.set_instance_name(instance_name); - request.mutable_blob_digest()->CopyFrom(digest); + request.mutable_blob_digest()->CopyFrom(blob_digest); bazel_re::SplitBlobResponse response{}; auto [ok, status] = WithRetry( [this, &response, &request]() { @@ -334,6 +334,11 @@ auto BazelCasClient::SplitBlob(std::string const& instance_name, return ProcessResponseContents<bazel_re::Digest>(response); } +auto BazelCasClient::BlobSplitSupport( + std::string const& instance_name) const noexcept -> bool { + return ::BlobSplitSupportCached(instance_name, stub_, &logger_); +} + template <class T_ForwardIter> auto BazelCasClient::FindMissingBlobs(std::string const& instance_name, T_ForwardIter const& start, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp index 762b4f37..77196022 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp @@ -129,12 +129,15 @@ class BazelCasClient { -> std::optional<BazelBlob>; /// @brief Split single blob into chunks - /// @param[in] instance_name Name of the CAS instance - /// @param[in] digest Blob digest to be splitted + /// @param[in] instance_name Name of the CAS instance + /// @param[in] blob_digest Blob digest to be splitted /// @return The chunk digests of the splitted blob [[nodiscard]] auto SplitBlob(std::string const& instance_name, - bazel_re::Digest const& digest) noexcept - -> std::optional<std::vector<bazel_re::Digest>>; + bazel_re::Digest const& blob_digest) + const noexcept -> std::optional<std::vector<bazel_re::Digest>>; + + [[nodiscard]] auto BlobSplitSupport( + std::string const& instance_name) const noexcept -> bool; private: std::unique_ptr<ByteStreamClient> stream_{}; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 223960cb..04c13589 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -154,9 +154,13 @@ auto BazelNetwork::IsAvailable(std::vector<bazel_re::Digest> const& digests) return cas_->FindMissingBlobs(instance_name_, digests); } -auto BazelNetwork::SplitBlob(bazel_re::Digest const& digest) const noexcept +auto BazelNetwork::SplitBlob(bazel_re::Digest const& blob_digest) const noexcept -> std::optional<std::vector<bazel_re::Digest>> { - return cas_->SplitBlob(instance_name_, digest); + return cas_->SplitBlob(instance_name_, blob_digest); +} + +auto BazelNetwork::BlobSplitSupport() const noexcept -> bool { + return cas_->BlobSplitSupport(instance_name_); } template <class T_Iter> diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index af48c7db..0808d2e9 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -75,8 +75,10 @@ class BazelNetwork { [[nodiscard]] auto IsAvailable(std::vector<bazel_re::Digest> const& digests) const noexcept -> std::vector<bazel_re::Digest>; - [[nodiscard]] auto SplitBlob(bazel_re::Digest const& digest) const noexcept - -> std::optional<std::vector<bazel_re::Digest>>; + [[nodiscard]] auto SplitBlob(bazel_re::Digest const& blob_digest) + const noexcept -> std::optional<std::vector<bazel_re::Digest>>; + + [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool; /// \brief Uploads blobs to CAS /// \param blobs The blobs to upload |