summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorSascha Roloff <sascha.roloff@huawei.com>2024-02-23 09:32:54 +0100
committerSascha Roloff <sascha.roloff@huawei.com>2024-02-26 17:16:21 +0100
commitd83c997ad5a866f4fbb38d4a81e7edf70a491db2 (patch)
tree7331b4d58d2c56adecb6e9879862cf354129d6af /src
parent4ae3f068372041f949538fb273113a4a1c665a0f (diff)
downloadjustbuild-d83c997ad5a866f4fbb38d4a81e7edf70a491db2.tar.gz
Refactor split and splice implementations.
Currently, the implementations of the split and splice operation are both hidden behind the Bazel API implementation. This was sufficient to implement splitting at the server and splicing at the client. In order to support the other direction of splitting at the client and splicing at the server while reusing their implementations, the code needs to be refactored. First, the functionality of split and splice are explicitly exposed at the general execution API interface and implemented in the sub APIs. Second, the implementations of split and splice are factored into a separate utils class.
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_api/common/execution_api.hpp20
-rw-r--r--src/buildtool/execution_api/execution_service/TARGETS30
-rw-r--r--src/buildtool/execution_api/execution_service/cas_server.cpp103
-rw-r--r--src/buildtool/execution_api/execution_service/cas_server.hpp4
-rw-r--r--src/buildtool/execution_api/execution_service/cas_utils.cpp155
-rw-r--r--src/buildtool/execution_api/execution_service/cas_utils.hpp45
-rw-r--r--src/buildtool/execution_api/local/TARGETS2
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp37
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.cpp61
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.hpp5
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp13
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp11
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_network.cpp8
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_network.hpp6
14 files changed, 364 insertions, 136 deletions
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp
index 43e16783..993f6d17 100644
--- a/src/buildtool/execution_api/common/execution_api.hpp
+++ b/src/buildtool/execution_api/common/execution_api.hpp
@@ -135,6 +135,26 @@ class IExecutionApi {
[[nodiscard]] virtual auto IsAvailable(
std::vector<ArtifactDigest> const& digests) const noexcept
-> std::vector<ArtifactDigest> = 0;
+
+ [[nodiscard]] virtual auto SplitBlob(ArtifactDigest const& /*blob_digest*/)
+ const noexcept -> std::optional<std::vector<ArtifactDigest>> {
+ return std::nullopt;
+ }
+
+ [[nodiscard]] virtual auto BlobSplitSupport() const noexcept -> bool {
+ return false;
+ }
+
+ [[nodiscard]] virtual auto SpliceBlob(
+ ArtifactDigest const& /*blob_digest*/,
+ std::vector<ArtifactDigest> const& /*chunk_digests*/) const noexcept
+ -> std::optional<ArtifactDigest> {
+ return std::nullopt;
+ }
+
+ [[nodiscard]] virtual auto BlobSpliceSupport() const noexcept -> bool {
+ return false;
+ }
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS
index 85068f0f..e494711c 100644
--- a/src/buildtool/execution_api/execution_service/TARGETS
+++ b/src/buildtool/execution_api/execution_service/TARGETS
@@ -58,12 +58,7 @@
, ["@", "fmt", "", "fmt"]
, ["src/buildtool/storage", "storage"]
, ["src/utils/cpp", "verify_hash"]
- , ["src/buildtool/common", "common"]
- , ["src/buildtool/file_system", "git_repo"]
- , ["src/buildtool/file_system", "object_type"]
- , ["src/buildtool/logging", "log_level"]
- , ["src/utils/cpp", "hex_string"]
- , "file_chunker"
+ , "cas_utils"
]
}
, "server_implementation":
@@ -151,4 +146,27 @@
, "stage": ["src", "buildtool", "execution_api", "execution_service"]
, "private-deps": [["@", "gsl", "", "gsl"]]
}
+, "cas_utils":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["cas_utils"]
+ , "hdrs": ["cas_utils.hpp"]
+ , "srcs": ["cas_utils.cpp"]
+ , "stage": ["src", "buildtool", "execution_api", "execution_service"]
+ , "deps":
+ [ ["@", "grpc", "", "grpc++"]
+ , ["src/buildtool/common", "bazel_types"]
+ , ["src/buildtool/storage", "storage"]
+ ]
+ , "private-deps":
+ [ "file_chunker"
+ , ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/compatibility", "compatibility"]
+ , ["src/buildtool/file_system", "git_repo"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/storage", "fs_utils"]
+ , ["src/utils/cpp", "hex_string"]
+ ]
+ }
}
diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp
index 49939c20..4fd78323 100644
--- a/src/buildtool/execution_api/execution_service/cas_server.cpp
+++ b/src/buildtool/execution_api/execution_service/cas_server.cpp
@@ -21,15 +21,10 @@
#include <vector>
#include "fmt/core.h"
-#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/compatibility/native_support.hpp"
-#include "src/buildtool/execution_api/execution_service/file_chunker.hpp"
-#include "src/buildtool/file_system/git_repo.hpp"
-#include "src/buildtool/file_system/object_type.hpp"
-#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/execution_api/execution_service/cas_utils.hpp"
#include "src/buildtool/storage/garbage_collector.hpp"
-#include "src/utils/cpp/hex_string.hpp"
#include "src/utils/cpp/verify_hash.hpp"
static constexpr std::size_t kGitSHA1Length = 42;
@@ -100,46 +95,6 @@ auto CASServiceImpl::CheckDigestConsistency(bazel_re::Digest const& ref,
return std::nullopt;
}
-auto CASServiceImpl::EnsureTreeInvariant(std::string const& data,
- std::string const& hash) const noexcept
- -> std::optional<std::string> {
- auto entries = GitRepo::ReadTreeData(
- data,
- NativeSupport::Unprefix(hash),
- [](auto const& /*unused*/) { return true; },
- /*is_hex_id=*/true);
- if (not entries) {
- auto str = fmt::format("Could not read tree data {}", hash);
- logger_.Emit(LogLevel::Error, str);
- return str;
- }
- for (auto const& entry : *entries) {
- for (auto const& item : entry.second) {
- auto digest = static_cast<bazel_re::Digest>(
- ArtifactDigest{ToHexString(entry.first),
- /*size is unknown*/ 0,
- IsTreeObject(item.type)});
- if (not(IsTreeObject(item.type)
- ? storage_->CAS().TreePath(digest)
- : storage_->CAS().BlobPath(digest, false))) {
- auto str = fmt::format(
- "Tree invariant violated {}: missing element {}",
- hash,
- digest.hash());
- logger_.Emit(LogLevel::Error, str);
- return str;
- }
- // The GitRepo::tree_entries_t data structure maps the object id to
- // a list of entries of that object in possibly multiple trees. It
- // is sufficient to check the existence of only one of these entries
- // to be sure that the object is in CAS since they all have the same
- // content.
- break;
- }
- }
- return std::nullopt;
-}
-
auto CASServiceImpl::BatchUpdateBlobs(
::grpc::ServerContext* /*context*/,
const ::bazel_re::BatchUpdateBlobsRequest* request,
@@ -166,7 +121,8 @@ auto CASServiceImpl::BatchUpdateBlobs(
if (NativeSupport::IsTree(hash)) {
// In native mode: for trees, check whether the tree invariant holds
// before storing the actual tree object.
- if (auto err = EnsureTreeInvariant(x.data(), hash)) {
+ if (auto err =
+ CASUtils::EnsureTreeInvariant(x.data(), hash, *storage_)) {
return ::grpc::Status{grpc::StatusCode::FAILED_PRECONDITION,
*err};
}
@@ -273,55 +229,20 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/,
logger_.Emit(LogLevel::Info, "SplitBlob({})", blob_digest.hash());
- // Check blob existence.
- auto path = std::optional<std::filesystem::path>{};
- if (NativeSupport::IsTree(blob_digest.hash())) {
- path = storage_->CAS().TreePath(blob_digest);
- }
- else {
- path = storage_->CAS().BlobPath(blob_digest, true);
- if (not path) {
- path = storage_->CAS().BlobPath(blob_digest, false);
- }
- }
- if (not path) {
- auto str =
- fmt::format("SplitBlob: blob not found {}", blob_digest.hash());
- logger_.Emit(LogLevel::Error, str);
- return ::grpc::Status{grpc::StatusCode::NOT_FOUND, str};
- }
-
- // Split blob into chunks, store each chunk in CAS, and collect chunk
- // digests.
- auto chunker = FileChunker{*path};
- if (not chunker.IsOpen()) {
- auto str = fmt::format("SplitBlob: could not open blob for reading");
+ // Split blob into chunks.
+ auto split_result = CASUtils::SplitBlob(blob_digest, *storage_);
+ if (std::holds_alternative<grpc::Status>(split_result)) {
+ auto status = std::get<grpc::Status>(split_result);
+ auto str = fmt::format("SplitBlob: {}", status.error_message());
logger_.Emit(LogLevel::Error, str);
- return ::grpc::Status{grpc::StatusCode::INTERNAL, str};
- }
-
- auto chunk_digests = std::vector<bazel_re::Digest>{};
- while (auto chunk = chunker.NextChunk()) {
- auto chunk_digest = storage_->CAS().StoreBlob(*chunk, false);
- if (not chunk_digest) {
- auto str =
- fmt::format("SplitBlob: could not store chunk of blob {}",
- blob_digest.hash());
- logger_.Emit(LogLevel::Error, str);
- return ::grpc::Status{grpc::StatusCode::RESOURCE_EXHAUSTED, str};
- }
- chunk_digests.emplace_back(*chunk_digest);
- }
- if (not chunker.Finished()) {
- auto str =
- fmt::format("SplitBlob: could split blob {}", blob_digest.hash());
- logger_.Emit(LogLevel::Error, str);
- return ::grpc::Status{grpc::StatusCode::INTERNAL, str};
+ return ::grpc::Status{status.error_code(), str};
}
+ auto chunk_digests = std::get<std::vector<bazel_re::Digest>>(split_result);
logger_.Emit(LogLevel::Debug, [&blob_digest, &chunk_digests]() {
std::stringstream ss{};
ss << "Split blob " << blob_digest.hash() << ":"
- << blob_digest.size_bytes() << " into [ ";
+ << blob_digest.size_bytes() << " into " << chunk_digests.size()
+ << " chunks: [ ";
for (auto const& chunk_digest : chunk_digests) {
ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes()
<< " ";
diff --git a/src/buildtool/execution_api/execution_service/cas_server.hpp b/src/buildtool/execution_api/execution_service/cas_server.hpp
index c9f7cff4..306c2833 100644
--- a/src/buildtool/execution_api/execution_service/cas_server.hpp
+++ b/src/buildtool/execution_api/execution_service/cas_server.hpp
@@ -147,10 +147,6 @@ class CASServiceImpl final
bazel_re::Digest const& computed)
const noexcept -> std::optional<std::string>;
- [[nodiscard]] auto EnsureTreeInvariant(
- std::string const& data,
- std::string const& hash) const noexcept -> std::optional<std::string>;
-
gsl::not_null<Storage const*> storage_ = &Storage::Instance();
Logger logger_{"execution-service"};
};
diff --git a/src/buildtool/execution_api/execution_service/cas_utils.cpp b/src/buildtool/execution_api/execution_service/cas_utils.cpp
new file mode 100644
index 00000000..3da56e28
--- /dev/null
+++ b/src/buildtool/execution_api/execution_service/cas_utils.cpp
@@ -0,0 +1,155 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/execution_api/execution_service/cas_utils.hpp"
+
+#include <fstream>
+
+#include "fmt/core.h"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/compatibility/native_support.hpp"
+#include "src/buildtool/execution_api/execution_service/file_chunker.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/git_repo.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/storage/fs_utils.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+auto CASUtils::EnsureTreeInvariant(std::string const& data,
+ std::string const& hash,
+ Storage const& storage) noexcept
+ -> std::optional<std::string> {
+ auto entries = GitRepo::ReadTreeData(
+ data,
+ NativeSupport::Unprefix(hash),
+ [](auto const& /*unused*/) { return true; },
+ /*is_hex_id=*/true);
+ if (not entries) {
+ return fmt::format("could not read tree data {}", hash);
+ }
+ for (auto const& entry : *entries) {
+ for (auto const& item : entry.second) {
+ auto digest = static_cast<bazel_re::Digest>(
+ ArtifactDigest{ToHexString(entry.first),
+ /*size is unknown*/ 0,
+ IsTreeObject(item.type)});
+ if (not(IsTreeObject(item.type)
+ ? storage.CAS().TreePath(digest)
+ : storage.CAS().BlobPath(digest, false))) {
+ return fmt::format(
+ "tree invariant violated {}: missing element {}",
+ hash,
+ digest.hash());
+ }
+ // The GitRepo::tree_entries_t data structure maps the object id to
+ // a list of entries of that object in possibly multiple trees. It
+ // is sufficient to check the existence of only one of these entries
+ // to be sure that the object is in CAS since they all have the same
+ // content.
+ break;
+ }
+ }
+ return std::nullopt;
+}
+
+auto CASUtils::SplitBlob(bazel_re::Digest const& blob_digest,
+ Storage const& storage) noexcept
+ -> std::variant<std::vector<bazel_re::Digest>, grpc::Status> {
+
+ // Check blob existence.
+ auto path = NativeSupport::IsTree(blob_digest.hash())
+ ? storage.CAS().TreePath(blob_digest)
+ : storage.CAS().BlobPath(blob_digest, false);
+ if (not path) {
+ return grpc::Status{
+ grpc::StatusCode::NOT_FOUND,
+ fmt::format("blob not found {}", blob_digest.hash())};
+ }
+
+ // Split blob into chunks, store each chunk in CAS, and collect chunk
+ // digests.
+ auto chunker = FileChunker{*path};
+ if (not chunker.IsOpen()) {
+ return grpc::Status{
+ grpc::StatusCode::INTERNAL,
+ fmt::format("could not open blob {}", blob_digest.hash())};
+ }
+ auto chunk_digests = std::vector<bazel_re::Digest>{};
+ while (auto chunk = chunker.NextChunk()) {
+ auto chunk_digest = storage.CAS().StoreBlob(*chunk, false);
+ if (not chunk_digest) {
+ return grpc::Status{grpc::StatusCode::INTERNAL,
+ fmt::format("could not store chunk of blob {}",
+ blob_digest.hash())};
+ }
+ chunk_digests.emplace_back(*chunk_digest);
+ }
+ if (not chunker.Finished()) {
+ return grpc::Status{
+ grpc::StatusCode::INTERNAL,
+ fmt::format("could not split blob {}", blob_digest.hash())};
+ }
+
+ return chunk_digests;
+}
+
+auto CASUtils::SpliceBlob(bazel_re::Digest const& blob_digest,
+ std::vector<bazel_re::Digest> const& chunk_digests,
+ Storage const& storage) noexcept
+ -> std::variant<bazel_re::Digest, grpc::Status> {
+
+ // Assemble blob from chunks.
+ auto tmp_dir = StorageUtils::CreateTypedTmpDir("splice");
+ auto tmp_file = tmp_dir->GetPath() / "blob";
+ {
+ std::ofstream tmp(tmp_file, std::ios::binary);
+ for (auto const& chunk_digest : chunk_digests) {
+ // Check chunk existence (only check file CAS).
+ auto path = storage.CAS().BlobPath(chunk_digest, false);
+ if (not path) {
+ return grpc::Status{
+ grpc::StatusCode::NOT_FOUND,
+ fmt::format("chunk not found {}", chunk_digest.hash())};
+ }
+ // Load chunk data.
+ auto chunk_data = FileSystemManager::ReadFile(*path);
+ if (not chunk_data) {
+ return grpc::Status{grpc::StatusCode::INTERNAL,
+ fmt::format("could read chunk data {}",
+ chunk_digest.hash())};
+ }
+ tmp << *chunk_data;
+ }
+ }
+
+ // Store resulting blob in according CAS.
+ auto const& hash = blob_digest.hash();
+ if (NativeSupport::IsTree(hash)) {
+ auto const& digest =
+ storage.CAS().StoreTree</* kOwner= */ true>(tmp_file);
+ if (not digest) {
+ return grpc::Status{grpc::StatusCode::INTERNAL,
+ fmt::format("could not store tree {}", hash)};
+ }
+ return *digest;
+ }
+
+ auto const& digest =
+ storage.CAS().StoreBlob</* kOwner= */ true>(tmp_file, false);
+ if (not digest) {
+ return grpc::Status{grpc::StatusCode::INTERNAL,
+ fmt::format("could not store blob {}", hash)};
+ }
+ return *digest;
+}
diff --git a/src/buildtool/execution_api/execution_service/cas_utils.hpp b/src/buildtool/execution_api/execution_service/cas_utils.hpp
new file mode 100644
index 00000000..b22260b8
--- /dev/null
+++ b/src/buildtool/execution_api/execution_service/cas_utils.hpp
@@ -0,0 +1,45 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP
+
+#include <optional>
+#include <string>
+#include <variant>
+#include <vector>
+
+#include "grpcpp/support/status.h"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/storage/storage.hpp"
+
+class CASUtils {
+ public:
+ [[nodiscard]] static auto EnsureTreeInvariant(
+ std::string const& data,
+ std::string const& hash,
+ Storage const& storage) noexcept -> std::optional<std::string>;
+
+ [[nodiscard]] static auto SplitBlob(bazel_re::Digest const& blob_digest,
+ Storage const& storage) noexcept
+ -> std::variant<std::vector<bazel_re::Digest>, grpc::Status>;
+
+ [[nodiscard]] static auto SpliceBlob(
+ bazel_re::Digest const& blob_digest,
+ std::vector<bazel_re::Digest> const& chunk_digests,
+ Storage const& storage) noexcept
+ -> std::variant<bazel_re::Digest, grpc::Status>;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP
diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS
index c8895c20..f4cfde76 100644
--- a/src/buildtool/execution_api/local/TARGETS
+++ b/src/buildtool/execution_api/local/TARGETS
@@ -26,6 +26,7 @@
, "deps":
[ ["@", "fmt", "", "fmt"]
, ["@", "gsl", "", "gsl"]
+ , ["@", "grpc", "", "grpc++"]
, ["src/buildtool/common", "config"]
, ["src/buildtool/storage", "storage"]
, ["src/buildtool/execution_api/common", "common"]
@@ -36,6 +37,7 @@
, ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
, ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/execution_api/execution_service", "cas_utils"]
]
, "stage": ["src", "buildtool", "execution_api", "local"]
, "private-deps":
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 1e3de092..5880475e 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -15,14 +15,18 @@
#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP
#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP
+#include <iterator>
#include <map>
#include <memory>
#include <optional>
+#include <sstream>
#include <string>
#include <unordered_map>
+#include <variant>
#include <vector>
#include "fmt/core.h"
+#include "grpcpp/support/status.h"
#include "gsl/gsl"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
@@ -30,6 +34,7 @@
#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/execution_service/cas_utils.hpp"
#include "src/buildtool/execution_api/git/git_api.hpp"
#include "src/buildtool/execution_api/local/local_action.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
@@ -454,6 +459,38 @@ class LocalApi final : public IExecutionApi {
return result;
}
+ [[nodiscard]] auto SpliceBlob(
+ ArtifactDigest const& blob_digest,
+ std::vector<ArtifactDigest> const& chunk_digests) const noexcept
+ -> std::optional<ArtifactDigest> final {
+ Logger::Log(LogLevel::Debug,
+ "SpliceBlob({}, {} chunks)",
+ blob_digest.hash(),
+ chunk_digests.size());
+ auto digests = std::vector<bazel_re::Digest>{};
+ digests.reserve(chunk_digests.size());
+ std::transform(
+ chunk_digests.cbegin(),
+ chunk_digests.cend(),
+ std::back_inserter(digests),
+ [](auto const& artifact_digest) {
+ return static_cast<bazel_re::Digest>(artifact_digest);
+ });
+ auto splice_result = CASUtils::SpliceBlob(
+ static_cast<bazel_re::Digest>(blob_digest), digests, *storage_);
+ if (std::holds_alternative<grpc::Status>(splice_result)) {
+ auto* status = std::get_if<grpc::Status>(&splice_result);
+ Logger::Log(LogLevel::Error, status->error_message());
+ return std::nullopt;
+ }
+ auto* digest = std::get_if<bazel_re::Digest>(&splice_result);
+ return ArtifactDigest{*digest};
+ }
+
+ [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final {
+ return true;
+ }
+
private:
std::optional<gsl::not_null<RepositoryConfig*>> repo_config_{};
gsl::not_null<Storage const*> storage_ = &Storage::Instance();
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
index 48cf3133..157016f7 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
@@ -17,7 +17,8 @@
#include <algorithm>
#include <atomic>
#include <cstdint>
-#include <fstream>
+#include <iterator>
+#include <sstream>
#include <unordered_map>
#include <unordered_set>
@@ -120,7 +121,7 @@ namespace {
// Fetch unknown chunks.
auto digest_set = std::unordered_set<bazel_re::Digest>{
- (*chunk_digests).begin(), (*chunk_digests).end()};
+ chunk_digests->begin(), chunk_digests->end()};
auto unique_digests =
std::vector<bazel_re::Digest>{digest_set.begin(), digest_set.end()};
auto missing_digests = ::IsAvailable(unique_digests, api);
@@ -129,30 +130,24 @@ namespace {
}
// Assemble blob from chunks.
- auto tmp_dir = StorageUtils::CreateTypedTmpDir("splice");
- auto tmp_file = tmp_dir->GetPath() / "blob";
- std::size_t total_size{};
- {
- std::ofstream tmp(tmp_file, std::ios::binary);
- for (auto const& chunk_digest : *chunk_digests) {
- auto info =
- Artifact::ObjectInfo{.digest = ArtifactDigest{chunk_digest},
- .type = ObjectType::File};
- auto chunk_data = api->RetrieveToMemory(info);
- if (not chunk_data) {
- Logger::Log(LogLevel::Error,
- "could not load blob chunk in memory: ",
- chunk_digest.hash());
- return false;
- }
- tmp << *chunk_data;
- total_size += chunk_data->size();
- }
+ auto artifact_digests = std::vector<ArtifactDigest>{};
+ artifact_digests.reserve(chunk_digests->size());
+ std::transform(chunk_digests->cbegin(),
+ chunk_digests->cend(),
+ std::back_inserter(artifact_digests),
+ [](auto const& digest) { return ArtifactDigest{digest}; });
+ auto digest = api->SpliceBlob(artifact_info.digest, artifact_digests);
+ if (not digest) {
+ // If blob splicing failed, fall back to regular fetching.
+ return ::RetrieveToCas({artifact_info.digest}, api, network, info_map);
}
Logger::Log(
LogLevel::Debug,
- [&artifact_info, &unique_digests, &missing_digests, &total_size]() {
+ [&artifact_info,
+ &unique_digests,
+ &missing_digests,
+ total_size = digest->size()]() {
auto missing_digest_set = std::unordered_set<bazel_re::Digest>{
missing_digests.begin(), missing_digests.end()};
std::uint64_t transmitted_bytes{0};
@@ -172,7 +167,7 @@ namespace {
artifact_info.ToString());
});
- return api->UploadFile(tmp_file, artifact_info.type);
+ return true;
}
} // namespace
@@ -584,3 +579,23 @@ auto BazelApi::CreateAction(
}
return result;
}
+
+[[nodiscard]] auto BazelApi::SplitBlob(ArtifactDigest const& blob_digest)
+ const noexcept -> std::optional<std::vector<ArtifactDigest>> {
+ auto chunk_digests =
+ network_->SplitBlob(static_cast<bazel_re::Digest>(blob_digest));
+ if (not chunk_digests) {
+ return std::nullopt;
+ }
+ auto artifact_digests = std::vector<ArtifactDigest>{};
+ artifact_digests.reserve(chunk_digests->size());
+ std::transform(chunk_digests->cbegin(),
+ chunk_digests->cend(),
+ std::back_inserter(artifact_digests),
+ [](auto const& digest) { return ArtifactDigest{digest}; });
+ return artifact_digests;
+}
+
+[[nodiscard]] auto BazelApi::BlobSplitSupport() const noexcept -> bool {
+ return network_->BlobSplitSupport();
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
index aae90907..2d514ecd 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
@@ -96,6 +96,11 @@ class BazelApi final : public IExecutionApi {
Artifact::ObjectInfo const& artifact_info) noexcept
-> std::optional<std::string> final;
+ [[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest)
+ const noexcept -> std::optional<std::vector<ArtifactDigest>> final;
+
+ [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final;
+
private:
std::shared_ptr<BazelNetwork> network_;
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp
index f193bf8b..473dd5ba 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp
@@ -93,7 +93,7 @@ namespace {
return blob_split_support_map[instance_name];
}
}
- auto supported = BlobSplitSupport(instance_name, stub);
+ auto supported = ::BlobSplitSupport(instance_name, stub);
logger->Emit(LogLevel::Debug,
"Blob split support for \"{}\": {}",
instance_name,
@@ -312,14 +312,14 @@ auto BazelCasClient::ReadSingleBlob(std::string const& instance_name,
}
auto BazelCasClient::SplitBlob(std::string const& instance_name,
- bazel_re::Digest const& digest) noexcept
- -> std::optional<std::vector<bazel_re::Digest>> {
+ bazel_re::Digest const& blob_digest)
+ const noexcept -> std::optional<std::vector<bazel_re::Digest>> {
if (not BlobSplitSupportCached(instance_name, stub_, &logger_)) {
return std::nullopt;
}
bazel_re::SplitBlobRequest request{};
request.set_instance_name(instance_name);
- request.mutable_blob_digest()->CopyFrom(digest);
+ request.mutable_blob_digest()->CopyFrom(blob_digest);
bazel_re::SplitBlobResponse response{};
auto [ok, status] = WithRetry(
[this, &response, &request]() {
@@ -334,6 +334,11 @@ auto BazelCasClient::SplitBlob(std::string const& instance_name,
return ProcessResponseContents<bazel_re::Digest>(response);
}
+auto BazelCasClient::BlobSplitSupport(
+ std::string const& instance_name) const noexcept -> bool {
+ return ::BlobSplitSupportCached(instance_name, stub_, &logger_);
+}
+
template <class T_ForwardIter>
auto BazelCasClient::FindMissingBlobs(std::string const& instance_name,
T_ForwardIter const& start,
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp
index 762b4f37..77196022 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp
@@ -129,12 +129,15 @@ class BazelCasClient {
-> std::optional<BazelBlob>;
/// @brief Split single blob into chunks
- /// @param[in] instance_name Name of the CAS instance
- /// @param[in] digest Blob digest to be splitted
+ /// @param[in] instance_name Name of the CAS instance
+ /// @param[in] blob_digest Blob digest to be splitted
/// @return The chunk digests of the splitted blob
[[nodiscard]] auto SplitBlob(std::string const& instance_name,
- bazel_re::Digest const& digest) noexcept
- -> std::optional<std::vector<bazel_re::Digest>>;
+ bazel_re::Digest const& blob_digest)
+ const noexcept -> std::optional<std::vector<bazel_re::Digest>>;
+
+ [[nodiscard]] auto BlobSplitSupport(
+ std::string const& instance_name) const noexcept -> bool;
private:
std::unique_ptr<ByteStreamClient> stream_{};
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp
index 223960cb..04c13589 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp
@@ -154,9 +154,13 @@ auto BazelNetwork::IsAvailable(std::vector<bazel_re::Digest> const& digests)
return cas_->FindMissingBlobs(instance_name_, digests);
}
-auto BazelNetwork::SplitBlob(bazel_re::Digest const& digest) const noexcept
+auto BazelNetwork::SplitBlob(bazel_re::Digest const& blob_digest) const noexcept
-> std::optional<std::vector<bazel_re::Digest>> {
- return cas_->SplitBlob(instance_name_, digest);
+ return cas_->SplitBlob(instance_name_, blob_digest);
+}
+
+auto BazelNetwork::BlobSplitSupport() const noexcept -> bool {
+ return cas_->BlobSplitSupport(instance_name_);
}
template <class T_Iter>
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp
index af48c7db..0808d2e9 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp
@@ -75,8 +75,10 @@ class BazelNetwork {
[[nodiscard]] auto IsAvailable(std::vector<bazel_re::Digest> const& digests)
const noexcept -> std::vector<bazel_re::Digest>;
- [[nodiscard]] auto SplitBlob(bazel_re::Digest const& digest) const noexcept
- -> std::optional<std::vector<bazel_re::Digest>>;
+ [[nodiscard]] auto SplitBlob(bazel_re::Digest const& blob_digest)
+ const noexcept -> std::optional<std::vector<bazel_re::Digest>>;
+
+ [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool;
/// \brief Uploads blobs to CAS
/// \param blobs The blobs to upload