summaryrefslogtreecommitdiff
path: root/src/buildtool/execution_api
diff options
context:
space:
mode:
authorMaksim Denisov <denisov.maksim@huawei.com>2024-07-23 18:20:44 +0200
committerMaksim Denisov <denisov.maksim@huawei.com>2024-08-30 17:17:09 +0200
commit0df9bfcbcda9f87097bd313819be0be2cf5fa892 (patch)
treebcffd45a05d8994b2e6e5bd9cd5b157eb69802c3 /src/buildtool/execution_api
parentd84ab0c8f1e0df651c2d1f6cd9f4bdc03c25932c (diff)
downloadjustbuild-0df9bfcbcda9f87097bd313819be0be2cf5fa892.tar.gz
Replace bazel_re::Digest in CASUtils
...with ArtifactDigest
Diffstat (limited to 'src/buildtool/execution_api')
-rw-r--r--src/buildtool/execution_api/execution_service/TARGETS2
-rw-r--r--src/buildtool/execution_api/execution_service/bytestream_server.cpp4
-rw-r--r--src/buildtool/execution_api/execution_service/cas_server.cpp41
-rw-r--r--src/buildtool/execution_api/execution_service/cas_utils.cpp76
-rw-r--r--src/buildtool/execution_api/execution_service/cas_utils.hpp18
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp35
6 files changed, 72 insertions, 104 deletions
diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS
index fb88cb05..ecdccf04 100644
--- a/src/buildtool/execution_api/execution_service/TARGETS
+++ b/src/buildtool/execution_api/execution_service/TARGETS
@@ -169,7 +169,7 @@
, "stage": ["src", "buildtool", "execution_api", "execution_service"]
, "deps":
[ ["@", "grpc", "", "grpc++"]
- , ["src/buildtool/common", "bazel_types"]
+ , ["src/buildtool/common", "common"]
, ["src/buildtool/storage", "storage"]
, ["src/utils/cpp", "expected"]
]
diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp
index 0f77542d..1b2aad14 100644
--- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp
+++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp
@@ -172,8 +172,8 @@ auto BytestreamServiceImpl::Write(
}
ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, true};
- if (auto err = CASUtils::EnsureTreeInvariant(
- static_cast<bazel_re::Digest>(dgst), *content, storage_)) {
+ if (auto err =
+ CASUtils::EnsureTreeInvariant(dgst, *content, storage_)) {
auto const str = fmt::format("Write: {}", *std::move(err));
logger_.Emit(LogLevel::Error, "{}", str);
return ::grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, str};
diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp
index a2a37e37..2c2260ac 100644
--- a/src/buildtool/execution_api/execution_service/cas_server.cpp
+++ b/src/buildtool/execution_api/execution_service/cas_server.cpp
@@ -145,12 +145,12 @@ auto CASServiceImpl::BatchUpdateBlobs(
auto* r = response->add_responses();
r->mutable_digest()->CopyFrom(x.digest());
- bool const is_tree = NativeSupport::IsTree(hash);
- if (is_tree) {
+ ArtifactDigest const digest{x.digest()};
+ if (digest.IsTree()) {
// In native mode: for trees, check whether the tree invariant holds
// before storing the actual tree object.
- if (auto err = CASUtils::EnsureTreeInvariant(
- x.digest(), x.data(), storage_)) {
+ if (auto err =
+ CASUtils::EnsureTreeInvariant(digest, x.data(), storage_)) {
auto const str =
fmt::format("BatchUpdateBlobs: {}", *std::move(err));
logger_.Emit(LogLevel::Error, "{}", str);
@@ -160,20 +160,20 @@ auto CASServiceImpl::BatchUpdateBlobs(
}
auto const cas_digest =
- is_tree
+ digest.IsTree()
? storage_.CAS().StoreTree(x.data())
: storage_.CAS().StoreBlob(x.data(), /*is_executable=*/false);
if (not cas_digest) {
auto const str =
fmt::format("BatchUpdateBlobs: could not upload {} {}",
- is_tree ? "tree" : "blob",
+ digest.IsTree() ? "tree" : "blob",
hash);
logger_.Emit(LogLevel::Error, "{}", str);
return ::grpc::Status{grpc::StatusCode::INTERNAL, str};
}
- if (auto err = CheckDigestConsistency(x.digest(), *cas_digest)) {
+ if (auto err = CheckDigestConsistency(digest, *cas_digest)) {
auto const str =
fmt::format("BatchUpdateBlobs: {}", *std::move(err));
logger_.Emit(LogLevel::Error, "{}", str);
@@ -278,11 +278,12 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/,
}
// Split blob into chunks.
+ ArtifactDigest const digest{blob_digest};
auto split_result = chunking_algorithm ==
::bazel_re::ChunkingAlgorithm_Value::
ChunkingAlgorithm_Value_IDENTITY
- ? CASUtils::SplitBlobIdentity(blob_digest, storage_)
- : CASUtils::SplitBlobFastCDC(blob_digest, storage_);
+ ? CASUtils::SplitBlobIdentity(digest, storage_)
+ : CASUtils::SplitBlobFastCDC(digest, storage_);
if (not split_result) {
auto const& status = split_result.error();
@@ -298,16 +299,18 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/,
<< blob_digest.size_bytes() << " into " << chunk_digests.size()
<< " chunks: [ ";
for (auto const& chunk_digest : chunk_digests) {
- ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes()
- << " ";
+ ss << chunk_digest.hash() << ":" << chunk_digest.size() << " ";
}
ss << "]";
return ss.str();
});
- std::copy(chunk_digests.cbegin(),
- chunk_digests.cend(),
- pb::back_inserter(response->mutable_chunk_digests()));
+ std::transform(chunk_digests.cbegin(),
+ chunk_digests.cend(),
+ pb::back_inserter(response->mutable_chunk_digests()),
+ [](ArtifactDigest const& digest) {
+ return static_cast<bazel_re::Digest>(digest);
+ });
return ::grpc::Status::OK;
}
@@ -343,7 +346,8 @@ auto CASServiceImpl::SpliceBlob(::grpc::ServerContext* /*context*/,
return ::grpc::Status{grpc::StatusCode::INTERNAL, str};
}
- auto chunk_digests = std::vector<bazel_re::Digest>{};
+ ArtifactDigest const digest{blob_digest};
+ auto chunk_digests = std::vector<ArtifactDigest>{};
chunk_digests.reserve(request->chunk_digests().size());
for (auto const& x : request->chunk_digests()) {
if (not IsValidHash(x.hash())) {
@@ -352,19 +356,18 @@ auto CASServiceImpl::SpliceBlob(::grpc::ServerContext* /*context*/,
logger_.Emit(LogLevel::Error, "{}", str);
return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str};
}
- chunk_digests.push_back(x);
+ chunk_digests.push_back(ArtifactDigest{x});
}
// Splice blob from chunks.
- auto splice_result =
- CASUtils::SpliceBlob(blob_digest, chunk_digests, storage_);
+ auto splice_result = CASUtils::SpliceBlob(digest, chunk_digests, storage_);
if (not splice_result) {
auto const& status = splice_result.error();
auto const str = fmt::format("SpliceBlob: {}", status.error_message());
logger_.Emit(LogLevel::Error, "{}", str);
return ::grpc::Status{status.error_code(), str};
}
- if (auto err = CheckDigestConsistency(blob_digest, *splice_result)) {
+ if (auto err = CheckDigestConsistency(digest, *splice_result)) {
auto const str = fmt::format("SpliceBlob: {}", *err);
logger_.Emit(LogLevel::Error, "{}", str);
return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str};
diff --git a/src/buildtool/execution_api/execution_service/cas_utils.cpp b/src/buildtool/execution_api/execution_service/cas_utils.cpp
index a01cadc6..a406bdec 100644
--- a/src/buildtool/execution_api/execution_service/cas_utils.cpp
+++ b/src/buildtool/execution_api/execution_service/cas_utils.cpp
@@ -14,8 +14,12 @@
#include "src/buildtool/execution_api/execution_service/cas_utils.hpp"
+#include <utility>
+
#include "fmt/core.h"
#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/storage/large_object_cas.hpp"
+#include "src/buildtool/storage/local_cas.hpp"
static auto ToGrpc(LargeObjectError&& error) noexcept -> grpc::Status {
switch (error.Code()) {
@@ -33,25 +37,24 @@ static auto ToGrpc(LargeObjectError&& error) noexcept -> grpc::Status {
return grpc::Status{grpc::StatusCode::INTERNAL, "an unknown error"};
}
-auto CASUtils::EnsureTreeInvariant(bazel_re::Digest const& digest,
+auto CASUtils::EnsureTreeInvariant(ArtifactDigest const& digest,
std::string const& tree_data,
Storage const& storage) noexcept
-> std::optional<std::string> {
- auto const a_digest = static_cast<ArtifactDigest>(digest);
- auto error = storage.CAS().CheckTreeInvariant(a_digest, tree_data);
+ auto error = storage.CAS().CheckTreeInvariant(digest, tree_data);
if (error) {
return std::move(*error).Message();
}
return std::nullopt;
}
-auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest,
+auto CASUtils::SplitBlobIdentity(ArtifactDigest const& blob_digest,
Storage const& storage) noexcept
- -> expected<std::vector<bazel_re::Digest>, grpc::Status> {
+ -> expected<std::vector<ArtifactDigest>, grpc::Status> {
// Check blob existence.
- auto const a_digest = static_cast<ArtifactDigest>(blob_digest);
- auto path = a_digest.IsTree() ? storage.CAS().TreePath(a_digest)
- : storage.CAS().BlobPath(a_digest, false);
+ auto path = blob_digest.IsTree()
+ ? storage.CAS().TreePath(blob_digest)
+ : storage.CAS().BlobPath(blob_digest, false);
if (not path) {
return unexpected{
grpc::Status{grpc::StatusCode::NOT_FOUND,
@@ -62,8 +65,8 @@ auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest,
// operation is stored in (file) CAS. This means for the native mode, if we
// return the identity of a tree, we need to put the tree data in file CAS
// and return the resulting digest.
- auto chunk_digests = std::vector<bazel_re::Digest>{};
- if (a_digest.IsTree()) {
+ auto chunk_digests = std::vector<ArtifactDigest>{};
+ if (blob_digest.IsTree()) {
auto tree_data = FileSystemManager::ReadFile(*path);
if (not tree_data) {
return unexpected{grpc::Status{
@@ -84,50 +87,31 @@ auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest,
return chunk_digests;
}
-auto CASUtils::SplitBlobFastCDC(bazel_re::Digest const& blob_digest,
+auto CASUtils::SplitBlobFastCDC(ArtifactDigest const& blob_digest,
Storage const& storage) noexcept
- -> expected<std::vector<bazel_re::Digest>, grpc::Status> {
+ -> expected<std::vector<ArtifactDigest>, grpc::Status> {
// Split blob into chunks:
- auto const a_digest = static_cast<ArtifactDigest>(blob_digest);
- auto split = a_digest.IsTree() ? storage.CAS().SplitTree(a_digest)
- : storage.CAS().SplitBlob(a_digest);
+ auto split = blob_digest.IsTree() ? storage.CAS().SplitTree(blob_digest)
+ : storage.CAS().SplitBlob(blob_digest);
- // Process result:
- if (split) {
- std::vector<bazel_re::Digest> result;
- result.reserve(split->size());
- std::transform(split->begin(),
- split->end(),
- std::back_inserter(result),
- [](ArtifactDigest const& digest) {
- return static_cast<bazel_re::Digest>(digest);
- });
- return result;
+ if (not split) {
+ return unexpected{ToGrpc(std::move(split).error())};
}
- // Process errors
- return unexpected{ToGrpc(std::move(split).error())};
+ return *std::move(split);
}
-auto CASUtils::SpliceBlob(bazel_re::Digest const& blob_digest,
- std::vector<bazel_re::Digest> const& chunk_digests,
+auto CASUtils::SpliceBlob(ArtifactDigest const& blob_digest,
+ std::vector<ArtifactDigest> const& chunk_digests,
Storage const& storage) noexcept
- -> expected<bazel_re::Digest, grpc::Status> {
- auto const a_digest = static_cast<ArtifactDigest>(blob_digest);
- std::vector<ArtifactDigest> a_parts;
- a_parts.reserve(chunk_digests.size());
- std::transform(chunk_digests.begin(),
- chunk_digests.end(),
- std::back_inserter(a_parts),
- [](auto const& digest) { return ArtifactDigest{digest}; });
-
+ -> expected<ArtifactDigest, grpc::Status> {
// Splice blob from chunks:
- auto splice = a_digest.IsTree()
- ? storage.CAS().SpliceTree(a_digest, a_parts)
- : storage.CAS().SpliceBlob(a_digest, a_parts, false);
+ auto splice =
+ blob_digest.IsTree()
+ ? storage.CAS().SpliceTree(blob_digest, chunk_digests)
+ : storage.CAS().SpliceBlob(blob_digest, chunk_digests, false);
- // Process result:
- if (splice) {
- return static_cast<bazel_re::Digest>(*splice);
+ if (not splice) {
+ return unexpected{ToGrpc(std::move(splice).error())};
}
- return unexpected{ToGrpc(std::move(splice).error())};
+ return *std::move(splice);
}
diff --git a/src/buildtool/execution_api/execution_service/cas_utils.hpp b/src/buildtool/execution_api/execution_service/cas_utils.hpp
index a7aec9ed..191983c0 100644
--- a/src/buildtool/execution_api/execution_service/cas_utils.hpp
+++ b/src/buildtool/execution_api/execution_service/cas_utils.hpp
@@ -20,32 +20,32 @@
#include <vector>
#include "grpcpp/support/status.h"
-#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/storage/storage.hpp"
#include "src/utils/cpp/expected.hpp"
class CASUtils {
public:
[[nodiscard]] static auto EnsureTreeInvariant(
- bazel_re::Digest const& digest,
+ ArtifactDigest const& digest,
std::string const& tree_data,
Storage const& storage) noexcept -> std::optional<std::string>;
[[nodiscard]] static auto SplitBlobIdentity(
- bazel_re::Digest const& blob_digest,
+ ArtifactDigest const& blob_digest,
Storage const& storage) noexcept
- -> expected<std::vector<bazel_re::Digest>, grpc::Status>;
+ -> expected<std::vector<ArtifactDigest>, grpc::Status>;
[[nodiscard]] static auto SplitBlobFastCDC(
- bazel_re::Digest const& blob_digest,
+ ArtifactDigest const& blob_digest,
Storage const& storage) noexcept
- -> expected<std::vector<bazel_re::Digest>, grpc::Status>;
+ -> expected<std::vector<ArtifactDigest>, grpc::Status>;
[[nodiscard]] static auto SpliceBlob(
- bazel_re::Digest const& blob_digest,
- std::vector<bazel_re::Digest> const& chunk_digests,
+ ArtifactDigest const& blob_digest,
+ std::vector<ArtifactDigest> const& chunk_digests,
Storage const& storage) noexcept
- -> expected<bazel_re::Digest, grpc::Status>;
+ -> expected<ArtifactDigest, grpc::Status>;
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 35b7b768..f1515418 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -331,9 +331,8 @@ class LocalApi final : public IExecutionApi {
[[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest)
const noexcept -> std::optional<std::vector<ArtifactDigest>> final {
Logger::Log(LogLevel::Debug, "SplitBlob({})", blob_digest.hash());
- auto split_result = CASUtils::SplitBlobFastCDC(
- static_cast<bazel_re::Digest>(blob_digest),
- *local_context_.storage);
+ auto split_result =
+ CASUtils::SplitBlobFastCDC(blob_digest, *local_context_.storage);
if (not split_result) {
Logger::Log(LogLevel::Error, split_result.error().error_message());
return std::nullopt;
@@ -345,20 +344,12 @@ class LocalApi final : public IExecutionApi {
<< blob_digest.size() << " into " << chunk_digests.size()
<< " chunks: [ ";
for (auto const& chunk_digest : chunk_digests) {
- ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes()
- << " ";
+ ss << chunk_digest.hash() << ":" << chunk_digest.size() << " ";
}
ss << "]";
return ss.str();
});
- auto artifact_digests = std::vector<ArtifactDigest>{};
- artifact_digests.reserve(chunk_digests.size());
- std::transform(
- chunk_digests.cbegin(),
- chunk_digests.cend(),
- std::back_inserter(artifact_digests),
- [](auto const& digest) { return ArtifactDigest{digest}; });
- return artifact_digests;
+ return *std::move(split_result);
}
[[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final {
@@ -373,24 +364,14 @@ class LocalApi final : public IExecutionApi {
"SpliceBlob({}, {} chunks)",
blob_digest.hash(),
chunk_digests.size());
- auto digests = std::vector<bazel_re::Digest>{};
- digests.reserve(chunk_digests.size());
- std::transform(
- chunk_digests.cbegin(),
- chunk_digests.cend(),
- std::back_inserter(digests),
- [](auto const& artifact_digest) {
- return static_cast<bazel_re::Digest>(artifact_digest);
- });
- auto splice_result =
- CASUtils::SpliceBlob(static_cast<bazel_re::Digest>(blob_digest),
- digests,
- *local_context_.storage);
+
+ auto splice_result = CASUtils::SpliceBlob(
+ blob_digest, chunk_digests, *local_context_.storage);
if (not splice_result) {
Logger::Log(LogLevel::Error, splice_result.error().error_message());
return std::nullopt;
}
- return ArtifactDigest{*std::move(splice_result)};
+ return *std::move(splice_result);
}
[[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final {