diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-07-23 18:20:44 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-08-30 17:17:09 +0200 |
commit | 0df9bfcbcda9f87097bd313819be0be2cf5fa892 (patch) | |
tree | bcffd45a05d8994b2e6e5bd9cd5b157eb69802c3 /src/buildtool/execution_api | |
parent | d84ab0c8f1e0df651c2d1f6cd9f4bdc03c25932c (diff) | |
download | justbuild-0df9bfcbcda9f87097bd313819be0be2cf5fa892.tar.gz |
Replace bazel_re::Digest in CASUtils
...with ArtifactDigest
Diffstat (limited to 'src/buildtool/execution_api')
6 files changed, 72 insertions, 104 deletions
diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS index fb88cb05..ecdccf04 100644 --- a/src/buildtool/execution_api/execution_service/TARGETS +++ b/src/buildtool/execution_api/execution_service/TARGETS @@ -169,7 +169,7 @@ , "stage": ["src", "buildtool", "execution_api", "execution_service"] , "deps": [ ["@", "grpc", "", "grpc++"] - , ["src/buildtool/common", "bazel_types"] + , ["src/buildtool/common", "common"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "expected"] ] diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index 0f77542d..1b2aad14 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -172,8 +172,8 @@ auto BytestreamServiceImpl::Write( } ArtifactDigest dgst{NativeSupport::Unprefix(*hash), 0, true}; - if (auto err = CASUtils::EnsureTreeInvariant( - static_cast<bazel_re::Digest>(dgst), *content, storage_)) { + if (auto err = + CASUtils::EnsureTreeInvariant(dgst, *content, storage_)) { auto const str = fmt::format("Write: {}", *std::move(err)); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, str}; diff --git a/src/buildtool/execution_api/execution_service/cas_server.cpp b/src/buildtool/execution_api/execution_service/cas_server.cpp index a2a37e37..2c2260ac 100644 --- a/src/buildtool/execution_api/execution_service/cas_server.cpp +++ b/src/buildtool/execution_api/execution_service/cas_server.cpp @@ -145,12 +145,12 @@ auto CASServiceImpl::BatchUpdateBlobs( auto* r = response->add_responses(); r->mutable_digest()->CopyFrom(x.digest()); - bool const is_tree = NativeSupport::IsTree(hash); - if (is_tree) { + ArtifactDigest const digest{x.digest()}; + if (digest.IsTree()) { // In native mode: for trees, check whether the tree invariant holds // before storing the actual tree object. - if (auto err = CASUtils::EnsureTreeInvariant( - x.digest(), x.data(), storage_)) { + if (auto err = + CASUtils::EnsureTreeInvariant(digest, x.data(), storage_)) { auto const str = fmt::format("BatchUpdateBlobs: {}", *std::move(err)); logger_.Emit(LogLevel::Error, "{}", str); @@ -160,20 +160,20 @@ auto CASServiceImpl::BatchUpdateBlobs( } auto const cas_digest = - is_tree + digest.IsTree() ? storage_.CAS().StoreTree(x.data()) : storage_.CAS().StoreBlob(x.data(), /*is_executable=*/false); if (not cas_digest) { auto const str = fmt::format("BatchUpdateBlobs: could not upload {} {}", - is_tree ? "tree" : "blob", + digest.IsTree() ? "tree" : "blob", hash); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } - if (auto err = CheckDigestConsistency(x.digest(), *cas_digest)) { + if (auto err = CheckDigestConsistency(digest, *cas_digest)) { auto const str = fmt::format("BatchUpdateBlobs: {}", *std::move(err)); logger_.Emit(LogLevel::Error, "{}", str); @@ -278,11 +278,12 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/, } // Split blob into chunks. + ArtifactDigest const digest{blob_digest}; auto split_result = chunking_algorithm == ::bazel_re::ChunkingAlgorithm_Value:: ChunkingAlgorithm_Value_IDENTITY - ? CASUtils::SplitBlobIdentity(blob_digest, storage_) - : CASUtils::SplitBlobFastCDC(blob_digest, storage_); + ? CASUtils::SplitBlobIdentity(digest, storage_) + : CASUtils::SplitBlobFastCDC(digest, storage_); if (not split_result) { auto const& status = split_result.error(); @@ -298,16 +299,18 @@ auto CASServiceImpl::SplitBlob(::grpc::ServerContext* /*context*/, << blob_digest.size_bytes() << " into " << chunk_digests.size() << " chunks: [ "; for (auto const& chunk_digest : chunk_digests) { - ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes() - << " "; + ss << chunk_digest.hash() << ":" << chunk_digest.size() << " "; } ss << "]"; return ss.str(); }); - std::copy(chunk_digests.cbegin(), - chunk_digests.cend(), - pb::back_inserter(response->mutable_chunk_digests())); + std::transform(chunk_digests.cbegin(), + chunk_digests.cend(), + pb::back_inserter(response->mutable_chunk_digests()), + [](ArtifactDigest const& digest) { + return static_cast<bazel_re::Digest>(digest); + }); return ::grpc::Status::OK; } @@ -343,7 +346,8 @@ auto CASServiceImpl::SpliceBlob(::grpc::ServerContext* /*context*/, return ::grpc::Status{grpc::StatusCode::INTERNAL, str}; } - auto chunk_digests = std::vector<bazel_re::Digest>{}; + ArtifactDigest const digest{blob_digest}; + auto chunk_digests = std::vector<ArtifactDigest>{}; chunk_digests.reserve(request->chunk_digests().size()); for (auto const& x : request->chunk_digests()) { if (not IsValidHash(x.hash())) { @@ -352,19 +356,18 @@ auto CASServiceImpl::SpliceBlob(::grpc::ServerContext* /*context*/, logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; } - chunk_digests.push_back(x); + chunk_digests.push_back(ArtifactDigest{x}); } // Splice blob from chunks. - auto splice_result = - CASUtils::SpliceBlob(blob_digest, chunk_digests, storage_); + auto splice_result = CASUtils::SpliceBlob(digest, chunk_digests, storage_); if (not splice_result) { auto const& status = splice_result.error(); auto const str = fmt::format("SpliceBlob: {}", status.error_message()); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{status.error_code(), str}; } - if (auto err = CheckDigestConsistency(blob_digest, *splice_result)) { + if (auto err = CheckDigestConsistency(digest, *splice_result)) { auto const str = fmt::format("SpliceBlob: {}", *err); logger_.Emit(LogLevel::Error, "{}", str); return ::grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, str}; diff --git a/src/buildtool/execution_api/execution_service/cas_utils.cpp b/src/buildtool/execution_api/execution_service/cas_utils.cpp index a01cadc6..a406bdec 100644 --- a/src/buildtool/execution_api/execution_service/cas_utils.cpp +++ b/src/buildtool/execution_api/execution_service/cas_utils.cpp @@ -14,8 +14,12 @@ #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" +#include <utility> + #include "fmt/core.h" #include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/large_object_cas.hpp" +#include "src/buildtool/storage/local_cas.hpp" static auto ToGrpc(LargeObjectError&& error) noexcept -> grpc::Status { switch (error.Code()) { @@ -33,25 +37,24 @@ static auto ToGrpc(LargeObjectError&& error) noexcept -> grpc::Status { return grpc::Status{grpc::StatusCode::INTERNAL, "an unknown error"}; } -auto CASUtils::EnsureTreeInvariant(bazel_re::Digest const& digest, +auto CASUtils::EnsureTreeInvariant(ArtifactDigest const& digest, std::string const& tree_data, Storage const& storage) noexcept -> std::optional<std::string> { - auto const a_digest = static_cast<ArtifactDigest>(digest); - auto error = storage.CAS().CheckTreeInvariant(a_digest, tree_data); + auto error = storage.CAS().CheckTreeInvariant(digest, tree_data); if (error) { return std::move(*error).Message(); } return std::nullopt; } -auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest, +auto CASUtils::SplitBlobIdentity(ArtifactDigest const& blob_digest, Storage const& storage) noexcept - -> expected<std::vector<bazel_re::Digest>, grpc::Status> { + -> expected<std::vector<ArtifactDigest>, grpc::Status> { // Check blob existence. - auto const a_digest = static_cast<ArtifactDigest>(blob_digest); - auto path = a_digest.IsTree() ? storage.CAS().TreePath(a_digest) - : storage.CAS().BlobPath(a_digest, false); + auto path = blob_digest.IsTree() + ? storage.CAS().TreePath(blob_digest) + : storage.CAS().BlobPath(blob_digest, false); if (not path) { return unexpected{ grpc::Status{grpc::StatusCode::NOT_FOUND, @@ -62,8 +65,8 @@ auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest, // operation is stored in (file) CAS. This means for the native mode, if we // return the identity of a tree, we need to put the tree data in file CAS // and return the resulting digest. - auto chunk_digests = std::vector<bazel_re::Digest>{}; - if (a_digest.IsTree()) { + auto chunk_digests = std::vector<ArtifactDigest>{}; + if (blob_digest.IsTree()) { auto tree_data = FileSystemManager::ReadFile(*path); if (not tree_data) { return unexpected{grpc::Status{ @@ -84,50 +87,31 @@ auto CASUtils::SplitBlobIdentity(bazel_re::Digest const& blob_digest, return chunk_digests; } -auto CASUtils::SplitBlobFastCDC(bazel_re::Digest const& blob_digest, +auto CASUtils::SplitBlobFastCDC(ArtifactDigest const& blob_digest, Storage const& storage) noexcept - -> expected<std::vector<bazel_re::Digest>, grpc::Status> { + -> expected<std::vector<ArtifactDigest>, grpc::Status> { // Split blob into chunks: - auto const a_digest = static_cast<ArtifactDigest>(blob_digest); - auto split = a_digest.IsTree() ? storage.CAS().SplitTree(a_digest) - : storage.CAS().SplitBlob(a_digest); + auto split = blob_digest.IsTree() ? storage.CAS().SplitTree(blob_digest) + : storage.CAS().SplitBlob(blob_digest); - // Process result: - if (split) { - std::vector<bazel_re::Digest> result; - result.reserve(split->size()); - std::transform(split->begin(), - split->end(), - std::back_inserter(result), - [](ArtifactDigest const& digest) { - return static_cast<bazel_re::Digest>(digest); - }); - return result; + if (not split) { + return unexpected{ToGrpc(std::move(split).error())}; } - // Process errors - return unexpected{ToGrpc(std::move(split).error())}; + return *std::move(split); } -auto CASUtils::SpliceBlob(bazel_re::Digest const& blob_digest, - std::vector<bazel_re::Digest> const& chunk_digests, +auto CASUtils::SpliceBlob(ArtifactDigest const& blob_digest, + std::vector<ArtifactDigest> const& chunk_digests, Storage const& storage) noexcept - -> expected<bazel_re::Digest, grpc::Status> { - auto const a_digest = static_cast<ArtifactDigest>(blob_digest); - std::vector<ArtifactDigest> a_parts; - a_parts.reserve(chunk_digests.size()); - std::transform(chunk_digests.begin(), - chunk_digests.end(), - std::back_inserter(a_parts), - [](auto const& digest) { return ArtifactDigest{digest}; }); - + -> expected<ArtifactDigest, grpc::Status> { // Splice blob from chunks: - auto splice = a_digest.IsTree() - ? storage.CAS().SpliceTree(a_digest, a_parts) - : storage.CAS().SpliceBlob(a_digest, a_parts, false); + auto splice = + blob_digest.IsTree() + ? storage.CAS().SpliceTree(blob_digest, chunk_digests) + : storage.CAS().SpliceBlob(blob_digest, chunk_digests, false); - // Process result: - if (splice) { - return static_cast<bazel_re::Digest>(*splice); + if (not splice) { + return unexpected{ToGrpc(std::move(splice).error())}; } - return unexpected{ToGrpc(std::move(splice).error())}; + return *std::move(splice); } diff --git a/src/buildtool/execution_api/execution_service/cas_utils.hpp b/src/buildtool/execution_api/execution_service/cas_utils.hpp index a7aec9ed..191983c0 100644 --- a/src/buildtool/execution_api/execution_service/cas_utils.hpp +++ b/src/buildtool/execution_api/execution_service/cas_utils.hpp @@ -20,32 +20,32 @@ #include <vector> #include "grpcpp/support/status.h" -#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/expected.hpp" class CASUtils { public: [[nodiscard]] static auto EnsureTreeInvariant( - bazel_re::Digest const& digest, + ArtifactDigest const& digest, std::string const& tree_data, Storage const& storage) noexcept -> std::optional<std::string>; [[nodiscard]] static auto SplitBlobIdentity( - bazel_re::Digest const& blob_digest, + ArtifactDigest const& blob_digest, Storage const& storage) noexcept - -> expected<std::vector<bazel_re::Digest>, grpc::Status>; + -> expected<std::vector<ArtifactDigest>, grpc::Status>; [[nodiscard]] static auto SplitBlobFastCDC( - bazel_re::Digest const& blob_digest, + ArtifactDigest const& blob_digest, Storage const& storage) noexcept - -> expected<std::vector<bazel_re::Digest>, grpc::Status>; + -> expected<std::vector<ArtifactDigest>, grpc::Status>; [[nodiscard]] static auto SpliceBlob( - bazel_re::Digest const& blob_digest, - std::vector<bazel_re::Digest> const& chunk_digests, + ArtifactDigest const& blob_digest, + std::vector<ArtifactDigest> const& chunk_digests, Storage const& storage) noexcept - -> expected<bazel_re::Digest, grpc::Status>; + -> expected<ArtifactDigest, grpc::Status>; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_EXECUTION_SERVICE_CAS_UTILS_HPP diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 35b7b768..f1515418 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -331,9 +331,8 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto SplitBlob(ArtifactDigest const& blob_digest) const noexcept -> std::optional<std::vector<ArtifactDigest>> final { Logger::Log(LogLevel::Debug, "SplitBlob({})", blob_digest.hash()); - auto split_result = CASUtils::SplitBlobFastCDC( - static_cast<bazel_re::Digest>(blob_digest), - *local_context_.storage); + auto split_result = + CASUtils::SplitBlobFastCDC(blob_digest, *local_context_.storage); if (not split_result) { Logger::Log(LogLevel::Error, split_result.error().error_message()); return std::nullopt; @@ -345,20 +344,12 @@ class LocalApi final : public IExecutionApi { << blob_digest.size() << " into " << chunk_digests.size() << " chunks: [ "; for (auto const& chunk_digest : chunk_digests) { - ss << chunk_digest.hash() << ":" << chunk_digest.size_bytes() - << " "; + ss << chunk_digest.hash() << ":" << chunk_digest.size() << " "; } ss << "]"; return ss.str(); }); - auto artifact_digests = std::vector<ArtifactDigest>{}; - artifact_digests.reserve(chunk_digests.size()); - std::transform( - chunk_digests.cbegin(), - chunk_digests.cend(), - std::back_inserter(artifact_digests), - [](auto const& digest) { return ArtifactDigest{digest}; }); - return artifact_digests; + return *std::move(split_result); } [[nodiscard]] auto BlobSplitSupport() const noexcept -> bool final { @@ -373,24 +364,14 @@ class LocalApi final : public IExecutionApi { "SpliceBlob({}, {} chunks)", blob_digest.hash(), chunk_digests.size()); - auto digests = std::vector<bazel_re::Digest>{}; - digests.reserve(chunk_digests.size()); - std::transform( - chunk_digests.cbegin(), - chunk_digests.cend(), - std::back_inserter(digests), - [](auto const& artifact_digest) { - return static_cast<bazel_re::Digest>(artifact_digest); - }); - auto splice_result = - CASUtils::SpliceBlob(static_cast<bazel_re::Digest>(blob_digest), - digests, - *local_context_.storage); + + auto splice_result = CASUtils::SpliceBlob( + blob_digest, chunk_digests, *local_context_.storage); if (not splice_result) { Logger::Log(LogLevel::Error, splice_result.error().error_message()); return std::nullopt; } - return ArtifactDigest{*std::move(splice_result)}; + return *std::move(splice_result); } [[nodiscard]] auto BlobSpliceSupport() const noexcept -> bool final { |