diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2025-02-21 16:18:47 +0100 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2025-02-27 09:03:30 +0100 |
commit | a8a167cb206f6c66e52ab4e92e0939e91b8dfed8 (patch) | |
tree | 7861406725739f45a98c1312ed45264bb10b9b35 /src/buildtool/execution_api/remote/bazel | |
parent | e1880bead60d433de2960104bd62cd9e27bfca17 (diff) | |
download | justbuild-a8a167cb206f6c66e52ab4e92e0939e91b8dfed8.tar.gz |
ArtifactBlob: Convert to a class
Diffstat (limited to 'src/buildtool/execution_api/remote/bazel')
6 files changed, 41 insertions, 37 deletions
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index 39000194..ea6b71b9 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -73,9 +73,9 @@ namespace { return false; } for (auto& blob : blobs) { - auto const info = back_map->GetReference(blob.digest); - blob.is_exec = - info.has_value() and IsExecutableObject(info.value()->type); + auto const info = back_map->GetReference(blob.GetDigest()); + blob.SetExecutable(info.has_value() and + IsExecutableObject(info.value()->type)); // Collect blob and upload to other CAS if transfer size reached. if (not UpdateContainerAndUpload( &container, @@ -239,7 +239,7 @@ auto BazelApi::CreateAction( auto const& type = artifacts_info[gpos].type; if (not FileSystemManager::WriteFileAs</*kSetEpochTime=*/true, /*kSetWritable=*/true>( - *blobs[pos].data, output_paths[gpos], type)) { + *blobs[pos].ReadContent(), output_paths[gpos], type)) { Logger::Log(LogLevel::Warning, "staging to output path {} failed.", output_paths[gpos].string()); @@ -486,7 +486,7 @@ auto BazelApi::CreateAction( -> std::optional<std::string> { auto reader = network_->CreateReader(); if (auto blob = reader.ReadSingleBlob(artifact_info.digest)) { - return *blob->data; + return *blob->ReadContent(); } return std::nullopt; } @@ -520,7 +520,7 @@ auto BazelApi::CreateAction( targets->reserve(digests.size()); for (auto blobs : reader.ReadIncrementally(&digests)) { for (auto const& blob : blobs) { - targets->emplace_back(*blob.data); + targets->emplace_back(*blob.ReadContent()); } } }); diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index ed9a00a5..20b4b750 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -46,7 +46,7 @@ namespace { [[nodiscard]] auto GetContentSize(ArtifactBlob const& blob) noexcept -> std::size_t { - return blob.data->size(); + return blob.GetContentSize(); } template <typename TRequest, @@ -382,15 +382,15 @@ auto BazelCasClient::UpdateSingleBlob(std::string const& instance_name, logger_.Emit(LogLevel::Trace, [&blob]() { std::ostringstream oss{}; oss << "upload single blob" << std::endl; - oss << fmt::format(" - {}", blob.digest.hash()) << std::endl; + oss << fmt::format(" - {}", blob.GetDigest().hash()) << std::endl; return oss.str(); }); if (not stream_->Write(instance_name, blob)) { logger_.Emit(LogLevel::Error, "Failed to write {}:{}", - blob.digest.hash(), - blob.digest.size()); + blob.GetDigest().hash(), + blob.GetDigest().size()); return false; } return true; @@ -599,8 +599,9 @@ auto BazelCasClient::BatchUpdateBlobs(std::string const& instance_name, request.set_instance_name(instance_name); auto& r = *request.add_requests(); - (*r.mutable_digest()) = ArtifactDigestFactory::ToBazel(blob.digest); - r.set_data(*blob.data); + (*r.mutable_digest()) = + ArtifactDigestFactory::ToBazel(blob.GetDigest()); + r.set_data(*blob.ReadContent()); return request; }; @@ -623,7 +624,7 @@ auto BazelCasClient::BatchUpdateBlobs(std::string const& instance_name, logger_.Emit( LogLevel::Warning, "BatchUpdateBlobs: Failed to prepare request for {}", - it->digest.hash()); + it->GetDigest().hash()); ++it; continue; } @@ -681,7 +682,7 @@ auto BazelCasClient::BatchUpdateBlobs(std::string const& instance_name, std::ostringstream oss{}; oss << "upload blobs" << std::endl; for (auto const& blob : blobs) { - oss << fmt::format(" - {}", blob.digest.hash()) << std::endl; + oss << fmt::format(" - {}", blob.GetDigest().hash()) << std::endl; } oss << "received blobs" << std::endl; for (auto const& digest : updated) { @@ -701,7 +702,8 @@ auto BazelCasClient::BatchUpdateBlobs(std::string const& instance_name, std::unordered_set<ArtifactBlob> missing_blobs; missing_blobs.reserve(missing); for (auto const& blob : blobs) { - auto bazel_digest = ArtifactDigestFactory::ToBazel(blob.digest); + auto bazel_digest = + ArtifactDigestFactory::ToBazel(blob.GetDigest()); if (not updated.contains(bazel_digest)) { missing_blobs.emplace(blob); } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index a11ff408..1a8e6d8f 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -90,7 +90,7 @@ auto BazelNetwork::DoUploadBlobs( // First upload all blobs that must use bytestream api because of their // size: for (auto it = blobs.begin(); it != blobs.end();) { - if (it->data->size() <= MessageLimits::kMaxGrpcLength) { + if (it->GetContentSize() <= MessageLimits::kMaxGrpcLength) { ++it; continue; } @@ -114,7 +114,7 @@ auto BazelNetwork::UploadBlobs(std::unordered_set<ArtifactBlob>&& blobs, bool skip_find_missing) noexcept -> bool { if (not skip_find_missing) { auto const back_map = BackMap<ArtifactDigest, ArtifactBlob>::Make( - &blobs, [](ArtifactBlob const& blob) { return blob.digest; }); + &blobs, [](ArtifactBlob const& blob) { return blob.GetDigest(); }); if (back_map == nullptr) { return false; } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp index ed637535..7fe90b72 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp @@ -67,7 +67,7 @@ auto BazelNetworkReader::ReadDirectory(ArtifactDigest const& digest) if (auto blob = ReadSingleBlob(digest)) { return BazelMsgFactory::MessageFromString<bazel_re::Directory>( - *blob->data); + *blob->ReadContent()); } Logger::Log( LogLevel::Debug, "Directory {} not found in CAS", digest.hash()); @@ -94,7 +94,7 @@ auto BazelNetworkReader::ReadGitTree(ArtifactDigest const& digest) } bool valid = std::all_of( blobs.begin(), blobs.end(), [](ArtifactBlob const& blob) { - return PathIsNonUpwards(*blob.data); + return PathIsNonUpwards(*blob.ReadContent()); }); if (not valid) { return false; @@ -104,7 +104,7 @@ auto BazelNetworkReader::ReadGitTree(ArtifactDigest const& digest) return true; }; - std::string const& content = *read_blob->data; + std::string const content = *read_blob->ReadContent(); return GitRepo::ReadTreeData(content, hash_function_.HashTreeData(content).Bytes(), check_symlinks, @@ -122,7 +122,7 @@ auto BazelNetworkReader::DumpRawTree(Artifact::ObjectInfo const& info, } try { - return std::invoke(dumper, *read_blob->data); + return std::invoke(dumper, *read_blob->ReadContent()); } catch (...) { return false; } @@ -193,7 +193,8 @@ auto BazelNetworkReader::BatchReadBlobs( // Map digests to blobs for further lookup: auto const back_map = BackMap<ArtifactDigest, ArtifactBlob>::Make( - &batched_blobs, [](ArtifactBlob const& blob) { return blob.digest; }); + &batched_blobs, + [](ArtifactBlob const& blob) { return blob.GetDigest(); }); if (back_map == nullptr) { return {}; @@ -235,12 +236,12 @@ auto BazelNetworkReader::BatchReadBlobs( auto BazelNetworkReader::Validate(ArtifactBlob const& blob) const noexcept -> bool { - auto rehashed = blob.digest.IsTree() + auto rehashed = blob.GetDigest().IsTree() ? ArtifactDigestFactory::HashDataAs<ObjectType::Tree>( - hash_function_, *blob.data) + hash_function_, *blob.ReadContent()) : ArtifactDigestFactory::HashDataAs<ObjectType::File>( - hash_function_, *blob.data); - return rehashed == blob.digest; + hash_function_, *blob.ReadContent()); + return rehashed == blob.GetDigest(); } auto BazelNetworkReader::GetMaxBatchTransferSize() const noexcept diff --git a/src/buildtool/execution_api/remote/bazel/bazel_response.cpp b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp index ee207ac7..a0a621a4 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_response.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp @@ -67,7 +67,7 @@ auto BazelResponse::ReadStringBlob(bazel_re::Digest const& id) noexcept if (digest.has_value()) { auto reader = network_->CreateReader(); if (auto blob = reader.ReadSingleBlob(*digest)) { - return *blob->data; + return *blob->ReadContent(); } } Logger::Log(LogLevel::Warning, @@ -240,11 +240,11 @@ auto BazelResponse::Populate() noexcept -> std::optional<std::string> { for (auto const& tree_blob : tree_blobs) { try { auto tree = BazelMsgFactory::MessageFromString<bazel_re::Tree>( - *tree_blob.data); + *tree_blob.ReadContent()); if (not tree) { return fmt::format( "BazelResponse: failed to create Tree for {}", - tree_blob.digest.hash()); + tree_blob.GetDigest().hash()); } // The server does not store the Directory messages it just @@ -266,7 +266,7 @@ auto BazelResponse::Populate() noexcept -> std::optional<std::string> { return fmt::format( "BazelResponse: unexpected failure gathering digest for " "{}:\n{}", - tree_blob.digest.hash(), + tree_blob.GetDigest().hash(), ex.what()); } ++pos; @@ -291,7 +291,7 @@ auto BazelResponse::UploadTreeMessageDirectories( if (not rootdir_blob) { return unexpected{std::move(rootdir_blob).error()}; } - auto const root_digest = rootdir_blob->digest; + auto const root_digest = rootdir_blob->GetDigest(); // store or upload rootdir blob, taking maximum transfer size into account if (not UpdateContainerAndUpload(&dir_blobs, *std::move(rootdir_blob), @@ -307,7 +307,7 @@ auto BazelResponse::UploadTreeMessageDirectories( if (not blob) { return unexpected{std::move(blob).error()}; } - auto const blob_digest = blob->digest; + auto const blob_digest = blob->GetDigest(); if (not UpdateContainerAndUpload(&dir_blobs, *std::move(blob), /*exception_is_fatal=*/false, diff --git a/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp index 432e31ab..99ad4e81 100644 --- a/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp @@ -137,14 +137,15 @@ class ByteStreamClient { auto writer = stub_->Write(&ctx, &response); auto const resource_name = ByteStreamUtils::WriteRequest::ToString( - instance_name, uuid, blob.digest); + instance_name, uuid, blob.GetDigest()); google::bytestream::WriteRequest request{}; request.set_resource_name(resource_name); request.mutable_data()->reserve(ByteStreamUtils::kChunkSize); + auto const data_to_read = blob.ReadContent(); auto const to_read = ::IncrementalReader::FromMemory( - ByteStreamUtils::kChunkSize, &*blob.data); + ByteStreamUtils::kChunkSize, data_to_read.get()); if (not to_read.has_value()) { logger_.Emit( LogLevel::Error, @@ -169,7 +170,7 @@ class ByteStreamClient { request.set_write_offset(static_cast<int>(pos)); request.set_finish_write(pos + chunk->size() >= - blob.data->size()); + blob.GetContentSize()); if (writer->Write(request)) { pos += chunk->size(); ++it; @@ -206,12 +207,12 @@ class ByteStreamClient { return false; } if (gsl::narrow<std::size_t>(response.committed_size()) != - blob.data->size()) { + blob.GetContentSize()) { logger_.Emit( LogLevel::Warning, "Commited size {} is different from the original one {}.", response.committed_size(), - blob.data->size()); + blob.GetContentSize()); return false; } return true; |