From 92bfbfa2f5058f1396237a99f65ac2b05e312310 Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Fri, 17 May 2024 22:57:34 +0200 Subject: Avoid memory usage duplication --- .../remote/bazel/bazel_cas_client.cpp | 36 +++++++++------------- .../remote/bazel/bazel_cas_client.hpp | 14 +++------ .../execution_api/remote/bazel/bazel_network.cpp | 16 +++++++--- 3 files changed, 30 insertions(+), 36 deletions(-) (limited to 'src') diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index 373d218e..716e3f53 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -21,7 +21,6 @@ #include #include "grpcpp/grpcpp.h" -#include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/retry.hpp" @@ -193,13 +192,6 @@ auto BazelCasClient::FindMissingBlobs( return FindMissingBlobs(instance_name, digests.begin(), digests.end()); } -auto BazelCasClient::BatchUpdateBlobs( - std::string const& instance_name, - std::vector::const_iterator const& begin, - std::vector::const_iterator const& end) noexcept -> std::size_t { - return DoBatchUpdateBlobs(instance_name, begin, end); -} - auto BazelCasClient::BatchReadBlobs( std::string const& instance_name, std::vector::const_iterator const& begin, @@ -492,12 +484,12 @@ auto BazelCasClient::FindMissingBlobs(std::string const& instance_name, return result; } -template -auto BazelCasClient::DoBatchUpdateBlobs(std::string const& instance_name, - T_OutputIter const& start, - T_OutputIter const& end) noexcept - -> std::size_t { - if (start == end) { +auto BazelCasClient::BatchUpdateBlobs( + std::string const& instance_name, + std::vector>::const_iterator const& begin, + std::vector>::const_iterator const& + end) noexcept -> std::size_t { + if (begin == end) { return 0; } std::vector result; @@ -505,15 +497,15 @@ auto BazelCasClient::DoBatchUpdateBlobs(std::string const& instance_name, auto requests = CreateBatchRequestsMaxSize( instance_name, - start, + begin, end, "BatchUpdateBlobs", - [this](bazel_re::BatchUpdateBlobsRequest* request, - BazelBlob const& x) { + [](bazel_re::BatchUpdateBlobsRequest* request, + BazelBlob const* x) { *(request->add_requests()) = - this->CreateUpdateBlobsSingleRequest(x); + BazelCasClient::CreateUpdateBlobsSingleRequest(*x); }); - result.reserve(std::distance(start, end)); + result.reserve(std::distance(begin, end)); auto batch_update_blobs = [this, &result](auto const& request) -> RetryResponse { bazel_re::BatchUpdateBlobsResponse response; @@ -557,11 +549,11 @@ auto BazelCasClient::DoBatchUpdateBlobs(std::string const& instance_name, } catch (...) { logger_.Emit(LogLevel::Error, "Caught exception in DoBatchUpdateBlobs"); } - logger_.Emit(LogLevel::Trace, [&start, &end, &result]() { + logger_.Emit(LogLevel::Trace, [begin, end, &result]() { std::ostringstream oss{}; oss << "upload blobs" << std::endl; - std::for_each(start, end, [&oss](auto const& blob) { - oss << fmt::format(" - {}", blob.digest.hash()) << std::endl; + std::for_each(begin, end, [&oss](BazelBlob const* blob) { + oss << fmt::format(" - {}", blob->digest.hash()) << std::endl; }); oss << "received blobs" << std::endl; std::for_each( diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp index 69db091f..4b669ab8 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp @@ -24,6 +24,7 @@ #include #include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h" +#include "gsl/gsl" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" @@ -63,9 +64,10 @@ class BazelCasClient { /// \returns The digests of blobs successfully updated [[nodiscard]] auto BatchUpdateBlobs( std::string const& instance_name, - std::vector::const_iterator const& begin, - std::vector::const_iterator const& end) noexcept - -> std::size_t; + std::vector>::const_iterator const& + begin, + std::vector>::const_iterator const& + end) noexcept -> std::size_t; /// \brief Read multiple blobs in batch transfer /// \param[in] instance_name Name of the CAS instance @@ -144,12 +146,6 @@ class BazelCasClient { T_OutputIter const& end) noexcept -> std::vector; - template - [[nodiscard]] auto DoBatchUpdateBlobs(std::string const& instance_name, - T_OutputIter const& start, - T_OutputIter const& end) noexcept - -> std::size_t; - template [[nodiscard]] auto CreateBatchRequestsMaxSize( std::string const& instance_name, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 2d434bde..325abd02 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -188,17 +188,23 @@ auto BazelNetwork::DoUploadBlobs(T_Iter const& first, // // The blobs belonging to the second group are uploaded via the // bytestream api. - std::vector sorted(first, last); + std::vector> sorted; + sorted.reserve(std::distance(first, last)); + std::transform( + first, last, std::back_inserter(sorted), [](BazelBlob const& b) { + return &b; + }); + auto it = std::stable_partition( - sorted.begin(), sorted.end(), [](auto const& x) { - return x.data.size() <= kMaxBatchTransferSize; + sorted.begin(), sorted.end(), [](BazelBlob const* x) { + return x->data.size() <= kMaxBatchTransferSize; }); auto digests_count = cas_->BatchUpdateBlobs(instance_name_, sorted.begin(), it); return digests_count == std::distance(sorted.begin(), it) && - std::all_of(it, sorted.end(), [this](auto const& x) { - return cas_->UpdateSingleBlob(instance_name_, x); + std::all_of(it, sorted.end(), [this](BazelBlob const* x) { + return cas_->UpdateSingleBlob(instance_name_, *x); }); } catch (...) { Logger::Log(LogLevel::Warning, "Unknown exception"); -- cgit v1.2.3