diff options
3 files changed, 5 insertions, 164 deletions
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp index a82d5c6e..6f125d67 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp @@ -15,6 +15,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include <algorithm> +#include <cstddef> #include <filesystem> #include <memory> #include <unordered_set> @@ -223,89 +224,3 @@ auto BazelNetworkReader::ReadOrdered(std::vector<ArtifactDigest> const& digests) } return artifacts; } - -auto BazelNetworkReader::ReadIncrementally( - gsl::not_null<std::vector<ArtifactDigest> const*> const& digests) - const noexcept -> IncrementalReader { - return IncrementalReader{*this, digests}; -} - -auto BazelNetworkReader::BatchReadBlobs( - std::vector<ArtifactDigest> const& digests) const noexcept - -> std::vector<ArtifactBlob> { - // Batch blobs: - auto const batched_blobs = cas_.BatchReadBlobs( - instance_name_, std::unordered_set(digests.begin(), digests.end())); - - // Map digests to blobs for further lookup: - auto const back_map = BackMap<ArtifactDigest, ArtifactBlob>::Make( - &batched_blobs, - [](ArtifactBlob const& blob) { return blob.GetDigest(); }); - - if (back_map == nullptr) { - return {}; - } - - // Restore the requested order: - std::vector<ArtifactBlob> artifacts; - artifacts.reserve(digests.size()); - for (ArtifactDigest const& digest : digests) { - if (auto value = back_map->GetReference(digest)) { - artifacts.emplace_back(*value.value()); - } - } - return artifacts; -} - -auto BazelNetworkReader::GetMaxBatchTransferSize() const noexcept - -> std::size_t { - return cas_.GetMaxBatchTransferSize(instance_name_); -} - -namespace { -[[nodiscard]] auto FindCurrentIterator( - std::size_t content_limit, - std::vector<ArtifactDigest>::const_iterator const& begin, - std::vector<ArtifactDigest>::const_iterator const& end) noexcept { - auto it = - std::find_if(begin, end, [content_limit](ArtifactDigest const& digest) { - auto const size = digest.size(); - return size == 0 or size > content_limit; - }); - if (it == begin and begin != end) { - ++it; - } - return it; -} -} // namespace - -BazelNetworkReader::IncrementalReader::Iterator::Iterator( - BazelNetworkReader const& owner, - std::vector<ArtifactDigest>::const_iterator begin, - std::vector<ArtifactDigest>::const_iterator end) noexcept - : owner_{owner}, begin_{begin}, end_{end} { - current_ = - FindCurrentIterator(owner_.GetMaxBatchTransferSize(), begin_, end_); -} - -auto BazelNetworkReader::IncrementalReader::Iterator::operator*() const noexcept - -> value_type { - if (begin_ != current_) { - if (std::distance(begin_, current_) > 1) { - std::vector<ArtifactDigest> request{begin_, current_}; - return owner_.BatchReadBlobs(request); - } - if (auto blob = owner_.ReadSingleBlob(*begin_)) { - return {std::move(*blob)}; - } - } - return {}; -} - -auto BazelNetworkReader::IncrementalReader::Iterator::operator++() noexcept - -> Iterator& { - begin_ = current_; - current_ = - FindCurrentIterator(owner_.GetMaxBatchTransferSize(), begin_, end_); - return *this; -} diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp index c11da691..f7915446 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp @@ -15,10 +15,8 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP -#include <cstddef> #include <filesystem> #include <functional> -#include <iterator> #include <optional> #include <string> #include <unordered_map> @@ -80,10 +78,6 @@ class BazelNetworkReader final { [[nodiscard]] auto ReadOrdered(std::vector<ArtifactDigest> const& digests) const noexcept -> std::vector<ArtifactBlob>; - [[nodiscard]] auto ReadIncrementally( - gsl::not_null<std::vector<ArtifactDigest> const*> const& digests) - const noexcept -> IncrementalReader; - private: using DirectoryMap = std::unordered_map<ArtifactDigest, bazel_re::Directory>; @@ -96,67 +90,6 @@ class BazelNetworkReader final { [[nodiscard]] auto MakeAuxiliaryMap( std::vector<bazel_re::Directory>&& full_tree) const noexcept -> std::optional<DirectoryMap>; - - [[nodiscard]] auto BatchReadBlobs( - std::vector<ArtifactDigest> const& digests) const noexcept - -> std::vector<ArtifactBlob>; - - [[nodiscard]] auto GetMaxBatchTransferSize() const noexcept -> std::size_t; -}; - -class BazelNetworkReader::IncrementalReader final { - public: - IncrementalReader( - BazelNetworkReader const& owner, - gsl::not_null<std::vector<ArtifactDigest> const*> digests) noexcept - : owner_(owner), digests_(*digests) {} - - class Iterator final { - public: - using value_type = std::vector<ArtifactBlob>; - using pointer = value_type*; - using reference = value_type&; - using difference_type = std::ptrdiff_t; - using iterator_category = std::forward_iterator_tag; - - Iterator(BazelNetworkReader const& owner, - std::vector<ArtifactDigest>::const_iterator begin, - std::vector<ArtifactDigest>::const_iterator end) noexcept; - - auto operator*() const noexcept -> value_type; - auto operator++() noexcept -> Iterator&; - - [[nodiscard]] friend auto operator==(Iterator const& lhs, - Iterator const& rhs) noexcept - -> bool { - return lhs.begin_ == rhs.begin_ and lhs.end_ == rhs.end_ and - lhs.current_ == rhs.current_; - } - - [[nodiscard]] friend auto operator!=(Iterator const& lhs, - Iterator const& rhs) noexcept - -> bool { - return not(lhs == rhs); - } - - private: - BazelNetworkReader const& owner_; - std::vector<ArtifactDigest>::const_iterator begin_; - std::vector<ArtifactDigest>::const_iterator end_; - std::vector<ArtifactDigest>::const_iterator current_; - }; - - [[nodiscard]] auto begin() const noexcept { - return Iterator{owner_, digests_.begin(), digests_.end()}; - } - - [[nodiscard]] auto end() const noexcept { - return Iterator{owner_, digests_.end(), digests_.end()}; - } - - private: - BazelNetworkReader const& owner_; - std::vector<ArtifactDigest> const& digests_; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP diff --git a/test/buildtool/execution_api/bazel/bazel_network.test.cpp b/test/buildtool/execution_api/bazel/bazel_network.test.cpp index 6b8f7940..78f396ff 100644 --- a/test/buildtool/execution_api/bazel/bazel_network.test.cpp +++ b/test/buildtool/execution_api/bazel/bazel_network.test.cpp @@ -88,16 +88,13 @@ TEST_CASE("Bazel network: write/read blobs", "[execution_api]") { REQUIRE(network.UploadBlobs({*foo, *bar, *baz})); // Read blobs in order - auto reader = network.CreateReader(); std::vector<ArtifactDigest> to_read{foo->GetDigest(), bar->GetDigest(), baz->GetDigest(), bar->GetDigest(), foo->GetDigest()}; - std::vector<ArtifactBlob> blobs{}; - for (auto next : reader.ReadIncrementally(&to_read)) { - blobs.insert(blobs.end(), next.begin(), next.end()); - } + std::vector<ArtifactBlob> const blobs = + network.CreateReader().ReadOrdered(to_read); // Check order maintained REQUIRE(blobs.size() == 5); @@ -163,12 +160,8 @@ TEST_CASE("Bazel network: read blobs with unknown size", "[execution_api]") { /*size_unknown=*/0}; // Read blobs - auto reader = network.CreateReader(); - std::vector<ArtifactDigest> to_read{foo_digest, bar_digest}; - std::vector<ArtifactBlob> blobs{}; - for (auto next : reader.ReadIncrementally(&to_read)) { - blobs.insert(blobs.end(), next.begin(), next.end()); - } + std::vector<ArtifactBlob> const blobs = + network.CreateReader().ReadOrdered({foo_digest, bar_digest}); // Check order maintained REQUIRE(blobs.size() == 2); |