diff options
7 files changed, 35 insertions, 221 deletions
diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS index 8eb64976..1ef91526 100644 --- a/src/buildtool/execution_api/bazel_msg/TARGETS +++ b/src/buildtool/execution_api/bazel_msg/TARGETS @@ -5,8 +5,8 @@ , "deps": [ ["src/buildtool/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] - , ["src/utils/cpp", "concepts"] , ["src/buildtool/common", "bazel_types"] + , ["src/utils/cpp", "transformed_range"] ] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] , "private-deps": diff --git a/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp b/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp index 41f30ddd..07c78550 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp @@ -16,216 +16,19 @@ #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_CONTAINER_HPP #include <cstddef> -#include <string> -#include <type_traits> #include <unordered_map> -#include <utility> +#include <utility> // std::pair via auto #include <vector> -#include "gsl/gsl" #include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" -#include "src/utils/cpp/concepts.hpp" - -namespace detail { - -// Interface for transforming iteratee for wrapped_iterators -template <class T_Iteratee, class T_Iterator> -struct wrapped_iterator_transform { - [[nodiscard]] virtual auto operator()(T_Iterator const&) const& noexcept - -> T_Iteratee const& = 0; - [[nodiscard]] auto operator()(T_Iterator const&) && noexcept = delete; -}; - -// Wrap iterator from read-only container with custom transform. This class -// represents a read-only iterable view with an implicit transform operation. -template <class T_Iteratee, - class T_Iterator, - derived_from<wrapped_iterator_transform<T_Iteratee, T_Iterator>> - T_Transform> -class wrapped_iterator { - public: - wrapped_iterator(T_Iterator it, T_Transform&& transform) noexcept - : it_{std::move(it)}, transform_{std::move(transform)} {} - wrapped_iterator(wrapped_iterator const& other) noexcept = default; - wrapped_iterator(wrapped_iterator&& other) noexcept = default; - ~wrapped_iterator() noexcept = default; - - auto operator=(wrapped_iterator const& other) noexcept - -> wrapped_iterator& = default; - auto operator=(wrapped_iterator&& other) noexcept - -> wrapped_iterator& = default; - - auto operator++() noexcept -> wrapped_iterator& { - ++it_; - return *this; - } - - auto operator++(int) noexcept -> wrapped_iterator { - wrapped_iterator r = *this; - ++(*this); - return r; - } - - [[nodiscard]] auto operator==(wrapped_iterator other) const noexcept - -> bool { - return it_ == other.it_; - } - [[nodiscard]] auto operator!=(wrapped_iterator other) const noexcept - -> bool { - return not(*this == other); - } - [[nodiscard]] auto operator*() const noexcept -> T_Iteratee const& { - return transform_(it_); - } - using difference_type = typename T_Iterator::difference_type; - using value_type = T_Iteratee; - using pointer = T_Iteratee const*; - using reference = T_Iteratee const&; - using iterator_category = std::forward_iterator_tag; - - private: - T_Iterator it_; - T_Transform transform_; -}; - -} // namespace detail +#include "src/utils/cpp/transformed_range.hpp" /// \brief Container for Blobs /// Can be used to iterate over digests or subset of blobs with certain digest. class BlobContainer { using underlaying_map_t = std::unordered_map<bazel_re::Digest, BazelBlob>; - using item_iterator = underlaying_map_t::const_iterator; - - // transform underlaying_map_t::value_type to BazelBlob - struct item_to_blob - : public detail::wrapped_iterator_transform<BazelBlob, item_iterator> { - public: - auto operator()(item_iterator const& it) const& noexcept - -> BazelBlob const& final { - return it->second; - } - }; public: - class iterator : public detail::wrapped_iterator<BazelBlob, - item_iterator, - item_to_blob> { - friend class BlobContainer; - explicit iterator(item_iterator const& it) noexcept - : wrapped_iterator{it, item_to_blob{}} {} - }; - - /// \brief Iterable read-only list for Digests - class DigestList { - friend class BlobContainer; - - // transform underlaying_map_t::value_type to Digest - struct item_to_digest - : public detail::wrapped_iterator_transform<bazel_re::Digest, - item_iterator> { - public: - auto operator()(item_iterator const& it) const& noexcept - -> bazel_re::Digest const& final { - return it->first; - } - }; - - public: - /// \brief Read-only iterator for DigestList - class iterator : public detail::wrapped_iterator<bazel_re::Digest, - item_iterator, - item_to_digest> { - public: - explicit iterator(item_iterator const& it) noexcept - : wrapped_iterator{it, item_to_digest{}} {} - }; - - /// \brief Obtain start iterator for DigestList - [[nodiscard]] auto begin() const noexcept -> iterator { - return iterator(blobs_->cbegin()); - } - - /// \brief Obtain end iterator for DigestList - [[nodiscard]] auto end() const noexcept -> iterator { - return iterator(blobs_->cend()); - } - - private: - gsl::not_null<underlaying_map_t const*> blobs_; - - explicit DigestList(underlaying_map_t const& blobs) noexcept - : blobs_{&blobs} {} - }; - - /// \brief Iterable read-only list for Blobs related to given Digests - class RelatedBlobList { - friend class BlobContainer; - using digest_iterator = std::vector<bazel_re::Digest>::const_iterator; - - // transform Digest to BazelBlob - struct digest_to_blob - : public detail::wrapped_iterator_transform<BazelBlob, - digest_iterator> { - public: - explicit digest_to_blob( - gsl::not_null<underlaying_map_t const*> blobs) noexcept - : blobs_{blobs} {} - digest_to_blob(digest_to_blob const& other) noexcept = default; - digest_to_blob(digest_to_blob&& other) noexcept = default; - ~digest_to_blob() noexcept = default; - - auto operator=(digest_to_blob const& other) noexcept - -> digest_to_blob& = default; - auto operator=(digest_to_blob&& other) noexcept - -> digest_to_blob& = default; - - auto operator()(digest_iterator const& it) const& noexcept - -> BazelBlob const& final { - try { - return blobs_->at(*it); - } catch (std::exception const&) { - return kEmpty; - } - } - - private: - static inline BazelBlob const kEmpty{bazel_re::Digest{}, - std::string{}, - /*is_exec=*/false}; - gsl::not_null<underlaying_map_t const*> blobs_; - }; - - public: - /// \brief Read-only iterator for RelatedBlobList - class iterator : public detail::wrapped_iterator<BazelBlob, - digest_iterator, - digest_to_blob> { - public: - iterator( - digest_iterator const& it, - gsl::not_null<underlaying_map_t const*> const& blobs) noexcept - : wrapped_iterator{it, digest_to_blob{blobs}} {} - }; - - /// \brief Obtain start iterator for RelatedBlobList - [[nodiscard]] auto begin() const noexcept -> iterator { - return iterator(digests_.cbegin(), blobs_); - } - - /// \brief Obtain end iterator for RelatedBlobList - [[nodiscard]] auto end() const noexcept -> iterator { - return iterator(digests_.cend(), blobs_); - } - - private: - std::vector<bazel_re::Digest> digests_; - gsl::not_null<underlaying_map_t const*> blobs_; - - RelatedBlobList(underlaying_map_t const& blobs, - std::vector<bazel_re::Digest> digests) noexcept - : digests_{std::move(digests)}, blobs_{&blobs} {} - }; - BlobContainer() noexcept = default; explicit BlobContainer(std::vector<BazelBlob> blobs) { blobs_.reserve(blobs.size()); @@ -254,27 +57,35 @@ class BlobContainer { return blobs_.contains(blob.digest); } + /// \brief Obtain iterable list of Blobs from container. + [[nodiscard]] auto Blobs() const noexcept { + auto converter = [](auto const& p) -> BazelBlob const& { + return p.second; + }; + return TransformedRange{ + blobs_.cbegin(), blobs_.cend(), std::move(converter)}; + } + /// \brief Obtain iterable list of Digests from container. - [[nodiscard]] auto Digests() const noexcept -> DigestList { - return DigestList{blobs_}; + [[nodiscard]] auto Digests() const noexcept { + auto converter = [](auto const& p) -> bazel_re::Digest const& { + return p.first; + }; + return TransformedRange{ + blobs_.cbegin(), blobs_.cend(), std::move(converter)}; } /// \brief Obtain iterable list of BazelBlobs related to Digests. /// \param[in] related Related Digests [[nodiscard]] auto RelatedBlobs( - std::vector<bazel_re::Digest> const& related) const noexcept - -> RelatedBlobList { - return RelatedBlobList{blobs_, related}; + std::vector<bazel_re::Digest> const& related) const noexcept { + auto converter = [this](auto const& digest) -> BazelBlob const& { + return blobs_.at(digest); + }; + return TransformedRange{ + related.begin(), related.end(), std::move(converter)}; }; - [[nodiscard]] auto begin() const noexcept -> iterator { - return iterator{blobs_.begin()}; - } - - [[nodiscard]] auto end() const noexcept -> iterator { - return iterator{blobs_.end()}; - } - private: underlaying_map_t blobs_{}; }; diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index b9be5aed..2f6a1195 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -270,7 +270,7 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto Upload(BlobContainer const& blobs, bool /*skip_find_missing*/) noexcept -> bool final { - for (auto const& blob : blobs) { + for (auto const& blob : blobs.Blobs()) { auto const is_tree = NativeSupport::IsTree(blob.digest.hash()); auto cas_digest = is_tree ? storage_->CAS().StoreTree(blob.data) diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index 4b89a23d..3b85198e 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -188,9 +188,11 @@ auto BazelCasClient::FindMissingBlobs( auto BazelCasClient::FindMissingBlobs( std::string const& instance_name, - BlobContainer::DigestList const& digests) noexcept + BlobContainer const& blob_container) noexcept -> std::vector<bazel_re::Digest> { - return FindMissingBlobs(instance_name, digests.begin(), digests.end()); + auto digests_range = blob_container.Digests(); + return FindMissingBlobs( + instance_name, digests_range.begin(), digests_range.end()); } auto BazelCasClient::BatchReadBlobs( diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp index 4b669ab8..b2b80a48 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp @@ -54,7 +54,7 @@ class BazelCasClient { /// \returns The digests of blobs not found in CAS [[nodiscard]] auto FindMissingBlobs( std::string const& instance_name, - BlobContainer::DigestList const& digests) noexcept + BlobContainer const& blob_container) noexcept -> std::vector<bazel_re::Digest>; /// \brief Upload multiple blobs in batch transfer diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index af0365ff..34fe49c7 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -102,12 +102,12 @@ auto BazelNetwork::DoUploadBlobs(T_Iter const& first, auto BazelNetwork::UploadBlobs(BlobContainer const& blobs, bool skip_find_missing) noexcept -> bool { if (skip_find_missing) { - return DoUploadBlobs(blobs.begin(), blobs.end()); + auto blob_range = blobs.Blobs(); + return DoUploadBlobs(blob_range.begin(), blob_range.end()); } // find digests of blobs missing in CAS - auto missing_digests = - cas_->FindMissingBlobs(instance_name_, blobs.Digests()); + auto missing_digests = cas_->FindMissingBlobs(instance_name_, blobs); if (not missing_digests.empty()) { // update missing blobs diff --git a/test/buildtool/execution_engine/executor/executor.test.cpp b/test/buildtool/execution_engine/executor/executor.test.cpp index 790e20ca..02836d07 100644 --- a/test/buildtool/execution_engine/executor/executor.test.cpp +++ b/test/buildtool/execution_engine/executor/executor.test.cpp @@ -176,8 +176,9 @@ class TestApi : public IExecutionApi { } auto Upload(BlobContainer const& blobs, bool /*unused*/) noexcept -> bool final { + auto blob_range = blobs.Blobs(); return std::all_of( - blobs.begin(), blobs.end(), [this](auto const& blob) { + blob_range.begin(), blob_range.end(), [this](auto const& blob) { return config_.artifacts[blob.data] .uploads // for local artifacts or config_.artifacts[blob.digest.hash()] |