diff options
Diffstat (limited to 'src/buildtool')
9 files changed, 38 insertions, 42 deletions
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index 9ea074ec..1470d62a 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -60,6 +60,7 @@ , ["src/buildtool/execution_api/common", "message_limits"] , ["src/buildtool/execution_api/utils", "outputscheck"] , ["src/buildtool/file_system", "object_type"] + , ["src/utils/cpp", "back_map"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "transformed_range"] diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.cpp b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp index 17ad10a2..1b01811e 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_action.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp @@ -16,6 +16,7 @@ #include <algorithm> #include <compare> +#include <functional> #include <utility> // std::move #include "gsl/gsl" @@ -52,7 +53,7 @@ BazelAction::BazelAction( auto BazelAction::Execute(Logger const* logger) noexcept -> IExecutionResponse::Ptr { - BazelBlobContainer blobs{}; + std::unordered_set<BazelBlob> blobs{}; auto do_cache = CacheEnabled(cache_flag_); auto action = CreateBundlesForAction(&blobs, root_digest_, not do_cache); if (not action) { @@ -138,7 +139,7 @@ auto BazelAction::Execute(Logger const* logger) noexcept return nullptr; } -auto BazelAction::CreateBundlesForAction(BazelBlobContainer* blobs, +auto BazelAction::CreateBundlesForAction(std::unordered_set<BazelBlob>* blobs, ArtifactDigest const& exec_dir, bool do_not_cache) const noexcept -> std::optional<bazel_re::Digest> { @@ -146,7 +147,7 @@ auto BazelAction::CreateBundlesForAction(BazelBlobContainer* blobs, std::optional<StoreFunc> store_blob = std::nullopt; if (blobs != nullptr) { store_blob = [&blobs](BazelBlob&& blob) { - blobs->Emplace(std::move(blob)); + blobs->emplace(std::move(blob)); }; } BazelMsgFactory::ActionDigestRequest request{ diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.hpp b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp index 909bb8fb..ae32a010 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_action.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp @@ -20,6 +20,7 @@ #include <memory> #include <optional> #include <string> +#include <unordered_set> #include <vector> #include "src/buildtool/common/artifact_digest.hpp" @@ -65,10 +66,10 @@ class BazelAction final : public IExecutionAction { std::map<std::string, std::string> const& env_vars, std::map<std::string, std::string> const& properties) noexcept; - [[nodiscard]] auto CreateBundlesForAction(BazelBlobContainer* blobs, - ArtifactDigest const& exec_dir, - bool do_not_cache) const noexcept - -> std::optional<bazel_re::Digest>; + [[nodiscard]] auto CreateBundlesForAction( + std::unordered_set<BazelBlob>* blobs, + ArtifactDigest const& exec_dir, + bool do_not_cache) const noexcept -> std::optional<bazel_re::Digest>; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index ae5285df..1c8fd96d 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -141,19 +141,19 @@ namespace { [[nodiscard]] auto ConvertToBazelBlobContainer( ArtifactBlobContainer&& container) noexcept - -> std::optional<BazelBlobContainer> { - std::vector<BazelBlob> blobs; + -> std::optional<std::unordered_set<BazelBlob>> { + std::unordered_set<BazelBlob> blobs; try { blobs.reserve(container.Size()); for (const auto& blob : container.Blobs()) { - blobs.emplace_back(ArtifactDigestFactory::ToBazel(blob.digest), - blob.data, - blob.is_exec); + blobs.emplace(ArtifactDigestFactory::ToBazel(blob.digest), + blob.data, + blob.is_exec); } } catch (...) { return std::nullopt; } - return BazelBlobContainer{std::move(blobs)}; + return blobs; } } // namespace @@ -531,7 +531,7 @@ auto BazelApi::CreateAction( bool skip_find_missing) const noexcept -> bool { auto bazel_blobs = ConvertToBazelBlobContainer(std::move(blobs)); - return bazel_blobs ? network_->UploadBlobs(std::move(*bazel_blobs), + return bazel_blobs ? network_->UploadBlobs(*std::move(bazel_blobs), skip_find_missing) : false; } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp index dff23491..5bd1d3b0 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp @@ -37,7 +37,6 @@ #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" -#include "src/utils/cpp/transformed_range.hpp" namespace { @@ -192,14 +191,6 @@ auto BazelCasClient::FindMissingBlobs( return FindMissingBlobs(instance_name, digests.begin(), digests.end()); } -auto BazelCasClient::FindMissingBlobs(std::string const& instance_name, - BazelBlobContainer const& blob_container) - const noexcept -> std::unordered_set<bazel_re::Digest> { - auto digests_range = blob_container.Digests(); - return FindMissingBlobs( - instance_name, digests_range.begin(), digests_range.end()); -} - auto BazelCasClient::BatchReadBlobs( std::string const& instance_name, std::vector<bazel_re::Digest>::const_iterator const& begin, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp index ad3210f4..c08adc92 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp @@ -57,15 +57,6 @@ class BazelCasClient { std::unordered_set<bazel_re::Digest> const& digests) const noexcept -> std::unordered_set<bazel_re::Digest>; - /// \brief Find missing blobs - /// \param[in] instance_name Name of the CAS instance - /// \param[in] digests The blob digests to search for - /// \returns The digests of blobs not found in CAS - [[nodiscard]] auto FindMissingBlobs( - std::string const& instance_name, - BazelBlobContainer const& blob_container) const noexcept - -> std::unordered_set<bazel_re::Digest>; - /// \brief Upload multiple blobs in batch transfer /// \param[in] instance_name Name of the CAS instance /// \param[in] begin Start of the blobs to upload diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 3f33136a..c8eb76df 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -19,10 +19,11 @@ #include <iterator> #include <utility> +#include "src/buildtool/execution_api/common/content_blob_container.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" -#include "src/utils/cpp/transformed_range.hpp" +#include "src/utils/cpp/back_map.hpp" BazelNetwork::BazelNetwork( std::string instance_name, @@ -108,21 +109,25 @@ auto BazelNetwork::DoUploadBlobs(TIter const& first, } } -auto BazelNetwork::UploadBlobs(BazelBlobContainer&& blobs, +auto BazelNetwork::UploadBlobs(std::unordered_set<BazelBlob>&& blobs, bool skip_find_missing) noexcept -> bool { if (skip_find_missing) { - auto blob_range = blobs.Blobs(); - return DoUploadBlobs(blob_range.begin(), blob_range.end()); + return DoUploadBlobs(blobs.begin(), blobs.end()); + } + + auto const back_map = BackMap<bazel_re::Digest, BazelBlob>::Make( + &blobs, [](BazelBlob const& blob) { return blob.digest; }); + if (not back_map.has_value()) { + return false; } // find digests of blobs missing in CAS - auto missing_digests_set = cas_->FindMissingBlobs(instance_name_, blobs); - std::vector missing_digests(missing_digests_set.begin(), - missing_digests_set.end()); + auto missing_digests = + cas_->FindMissingBlobs(instance_name_, back_map->GetKeys()); if (not missing_digests.empty()) { // update missing blobs - auto missing_blobs = blobs.RelatedBlobs(missing_digests); + auto missing_blobs = back_map->GetValues(missing_digests); return DoUploadBlobs(missing_blobs.begin(), missing_blobs.end()); } return true; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index 2e99de9f..cd4dd1dc 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -72,7 +72,7 @@ class BazelNetwork { /// \param blobs The blobs to upload /// \param skip_find_missing Skip finding missing blobs, just upload all /// \returns True if upload was successful, false otherwise - [[nodiscard]] auto UploadBlobs(BazelBlobContainer&& blobs, + [[nodiscard]] auto UploadBlobs(std::unordered_set<BazelBlob>&& blobs, bool skip_find_missing = false) noexcept -> bool; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_response.cpp b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp index 3aaecb88..2a06985d 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_response.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp @@ -20,6 +20,7 @@ #include <filesystem> #include <functional> #include <iterator> +#include <unordered_set> #include <vector> #include "fmt/core.h" @@ -40,6 +41,7 @@ #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" #include "src/utils/cpp/path.hpp" +#include "src/utils/cpp/transformed_range.hpp" namespace { @@ -276,7 +278,11 @@ auto BazelResponse::UploadTreeMessageDirectories( bazel_re::Tree const& tree) const -> expected<ArtifactDigest, std::string> { auto const upload_callback = [&network = *network_](BazelBlobContainer&& blobs) -> bool { - return network.UploadBlobs(std::move(blobs)); + std::unordered_set<BazelBlob> bazel_blobs; + for (auto const& blob : blobs.Blobs()) { + bazel_blobs.emplace(blob); + } + return network.UploadBlobs(std::move(bazel_blobs)); }; auto const hash_function = network_->GetHashFunction(); BazelBlobContainer dir_blobs{}; |