summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_api/common/common_api.hpp7
-rw-r--r--src/buildtool/execution_api/common/execution_api.hpp5
-rw-r--r--src/buildtool/execution_api/git/git_api.hpp11
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp11
-rw-r--r--src/buildtool/execution_api/remote/TARGETS1
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.cpp27
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.hpp5
-rw-r--r--src/buildtool/execution_api/serve/mr_git_api.hpp6
-rw-r--r--src/buildtool/execution_api/serve/mr_local_api.cpp7
-rw-r--r--src/buildtool/execution_api/serve/mr_local_api.hpp6
-rw-r--r--src/buildtool/execution_engine/executor/TARGETS1
-rw-r--r--src/buildtool/execution_engine/executor/executor.hpp90
12 files changed, 89 insertions, 88 deletions
diff --git a/src/buildtool/execution_api/common/common_api.hpp b/src/buildtool/execution_api/common/common_api.hpp
index 991b9875..d5daa3dd 100644
--- a/src/buildtool/execution_api/common/common_api.hpp
+++ b/src/buildtool/execution_api/common/common_api.hpp
@@ -21,6 +21,7 @@
#include <iterator>
#include <optional>
#include <unordered_map>
+#include <unordered_set>
#include <vector>
#include "gsl/gsl"
@@ -39,7 +40,7 @@
/// to some given original type.
template <typename T>
struct MissingArtifactsInfo {
- std::vector<ArtifactDigest> digests;
+ std::unordered_set<ArtifactDigest> digests;
std::unordered_map<ArtifactDigest, T> back_map;
};
@@ -69,7 +70,7 @@ template <typename TValue, typename TIterator>
TIterator const& end,
typename std::function<ArtifactDigest(TValue const&)> const&
converter) noexcept -> std::optional<MissingArtifactsInfo<TValue>> {
- std::vector<ArtifactDigest> digests;
+ std::unordered_set<ArtifactDigest> digests;
digests.reserve(std::distance(begin, end));
MissingArtifactsInfo<TValue> res{};
for (auto it = begin; it != end; ++it) {
@@ -77,7 +78,7 @@ template <typename TValue, typename TIterator>
auto const inserted =
res.back_map.insert({std::invoke(converter, *it), *it});
if (inserted.second) {
- digests.emplace_back(inserted.first->first);
+ digests.emplace(inserted.first->first);
}
} catch (...) {
return std::nullopt;
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp
index 02c78bcc..b868b57d 100644
--- a/src/buildtool/execution_api/common/execution_api.hpp
+++ b/src/buildtool/execution_api/common/execution_api.hpp
@@ -21,6 +21,7 @@
#include <memory>
#include <optional>
#include <string>
+#include <unordered_set>
#include <vector>
#include "src/buildtool/common/artifact.hpp" // Artifact::ObjectInfo
@@ -129,8 +130,8 @@ class IExecutionApi {
ArtifactDigest const& digest) const noexcept -> bool = 0;
[[nodiscard]] virtual auto IsAvailable(
- std::vector<ArtifactDigest> const& digests) const noexcept
- -> std::vector<ArtifactDigest> = 0;
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> = 0;
[[nodiscard]] virtual auto SplitBlob(ArtifactDigest const& /*blob_digest*/)
const noexcept -> std::optional<std::vector<ArtifactDigest>> {
diff --git a/src/buildtool/execution_api/git/git_api.hpp b/src/buildtool/execution_api/git/git_api.hpp
index a6ad4196..7fd21005 100644
--- a/src/buildtool/execution_api/git/git_api.hpp
+++ b/src/buildtool/execution_api/git/git_api.hpp
@@ -24,6 +24,7 @@
#include <optional>
#include <string>
#include <unordered_map>
+#include <unordered_set>
#include <utility>
#include <vector>
@@ -327,12 +328,14 @@ class GitApi final : public IExecutionApi {
.has_value();
}
- [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests)
- const noexcept -> std::vector<ArtifactDigest> override {
- std::vector<ArtifactDigest> result;
+ [[nodiscard]] auto IsAvailable(
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> override {
+ std::unordered_set<ArtifactDigest> result;
+ result.reserve(digests.size());
for (auto const& digest : digests) {
if (not IsAvailable(digest)) {
- result.push_back(digest);
+ result.emplace(digest);
}
}
return result;
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 7ee74be9..f1e448a0 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -27,6 +27,7 @@
#include <sstream>
#include <string>
#include <unordered_map>
+#include <unordered_set>
#include <utility> // std::move
#include <vector>
@@ -311,12 +312,14 @@ class LocalApi final : public IExecutionApi {
return found;
}
- [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests)
- const noexcept -> std::vector<ArtifactDigest> final {
- std::vector<ArtifactDigest> result;
+ [[nodiscard]] auto IsAvailable(
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> final {
+ std::unordered_set<ArtifactDigest> result;
+ result.reserve(digests.size());
for (auto const& digest : digests) {
if (not IsAvailable(digest)) {
- result.push_back(digest);
+ result.emplace(digest);
}
}
return result;
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS
index f61324bd..9ea074ec 100644
--- a/src/buildtool/execution_api/remote/TARGETS
+++ b/src/buildtool/execution_api/remote/TARGETS
@@ -84,7 +84,6 @@
, "stage": ["src", "buildtool", "execution_api", "remote"]
, "private-deps":
[ "bazel_network"
- , ["@", "fmt", "", "fmt"]
, ["src/buildtool/common", "artifact_digest_factory"]
, ["src/buildtool/common", "bazel_types"]
, ["src/buildtool/common", "protocol_traits"]
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
index ed5247e5..66f81d01 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
@@ -117,14 +117,15 @@ namespace {
}
// Fetch unknown chunks.
- auto digest_set = std::unordered_set<ArtifactDigest>{chunk_digests->begin(),
- chunk_digests->end()};
- auto unique_digests =
- std::vector<ArtifactDigest>{digest_set.begin(), digest_set.end()};
-
- auto missing_artifact_digests = other_api.IsAvailable(unique_digests);
- if (not ::RetrieveToCas(
- missing_artifact_digests, other_api, network, info_map)) {
+ auto missing_artifact_digests = other_api.IsAvailable(
+ std::unordered_set(chunk_digests->begin(), chunk_digests->end()));
+
+ std::vector<ArtifactDigest> missing_digests;
+ missing_digests.reserve(missing_artifact_digests.size());
+ std::move(missing_artifact_digests.begin(),
+ missing_artifact_digests.end(),
+ std::back_inserter(missing_digests));
+ if (not ::RetrieveToCas(missing_digests, other_api, network, info_map)) {
return false;
}
@@ -570,8 +571,8 @@ auto BazelApi::CreateAction(
}
[[nodiscard]] auto BazelApi::IsAvailable(
- std::vector<ArtifactDigest> const& digests) const noexcept
- -> std::vector<ArtifactDigest> {
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> {
auto const back_map = BackMap<bazel_re::Digest, ArtifactDigest>::Make(
&digests, ArtifactDigestFactory::ToBazel);
if (not back_map.has_value()) {
@@ -579,11 +580,7 @@ auto BazelApi::CreateAction(
}
auto const bazel_result = network_->IsAvailable(back_map->GetKeys());
- auto missing = back_map->GetValues(bazel_result);
- std::vector<ArtifactDigest> result;
- result.reserve(missing.size());
- std::move(missing.begin(), missing.end(), std::back_inserter(result));
- return result;
+ return back_map->GetValues(bazel_result);
}
[[nodiscard]] auto BazelApi::SplitBlob(ArtifactDigest const& blob_digest)
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
index e45e0cdf..de4662db 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
@@ -102,8 +102,9 @@ class BazelApi final : public IExecutionApi {
[[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept
-> bool final;
- [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests)
- const noexcept -> std::vector<ArtifactDigest> final;
+ [[nodiscard]] auto IsAvailable(
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> final;
[[nodiscard]] auto RetrieveToMemory(
Artifact::ObjectInfo const& artifact_info) const noexcept
diff --git a/src/buildtool/execution_api/serve/mr_git_api.hpp b/src/buildtool/execution_api/serve/mr_git_api.hpp
index 31f723bf..b03bfcf1 100644
--- a/src/buildtool/execution_api/serve/mr_git_api.hpp
+++ b/src/buildtool/execution_api/serve/mr_git_api.hpp
@@ -16,9 +16,11 @@
#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP
#include <filesystem>
+#include <functional>
#include <map>
#include <optional>
#include <string>
+#include <unordered_set>
#include <vector>
#include "gsl/gsl"
@@ -122,8 +124,8 @@ class MRGitApi final : public IExecutionApi {
/// \brief Not implemented.
[[nodiscard]] auto IsAvailable(
- std::vector<ArtifactDigest> const& /*digests*/) const noexcept
- -> std::vector<ArtifactDigest> final {
+ std::unordered_set<ArtifactDigest> const& /*digests*/) const noexcept
+ -> std::unordered_set<ArtifactDigest> final {
// Not supported.
return {};
}
diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp
index 118fd564..5449b245 100644
--- a/src/buildtool/execution_api/serve/mr_local_api.cpp
+++ b/src/buildtool/execution_api/serve/mr_local_api.cpp
@@ -14,6 +14,7 @@
#include "src/buildtool/execution_api/serve/mr_local_api.hpp"
+#include <functional>
#include <utility>
#include "src/buildtool/common/protocol_traits.hpp"
@@ -123,8 +124,8 @@ auto MRLocalApi::IsAvailable(ArtifactDigest const& digest) const noexcept
return compat_local_api_->IsAvailable(digest);
}
-auto MRLocalApi::IsAvailable(std::vector<ArtifactDigest> const& digests)
- const noexcept -> std::vector<ArtifactDigest> {
+auto MRLocalApi::IsAvailable(std::unordered_set<ArtifactDigest> const& digests)
+ const noexcept -> std::unordered_set<ArtifactDigest> {
// This method can legitimately be called with both native and
// compatible digests when in compatible mode, therefore we need to
// interrogate the hash type of the input.
@@ -134,7 +135,7 @@ auto MRLocalApi::IsAvailable(std::vector<ArtifactDigest> const& digests)
return {}; // nothing to do
}
// native digests get dispatched to native local api
- if (ProtocolTraits::IsNative(digests[0].GetHashType())) {
+ if (ProtocolTraits::IsNative(digests.begin()->GetHashType())) {
return native_local_api_->IsAvailable(digests);
}
// compatible digests get dispatched to compatible local api
diff --git a/src/buildtool/execution_api/serve/mr_local_api.hpp b/src/buildtool/execution_api/serve/mr_local_api.hpp
index f930bc46..c548241e 100644
--- a/src/buildtool/execution_api/serve/mr_local_api.hpp
+++ b/src/buildtool/execution_api/serve/mr_local_api.hpp
@@ -19,6 +19,7 @@
#include <map>
#include <optional>
#include <string>
+#include <unordered_set>
#include <vector>
#include "gsl/gsl"
@@ -125,8 +126,9 @@ class MRLocalApi final : public IExecutionApi {
/// \note The caller is responsible for passing vectors with digests of the
/// same type. For simplicity, this method takes the first digest of the
/// vector as representative for figuring out hash function type.
- [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests)
- const noexcept -> std::vector<ArtifactDigest> final;
+ [[nodiscard]] auto IsAvailable(
+ std::unordered_set<ArtifactDigest> const& digests) const noexcept
+ -> std::unordered_set<ArtifactDigest> final;
private:
// retain local context references to have direct access to storages
diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS
index 39cad1c0..ff652d75 100644
--- a/src/buildtool/execution_engine/executor/TARGETS
+++ b/src/buildtool/execution_engine/executor/TARGETS
@@ -32,6 +32,7 @@
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/progress_reporting", "progress"]
, ["src/buildtool/progress_reporting", "task_tracker"]
+ , ["src/utils/cpp", "back_map"]
, ["src/utils/cpp", "expected"]
, ["src/utils/cpp", "hex_string"]
, ["src/utils/cpp", "path_rebase"]
diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp
index 0ec7dfe6..af0b759e 100644
--- a/src/buildtool/execution_engine/executor/executor.hpp
+++ b/src/buildtool/execution_engine/executor/executor.hpp
@@ -65,6 +65,7 @@
#include "src/buildtool/logging/logger.hpp"
#include "src/buildtool/progress_reporting/progress.hpp"
#include "src/buildtool/progress_reporting/task_tracker.hpp"
+#include "src/utils/cpp/back_map.hpp"
#include "src/utils/cpp/expected.hpp"
#include "src/utils/cpp/hex_string.hpp"
#include "src/utils/cpp/path_rebase.hpp"
@@ -293,26 +294,17 @@ class ExecutorImpl {
GitTree const& tree) noexcept
-> bool {
// create list of digests for batch check of CAS availability
- std::vector<ArtifactDigest> digests;
- std::unordered_map<ArtifactDigest, gsl::not_null<GitTreeEntryPtr>>
- entry_map;
- for (auto const& [path, entry] : tree) {
- // Since GitTrees are processed here, HashFunction::Type::GitSHA1 is
- // used
- auto digest =
- ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1,
- entry->Hash(),
- *entry->Size(),
- entry->IsTree());
- if (not digest) {
- return false;
- }
- digests.emplace_back(*digest);
- try {
- entry_map.emplace(*std::move(digest), entry);
- } catch (...) {
- return false;
- }
+ using ElementType = typename GitTree::entries_t::value_type;
+ auto const back_map = BackMap<ArtifactDigest, ElementType>::Make(
+ &tree, [](ElementType const& entry) {
+ return ArtifactDigestFactory::Create(
+ HashFunction::Type::GitSHA1,
+ entry.second->Hash(),
+ *entry.second->Size(),
+ entry.second->IsTree());
+ });
+ if (not back_map.has_value()) {
+ return false;
}
Logger::Log(LogLevel::Trace, [&tree]() {
@@ -327,44 +319,42 @@ class ExecutorImpl {
});
// find missing digests
- auto missing_digests = api.IsAvailable(digests);
+ auto const missing_digests = api.IsAvailable(back_map->GetKeys());
+ auto const missing_entries =
+ back_map->IterateReferences(&missing_digests);
// process missing trees
- for (auto const& digest : missing_digests) {
- if (auto it = entry_map.find(digest); it != entry_map.end()) {
- auto const& entry = it->second;
- if (entry->IsTree()) {
- auto const& tree = entry->Tree();
- if (not tree or not VerifyOrUploadTree(api, *tree)) {
- return false;
- }
+ for (auto const& [_, value] : missing_entries) {
+ auto const entry = value->second;
+ if (entry->IsTree()) {
+ auto const& tree = entry->Tree();
+ if (not tree or not VerifyOrUploadTree(api, *tree)) {
+ return false;
}
}
}
// upload missing entries (blobs or trees)
ArtifactBlobContainer container;
- for (auto const& digest : missing_digests) {
- if (auto it = entry_map.find(digest); it != entry_map.end()) {
- auto const& entry = it->second;
- auto content = entry->RawData();
- if (not content) {
- return false;
- }
- // store and/or upload blob, taking into account the maximum
- // transfer size
- if (not UpdateContainerAndUpload<ArtifactDigest>(
- &container,
- ArtifactBlob{digest,
- std::move(*content),
- IsExecutableObject(entry->Type())},
- /*exception_is_fatal=*/true,
- [&api](ArtifactBlobContainer&& blobs) {
- return api.Upload(std::move(blobs),
- /*skip_find_missing=*/true);
- })) {
- return false;
- }
+ for (auto const& [digest, value] : missing_entries) {
+ auto const entry = value->second;
+ auto content = entry->RawData();
+ if (not content.has_value()) {
+ return false;
+ }
+ // store and/or upload blob, taking into account the maximum
+ // transfer size
+ if (not UpdateContainerAndUpload<ArtifactDigest>(
+ &container,
+ ArtifactBlob{*digest,
+ std::move(*content),
+ IsExecutableObject(entry->Type())},
+ /*exception_is_fatal=*/true,
+ [&api](ArtifactBlobContainer&& blobs) {
+ return api.Upload(std::move(blobs),
+ /*skip_find_missing=*/true);
+ })) {
+ return false;
}
}
// upload remaining blobs
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 20:57:14 +0000