summaryrefslogtreecommitdiff
path: root/src/buildtool/execution_api/common
diff options
context:
space:
mode:
Diffstat (limited to 'src/buildtool/execution_api/common')
-rw-r--r--src/buildtool/execution_api/common/TARGETS10
-rw-r--r--src/buildtool/execution_api/common/common_api.cpp59
-rw-r--r--src/buildtool/execution_api/common/common_api.hpp62
3 files changed, 67 insertions, 64 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS
index df50ecaf..2687aec7 100644
--- a/src/buildtool/execution_api/common/TARGETS
+++ b/src/buildtool/execution_api/common/TARGETS
@@ -90,21 +90,21 @@
, "hdrs": ["common_api.hpp"]
, "srcs": ["common_api.cpp"]
, "deps":
- [ "blob_tree"
+ [ "artifact_blob_container"
+ , "blob_tree"
, "common"
- , "content_blob_container"
- , "message_limits"
, ["@", "gsl", "", "gsl"]
, ["src/buildtool/common", "common"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
, ["src/buildtool/execution_api/bazel_msg", "directory_tree"]
- , ["src/buildtool/logging", "log_level"]
, ["src/buildtool/logging", "logging"]
]
, "stage": ["src", "buildtool", "execution_api", "common"]
, "private-deps":
- [ "artifact_blob_container"
+ [ "content_blob_container"
+ , "message_limits"
, ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/logging", "log_level"]
, ["src/buildtool/file_system", "object_type"]
]
}
diff --git a/src/buildtool/execution_api/common/common_api.cpp b/src/buildtool/execution_api/common/common_api.cpp
index 8bd8f0e0..3a713fc4 100644
--- a/src/buildtool/execution_api/common/common_api.cpp
+++ b/src/buildtool/execution_api/common/common_api.cpp
@@ -28,8 +28,10 @@
#include <utility>
#include "fmt/core.h"
-#include "src/buildtool/execution_api/common/artifact_blob_container.hpp"
+#include "src/buildtool/execution_api/common/content_blob_container.hpp"
+#include "src/buildtool/execution_api/common/message_limits.hpp"
#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/log_level.hpp"
auto CommonRetrieveToFds(
std::vector<Artifact::ObjectInfo> const& artifacts_info,
@@ -123,7 +125,7 @@ auto CommonUploadBlobTree(BlobTreePtr const& blob_tree,
}
// Optimize store & upload by taking into account the maximum
// transfer size.
- if (not UpdateContainerAndUpload<ArtifactDigest>(
+ if (not UpdateContainerAndUpload(
&container,
node->Blob(),
/*exception_is_fatal=*/false,
@@ -148,7 +150,7 @@ auto CommonUploadTreeCompatible(
// Store and upload blobs, taking into account the maximum transfer size.
auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
build_root, resolve_links, [&blobs, &api](ArtifactBlob&& blob) {
- return UpdateContainerAndUpload<ArtifactDigest>(
+ return UpdateContainerAndUpload(
&blobs,
std::move(blob),
/*exception_is_fatal=*/false,
@@ -202,3 +204,54 @@ auto CommonUploadTreeNative(IExecutionApi const& api,
}
return tree_blob.digest;
}
+
+auto UpdateContainerAndUpload(
+ gsl::not_null<std::unordered_set<ArtifactBlob>*> const& container,
+ ArtifactBlob&& blob,
+ bool exception_is_fatal,
+ std::function<bool(std::unordered_set<ArtifactBlob>&&)> const& uploader,
+ Logger const* logger) noexcept -> bool {
+ // Optimize upload of blobs with respect to the maximum transfer limit, such
+ // that we never store unnecessarily more data in the container than we need
+ // per remote transfer.
+ try {
+ if (blob.data->size() > kMaxBatchTransferSize) {
+ // large blobs use individual stream upload
+ if (not uploader(
+ std::unordered_set<ArtifactBlob>{{std::move(blob)}})) {
+ return false;
+ }
+ }
+ else {
+ if (not container->contains(blob)) {
+ std::size_t content_size = 0;
+ for (auto const& blob : *container) {
+ content_size += blob.data->size();
+ }
+
+ if (content_size + blob.data->size() > kMaxBatchTransferSize) {
+ // swap away from original container to allow move during
+ // upload
+ std::unordered_set<ArtifactBlob> tmp_container{};
+ std::swap(*container, tmp_container);
+ // if we would surpass the transfer limit, upload the
+ // current container and clear it before adding more blobs
+ if (not uploader(std::move(tmp_container))) {
+ return false;
+ }
+ }
+ // add current blob to container
+ container->emplace(std::move(blob));
+ }
+ }
+ } catch (std::exception const& ex) {
+ if (exception_is_fatal) {
+ Logger::Log(logger,
+ LogLevel::Error,
+ "failed to emplace blob with\n:{}",
+ ex.what());
+ }
+ return false;
+ }
+ return true; // success!
+}
diff --git a/src/buildtool/execution_api/common/common_api.hpp b/src/buildtool/execution_api/common/common_api.hpp
index 5060c086..04997b5f 100644
--- a/src/buildtool/execution_api/common/common_api.hpp
+++ b/src/buildtool/execution_api/common/common_api.hpp
@@ -15,9 +15,7 @@
#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP
#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP
-#include <cstddef>
#include <cstdio>
-#include <exception>
#include <functional>
#include <iterator>
#include <optional>
@@ -30,11 +28,9 @@
#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp"
+#include "src/buildtool/execution_api/common/artifact_blob_container.hpp"
#include "src/buildtool/execution_api/common/blob_tree.hpp"
-#include "src/buildtool/execution_api/common/content_blob_container.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
-#include "src/buildtool/execution_api/common/message_limits.hpp"
-#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
/// \brief Stores a list of missing artifact digests, as well as a back-mapping
@@ -121,57 +117,11 @@ template <typename TValue, typename TIterator>
/// \param logger Use this instance for any logging. If nullptr, use the default
/// logger. This value is used only if exception_is_fatal==true.
/// \returns Returns true on success, false otherwise (failures or exceptions).
-template <typename TDigest>
-auto UpdateContainerAndUpload(
- gsl::not_null<std::unordered_set<ContentBlob<TDigest>>*> const& container,
- ContentBlob<TDigest>&& blob,
+[[nodiscard]] auto UpdateContainerAndUpload(
+ gsl::not_null<std::unordered_set<ArtifactBlob>*> const& container,
+ ArtifactBlob&& blob,
bool exception_is_fatal,
- std::function<bool(std::unordered_set<ContentBlob<TDigest>>&&)> const&
- uploader,
- Logger const* logger = nullptr) noexcept -> bool {
- // Optimize upload of blobs with respect to the maximum transfer limit, such
- // that we never store unnecessarily more data in the container than we need
- // per remote transfer.
- try {
- if (blob.data->size() > kMaxBatchTransferSize) {
- // large blobs use individual stream upload
- if (not uploader(std::unordered_set<ContentBlob<TDigest>>{
- {std::move(blob)}})) {
- return false;
- }
- }
- else {
- if (not container->contains(blob)) {
- std::size_t content_size = 0;
- for (auto const& blob : *container) {
- content_size += blob.data->size();
- }
-
- if (content_size + blob.data->size() > kMaxBatchTransferSize) {
- // swap away from original container to allow move during
- // upload
- std::unordered_set<ContentBlob<TDigest>> tmp_container{};
- std::swap(*container, tmp_container);
- // if we would surpass the transfer limit, upload the
- // current container and clear it before adding more blobs
- if (not uploader(std::move(tmp_container))) {
- return false;
- }
- }
- // add current blob to container
- container->emplace(std::move(blob));
- }
- }
- } catch (std::exception const& ex) {
- if (exception_is_fatal) {
- Logger::Log(logger,
- LogLevel::Error,
- "failed to emplace blob with\n:{}",
- ex.what());
- }
- return false;
- }
- return true; // success!
-}
+ std::function<bool(std::unordered_set<ArtifactBlob>&&)> const& uploader,
+ Logger const* logger = nullptr) noexcept -> bool;
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 06:54:21 +0000