From 7b62c8274bea6b55a36b425b670a341c8093de08 Mon Sep 17 00:00:00 2001
From: Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com>
Date: Fri, 24 May 2024 14:26:04 +0200
Subject: blob containers: Store and upload taking into account content size

Update logic populating containers to use the new method which
is aware of the maximum transfer limit.
---
 src/buildtool/execution_api/git/git_api.hpp | 51 +++++++++++++++++------------
 1 file changed, 30 insertions(+), 21 deletions(-)

(limited to 'src/buildtool/execution_api/git/git_api.hpp')

diff --git a/src/buildtool/execution_api/git/git_api.hpp b/src/buildtool/execution_api/git/git_api.hpp
index 914d4c19..b3b4daa3 100644
--- a/src/buildtool/execution_api/git/git_api.hpp
+++ b/src/buildtool/execution_api/git/git_api.hpp
@@ -23,8 +23,10 @@
 #include <vector>
 
 #include "gsl/gsl"
+#include "src/buildtool/common/artifact_digest.hpp"
 #include "src/buildtool/common/repository_config.hpp"
 #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/common/artifact_blob_container.hpp"
 #include "src/buildtool/execution_api/common/common_api.hpp"
 #include "src/buildtool/execution_api/common/execution_api.hpp"
 #include "src/buildtool/logging/log_level.hpp"
@@ -220,25 +222,28 @@ class GitApi final : public IExecutionApi {
                         }
                     }
                     else {
-                        content = entry->RawData();
-                        if (not content) {
+                        auto const& entry_content = entry->RawData();
+                        if (not entry_content) {
                             return false;
                         }
-                        auto digest =
-                            ArtifactDigest::Create<ObjectType::File>(*content);
-                        try {
-                            tree_deps_only_blobs.Emplace(ArtifactBlob{
-                                digest,
-                                *content,
-                                IsExecutableObject(entry->Type())});
-                        } catch (std::exception const& ex) {
-                            Logger::Log(LogLevel::Error,
-                                        "failed to emplace blob: ",
-                                        ex.what());
+                        auto digest = ArtifactDigest::Create<ObjectType::File>(
+                            *entry_content);
+                        // Collect blob and upload to remote CAS if transfer
+                        // size reached.
+                        if (not UpdateContainerAndUpload<ArtifactDigest>(
+                                &tree_deps_only_blobs,
+                                ArtifactBlob{std::move(digest),
+                                             *entry_content,
+                                             IsExecutableObject(entry->Type())},
+                                /*exception_is_fatal=*/true,
+                                [&api](ArtifactBlobContainer&& blobs) -> bool {
+                                    return api->Upload(std::move(blobs));
+                                })) {
                             return false;
                         }
                     }
                 }
+                // Upload remaining blobs.
                 if (not api->Upload(std::move(tree_deps_only_blobs))) {
                     return false;
                 }
@@ -256,18 +261,22 @@ class GitApi final : public IExecutionApi {
                     ? ArtifactDigest::Create<ObjectType::Tree>(*content)
                     : ArtifactDigest::Create<ObjectType::File>(*content);
 
-            try {
-                container.Emplace(ArtifactBlob{std::move(digest),
-                                               *content,
-                                               IsExecutableObject(info.type)});
-            } catch (std::exception const& ex) {
-                Logger::Log(
-                    LogLevel::Error, "failed to emplace blob: ", ex.what());
+            // Collect blob and upload to remote CAS if transfer size reached.
+            if (not UpdateContainerAndUpload<ArtifactDigest>(
+                    &container,
+                    ArtifactBlob{std::move(digest),
+                                 std::move(*content),
+                                 IsExecutableObject(info.type)},
+                    /*exception_is_fatal=*/true,
+                    [&api](ArtifactBlobContainer&& blobs) {
+                        return api->Upload(std::move(blobs),
+                                           /*skip_find_missing=*/true);
+                    })) {
                 return false;
             }
         }
 
-        // Upload blobs to remote CAS.
+        // Upload remaining blobs to remote CAS.
         return api->Upload(std::move(container), /*skip_find_missing=*/true);
     }
 
-- 
cgit v1.2.3