From 93cb5e3de9e142ceb6ef2baafa2046e917540563 Mon Sep 17 00:00:00 2001
From: Klaus Aehlig <klaus.aehlig@huawei.com>
Date: Thu, 26 Oct 2023 10:39:00 +0200
Subject: Fetch export targets sequentially when creating local backup

Typically, the number of export targets is small compared to the
size of the respective export targets. Moreover, export targets are
often chained, resulting in overlapping artifacts that we want to
fetch only once. Therefore, fetch export targets sequentially, giving
room to introduce parallelism in the individual fetch steps later.

While there, also log the beginning of the artifact synchronisation
for the export targets.
---
 src/buildtool/main/main.cpp | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

(limited to 'src')

diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 880c0e85..819d7470 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -836,32 +836,34 @@ void WriteTargetCacheEntries(
     std::unordered_map<TargetCacheKey, AnalysedTargetPtr> const& cache_targets,
     std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
         extra_infos,
-    std::size_t jobs,
+    std::size_t /* jobs */,
     gsl::not_null<IExecutionApi*> const& local_api,
     gsl::not_null<IExecutionApi*> const& remote_api) {
-    auto ts = TaskSystem{jobs};
+    if (!cache_targets.empty()) {
+        Logger::Log(LogLevel::Info,
+                    "Backing up artifacts of {} export targets",
+                    cache_targets.size());
+    }
     auto downloader = [&local_api, &remote_api](auto infos) {
         return remote_api->RetrieveToCas(infos, local_api);
     };
     for (auto const& [key, target] : cache_targets) {
-        ts.QueueTask(
-            [&key = key, &target = target, &extra_infos, &downloader]() {
-                if (auto entry =
-                        TargetCacheEntry::FromTarget(target, extra_infos)) {
-                    if (not Storage::Instance().TargetCache().Store(
-                            key, *entry, downloader)) {
-                        Logger::Log(LogLevel::Warning,
-                                    "Failed writing target cache entry for {}",
-                                    key.Id().ToString());
-                    }
-                }
-                else {
-                    Logger::Log(LogLevel::Warning,
-                                "Failed creating target cache entry for {}",
-                                key.Id().ToString());
-                }
-            });
+        if (auto entry = TargetCacheEntry::FromTarget(target, extra_infos)) {
+            if (not Storage::Instance().TargetCache().Store(
+                    key, *entry, downloader)) {
+                Logger::Log(LogLevel::Warning,
+                            "Failed writing target cache entry for {}",
+                            key.Id().ToString());
+            }
+        }
+        else {
+            Logger::Log(LogLevel::Warning,
+                        "Failed creating target cache entry for {}",
+                        key.Id().ToString());
+        }
     }
+    Logger::Log(LogLevel::Debug,
+                "Finished backing up artifacts of export targets");
 }
 
 #endif  // BOOTSTRAP_BUILD_TOOL
-- 
cgit v1.2.3