just-mr: rework progress reporting and statistics

To avoid unnecessary work, just-mr uses on-disk caches, including
for the mapping of a distdir to the corresponding git tree. This,
however, implies that the number of repositories that are actually
considered varies: in order to fetch a distdir repository, all
involved archives have to be fetched, but if we have a cache hit
none of them is even looked at.

So, in order to have a consistent reporting only count top-level
targets (i.e., the reachable repositories) in the statistics,
not the archives implicitly contained in a distdir, nor low-level
sub tasks. The actual fetch acitvity is shown separately by the
task tracker.

16 files changed, 141 insertions, 245 deletions

diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 53711136..965c3bf0 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -442,51 +442,11 @@ void SetupLogging(MultiRepoLogArguments const& clargs) {
     return config;
 }
 
-/// \brief Add repos from a given repo list to the reported repos set if its
-/// type is distdir. Ignore any missing or wrong info.
-void AddDistdirReportedRepos(
-    ExpressionPtr const& repos,
-    std::string const& repo_name,
-    std::shared_ptr<std::unordered_set<std::string>> const&
-        reported_repos_set) {
-    auto repo_desc_key = repos->Get(repo_name, Expression::none_t{});
-    auto repo_desc = repo_desc_key->Get("repository", Expression::none_t{});
-    auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc, repos);
-    if (not resolved_repo_desc) {
-        return;
-    }
-    auto repo_type = (*resolved_repo_desc)->Get("type", Expression::none_t{});
-    if (not repo_type->IsString()) {
-        return;
-    }
-    // check if the repo is distdir
-    auto repo_type_str = repo_type->String();
-    if (repo_type_str != "distdir") {
-        return;
-    }
-    // get distdir list
-    auto repo_desc_repositories =
-        repo_desc->Get("repositories", Expression::none_t{});
-    if (not repo_desc_repositories->IsList()) {
-        return;
-    }
-    auto distdir_repos = repo_desc_repositories->List();
-    // add to reported set
-    for (auto const& entry : distdir_repos) {
-        if (entry->IsString()) {
-            reported_repos_set->insert(entry->String());
-        }
-    }
-}
-
 /// \brief Get the repo dependency closure for a given main repository.
 /// For progress reporting we include
-void ReachableRepositories(
-    ExpressionPtr const& repos,
-    std::string const& main,
-    std::shared_ptr<SetupRepos> const& setup_repos,
-    std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
-        nullptr) {
+void ReachableRepositories(ExpressionPtr const& repos,
+                           std::string const& main,
+                           std::shared_ptr<SetupRepos> const& setup_repos) {
     // use temporary sets to avoid duplicates
     std::unordered_set<std::string> include_repos_set{};
     if (repos->IsMap()) {
@@ -496,12 +456,6 @@ void ReachableRepositories(
                 if (not include_repos_set.contains(repo_name)) {
                     // if not found, add it and repeat for its bindings
                     include_repos_set.insert(repo_name);
-                    // add to reported repos, if needed
-                    if (reported_repos_set) {
-                        reported_repos_set->insert(repo_name);
-                        AddDistdirReportedRepos(
-                            repos, repo_name, reported_repos_set);
-                    }
                     // check bindings
                     auto repos_repo_name =
                         repos->Get(repo_name, Expression::none_t{});
@@ -533,12 +487,6 @@ void ReachableRepositories(
                     if (layer_val.IsNotNull() and layer_val->IsString()) {
                         auto repo_name = layer_val->String();
                         setup_repos_set.insert(repo_name);
-                        // add to reported repos, if needed
-                        if (reported_repos_set) {
-                            reported_repos_set->insert(repo_name);
-                            AddDistdirReportedRepos(
-                                repos, repo_name, reported_repos_set);
-                        }
                     }
                 }
             }
@@ -562,20 +510,10 @@ void ReachableRepositories(
 
 void DefaultReachableRepositories(
     ExpressionPtr const& repos,
-    std::shared_ptr<SetupRepos> const& setup_repos,
-    std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
-        nullptr) {
+    std::shared_ptr<SetupRepos> const& setup_repos) {
     if (repos.IsNotNull() and repos->IsMap()) {
         setup_repos->to_setup = repos->Map().Keys();
         setup_repos->to_include = setup_repos->to_setup;
-        // get reported repos
-        if (reported_repos_set) {
-            *reported_repos_set = std::unordered_set<std::string>(
-                setup_repos->to_setup.begin(), setup_repos->to_setup.end());
-            for (auto const& repo : setup_repos->to_setup) {
-                AddDistdirReportedRepos(repos, repo, reported_repos_set);
-            }
-        }
     }
 }
 
@@ -804,19 +742,8 @@ void DefaultReachableRepositories(
     auto repo_fetch_map =
         CreateRepoFetchMap(&content_cas_map, *fetch_dir, arguments.common.jobs);
 
-    // set up map for progress tracing
-    auto& repo_set = JustMRProgress::Instance().RepositorySet();
-    repo_set.clear();
-    repo_set.reserve(nr);
-    for (auto const& repo : repos_to_fetch) {
-        auto distfile = (repo.archive.distfile
-                             ? repo.archive.distfile.value()
-                             : std::filesystem::path(repo.archive.fetch_url)
-                                   .filename()
-                                   .string());
-        repo_set.emplace(std::move(distfile));
-    }
     // set up progress observer
+    JustMRProgress::Instance().SetTotal(repos_to_fetch.size());
     std::atomic<bool> done{false};
     std::condition_variable cv{};
     auto reporter = JustMRProgressReporter::Reporter();
@@ -1010,15 +937,8 @@ void DefaultReachableRepositories(
     auto git_update_map =
         CreateGitUpdateMap(git_repo->GetGitCAS(), arguments.common.jobs);
 
-    // set up map for progress tracing
-    auto& repo_set = JustMRProgress::Instance().RepositorySet();
-    repo_set.clear();
-    repo_set.reserve(nr);
-    for (auto const& repo : repos_to_update) {
-        auto id = fmt::format("{}:{}", repo.first, repo.second);
-        repo_set.emplace(std::move(id));
-    }
     // set up progress observer
+    JustMRProgress::Instance().SetTotal(repos_to_update.size());
     std::atomic<bool> done{false};
     std::condition_variable cv{};
     auto reporter = JustMRProgressReporter::Reporter();
@@ -1105,8 +1025,7 @@ void DefaultReachableRepositories(
         mr_config["main"] = *main;
     }
     // get default repos to setup and to include
-    auto repos_to_report = std::make_shared<std::unordered_set<std::string>>();
-    DefaultReachableRepositories(repos, setup_repos, repos_to_report);
+    DefaultReachableRepositories(repos, setup_repos);
     // check if main is to be taken as first repo name lexicographically
     if (not main and not setup_repos->to_setup.empty()) {
         main = *std::min_element(setup_repos->to_setup.begin(),
@@ -1114,16 +1033,9 @@ void DefaultReachableRepositories(
     }
     // final check on which repos are to be set up
     if (main and not arguments.setup.sub_all) {
-        ReachableRepositories(repos, *main, setup_repos, repos_to_report);
+        ReachableRepositories(repos, *main, setup_repos);
     }
 
-    // report progress
-    auto nr = repos_to_report->size();
-    Logger::Log(LogLevel::Info,
-                "Found {} {} to set up",
-                nr,
-                nr == 1 ? "repository" : "repositories");
-
     // setup the required async maps
     auto crit_git_op_ptr = std::make_shared<CriticalGitOpGuard>();
     auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
@@ -1162,14 +1074,11 @@ void DefaultReachableRepositories(
                                                     &tree_id_git_map,
                                                     arguments.common.jobs);
 
-    // set up map for progress tracing
-    auto& repo_set = JustMRProgress::Instance().RepositorySet();
-    repo_set.clear();
-    repo_set.reserve(nr);
-    for (auto const& repo : *repos_to_report) {
-        repo_set.emplace(repo);
-    }
     // set up progress observer
+    Logger::Log(LogLevel::Info,
+                "Found {} repositories to set up",
+                setup_repos->to_setup.size());
+    JustMRProgress::Instance().SetTotal(setup_repos->to_setup.size());
     std::atomic<bool> done{false};
     std::condition_variable cv{};
     auto reporter = JustMRProgressReporter::Reporter();
diff --git a/src/other_tools/just_mr/progress_reporting/progress.hpp b/src/other_tools/just_mr/progress_reporting/progress.hpp
index 5d641673..befceb4b 100644
--- a/src/other_tools/just_mr/progress_reporting/progress.hpp
+++ b/src/other_tools/just_mr/progress_reporting/progress.hpp
@@ -34,16 +34,13 @@ class JustMRProgress {
         return task_tracker_;
     }
 
-    // Return a reference to the repository set. It is the responsibility of the
-    // caller to ensure that access only happens in a single-threaded context.
-    [[nodiscard]] auto RepositorySet() noexcept
-        -> std::unordered_set<std::string>& {
-        return repo_set_;
-    }
+    [[nodiscard]] auto GetTotal() const noexcept -> int { return total_; }
+
+    void SetTotal(int total) noexcept { total_ = total; }
 
   private:
     ::TaskTracker task_tracker_{};
-    std::unordered_set<std::string> repo_set_{};
+    int total_{};
 };
 
 #endif  // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_PROGRESS_REPORTING_PROGRESS_HPP
diff --git a/src/other_tools/just_mr/progress_reporting/progress_reporter.cpp b/src/other_tools/just_mr/progress_reporting/progress_reporter.cpp
index 8f313714..21a3db3c 100644
--- a/src/other_tools/just_mr/progress_reporting/progress_reporter.cpp
+++ b/src/other_tools/just_mr/progress_reporting/progress_reporter.cpp
@@ -27,30 +27,21 @@
 
 auto JustMRProgressReporter::Reporter() noexcept -> progress_reporter_t {
     return BaseProgressReporter::Reporter([]() {
-        int total =
-            gsl::narrow<int>(JustMRProgress::Instance().RepositorySet().size());
-        // Note: order of stats queries matters!
-        auto const& sample = JustMRProgress::Instance().TaskTracker().Sample();
+        int total = JustMRProgress::Instance().GetTotal();
         auto const& stats = JustMRStatistics::Instance();
         int local = stats.LocalPathsCounter();
         int cached = stats.CacheHitsCounter();
         int run = stats.ExecutedCounter();
-        int queued = stats.QueuedCounter();
-        int active = queued - run;
+        auto active = JustMRProgress::Instance().TaskTracker().Active();
+        auto sample = JustMRProgress::Instance().TaskTracker().Sample();
         std::string msg;
-        if (active > 0 and !sample.empty()) {
-            auto const& repo_set = JustMRProgress::Instance().RepositorySet();
-            if (repo_set.find(sample) != repo_set.end()) {
-                msg = fmt::format(
-                    "{} local roots, {} cached, {} run, {} processing "
-                    "({}{})",
-                    local,
-                    cached,
-                    run,
-                    active,
-                    nlohmann::json(sample).dump(),
-                    active > 1 ? ", ..." : "");
-            }
+        msg = fmt::format("{} local, {} cached, {} done", local, cached, run);
+        if ((active > 0) && !sample.empty()) {
+            msg = fmt::format("{}; {} fetches ({}{})",
+                              msg,
+                              active,
+                              nlohmann::json(sample).dump(),
+                              active > 1 ? ", ..." : "");
         }
         constexpr int kOneHundred{100};
         int total_work = total - cached - local;
diff --git a/src/other_tools/just_mr/progress_reporting/statistics.hpp b/src/other_tools/just_mr/progress_reporting/statistics.hpp
index efdce7f5..846d628b 100644
--- a/src/other_tools/just_mr/progress_reporting/statistics.hpp
+++ b/src/other_tools/just_mr/progress_reporting/statistics.hpp
@@ -27,12 +27,10 @@ class JustMRStatistics {
     void Reset() noexcept {
         num_local_paths_ = 0;
         num_cache_hits_ = 0;
-        num_queued_ = 0;
         num_executed_ = 0;
     }
     void IncrementLocalPathsCounter() noexcept { ++num_local_paths_; }
     void IncrementCacheHitsCounter() noexcept { ++num_cache_hits_; }
-    void IncrementQueuedCounter() noexcept { ++num_queued_; }
     void IncrementExecutedCounter() noexcept { ++num_executed_; }
 
     [[nodiscard]] auto LocalPathsCounter() const noexcept -> int {
@@ -41,9 +39,6 @@ class JustMRStatistics {
     [[nodiscard]] auto CacheHitsCounter() const noexcept -> int {
         return num_cache_hits_;
     }
-    [[nodiscard]] auto QueuedCounter() const noexcept -> int {
-        return num_queued_;
-    }
     [[nodiscard]] auto ExecutedCounter() const noexcept -> int {
         return num_executed_;
     }
@@ -51,7 +46,6 @@ class JustMRStatistics {
   private:
     std::atomic<int> num_local_paths_{};  // roots that are actual paths
     std::atomic<int> num_cache_hits_{};   // no-ops
-    std::atomic<int> num_queued_{};       // actual work queued
     std::atomic<int> num_executed_{};     // actual work done
 };
 
diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp
index 1220d50f..4a25b15c 100644
--- a/src/other_tools/ops_maps/content_cas_map.cpp
+++ b/src/other_tools/ops_maps/content_cas_map.cpp
@@ -55,11 +55,6 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
                                                   auto logger,
                                                   auto /*unused*/,
                                                   auto const& key) {
-        // start work reporting, but only if part of a distdir
-        if (key.origin_from_distdir) {
-            JustMRProgress::Instance().TaskTracker().Start(key.origin);
-            JustMRStatistics::Instance().IncrementQueuedCounter();
-        }
         // check if content already in CAS
         auto const& cas = Storage::Instance().CAS();
         auto digest = ArtifactDigest(key.content, 0, false);
@@ -67,6 +62,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
             (*setter)(true);
             return;
         }
+        JustMRProgress::Instance().TaskTracker().Start(key.origin);
         // add distfile to CAS
         auto repo_distfile =
             (key.distfile
@@ -75,6 +71,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
         JustMR::Utils::AddDistfileToCAS(repo_distfile, just_mr_paths);
         // check if content is in CAS now
         if (cas.BlobPath(digest, /*is_executable=*/false)) {
+            JustMRProgress::Instance().TaskTracker().Stop(key.origin);
             (*setter)(true);
             return;
         }
@@ -131,12 +128,8 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
                       /*fatal=*/true);
             return;
         }
+        JustMRProgress::Instance().TaskTracker().Stop(key.origin);
         (*setter)(true);
-        // report work done, but only if part of a distdir
-        if (key.origin_from_distdir) {
-            JustMRProgress::Instance().TaskTracker().Stop(key.origin);
-            JustMRStatistics::Instance().IncrementExecutedCounter();
-        }
     };
     return AsyncMapConsumer<ArchiveContent, bool>(ensure_in_cas, jobs);
 }
diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp
index 8c98fb09..c4305e69 100644
--- a/src/other_tools/ops_maps/git_update_map.cpp
+++ b/src/other_tools/ops_maps/git_update_map.cpp
@@ -28,10 +28,6 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, std::size_t jobs)
                                     auto logger,
                                     auto /* unused */,
                                     auto const& key) {
-        // start progress trace for Git repo
-        auto id = fmt::format("{}:{}", key.first, key.second);
-        JustMRProgress::Instance().TaskTracker().Start(id);
-        JustMRStatistics::Instance().IncrementQueuedCounter();
         // perform git update commit
         auto git_repo = GitRepoRemote::Open(git_cas);  // wrap the tmp odb
         if (not git_repo) {
@@ -57,15 +53,16 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, std::size_t jobs)
                     fatal);
             });
         // update commit
+        auto id = fmt::format("{}:{}", key.first, key.second);
+        JustMRProgress::Instance().TaskTracker().Start(id);
         auto new_commit = git_repo->UpdateCommitViaTmpRepo(
             tmp_dir->GetPath(), key.first, key.second, wrapped_logger);
+        JustMRProgress::Instance().TaskTracker().Stop(id);
         if (not new_commit) {
             return;
         }
-        (*setter)(new_commit->c_str());
-        // stop progress trace for Git repo
-        JustMRProgress::Instance().TaskTracker().Stop(id);
         JustMRStatistics::Instance().IncrementExecutedCounter();
+        (*setter)(new_commit->c_str());
     };
     return AsyncMapConsumer<StringPair, std::string>(update_commits, jobs);
 }
diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp
index 1c903fe3..5d58e4b0 100644
--- a/src/other_tools/ops_maps/repo_fetch_map.cpp
+++ b/src/other_tools/ops_maps/repo_fetch_map.cpp
@@ -39,9 +39,6 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
         auto tree_id_file = JustMR::Utils::GetArchiveTreeIDFile(
             key.repo_type, key.archive.content);
         if (not FileSystemManager::Exists(tree_id_file)) {
-            // start work reporting
-            JustMRProgress::Instance().TaskTracker().Start(key.archive.origin);
-            JustMRStatistics::Instance().IncrementQueuedCounter();
             // make sure content is in CAS
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
@@ -77,10 +74,8 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
                             return;
                         }
                         // success
-                        (*setter)(true);
-                        // report work done
-                        JustMRProgress::Instance().TaskTracker().Stop(origin);
                         JustMRStatistics::Instance().IncrementExecutedCounter();
+                        (*setter)(true);
                     }
                     else {
                         (*logger)(
@@ -123,9 +118,8 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
                     return;
                 }
                 // success
-                (*setter)(true);
-                // report cache hit
                 JustMRStatistics::Instance().IncrementCacheHitsCounter();
+                (*setter)(true);
             }
             else {
                 (*logger)(fmt::format("Content {} could not be found in CAS",
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 5a8dcb80..19aff9eb 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -103,10 +103,16 @@ void GitCheckout(ExpressionPtr const& repo_desc,
         ts,
         {std::move(git_repo_info)},
         [repos = std::move(repos), repo_name, setter](auto const& values) {
-            auto ws_root = *values[0];
+            auto ws_root = values[0]->first;
             nlohmann::json cfg({});
             cfg["workspace_root"] = ws_root;
             SetReposTakeOver(&cfg, repos, repo_name);
+            if (values[0]->second) {
+                JustMRStatistics::Instance().IncrementCacheHitsCounter();
+            }
+            else {
+                JustMRStatistics::Instance().IncrementExecutedCounter();
+            }
             (*setter)(std::move(cfg));
         },
         [logger, repo_name](auto const& msg, bool fatal) {
@@ -188,10 +194,16 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
         ts,
         {std::move(archive_repo_info)},
         [repos = std::move(repos), repo_name, setter](auto const& values) {
-            auto ws_root = *values[0];
+            auto ws_root = values[0]->first;
             nlohmann::json cfg({});
             cfg["workspace_root"] = ws_root;
             SetReposTakeOver(&cfg, repos, repo_name);
+            if (values[0]->second) {
+                JustMRStatistics::Instance().IncrementCacheHitsCounter();
+            }
+            else {
+                JustMRStatistics::Instance().IncrementExecutedCounter();
+            }
             (*setter)(std::move(cfg));
         },
         [logger, repo_name, repo_type](auto const& msg, bool fatal) {
@@ -236,9 +248,6 @@ void FileCheckout(ExpressionPtr const& repo_desc,
         repo_desc_pragma ? repo_desc_pragma->get()->At("to_git") : std::nullopt;
     if (pragma_to_git and pragma_to_git->get()->IsBool() and
         pragma_to_git->get()->Bool()) {
-        // start work reporting
-        JustMRProgress::Instance().TaskTracker().Start(repo_name);
-        JustMRStatistics::Instance().IncrementQueuedCounter();
         // get the WS root as git tree
         fpath_git_map->ConsumeAfterKeysReady(
             ts,
@@ -250,8 +259,7 @@ void FileCheckout(ExpressionPtr const& repo_desc,
                 SetReposTakeOver(&cfg, repos, repo_name);
                 (*setter)(std::move(cfg));
                 // report work done
-                JustMRProgress::Instance().TaskTracker().Stop(repo_name);
-                JustMRStatistics::Instance().IncrementExecutedCounter();
+                JustMRStatistics::Instance().IncrementLocalPathsCounter();
             },
             [logger, repo_name](auto const& msg, bool fatal) {
                 (*logger)(fmt::format("While setting the workspace root for "
@@ -438,10 +446,16 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
         ts,
         {std::move(distdir_info)},
         [repos = std::move(repos), repo_name, setter](auto const& values) {
-            auto ws_root = *values[0];
+            auto ws_root = values[0]->first;
             nlohmann::json cfg({});
             cfg["workspace_root"] = ws_root;
             SetReposTakeOver(&cfg, repos, repo_name);
+            if (values[0]->second) {
+                JustMRStatistics::Instance().IncrementCacheHitsCounter();
+            }
+            else {
+                JustMRStatistics::Instance().IncrementExecutedCounter();
+            }
             (*setter)(std::move(cfg));
         },
         [logger, repo_name](auto const& msg, bool fatal) {
@@ -530,10 +544,16 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
         ts,
         {std::move(tree_id_info)},
         [repos = std::move(repos), repo_name, setter](auto const& values) {
-            auto ws_root = *values[0];
+            auto ws_root = values[0]->first;
             nlohmann::json cfg({});
             cfg["workspace_root"] = ws_root;
             SetReposTakeOver(&cfg, repos, repo_name);
+            if (values[0]->second) {
+                JustMRStatistics::Instance().IncrementCacheHitsCounter();
+            }
+            else {
+                JustMRStatistics::Instance().IncrementExecutedCounter();
+            }
             (*setter)(std::move(cfg));
         },
         [logger, repo_name](auto const& msg, bool fatal) {
@@ -575,6 +595,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
             // no repository checkout required
             nlohmann::json cfg({});
             SetReposTakeOver(&cfg, repos, key);
+            JustMRStatistics::Instance().IncrementLocalPathsCounter();
             (*setter)(std::move(cfg));
         }
         else {
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 5e63e699..fae205ed 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -59,9 +59,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
         return;
     }
     if (not is_commit_present.value()) {
-        // start work reporting
         JustMRProgress::Instance().TaskTracker().Start(repo_info.origin);
-        JustMRStatistics::Instance().IncrementQueuedCounter();
         // if commit not there, fetch it
         auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("fetch");
         if (not tmp_dir) {
@@ -147,11 +145,10 @@ void EnsureCommit(GitRepoInfo const& repo_info,
                     return;
                 }
                 // set the workspace root
-                (*ws_setter)(
-                    nlohmann::json::array({"git tree", *subtree, repo_root}));
-                // report work done
                 JustMRProgress::Instance().TaskTracker().Stop(repo_info.origin);
-                JustMRStatistics::Instance().IncrementExecutedCounter();
+                (*ws_setter)(std::pair(
+                    nlohmann::json::array({"git tree", *subtree, repo_root}),
+                    false));
             },
             [logger, target_path = repo_root](auto const& msg, bool fatal) {
                 (*logger)(fmt::format("While running critical Git op "
@@ -176,9 +173,8 @@ void EnsureCommit(GitRepoInfo const& repo_info,
             return;
         }
         // set the workspace root
-        (*ws_setter)(nlohmann::json::array({"git tree", *subtree, repo_root}));
-        // report cache hit
-        JustMRStatistics::Instance().IncrementCacheHitsCounter();
+        (*ws_setter)(std::pair(
+            nlohmann::json::array({"git tree", *subtree, repo_root}), true));
     }
 }
 
@@ -252,5 +248,6 @@ auto CreateCommitGitMap(
                           fatal);
             });
     };
-    return AsyncMapConsumer<GitRepoInfo, nlohmann::json>(commit_to_git, jobs);
+    return AsyncMapConsumer<GitRepoInfo, std::pair<nlohmann::json, bool>>(
+        commit_to_git, jobs);
 }
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index 992f876c..413d83ac 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -16,6 +16,7 @@
 #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
 
 #include <string>
+#include <utility>
 
 #include "nlohmann/json.hpp"
 #include "src/other_tools/just_mr/utils.hpp"
@@ -49,8 +50,10 @@ struct hash<GitRepoInfo> {
 };
 }  // namespace std
 
-/// \brief Maps a Git repository commit hash to its tree workspace root.
-using CommitGitMap = AsyncMapConsumer<GitRepoInfo, nlohmann::json>;
+/// \brief Maps a Git repository commit hash to its tree workspace root,
+/// together with the information whether it was a cache hit.
+using CommitGitMap =
+    AsyncMapConsumer<GitRepoInfo, std::pair<nlohmann::json, bool>>;
 
 [[nodiscard]] auto CreateCommitGitMap(
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index f36f3142..80072fee 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -114,12 +114,12 @@ auto CreateContentGitMap(
                         return;
                     }
                     // set the workspace root
-                    (*setter)(nlohmann::json::array(
-                        {"git tree",
-                         *subtree_hash,
-                         JustMR::Utils::GetGitCacheRoot().string()}));
-                    // report cache hit
-                    JustMRStatistics::Instance().IncrementCacheHitsCounter();
+                    (*setter)(std::pair(
+                        nlohmann::json::array(
+                            {"git tree",
+                             *subtree_hash,
+                             JustMR::Utils::GetGitCacheRoot().string()}),
+                        true));
                 },
                 [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
                     auto const& msg, bool fatal) {
@@ -132,9 +132,6 @@ auto CreateContentGitMap(
                 });
         }
         else {
-            // start work reporting;
-            JustMRProgress::Instance().TaskTracker().Start(key.archive.origin);
-            JustMRStatistics::Instance().IncrementQueuedCounter();
             // do the fetch and import_to_git
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
@@ -242,15 +239,13 @@ auto CreateContentGitMap(
                                 return;
                             }
                             // set the workspace root
-                            (*setter)(nlohmann::json::array(
-                                {"git tree",
-                                 *subtree_hash,
-                                 JustMR::Utils::GetGitCacheRoot().string()}));
-                            // report work done
-                            JustMRProgress::Instance().TaskTracker().Stop(
-                                origin);
-                            JustMRStatistics::Instance()
-                                .IncrementExecutedCounter();
+                            (*setter)(
+                                std::pair(nlohmann::json::array(
+                                              {"git tree",
+                                               *subtree_hash,
+                                               JustMR::Utils::GetGitCacheRoot()
+                                                   .string()}),
+                                          false));
                         },
                         [logger, target_path = tmp_dir->GetPath()](
                             auto const& msg, bool fatal) {
@@ -271,6 +266,6 @@ auto CreateContentGitMap(
                 });
         }
     };
-    return AsyncMapConsumer<ArchiveRepoInfo, nlohmann::json>(gitify_content,
-                                                             jobs);
+    return AsyncMapConsumer<ArchiveRepoInfo, std::pair<nlohmann::json, bool>>(
+        gitify_content, jobs);
 }
diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp
index df545b13..d2aeb22d 100644
--- a/src/other_tools/root_maps/content_git_map.hpp
+++ b/src/other_tools/root_maps/content_git_map.hpp
@@ -15,11 +15,15 @@
 #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP
 #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP
 
+#include <utility>
+
 #include "src/other_tools/ops_maps/content_cas_map.hpp"
 #include "src/other_tools/ops_maps/import_to_git_map.hpp"
 
-/// \brief Maps the content of an archive to the resulting Git tree WS root.
-using ContentGitMap = AsyncMapConsumer<ArchiveRepoInfo, nlohmann::json>;
+/// \brief Maps the content of an archive to the resulting Git tree WS root,
+/// togehter with the information whether it was a cache hit.
+using ContentGitMap =
+    AsyncMapConsumer<ArchiveRepoInfo, std::pair<nlohmann::json, bool>>;
 
 [[nodiscard]] auto CreateContentGitMap(
     gsl::not_null<ContentCASMap*> const& content_cas_map,
@@ -27,4 +31,4 @@ using ContentGitMap = AsyncMapConsumer<ArchiveRepoInfo, nlohmann::json>;
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     std::size_t jobs) -> ContentGitMap;
 
-#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP
\ No newline at end of file
+#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_CONTENT_GIT_MAP_HPP
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index ef21b7cc..403247e4 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -99,12 +99,12 @@ auto CreateDistdirGitMap(
                     }
                     // subdir is ".", so no need to deal with the Git cache
                     // set the workspace root
-                    (*setter)(nlohmann::json::array(
-                        {"git tree",
-                         distdir_tree_id,
-                         JustMR::Utils::GetGitCacheRoot().string()}));
-                    // report cache hit
-                    JustMRStatistics::Instance().IncrementCacheHitsCounter();
+                    (*setter)(std::pair(
+                        nlohmann::json::array(
+                            {"git tree",
+                             distdir_tree_id,
+                             JustMR::Utils::GetGitCacheRoot().string()}),
+                        true));
                 },
                 [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
                     auto const& msg, bool fatal) {
@@ -117,9 +117,6 @@ auto CreateDistdirGitMap(
                 });
         }
         else {
-            // start work reporting
-            JustMRProgress::Instance().TaskTracker().Start(key.origin);
-            JustMRStatistics::Instance().IncrementQueuedCounter();
             // fetch the gathered distdir repos into CAS
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
@@ -180,15 +177,13 @@ auto CreateDistdirGitMap(
                                 return;
                             }
                             // set the workspace root
-                            (*setter)(nlohmann::json::array(
-                                {"git tree",
-                                 distdir_tree_id,
-                                 JustMR::Utils::GetGitCacheRoot().string()}));
-                            // report work done
-                            JustMRProgress::Instance().TaskTracker().Stop(
-                                origin);
-                            JustMRStatistics::Instance()
-                                .IncrementExecutedCounter();
+                            (*setter)(
+                                std::pair(nlohmann::json::array(
+                                              {"git tree",
+                                               distdir_tree_id,
+                                               JustMR::Utils::GetGitCacheRoot()
+                                                   .string()}),
+                                          false));
                         },
                         [logger, target_path = tmp_dir->GetPath()](
                             auto const& msg, bool fatal) {
@@ -209,5 +204,6 @@ auto CreateDistdirGitMap(
                 });
         }
     };
-    return AsyncMapConsumer<DistdirInfo, nlohmann::json>(distdir_to_git, jobs);
+    return AsyncMapConsumer<DistdirInfo, std::pair<nlohmann::json, bool>>(
+        distdir_to_git, jobs);
 }
diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp
index 8f7796e3..c440253b 100644
--- a/src/other_tools/root_maps/distdir_git_map.hpp
+++ b/src/other_tools/root_maps/distdir_git_map.hpp
@@ -15,6 +15,8 @@
 #ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
 #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
 
+#include <utility>
+
 #include "nlohmann/json.hpp"
 #include "src/other_tools/ops_maps/content_cas_map.hpp"
 #include "src/other_tools/ops_maps/import_to_git_map.hpp"
@@ -33,8 +35,10 @@ struct DistdirInfo {
 };
 
 /// \brief Maps a list of repositories belonging to a distdir to its
-/// corresponding workspace root.
-using DistdirGitMap = AsyncMapConsumer<DistdirInfo, nlohmann::json>;
+/// corresponding workspace root and indication whether this was a cache
+/// hit.
+using DistdirGitMap =
+    AsyncMapConsumer<DistdirInfo, std::pair<nlohmann::json, bool>>;
 
 [[nodiscard]] auto CreateDistdirGitMap(
     gsl::not_null<ContentCASMap*> const& content_cas_map,
@@ -52,4 +56,4 @@ struct hash<DistdirInfo> {
 };
 }  // namespace std
 
-#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
\ No newline at end of file
+#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp
index 2eb368c6..2003ec7e 100644
--- a/src/other_tools/root_maps/tree_id_git_map.cpp
+++ b/src/other_tools/root_maps/tree_id_git_map.cpp
@@ -54,13 +54,13 @@ void KeepCommitAndSetRoot(
                 return;
             }
             // set the workspace root
-            (*ws_setter)(nlohmann::json::array(
-                {"git tree",
-                 tree_id_info.hash,
-                 JustMR::Utils::GetGitCacheRoot().string()}));
-            // report work done
-            JustMRProgress::Instance().TaskTracker().Stop(tree_id_info.origin);
-            JustMRStatistics::Instance().IncrementExecutedCounter();
+            JustMRProgress::Instance().TaskTracker().Start(tree_id_info.origin);
+            (*ws_setter)(
+                std::pair(nlohmann::json::array(
+                              {"git tree",
+                               tree_id_info.hash,
+                               JustMR::Utils::GetGitCacheRoot().string()}),
+                          false));
         },
         [logger, commit, target_path = tmp_dir->GetPath()](auto const& msg,
                                                            bool fatal) {
@@ -133,9 +133,7 @@ auto CreateTreeIdGitMap(
                     return;
                 }
                 if (not *tree_found) {
-                    // start work reporting;
                     JustMRProgress::Instance().TaskTracker().Start(key.origin);
-                    JustMRStatistics::Instance().IncrementQueuedCounter();
                     // create temporary location for command execution root
                     auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("git-tree");
                     if (not tmp_dir) {
@@ -309,12 +307,12 @@ auto CreateTreeIdGitMap(
                 }
                 else {
                     // tree found, so return the git tree root as-is
-                    (*setter)(nlohmann::json::array(
-                        {"git tree",
-                         key.hash,
-                         JustMR::Utils::GetGitCacheRoot().string()}));
-                    // report cache hit
-                    JustMRStatistics::Instance().IncrementCacheHitsCounter();
+                    (*setter)(std::pair(
+                        nlohmann::json::array(
+                            {"git tree",
+                             key.hash,
+                             JustMR::Utils::GetGitCacheRoot().string()}),
+                        true));
                 }
             },
             [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
@@ -327,5 +325,6 @@ auto CreateTreeIdGitMap(
                           fatal);
             });
     };
-    return AsyncMapConsumer<TreeIdInfo, nlohmann::json>(tree_to_git, jobs);
+    return AsyncMapConsumer<TreeIdInfo, std::pair<nlohmann::json, bool>>(
+        tree_to_git, jobs);
 }
diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp
index 01eed06a..414f6ebd 100644
--- a/src/other_tools/root_maps/tree_id_git_map.hpp
+++ b/src/other_tools/root_maps/tree_id_git_map.hpp
@@ -16,6 +16,7 @@
 #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP
 
 #include <string>
+#include <utility>
 
 #include "nlohmann/json.hpp"
 #include "src/other_tools/ops_maps/critical_git_op_map.hpp"
@@ -43,12 +44,13 @@ struct hash<TreeIdInfo> {
 }  // namespace std
 
 /// \brief Maps a known tree provided through a generic command to its
-/// workspace root.
-using TreeIdGitMap = AsyncMapConsumer<TreeIdInfo, nlohmann::json>;
+/// workspace root and the information whether it was a cache it.
+using TreeIdGitMap =
+    AsyncMapConsumer<TreeIdInfo, std::pair<nlohmann::json, bool>>;
 
 [[nodiscard]] auto CreateTreeIdGitMap(
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     std::vector<std::string> const& launcher,
     std::size_t jobs) -> TreeIdGitMap;
 
-#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP
\ No newline at end of file
+#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP