Just-MR: Add progress reporting for setup command

This includes also the setup-env command, as well as all ~just~
known subcommands that require a just-mr setup step.

Co-authored-by: Sascha Roloff <sascha.roloff@huawei.com>

8 files changed, 166 insertions, 12 deletions

diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index b3996909..9a73f4f2 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -12,6 +12,8 @@
     , ["src/other_tools/ops_maps", "repo_fetch_map"]
     , ["src/other_tools/ops_maps", "git_update_map"]
     , ["src/other_tools/repo_map", "repos_to_setup_map"]
+    , ["src/buildtool/progress_reporting", "base_progress_reporter"]
+    , ["src/buildtool/progress_reporting", "progress"]
     , ["@", "json", "", "json"]
     , ["src/other_tools/just_mr/progress_reporting", "progress"]
     , ["src/other_tools/just_mr/progress_reporting", "statistics"]
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index d1675bdd..5f5053e1 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -440,9 +440,51 @@ void SetupLogging(MultiRepoLogArguments const& clargs) {
     return config;
 }
 
-void ReachableRepositories(ExpressionPtr const& repos,
-                           std::string const& main,
-                           std::shared_ptr<SetupRepos> const& setup_repos) {
+/// \brief Add repos from a given repo list to the reported repos set if its
+/// type is distdir. Ignore any missing or wrong info.
+void AddDistdirReportedRepos(
+    ExpressionPtr const& repos,
+    std::string const& repo_name,
+    std::shared_ptr<std::unordered_set<std::string>> const&
+        reported_repos_set) {
+    auto repo_desc_key = repos->Get(repo_name, Expression::none_t{});
+    auto repo_desc = repo_desc_key->Get("repository", Expression::none_t{});
+    auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc, repos);
+    if (not resolved_repo_desc) {
+        return;
+    }
+    auto repo_type = (*resolved_repo_desc)->Get("type", Expression::none_t{});
+    if (not repo_type->IsString()) {
+        return;
+    }
+    // check if the repo is distdir
+    auto repo_type_str = repo_type->String();
+    if (repo_type_str != "distdir") {
+        return;
+    }
+    // get distdir list
+    auto repo_desc_repositories =
+        repo_desc->Get("repositories", Expression::none_t{});
+    if (not repo_desc_repositories->IsList()) {
+        return;
+    }
+    auto distdir_repos = repo_desc_repositories->List();
+    // add to reported set
+    for (auto const& entry : distdir_repos) {
+        if (entry->IsString()) {
+            reported_repos_set->insert(entry->String());
+        }
+    }
+}
+
+/// \brief Get the repo dependency closure for a given main repository.
+/// For progress reporting we include
+void ReachableRepositories(
+    ExpressionPtr const& repos,
+    std::string const& main,
+    std::shared_ptr<SetupRepos> const& setup_repos,
+    std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
+        nullptr) {
     // use temporary sets to avoid duplicates
     std::unordered_set<std::string> include_repos_set{};
     if (repos->IsMap()) {
@@ -452,6 +494,13 @@ void ReachableRepositories(ExpressionPtr const& repos,
                 if (not include_repos_set.contains(repo_name)) {
                     // if not found, add it and repeat for its bindings
                     include_repos_set.insert(repo_name);
+                    // add to reported repos, if needed
+                    if (reported_repos_set) {
+                        reported_repos_set->insert(repo_name);
+                        AddDistdirReportedRepos(
+                            repos, repo_name, reported_repos_set);
+                    }
+                    // check bindings
                     auto repos_repo_name =
                         repos->Get(repo_name, Expression::none_t{});
                     if (not repos_repo_name.IsNotNull()) {
@@ -480,7 +529,14 @@ void ReachableRepositories(ExpressionPtr const& repos,
                     auto layer_val =
                         repos_repo->Get(layer, Expression::none_t{});
                     if (layer_val.IsNotNull() and layer_val->IsString()) {
-                        setup_repos_set.insert(layer_val->String());
+                        auto repo_name = layer_val->String();
+                        setup_repos_set.insert(repo_name);
+                        // add to reported repos, if needed
+                        if (reported_repos_set) {
+                            reported_repos_set->insert(repo_name);
+                            AddDistdirReportedRepos(
+                                repos, repo_name, reported_repos_set);
+                        }
                     }
                 }
             }
@@ -504,10 +560,20 @@ void ReachableRepositories(ExpressionPtr const& repos,
 
 void DefaultReachableRepositories(
     ExpressionPtr const& repos,
-    std::shared_ptr<SetupRepos> const& setup_repos) {
+    std::shared_ptr<SetupRepos> const& setup_repos,
+    std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
+        nullptr) {
     if (repos.IsNotNull() and repos->IsMap()) {
         setup_repos->to_setup = repos->Map().Keys();
         setup_repos->to_include = setup_repos->to_setup;
+        // get reported repos
+        if (reported_repos_set) {
+            *reported_repos_set = std::unordered_set<std::string>(
+                setup_repos->to_setup.begin(), setup_repos->to_setup.end());
+            for (auto const& repo : setup_repos->to_setup) {
+                AddDistdirReportedRepos(repos, repo, reported_repos_set);
+            }
+        }
     }
 }
 
@@ -994,6 +1060,8 @@ void DefaultReachableRepositories(
                                   CommandLineArguments const& arguments,
                                   bool interactive)
     -> std::optional<std::filesystem::path> {
+    // provide report
+    Logger::Log(LogLevel::Info, "Performing repositories setup");
     // set anchor dir to setup_root; current dir will be reverted when anchor
     // goes out of scope
     auto cwd_anchor = FileSystemManager::ChangeDirectory(
@@ -1027,7 +1095,8 @@ void DefaultReachableRepositories(
         mr_config["main"] = *main;
     }
     // get default repos to setup and to include
-    DefaultReachableRepositories(repos, setup_repos);
+    auto repos_to_report = std::make_shared<std::unordered_set<std::string>>();
+    DefaultReachableRepositories(repos, setup_repos, repos_to_report);
     // check if main is to be taken as first repo name lexicographically
     if (not main and not setup_repos->to_setup.empty()) {
         main = *std::min_element(setup_repos->to_setup.begin(),
@@ -1035,9 +1104,16 @@ void DefaultReachableRepositories(
     }
     // final check on which repos are to be set up
     if (main and not arguments.setup.sub_all) {
-        ReachableRepositories(repos, *main, setup_repos);
+        ReachableRepositories(repos, *main, setup_repos, repos_to_report);
     }
 
+    // report progress
+    auto nr = repos_to_report->size();
+    Logger::Log(LogLevel::Info,
+                "Found {} {} to check out",
+                nr,
+                nr == 1 ? "repository" : "repositories");
+
     // setup the required async maps
     auto crit_git_op_ptr = std::make_shared<CriticalGitOpGuard>();
     auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
@@ -1071,6 +1147,20 @@ void DefaultReachableRepositories(
                                                     &distdir_git_map,
                                                     arguments.common.jobs);
 
+    // set up map for progress tracing
+    auto& repo_set = JustMRProgress::Instance().RepositorySet();
+    repo_set.clear();
+    repo_set.reserve(nr);
+    for (auto const& repo : *repos_to_report) {
+        repo_set.emplace(repo);
+    }
+    // set up progress observer
+    std::atomic<bool> done{false};
+    std::condition_variable cv{};
+    auto reporter = JustMRProgressReporter::Reporter();
+    auto observer =
+        std::thread([reporter, &done, &cv]() { reporter(&done, &cv); });
+
     // Populate workspace_root and TAKE_OVER fields
     bool failed{false};
     {
@@ -1123,6 +1213,12 @@ void DefaultReachableRepositories(
                 failed = failed or fatal;
             });
     }
+
+    // close progress observer
+    done = true;
+    cv.notify_all();
+    observer.join();
+
     if (failed) {
         return std::nullopt;
     }
@@ -1336,6 +1432,9 @@ auto main(int argc, char* argv[]) -> int {
             if (not mr_config_path) {
                 return kExitSetupError;
             }
+            // report success
+            Logger::Log(LogLevel::Info, "Setup completed");
+            // print config file to stdout
             std::cout << mr_config_path->string() << std::endl;
             return kExitSuccess;
         }
diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS
index 905e09c8..5e89d85c 100644
--- a/src/other_tools/repo_map/TARGETS
+++ b/src/other_tools/repo_map/TARGETS
@@ -11,6 +11,9 @@
     , ["src/buildtool/build_engine/expression", "expression"]
     ]
   , "stage": ["src", "other_tools", "repo_map"]
-  , "private-deps": [["src/other_tools/just_mr", "utils"]]
+  , "private-deps":
+    [ ["src/other_tools/just_mr/progress_reporting", "progress"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
+    ]
   }
 }
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index bd34a231..36428668 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -14,7 +14,8 @@
 
 #include "src/other_tools/repo_map/repos_to_setup_map.hpp"
 
-#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 
 namespace {
 
@@ -232,6 +233,9 @@ void FileCheckout(ExpressionPtr const& repo_desc,
         repo_desc_pragma ? repo_desc_pragma->get()->At("to_git") : std::nullopt;
     if (pragma_to_git and pragma_to_git->get()->IsBool() and
         pragma_to_git->get()->Bool()) {
+        // start work reporting
+        JustMRProgress::Instance().TaskTracker().Start(repo_name);
+        JustMRStatistics::Instance().IncrementQueuedCounter();
         // get the WS root as git tree
         fpath_git_map->ConsumeAfterKeysReady(
             ts,
@@ -242,6 +246,9 @@ void FileCheckout(ExpressionPtr const& repo_desc,
                 cfg["workspace_root"] = ws_root;
                 SetReposTakeOver(&cfg, repos, repo_name);
                 (*setter)(std::move(cfg));
+                // report work done
+                JustMRProgress::Instance().TaskTracker().Stop(repo_name);
+                JustMRStatistics::Instance().IncrementExecutedCounter();
             },
             [logger, repo_name](auto const& msg, bool fatal) {
                 (*logger)(fmt::format("While setting the workspace root for "
@@ -258,6 +265,8 @@ void FileCheckout(ExpressionPtr const& repo_desc,
             nlohmann::json::array({"file", fpath.string()});  // explicit array
         SetReposTakeOver(&cfg, repos, repo_name);
         (*setter)(std::move(cfg));
+        // report local path
+        JustMRStatistics::Instance().IncrementLocalPathsCounter();
     }
 }
 
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index d609b7d3..573f2d8f 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -10,13 +10,14 @@
     ]
   , "stage": ["src", "other_tools", "root_maps"]
   , "private-deps":
-    [ ["src/other_tools/just_mr", "utils"]
-    , ["src/other_tools/ops_maps", "critical_git_op_map"]
+    [ ["src/other_tools/ops_maps", "critical_git_op_map"]
     , ["src/buildtool/execution_api/common", "common"]
     , ["src/buildtool/execution_api/local", "config"]
     , ["src/buildtool/execution_api/local", "local"]
     , ["src/utils/cpp", "tmp_dir"]
     , ["src/buildtool/file_system", "file_storage"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
     ]
   }
 , "commit_git_map":
@@ -34,6 +35,8 @@
   , "private-deps":
     [ ["src/other_tools/git_operations", "git_repo_remote"]
     , ["src/utils/cpp", "tmp_dir"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
     ]
   }
 , "fpath_git_map":
@@ -66,6 +69,8 @@
     [ ["src/other_tools/utils", "archive_ops"]
     , ["src/buildtool/execution_api/local", "local"]
     , ["src/buildtool/file_system", "file_storage"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
     ]
   }
 }
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 5d34182b..18adc4fa 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -17,6 +17,8 @@
 #include <algorithm>
 
 #include "src/other_tools/git_operations/git_repo_remote.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/utils/cpp/tmp_dir.hpp"
 
 namespace {
@@ -179,6 +181,10 @@ void EnsureCommit(GitRepoInfo const& repo_info,
                                "Keep referenced tree alive"  // message
                            },
                            GitOpType::KEEP_TAG};
+        // start work reporting
+        JustMRProgress::Instance().TaskTracker().Start(repo_info.origin);
+        JustMRStatistics::Instance().IncrementQueuedCounter();
+        // do the fetch
         critical_git_op_map->ConsumeAfterKeysReady(
             ts,
             {std::move(op_key)},
@@ -217,6 +223,9 @@ void EnsureCommit(GitRepoInfo const& repo_info,
                 // set the workspace root
                 (*ws_setter)(
                     nlohmann::json::array({"git tree", *subtree, repo_root}));
+                // report work done
+                JustMRProgress::Instance().TaskTracker().Stop(repo_info.origin);
+                JustMRStatistics::Instance().IncrementExecutedCounter();
             },
             [logger, target_path = repo_root](auto const& msg, bool fatal) {
                 (*logger)(fmt::format("While running critical Git op "
@@ -242,5 +251,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
         }
         // set the workspace root
         (*ws_setter)(nlohmann::json::array({"git tree", *subtree, repo_root}));
+        // report cache hit
+        JustMRStatistics::Instance().IncrementCacheHitsCounter();
     }
 }
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index 11adf136..584fabc7 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -16,6 +16,8 @@
 
 #include "src/buildtool/execution_api/local/local_cas.hpp"
 #include "src/buildtool/file_system/file_storage.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/other_tools/utils/archive_ops.hpp"
 
 namespace {
@@ -116,6 +118,8 @@ auto CreateContentGitMap(
                         {"git tree",
                          *subtree_hash,
                          JustMR::Utils::GetGitCacheRoot().string()}));
+                    // report cache hit
+                    JustMRStatistics::Instance().IncrementCacheHitsCounter();
                 },
                 [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
                     auto const& msg, bool fatal) {
@@ -128,6 +132,9 @@ auto CreateContentGitMap(
                 });
         }
         else {
+            // start work reporting;
+            JustMRProgress::Instance().TaskTracker().Start(key.archive.origin);
+            JustMRStatistics::Instance().IncrementQueuedCounter();
             // do the fetch and import_to_git
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
@@ -136,6 +143,7 @@ auto CreateContentGitMap(
                  repo_type = key.repo_type,
                  content_id = key.archive.content,
                  subdir = key.subdir,
+                 origin = key.archive.origin,
                  import_to_git_map,
                  ts,
                  setter,
@@ -175,6 +183,7 @@ auto CreateContentGitMap(
                         [tmp_dir,  // keep tmp_dir alive
                          archive_tree_id_file,
                          subdir,
+                         origin,
                          setter,
                          logger](auto const& values) {
                             // check for errors
@@ -236,6 +245,11 @@ auto CreateContentGitMap(
                                 {"git tree",
                                  *subtree_hash,
                                  JustMR::Utils::GetGitCacheRoot().string()}));
+                            // report work done
+                            JustMRProgress::Instance().TaskTracker().Stop(
+                                origin);
+                            JustMRStatistics::Instance()
+                                .IncrementExecutedCounter();
                         },
                         [logger, target_path = tmp_dir->GetPath()](
                             auto const& msg, bool fatal) {
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index 36914118..82e86303 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -20,7 +20,8 @@
 #include "src/buildtool/execution_api/local/config.hpp"
 #include "src/buildtool/execution_api/local/local_cas.hpp"
 #include "src/buildtool/file_system/file_storage.hpp"
-#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/other_tools/ops_maps/content_cas_map.hpp"
 #include "src/other_tools/ops_maps/critical_git_op_map.hpp"
 #include "src/utils/cpp/tmp_dir.hpp"
@@ -114,6 +115,9 @@ auto CreateDistdirGitMap(
                 });
         }
         else {
+            // start work reporting
+            JustMRProgress::Instance().TaskTracker().Start(key.origin);
+            JustMRStatistics::Instance().IncrementQueuedCounter();
             // fetch the gathered distdir repos into CAS
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
@@ -121,6 +125,7 @@ auto CreateDistdirGitMap(
                 [distdir_tree_id_file,
                  content_id = key.content_id,
                  content_list = key.content_list,
+                 origin = key.origin,
                  import_to_git_map,
                  ts,
                  setter,
@@ -151,6 +156,7 @@ auto CreateDistdirGitMap(
                         {std::move(c_info)},
                         [tmp_dir,  // keep tmp_dir alive
                          distdir_tree_id_file,
+                         origin,
                          setter,
                          logger](auto const& values) {
                             // check for errors
@@ -176,6 +182,11 @@ auto CreateDistdirGitMap(
                                 {"git tree",
                                  distdir_tree_id,
                                  JustMR::Utils::GetGitCacheRoot().string()}));
+                            // report work done
+                            JustMRProgress::Instance().TaskTracker().Stop(
+                                origin);
+                            JustMRStatistics::Instance()
+                                .IncrementExecutedCounter();
                         },
                         [logger, target_path = tmp_dir->GetPath()](
                             auto const& msg, bool fatal) {