Just-MR: Add progress reporting for update command

Co-authored-by: Sascha Roloff <sascha.roloff@huawei.com>

4 files changed, 59 insertions, 1 deletions

diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index 478a65b0..b3996909 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -13,6 +13,9 @@
     , ["src/other_tools/ops_maps", "git_update_map"]
     , ["src/other_tools/repo_map", "repos_to_setup_map"]
     , ["@", "json", "", "json"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"]
     ]
   , "stage": ["src", "other_tools", "just_mr"]
   , "private-ldflags":
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index f7e5ae1f..cffaf8e1 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <utility>
+
 #include <nlohmann/json.hpp>
 #include <unistd.h>
 
@@ -22,6 +24,9 @@
 #include "src/buildtool/main/version.hpp"
 #include "src/other_tools/just_mr/cli.hpp"
 #include "src/other_tools/just_mr/exit_codes.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/other_tools/ops_maps/git_update_map.hpp"
 #include "src/other_tools/ops_maps/repo_fetch_map.hpp"
 #include "src/other_tools/repo_map/repos_to_setup_map.hpp"
@@ -747,8 +752,14 @@ void DefaultReachableRepositories(
 [[nodiscard]] auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config,
                                    CommandLineArguments const& arguments)
     -> int {
+    // provide report
+    Logger::Log(LogLevel::Info, "Performing repositories update");
+
     // Check trivial case
     if (arguments.update.repos_to_update.empty()) {
+        // report success
+        Logger::Log(LogLevel::Info, "No update needed");
+        // print config file
         std::cout << config->ToJson().dump(2) << std::endl;
         return kExitSuccess;
     }
@@ -871,11 +882,36 @@ void DefaultReachableRepositories(
                     tmp_dir->GetPath().string());
         return kExitUpdateError;
     }
+
+    // report progress
+    auto nr = repos_to_update.size();
+    Logger::Log(LogLevel::Info,
+                "Discovered {} Git {} to update",
+                nr,
+                nr == 1 ? "repository" : "repositories");
+
     // Initialize resulting config to be updated
     auto mr_config = config->ToJson();
-    // Create and call git commit update map
+    // Create async map
     auto git_update_map =
         CreateGitUpdateMap(git_repo->GetGitCAS(), arguments.common.jobs);
+
+    // set up map for progress tracing
+    auto& repo_set = JustMRProgress::Instance().RepositorySet();
+    repo_set.clear();
+    repo_set.reserve(nr);
+    for (auto const& repo : repos_to_update) {
+        auto id = fmt::format("{}:{}", repo.first, repo.second);
+        repo_set.emplace(std::move(id));
+    }
+    // set up progress observer
+    std::atomic<bool> done{false};
+    std::condition_variable cv{};
+    auto reporter = JustMRProgressReporter::Reporter();
+    auto observer =
+        std::thread([reporter, &done, &cv]() { reporter(&done, &cv); });
+
+    // do the update
     bool failed{false};
     {
         TaskSystem ts{arguments.common.jobs};
@@ -899,9 +935,17 @@ void DefaultReachableRepositories(
                 failed = failed or fatal;
             });
     }
+
+    // close progress observer
+    done = true;
+    cv.notify_all();
+    observer.join();
+
     if (failed) {
         return kExitUpdateError;
     }
+    // report success
+    Logger::Log(LogLevel::Info, "Update completed");
     // print mr_config to stdout
     std::cout << mr_config.dump(2) << std::endl;
     return kExitSuccess;
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 5c99658a..07869b59 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -44,6 +44,8 @@
     [ ["src/other_tools/just_mr", "utils"]
     , ["src/buildtool/execution_api/local", "config"]
     , ["src/utils/cpp", "tmp_dir"]
+    , ["src/other_tools/just_mr/progress_reporting", "statistics"]
+    , ["src/other_tools/just_mr/progress_reporting", "progress"]
     ]
   }
 , "content_cas_map":
diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp
index c68d6f94..8c98fb09 100644
--- a/src/other_tools/ops_maps/git_update_map.cpp
+++ b/src/other_tools/ops_maps/git_update_map.cpp
@@ -16,6 +16,8 @@
 
 #include "fmt/core.h"
 #include "src/buildtool/execution_api/local/config.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/other_tools/just_mr/utils.hpp"
 #include "src/utils/cpp/tmp_dir.hpp"
 
@@ -26,6 +28,10 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, std::size_t jobs)
                                     auto logger,
                                     auto /* unused */,
                                     auto const& key) {
+        // start progress trace for Git repo
+        auto id = fmt::format("{}:{}", key.first, key.second);
+        JustMRProgress::Instance().TaskTracker().Start(id);
+        JustMRStatistics::Instance().IncrementQueuedCounter();
         // perform git update commit
         auto git_repo = GitRepoRemote::Open(git_cas);  // wrap the tmp odb
         if (not git_repo) {
@@ -57,6 +63,9 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, std::size_t jobs)
             return;
         }
         (*setter)(new_commit->c_str());
+        // stop progress trace for Git repo
+        JustMRProgress::Instance().TaskTracker().Stop(id);
+        JustMRStatistics::Instance().IncrementExecutedCounter();
     };
     return AsyncMapConsumer<StringPair, std::string>(update_commits, jobs);
 }