summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/TARGETS2
-rw-r--r--src/other_tools/just_mr/main.cpp113
-rw-r--r--src/other_tools/repo_map/TARGETS5
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp11
-rw-r--r--src/other_tools/root_maps/TARGETS9
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp11
-rw-r--r--src/other_tools/root_maps/content_git_map.cpp14
-rw-r--r--src/other_tools/root_maps/distdir_git_map.cpp13
8 files changed, 166 insertions, 12 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index b3996909..9a73f4f2 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -12,6 +12,8 @@
, ["src/other_tools/ops_maps", "repo_fetch_map"]
, ["src/other_tools/ops_maps", "git_update_map"]
, ["src/other_tools/repo_map", "repos_to_setup_map"]
+ , ["src/buildtool/progress_reporting", "base_progress_reporter"]
+ , ["src/buildtool/progress_reporting", "progress"]
, ["@", "json", "", "json"]
, ["src/other_tools/just_mr/progress_reporting", "progress"]
, ["src/other_tools/just_mr/progress_reporting", "statistics"]
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index d1675bdd..5f5053e1 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -440,9 +440,51 @@ void SetupLogging(MultiRepoLogArguments const& clargs) {
return config;
}
-void ReachableRepositories(ExpressionPtr const& repos,
- std::string const& main,
- std::shared_ptr<SetupRepos> const& setup_repos) {
+/// \brief Add repos from a given repo list to the reported repos set if its
+/// type is distdir. Ignore any missing or wrong info.
+void AddDistdirReportedRepos(
+ ExpressionPtr const& repos,
+ std::string const& repo_name,
+ std::shared_ptr<std::unordered_set<std::string>> const&
+ reported_repos_set) {
+ auto repo_desc_key = repos->Get(repo_name, Expression::none_t{});
+ auto repo_desc = repo_desc_key->Get("repository", Expression::none_t{});
+ auto resolved_repo_desc = JustMR::Utils::ResolveRepo(repo_desc, repos);
+ if (not resolved_repo_desc) {
+ return;
+ }
+ auto repo_type = (*resolved_repo_desc)->Get("type", Expression::none_t{});
+ if (not repo_type->IsString()) {
+ return;
+ }
+ // check if the repo is distdir
+ auto repo_type_str = repo_type->String();
+ if (repo_type_str != "distdir") {
+ return;
+ }
+ // get distdir list
+ auto repo_desc_repositories =
+ repo_desc->Get("repositories", Expression::none_t{});
+ if (not repo_desc_repositories->IsList()) {
+ return;
+ }
+ auto distdir_repos = repo_desc_repositories->List();
+ // add to reported set
+ for (auto const& entry : distdir_repos) {
+ if (entry->IsString()) {
+ reported_repos_set->insert(entry->String());
+ }
+ }
+}
+
+/// \brief Get the repo dependency closure for a given main repository.
+/// For progress reporting we include
+void ReachableRepositories(
+ ExpressionPtr const& repos,
+ std::string const& main,
+ std::shared_ptr<SetupRepos> const& setup_repos,
+ std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
+ nullptr) {
// use temporary sets to avoid duplicates
std::unordered_set<std::string> include_repos_set{};
if (repos->IsMap()) {
@@ -452,6 +494,13 @@ void ReachableRepositories(ExpressionPtr const& repos,
if (not include_repos_set.contains(repo_name)) {
// if not found, add it and repeat for its bindings
include_repos_set.insert(repo_name);
+ // add to reported repos, if needed
+ if (reported_repos_set) {
+ reported_repos_set->insert(repo_name);
+ AddDistdirReportedRepos(
+ repos, repo_name, reported_repos_set);
+ }
+ // check bindings
auto repos_repo_name =
repos->Get(repo_name, Expression::none_t{});
if (not repos_repo_name.IsNotNull()) {
@@ -480,7 +529,14 @@ void ReachableRepositories(ExpressionPtr const& repos,
auto layer_val =
repos_repo->Get(layer, Expression::none_t{});
if (layer_val.IsNotNull() and layer_val->IsString()) {
- setup_repos_set.insert(layer_val->String());
+ auto repo_name = layer_val->String();
+ setup_repos_set.insert(repo_name);
+ // add to reported repos, if needed
+ if (reported_repos_set) {
+ reported_repos_set->insert(repo_name);
+ AddDistdirReportedRepos(
+ repos, repo_name, reported_repos_set);
+ }
}
}
}
@@ -504,10 +560,20 @@ void ReachableRepositories(ExpressionPtr const& repos,
void DefaultReachableRepositories(
ExpressionPtr const& repos,
- std::shared_ptr<SetupRepos> const& setup_repos) {
+ std::shared_ptr<SetupRepos> const& setup_repos,
+ std::shared_ptr<std::unordered_set<std::string>> const& reported_repos_set =
+ nullptr) {
if (repos.IsNotNull() and repos->IsMap()) {
setup_repos->to_setup = repos->Map().Keys();
setup_repos->to_include = setup_repos->to_setup;
+ // get reported repos
+ if (reported_repos_set) {
+ *reported_repos_set = std::unordered_set<std::string>(
+ setup_repos->to_setup.begin(), setup_repos->to_setup.end());
+ for (auto const& repo : setup_repos->to_setup) {
+ AddDistdirReportedRepos(repos, repo, reported_repos_set);
+ }
+ }
}
}
@@ -994,6 +1060,8 @@ void DefaultReachableRepositories(
CommandLineArguments const& arguments,
bool interactive)
-> std::optional<std::filesystem::path> {
+ // provide report
+ Logger::Log(LogLevel::Info, "Performing repositories setup");
// set anchor dir to setup_root; current dir will be reverted when anchor
// goes out of scope
auto cwd_anchor = FileSystemManager::ChangeDirectory(
@@ -1027,7 +1095,8 @@ void DefaultReachableRepositories(
mr_config["main"] = *main;
}
// get default repos to setup and to include
- DefaultReachableRepositories(repos, setup_repos);
+ auto repos_to_report = std::make_shared<std::unordered_set<std::string>>();
+ DefaultReachableRepositories(repos, setup_repos, repos_to_report);
// check if main is to be taken as first repo name lexicographically
if (not main and not setup_repos->to_setup.empty()) {
main = *std::min_element(setup_repos->to_setup.begin(),
@@ -1035,9 +1104,16 @@ void DefaultReachableRepositories(
}
// final check on which repos are to be set up
if (main and not arguments.setup.sub_all) {
- ReachableRepositories(repos, *main, setup_repos);
+ ReachableRepositories(repos, *main, setup_repos, repos_to_report);
}
+ // report progress
+ auto nr = repos_to_report->size();
+ Logger::Log(LogLevel::Info,
+ "Found {} {} to check out",
+ nr,
+ nr == 1 ? "repository" : "repositories");
+
// setup the required async maps
auto crit_git_op_ptr = std::make_shared<CriticalGitOpGuard>();
auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
@@ -1071,6 +1147,20 @@ void DefaultReachableRepositories(
&distdir_git_map,
arguments.common.jobs);
+ // set up map for progress tracing
+ auto& repo_set = JustMRProgress::Instance().RepositorySet();
+ repo_set.clear();
+ repo_set.reserve(nr);
+ for (auto const& repo : *repos_to_report) {
+ repo_set.emplace(repo);
+ }
+ // set up progress observer
+ std::atomic<bool> done{false};
+ std::condition_variable cv{};
+ auto reporter = JustMRProgressReporter::Reporter();
+ auto observer =
+ std::thread([reporter, &done, &cv]() { reporter(&done, &cv); });
+
// Populate workspace_root and TAKE_OVER fields
bool failed{false};
{
@@ -1123,6 +1213,12 @@ void DefaultReachableRepositories(
failed = failed or fatal;
});
}
+
+ // close progress observer
+ done = true;
+ cv.notify_all();
+ observer.join();
+
if (failed) {
return std::nullopt;
}
@@ -1336,6 +1432,9 @@ auto main(int argc, char* argv[]) -> int {
if (not mr_config_path) {
return kExitSetupError;
}
+ // report success
+ Logger::Log(LogLevel::Info, "Setup completed");
+ // print config file to stdout
std::cout << mr_config_path->string() << std::endl;
return kExitSuccess;
}
diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS
index 905e09c8..5e89d85c 100644
--- a/src/other_tools/repo_map/TARGETS
+++ b/src/other_tools/repo_map/TARGETS
@@ -11,6 +11,9 @@
, ["src/buildtool/build_engine/expression", "expression"]
]
, "stage": ["src", "other_tools", "repo_map"]
- , "private-deps": [["src/other_tools/just_mr", "utils"]]
+ , "private-deps":
+ [ ["src/other_tools/just_mr/progress_reporting", "progress"]
+ , ["src/other_tools/just_mr/progress_reporting", "statistics"]
+ ]
}
}
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index bd34a231..36428668 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -14,7 +14,8 @@
#include "src/other_tools/repo_map/repos_to_setup_map.hpp"
-#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
namespace {
@@ -232,6 +233,9 @@ void FileCheckout(ExpressionPtr const& repo_desc,
repo_desc_pragma ? repo_desc_pragma->get()->At("to_git") : std::nullopt;
if (pragma_to_git and pragma_to_git->get()->IsBool() and
pragma_to_git->get()->Bool()) {
+ // start work reporting
+ JustMRProgress::Instance().TaskTracker().Start(repo_name);
+ JustMRStatistics::Instance().IncrementQueuedCounter();
// get the WS root as git tree
fpath_git_map->ConsumeAfterKeysReady(
ts,
@@ -242,6 +246,9 @@ void FileCheckout(ExpressionPtr const& repo_desc,
cfg["workspace_root"] = ws_root;
SetReposTakeOver(&cfg, repos, repo_name);
(*setter)(std::move(cfg));
+ // report work done
+ JustMRProgress::Instance().TaskTracker().Stop(repo_name);
+ JustMRStatistics::Instance().IncrementExecutedCounter();
},
[logger, repo_name](auto const& msg, bool fatal) {
(*logger)(fmt::format("While setting the workspace root for "
@@ -258,6 +265,8 @@ void FileCheckout(ExpressionPtr const& repo_desc,
nlohmann::json::array({"file", fpath.string()}); // explicit array
SetReposTakeOver(&cfg, repos, repo_name);
(*setter)(std::move(cfg));
+ // report local path
+ JustMRStatistics::Instance().IncrementLocalPathsCounter();
}
}
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index d609b7d3..573f2d8f 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -10,13 +10,14 @@
]
, "stage": ["src", "other_tools", "root_maps"]
, "private-deps":
- [ ["src/other_tools/just_mr", "utils"]
- , ["src/other_tools/ops_maps", "critical_git_op_map"]
+ [ ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_api/local", "config"]
, ["src/buildtool/execution_api/local", "local"]
, ["src/utils/cpp", "tmp_dir"]
, ["src/buildtool/file_system", "file_storage"]
+ , ["src/other_tools/just_mr/progress_reporting", "progress"]
+ , ["src/other_tools/just_mr/progress_reporting", "statistics"]
]
}
, "commit_git_map":
@@ -34,6 +35,8 @@
, "private-deps":
[ ["src/other_tools/git_operations", "git_repo_remote"]
, ["src/utils/cpp", "tmp_dir"]
+ , ["src/other_tools/just_mr/progress_reporting", "progress"]
+ , ["src/other_tools/just_mr/progress_reporting", "statistics"]
]
}
, "fpath_git_map":
@@ -66,6 +69,8 @@
[ ["src/other_tools/utils", "archive_ops"]
, ["src/buildtool/execution_api/local", "local"]
, ["src/buildtool/file_system", "file_storage"]
+ , ["src/other_tools/just_mr/progress_reporting", "progress"]
+ , ["src/other_tools/just_mr/progress_reporting", "statistics"]
]
}
}
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 5d34182b..18adc4fa 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -17,6 +17,8 @@
#include <algorithm>
#include "src/other_tools/git_operations/git_repo_remote.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
namespace {
@@ -179,6 +181,10 @@ void EnsureCommit(GitRepoInfo const& repo_info,
"Keep referenced tree alive" // message
},
GitOpType::KEEP_TAG};
+ // start work reporting
+ JustMRProgress::Instance().TaskTracker().Start(repo_info.origin);
+ JustMRStatistics::Instance().IncrementQueuedCounter();
+ // do the fetch
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
@@ -217,6 +223,9 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// set the workspace root
(*ws_setter)(
nlohmann::json::array({"git tree", *subtree, repo_root}));
+ // report work done
+ JustMRProgress::Instance().TaskTracker().Stop(repo_info.origin);
+ JustMRStatistics::Instance().IncrementExecutedCounter();
},
[logger, target_path = repo_root](auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
@@ -242,5 +251,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
}
// set the workspace root
(*ws_setter)(nlohmann::json::array({"git tree", *subtree, repo_root}));
+ // report cache hit
+ JustMRStatistics::Instance().IncrementCacheHitsCounter();
}
}
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index 11adf136..584fabc7 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -16,6 +16,8 @@
#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/buildtool/file_system/file_storage.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/other_tools/utils/archive_ops.hpp"
namespace {
@@ -116,6 +118,8 @@ auto CreateContentGitMap(
{"git tree",
*subtree_hash,
JustMR::Utils::GetGitCacheRoot().string()}));
+ // report cache hit
+ JustMRStatistics::Instance().IncrementCacheHitsCounter();
},
[logger, target_path = JustMR::Utils::GetGitCacheRoot()](
auto const& msg, bool fatal) {
@@ -128,6 +132,9 @@ auto CreateContentGitMap(
});
}
else {
+ // start work reporting;
+ JustMRProgress::Instance().TaskTracker().Start(key.archive.origin);
+ JustMRStatistics::Instance().IncrementQueuedCounter();
// do the fetch and import_to_git
content_cas_map->ConsumeAfterKeysReady(
ts,
@@ -136,6 +143,7 @@ auto CreateContentGitMap(
repo_type = key.repo_type,
content_id = key.archive.content,
subdir = key.subdir,
+ origin = key.archive.origin,
import_to_git_map,
ts,
setter,
@@ -175,6 +183,7 @@ auto CreateContentGitMap(
[tmp_dir, // keep tmp_dir alive
archive_tree_id_file,
subdir,
+ origin,
setter,
logger](auto const& values) {
// check for errors
@@ -236,6 +245,11 @@ auto CreateContentGitMap(
{"git tree",
*subtree_hash,
JustMR::Utils::GetGitCacheRoot().string()}));
+ // report work done
+ JustMRProgress::Instance().TaskTracker().Stop(
+ origin);
+ JustMRStatistics::Instance()
+ .IncrementExecutedCounter();
},
[logger, target_path = tmp_dir->GetPath()](
auto const& msg, bool fatal) {
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index 36914118..82e86303 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -20,7 +20,8 @@
#include "src/buildtool/execution_api/local/config.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/buildtool/file_system/file_storage.hpp"
-#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
@@ -114,6 +115,9 @@ auto CreateDistdirGitMap(
});
}
else {
+ // start work reporting
+ JustMRProgress::Instance().TaskTracker().Start(key.origin);
+ JustMRStatistics::Instance().IncrementQueuedCounter();
// fetch the gathered distdir repos into CAS
content_cas_map->ConsumeAfterKeysReady(
ts,
@@ -121,6 +125,7 @@ auto CreateDistdirGitMap(
[distdir_tree_id_file,
content_id = key.content_id,
content_list = key.content_list,
+ origin = key.origin,
import_to_git_map,
ts,
setter,
@@ -151,6 +156,7 @@ auto CreateDistdirGitMap(
{std::move(c_info)},
[tmp_dir, // keep tmp_dir alive
distdir_tree_id_file,
+ origin,
setter,
logger](auto const& values) {
// check for errors
@@ -176,6 +182,11 @@ auto CreateDistdirGitMap(
{"git tree",
distdir_tree_id,
JustMR::Utils::GetGitCacheRoot().string()}));
+ // report work done
+ JustMRProgress::Instance().TaskTracker().Stop(
+ origin);
+ JustMRStatistics::Instance()
+ .IncrementExecutedCounter();
},
[logger, target_path = tmp_dir->GetPath()](
auto const& msg, bool fatal) {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-05 15:45:57 +0000