diff options
author | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2023-03-22 14:23:24 +0100 |
---|---|---|
committer | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2023-03-23 09:50:43 +0100 |
commit | d6eb885a3f205e00a3be9f2ae2e8f510cae15d8a (patch) | |
tree | dabb8e10973ae1d5f21f4c01bda72b3d1b68402d | |
parent | e0c38d7083e6c972a432bfff3a9863f251f2170c (diff) | |
download | justbuild-d6eb885a3f205e00a3be9f2ae2e8f510cae15d8a.tar.gz |
just-mr: Add git binary option
In the rare cases that we need to shell out to git, let the user
configure what binary to use. Option resolves in the same way as
the just executable, including allowing it to be set via just-mrrc.
Updates all cases of shelling out to git (fetch and commit update).
Update just-mr and just-mrrc docs accordingly.
-rw-r--r-- | share/man/just-mr.1.org | 5 | ||||
-rw-r--r-- | share/man/just-mrrc.5.org | 5 | ||||
-rw-r--r-- | src/other_tools/git_operations/git_repo_remote.cpp | 6 | ||||
-rw-r--r-- | src/other_tools/git_operations/git_repo_remote.hpp | 2 | ||||
-rw-r--r-- | src/other_tools/just_mr/cli.hpp | 5 | ||||
-rw-r--r-- | src/other_tools/just_mr/main.cpp | 38 | ||||
-rw-r--r-- | src/other_tools/just_mr/utils.hpp | 1 | ||||
-rw-r--r-- | src/other_tools/ops_maps/git_update_map.cpp | 12 | ||||
-rw-r--r-- | src/other_tools/ops_maps/git_update_map.hpp | 1 | ||||
-rw-r--r-- | src/other_tools/ops_maps/import_to_git_map.cpp | 15 | ||||
-rw-r--r-- | src/other_tools/ops_maps/import_to_git_map.hpp | 1 | ||||
-rw-r--r-- | src/other_tools/root_maps/commit_git_map.cpp | 28 | ||||
-rw-r--r-- | src/other_tools/root_maps/commit_git_map.hpp | 1 | ||||
-rw-r--r-- | src/other_tools/root_maps/tree_id_git_map.cpp | 16 | ||||
-rw-r--r-- | src/other_tools/root_maps/tree_id_git_map.hpp | 1 | ||||
-rw-r--r-- | test/other_tools/git_operations/git_repo_remote.test.cpp | 22 |
16 files changed, 118 insertions, 41 deletions
diff --git a/share/man/just-mr.1.org b/share/man/just-mr.1.org index e30fe682..a166353e 100644 --- a/share/man/just-mr.1.org +++ b/share/man/just-mr.1.org @@ -103,6 +103,11 @@ See *just-mr-repository-config(5)* for more details on the input format. Path to the just-mrrc file to use. See *just-mrrc(5)* for more details.\\ Default: file path ~".just-mrrc"~ in the user's home directory. + *--git* PATH\\ + Path to the git binary in ~PATH~ or path to the git binary. Used in the rare + instances when shelling out to git is needed.\\ + Default: ~"git"~.\\ + *--norc*\\ Option to prevent reading any *just-mrrc(5)* file. diff --git a/share/man/just-mrrc.5.org b/share/man/just-mrrc.5.org index c6396f56..07c28481 100644 --- a/share/man/just-mrrc.5.org +++ b/share/man/just-mrrc.5.org @@ -62,6 +62,10 @@ The just-mrrc is given by a JSON object. path to the ~just~ binary to use for execution, if ~just-mr~ is used as a launcher. +- The value for the key ~"git"~ is a single location object, specifying the + path to the git binary to use in the instances when ~just-mr~ needs to shell + out. + - The value for the key ~"local launcher"~, if given, is list of strings setting the default for local launcher for ~just-mr~; command-line arguments take precedence over the value in the @@ -99,6 +103,7 @@ An example just-mrrc file could look like the following: , "log files": [{"root": "home", "path": ".log/just/latest-invocation"}] , "distdirs": [{"root": "home", "path": ".distfiles"}] , "just": {"root": "system", "path": "usr/bin/just"} +, "git": {"root": "system", "path": "usr/bin/git"} , "just args": { "build": ["-r", "10.0.0.1:8980", "--remote-execution-property", "OS:Linux"] , "install": ["-r", "10.0.0.1:8980", "--remote-execution-property", "OS:Linux"] diff --git a/src/other_tools/git_operations/git_repo_remote.cpp b/src/other_tools/git_operations/git_repo_remote.cpp index ab9d7ce2..8527a13c 100644 --- a/src/other_tools/git_operations/git_repo_remote.cpp +++ b/src/other_tools/git_operations/git_repo_remote.cpp @@ -390,6 +390,7 @@ auto GitRepoRemote::UpdateCommitViaTmpRepo( std::filesystem::path const& tmp_dir, std::string const& repo_url, std::string const& branch, + std::string const& git_bin, std::vector<std::string> const& launcher, anon_logger_ptr const& logger) const noexcept -> std::optional<std::string> { @@ -430,7 +431,7 @@ auto GitRepoRemote::UpdateCommitViaTmpRepo( } // default to shelling out to git for non-explicitly supported protocols auto cmdline = launcher; - cmdline.insert(cmdline.end(), {"git", "ls-remote", repo_url, branch}); + cmdline.insert(cmdline.end(), {git_bin, "ls-remote", repo_url, branch}); // set up the system command SystemCommand system{repo_url}; auto const command_output = @@ -494,6 +495,7 @@ auto GitRepoRemote::UpdateCommitViaTmpRepo( auto GitRepoRemote::FetchViaTmpRepo(std::filesystem::path const& tmp_dir, std::string const& repo_url, std::optional<std::string> const& branch, + std::string const& git_bin, std::vector<std::string> const& launcher, anon_logger_ptr const& logger) noexcept -> bool { @@ -550,7 +552,7 @@ auto GitRepoRemote::FetchViaTmpRepo(std::filesystem::path const& tmp_dir, // file. This however does not imply automatically that fetches // might not internally wait for each other through other means. cmdline.insert(cmdline.end(), - {"git", + {git_bin, "fetch", "--no-auto-gc", "--no-write-fetch-head", diff --git a/src/other_tools/git_operations/git_repo_remote.hpp b/src/other_tools/git_operations/git_repo_remote.hpp index 9b68ada9..cfe1736d 100644 --- a/src/other_tools/git_operations/git_repo_remote.hpp +++ b/src/other_tools/git_operations/git_repo_remote.hpp @@ -87,6 +87,7 @@ class GitRepoRemote : public GitRepo { std::filesystem::path const& tmp_dir, std::string const& repo_url, std::string const& branch, + std::string const& git_bin, std::vector<std::string> const& launcher, anon_logger_ptr const& logger) const noexcept -> std::optional<std::string>; @@ -104,6 +105,7 @@ class GitRepoRemote : public GitRepo { [[nodiscard]] auto FetchViaTmpRepo(std::filesystem::path const& tmp_dir, std::string const& repo_url, std::optional<std::string> const& branch, + std::string const& git_bin, std::vector<std::string> const& launcher, anon_logger_ptr const& logger) noexcept -> bool; diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp index 72e6f8a4..4e363460 100644 --- a/src/other_tools/just_mr/cli.hpp +++ b/src/other_tools/just_mr/cli.hpp @@ -41,6 +41,7 @@ struct MultiRepoCommonArguments { std::optional<std::filesystem::path> just_path{std::nullopt}; std::optional<std::string> main{std::nullopt}; std::optional<std::filesystem::path> rc_path{std::nullopt}; + std::optional<std::filesystem::path> git_path{std::nullopt}; bool norc{false}; std::size_t jobs{std::max(1U, std::thread::hardware_concurrency())}; }; @@ -148,6 +149,10 @@ static inline void SetupMultiRepoCommonArguments( }, "Use just-mrrc file from custom path.") ->type_name("RCFILE"); + app->add_option("--git", + clargs->git_path, + "Path to the git binary. (Default: \"git\")") + ->type_name("PATH"); app->add_flag("--norc", clargs->norc, "Do not use any just-mrrc file."); app->add_option("-j, --jobs", clargs->jobs, diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index 9f2a47ce..290d6bcf 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -379,6 +379,14 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { clargs->common.just_path = just->first; } } + // read git binary path; overwritten if user provided it already + if (not clargs->common.git_path) { + auto git = ReadLocation(rc_config["git"], + clargs->common.just_mr_paths->workspace_root); + if (git) { + clargs->common.git_path = git->first; + } + } // read additional just args; user can append, but does not overwrite auto just_args = rc_config["just args"]; if (just_args.IsNotNull()) { @@ -1007,9 +1015,11 @@ void DefaultReachableRepositories( // Initialize resulting config to be updated auto mr_config = config->ToJson(); // Create async map - auto git_update_map = CreateGitUpdateMap(git_repo->GetGitCAS(), - *arguments.common.local_launcher, - arguments.common.jobs); + auto git_update_map = + CreateGitUpdateMap(git_repo->GetGitCAS(), + arguments.common.git_path->string(), + *arguments.common.local_launcher, + arguments.common.jobs); // set up progress observer JustMRProgress::Instance().SetTotal(repos_to_update.size()); @@ -1118,13 +1128,16 @@ void DefaultReachableRepositories( arguments.common.jobs); auto import_to_git_map = CreateImportToGitMap(&critical_git_op_map, + arguments.common.git_path->string(), *arguments.common.local_launcher, arguments.common.jobs); - auto commit_git_map = CreateCommitGitMap(&critical_git_op_map, - arguments.common.just_mr_paths, - *arguments.common.local_launcher, - arguments.common.jobs); + auto commit_git_map = + CreateCommitGitMap(&critical_git_op_map, + arguments.common.just_mr_paths, + arguments.common.git_path->string(), + *arguments.common.local_launcher, + arguments.common.jobs); auto content_git_map = CreateContentGitMap(&content_cas_map, &import_to_git_map, &critical_git_op_map, @@ -1137,9 +1150,11 @@ void DefaultReachableRepositories( &import_to_git_map, &critical_git_op_map, arguments.common.jobs); - auto tree_id_git_map = CreateTreeIdGitMap(&critical_git_op_map, - *arguments.common.local_launcher, - arguments.common.jobs); + auto tree_id_git_map = + CreateTreeIdGitMap(&critical_git_op_map, + arguments.common.git_path->string(), + *arguments.common.local_launcher, + arguments.common.jobs); auto repos_to_setup_map = CreateReposToSetupMap(config, main, interactive, @@ -1361,6 +1376,9 @@ auto main(int argc, char* argv[]) -> int { if (not arguments.common.just_path) { arguments.common.just_path = kDefaultJustPath; } + if (not arguments.common.git_path) { + arguments.common.git_path = kDefaultGitPath; + } bool forward_build_root = true; if (not arguments.common.just_mr_paths->root) { forward_build_root = false; diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index dddd4b92..0d3dbf1c 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -29,6 +29,7 @@ std::unordered_set<std::string> const kLocationTypes{"workspace", "home", "system"}; auto const kDefaultJustPath = "just"; +auto const kDefaultGitPath = "git"; auto const kDefaultRCPath = StorageConfig::GetUserHome() / ".just-mrrc"; auto const kDefaultBuildRoot = StorageConfig::kDefaultBuildRoot; auto const kDefaultCheckoutLocationsFile = diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp index d399ff40..cc40aef6 100644 --- a/src/other_tools/ops_maps/git_update_map.cpp +++ b/src/other_tools/ops_maps/git_update_map.cpp @@ -22,13 +22,14 @@ #include "src/utils/cpp/tmp_dir.hpp" auto CreateGitUpdateMap(GitCASPtr const& git_cas, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> GitUpdateMap { - auto update_commits = [git_cas, launcher](auto /* unused */, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + auto update_commits = [git_cas, git_bin, launcher](auto /* unused */, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { // perform git update commit auto git_repo = GitRepoRemote::Open(git_cas); // wrap the tmp odb if (not git_repo) { @@ -59,6 +60,7 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, auto new_commit = git_repo->UpdateCommitViaTmpRepo(tmp_dir->GetPath(), key.first, key.second, + git_bin, launcher, wrapped_logger); JustMRProgress::Instance().TaskTracker().Stop(id); diff --git a/src/other_tools/ops_maps/git_update_map.hpp b/src/other_tools/ops_maps/git_update_map.hpp index 93a5da30..abd0aee9 100644 --- a/src/other_tools/ops_maps/git_update_map.hpp +++ b/src/other_tools/ops_maps/git_update_map.hpp @@ -38,6 +38,7 @@ struct hash<StringPair> { } // namespace std [[nodiscard]] auto CreateGitUpdateMap(GitCASPtr const& git_cas, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> GitUpdateMap; diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp index 2a3ddee2..c603853b 100644 --- a/src/other_tools/ops_maps/import_to_git_map.cpp +++ b/src/other_tools/ops_maps/import_to_git_map.cpp @@ -85,13 +85,15 @@ void KeepCommitAndSetTree( auto CreateImportToGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> ImportToGitMap { - auto import_to_git = [critical_git_op_map, launcher](auto ts, - auto setter, - auto logger, - auto /*unused*/, - auto const& key) { + auto import_to_git = [critical_git_op_map, git_bin, launcher]( + auto ts, + auto setter, + auto logger, + auto /*unused*/, + auto const& key) { // Perform initial commit at location: init + add . + commit GitOpKey op_key = { { @@ -107,6 +109,7 @@ auto CreateImportToGitMap( {std::move(op_key)}, [critical_git_op_map, target_path = key.target_path, + git_bin, launcher, ts, setter, @@ -136,6 +139,7 @@ auto CreateImportToGitMap( commit = *op_result.result, target_path, git_cas = op_result.git_cas, + git_bin, launcher, ts, setter, @@ -183,6 +187,7 @@ auto CreateImportToGitMap( tmp_dir->GetPath(), target_path.string(), std::nullopt, + git_bin, launcher, wrapped_logger)) { return; diff --git a/src/other_tools/ops_maps/import_to_git_map.hpp b/src/other_tools/ops_maps/import_to_git_map.hpp index 77f0983a..ee0b3c8f 100644 --- a/src/other_tools/ops_maps/import_to_git_map.hpp +++ b/src/other_tools/ops_maps/import_to_git_map.hpp @@ -59,6 +59,7 @@ using ImportToGitMap = [[nodiscard]] auto CreateImportToGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> ImportToGitMap; diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index 3500245b..48cf4d8f 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -36,6 +36,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, std::filesystem::path const& repo_root, GitCASPtr const& git_cas, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::string const& git_bin, std::vector<std::string> const& launcher, gsl::not_null<TaskSystem*> const& ts, CommitGitMap::SetterPtr const& ws_setter, @@ -77,6 +78,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (not git_repo->FetchViaTmpRepo(tmp_dir->GetPath(), repo_info.repo_url, repo_info.branch, + git_bin, launcher, wrapped_logger)) { return; @@ -186,14 +188,17 @@ void EnsureCommit(GitRepoInfo const& repo_info, auto CreateCommitGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, JustMR::PathsPtr const& just_mr_paths, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> CommitGitMap { - auto commit_to_git = [critical_git_op_map, just_mr_paths, launcher]( - auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + auto commit_to_git = [critical_git_op_map, + just_mr_paths, + git_bin, + launcher](auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { // get root for repo (making sure that if repo is a path, it is // absolute) std::string fetch_repo = key.repo_url; @@ -217,8 +222,14 @@ auto CreateCommitGitMap( critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [key, repo_root, critical_git_op_map, launcher, ts, setter, logger]( - auto const& values) { + [key, + repo_root, + critical_git_op_map, + git_bin, + launcher, + ts, + setter, + logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { @@ -240,6 +251,7 @@ auto CreateCommitGitMap( repo_root, op_result.git_cas, critical_git_op_map, + git_bin, launcher, ts, setter, diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index c021382c..352c094e 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -58,6 +58,7 @@ using CommitGitMap = [[nodiscard]] auto CreateCommitGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, JustMR::PathsPtr const& just_mr_paths, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> CommitGitMap; diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index a1f6b25c..d73057c4 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -79,13 +79,15 @@ void KeepCommitAndSetRoot( auto CreateTreeIdGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> TreeIdGitMap { - auto tree_to_git = [critical_git_op_map, launcher](auto ts, - auto setter, - auto logger, - auto /*unused*/, - auto const& key) { + auto tree_to_git = [critical_git_op_map, git_bin, launcher]( + auto ts, + auto setter, + auto logger, + auto /*unused*/, + auto const& key) { // first, check whether tree exists already in CAS // ensure Git cache // define Git operation to be done @@ -100,7 +102,7 @@ auto CreateTreeIdGitMap( critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [critical_git_op_map, launcher, key, ts, setter, logger]( + [critical_git_op_map, git_bin, launcher, key, ts, setter, logger]( auto const& values) { GitOpValue op_result = *values[0]; // check flag @@ -190,6 +192,7 @@ auto CreateTreeIdGitMap( cmdline, command_output, key, + git_bin, launcher, ts, setter, @@ -296,6 +299,7 @@ auto CreateTreeIdGitMap( tmp_dir->GetPath(), target_path.string(), std::nullopt, + git_bin, launcher, wrapped_logger)) { return; diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp index 414f6ebd..7e14e4fc 100644 --- a/src/other_tools/root_maps/tree_id_git_map.hpp +++ b/src/other_tools/root_maps/tree_id_git_map.hpp @@ -50,6 +50,7 @@ using TreeIdGitMap = [[nodiscard]] auto CreateTreeIdGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::string const& git_bin, std::vector<std::string> const& launcher, std::size_t jobs) -> TreeIdGitMap; diff --git a/test/other_tools/git_operations/git_repo_remote.test.cpp b/test/other_tools/git_operations/git_repo_remote.test.cpp index ae3f2e05..5e4f9f9e 100644 --- a/test/other_tools/git_operations/git_repo_remote.test.cpp +++ b/test/other_tools/git_operations/git_repo_remote.test.cpp @@ -248,8 +248,12 @@ TEST_CASE("Single-threaded fake repository operations", "[git_repo_remote]") { auto tmp_path_fetch_all = TestUtils::GetRepoPath(); REQUIRE(FileSystemManager::CreateDirectory(tmp_path_fetch_all)); // fetch all with base refspecs - REQUIRE(repo_fetch_all->FetchViaTmpRepo( - tmp_path_fetch_all, *repo_path, std::nullopt, {}, logger)); + REQUIRE(repo_fetch_all->FetchViaTmpRepo(tmp_path_fetch_all, + *repo_path, + std::nullopt, + "git", + {}, + logger)); // check commit is there after fetch CHECK(*repo_fetch_all->CheckCommitExists(kRootCommit, logger)); @@ -271,8 +275,13 @@ TEST_CASE("Single-threaded fake repository operations", "[git_repo_remote]") { REQUIRE( FileSystemManager::CreateDirectory(tmp_path_fetch_wRefspec)); // fetch all - REQUIRE(repo_fetch_wRefspec->FetchViaTmpRepo( - tmp_path_fetch_wRefspec, *repo_path, "master", {}, logger)); + REQUIRE( + repo_fetch_wRefspec->FetchViaTmpRepo(tmp_path_fetch_wRefspec, + *repo_path, + "master", + "git", + {}, + logger)); // check commit is there after fetch CHECK(*repo_fetch_wRefspec->CheckCommitExists(kRootCommit, logger)); @@ -290,7 +299,7 @@ TEST_CASE("Single-threaded fake repository operations", "[git_repo_remote]") { REQUIRE(FileSystemManager::CreateDirectory(tmp_path_commit_upd)); // do remote ls auto fetched_commit = repo_commit_upd->UpdateCommitViaTmpRepo( - tmp_path_commit_upd, *repo_path, "master", {}, logger); + tmp_path_commit_upd, *repo_path, "master", "git", {}, logger); REQUIRE(fetched_commit); CHECK(*fetched_commit == kRootCommit); @@ -358,6 +367,7 @@ TEST_CASE("Multi-threaded fake repository operations", "[git_repo_remote]") { target_repo->FetchViaTmpRepo(tmp_path_fetch_all, *remote_repo_path, std::nullopt, + "git", {}, logger)); } break; @@ -372,6 +382,7 @@ TEST_CASE("Multi-threaded fake repository operations", "[git_repo_remote]") { tmp_path_fetch_wRefspec, *remote_repo_path, "master", + "git", {}, logger)); } break; @@ -386,6 +397,7 @@ TEST_CASE("Multi-threaded fake repository operations", "[git_repo_remote]") { tmp_path_commit_upd, *remote_repo_path, "master", + "git", {}, logger); |