From f821e6b70c59037384ac6afb3a44517fe46953e6 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 12 Sep 2023 17:59:20 +0200 Subject: just serve: add remote execution endpoint and --fetch-absent option The serve service will communicate with this endpoint when needed, as well as ensure artifacts it provides are synced with the remote execution CAS, if requested by the client. If just-mr is given the --fetch-absent option, it Always produce present roots irrespective of the 'absent' pragma. For Git repositories marked with the 'absent' pragma, first try to fetch any commit trees provided by the serve endpoint from the execution endpoint CAS, before reverting to a network fetch. Co-authored-by: Klaus Aehlig Co-authored-by: Alberto Sartori --- src/other_tools/repo_map/repos_to_setup_map.cpp | 42 ++++++++++++++++--------- src/other_tools/repo_map/repos_to_setup_map.hpp | 1 + 2 files changed, 28 insertions(+), 15 deletions(-) (limited to 'src/other_tools/repo_map') diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index 5aefe3b1..f51e48ec 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -150,6 +150,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, std::string const& repo_name, std::string const& repo_type, gsl::not_null const& content_git_map, + bool fetch_absent, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { @@ -224,7 +225,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, .repo_type = repo_type, .subdir = subdir.empty() ? "." : subdir.string(), .pragma_special = pragma_special_value, - .absent = pragma_absent_value}; + .absent = not fetch_absent and pragma_absent_value}; // get the WS root as git tree content_git_map->ConsumeAfterKeysReady( ts, @@ -258,6 +259,7 @@ void FileCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& fpath_git_map, + bool fetch_absent, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { @@ -305,9 +307,10 @@ void FileCheckout(ExpressionPtr const& repo_desc, pragma_absent->get()->IsBool() and pragma_absent->get()->Bool(); // get the WS root as git tree - FpathInfo fpath_info = {.fpath = fpath, - .pragma_special = pragma_special_value, - .absent = pragma_absent_value}; + FpathInfo fpath_info = { + .fpath = fpath, + .pragma_special = pragma_special_value, + .absent = not fetch_absent and pragma_absent_value}; fpath_git_map->ConsumeAfterKeysReady( ts, {std::move(fpath_info)}, @@ -349,6 +352,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& distdir_git_map, + bool fetch_absent, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { @@ -509,11 +513,12 @@ void DistdirCheckout(ExpressionPtr const& repo_desc, HashFunction::ComputeBlobHash(nlohmann::json(*distdir_content).dump()) .HexString(); // get the WS root as git tree - DistdirInfo distdir_info = {.content_id = distdir_content_id, - .content_list = distdir_content, - .repos_to_fetch = dist_repos_to_fetch, - .origin = repo_name, - .absent = pragma_absent_value}; + DistdirInfo distdir_info = { + .content_id = distdir_content_id, + .content_list = distdir_content, + .repos_to_fetch = dist_repos_to_fetch, + .origin = repo_name, + .absent = not fetch_absent and pragma_absent_value}; distdir_git_map->ConsumeAfterKeysReady( ts, {std::move(distdir_info)}, @@ -545,6 +550,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc, ExpressionPtr&& repos, std::string const& repo_name, gsl::not_null const& tree_id_git_map, + bool fetch_absent, gsl::not_null const& ts, ReposToSetupMap::SetterPtr const& setter, ReposToSetupMap::LoggerPtr const& logger) { @@ -628,7 +634,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc, .env_vars = std::move(env), .command = std::move(cmd), .ignore_special = pragma_special_value == PragmaSpecial::Ignore, - .absent = pragma_absent_value}; + .absent = not fetch_absent and pragma_absent_value}; // get the WS root as git tree tree_id_git_map->ConsumeAfterKeysReady( ts, @@ -665,6 +671,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, gsl::not_null const& fpath_git_map, gsl::not_null const& distdir_git_map, gsl::not_null const& tree_id_git_map, + bool fetch_absent, std::size_t jobs) -> ReposToSetupMap { auto setup_repo = [config, main, @@ -673,11 +680,12 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, content_git_map, fpath_git_map, distdir_git_map, - tree_id_git_map](auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + tree_id_git_map, + fetch_absent](auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { auto repos = (*config)["repositories"]; if (main && (key == *main) && interactive) { // no repository checkout required @@ -765,6 +773,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, key, repo_type_str, content_git_map, + fetch_absent, ts, setter, wrapped_logger); @@ -775,6 +784,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, std::move(repos), key, fpath_git_map, + fetch_absent, ts, setter, wrapped_logger); @@ -785,6 +795,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, std::move(repos), key, distdir_git_map, + fetch_absent, ts, setter, wrapped_logger); @@ -795,6 +806,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, std::move(repos), key, tree_id_git_map, + fetch_absent, ts, setter, wrapped_logger); diff --git a/src/other_tools/repo_map/repos_to_setup_map.hpp b/src/other_tools/repo_map/repos_to_setup_map.hpp index 3306800d..1fbf9687 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.hpp +++ b/src/other_tools/repo_map/repos_to_setup_map.hpp @@ -34,6 +34,7 @@ auto CreateReposToSetupMap(std::shared_ptr const& config, gsl::not_null const& fpath_git_map, gsl::not_null const& distdir_git_map, gsl::not_null const& tree_id_git_map, + bool fetch_absent, std::size_t jobs) -> ReposToSetupMap; #endif // INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP -- cgit v1.2.3