diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-06-11 16:07:50 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-06-18 12:05:10 +0200 |
commit | ec447408d17c530ce2023d8148dd34a6d8138535 (patch) | |
tree | c644991efe281d3a7fa91451fa2d4fa70884f763 /src | |
parent | 651af149098289b60cdfb062d36eb435d8e49c52 (diff) | |
download | justbuild-ec447408d17c530ce2023d8148dd34a6d8138535.tar.gz |
Pass ServeApi to just-mr maps by reference
...instead of using singleton calls.
Diffstat (limited to 'src')
23 files changed, 237 insertions, 197 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index be03021d..ff2af01f 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -110,6 +110,7 @@ , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "fmt", "", "fmt"] + , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] @@ -125,6 +126,8 @@ , "setup_utils" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/serve_api/remote", "config"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] } , "update": @@ -157,6 +160,7 @@ , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "json", "", "json"] + , ["@", "gsl", "", "gsl"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "fs_utils"] @@ -177,6 +181,8 @@ , "setup_utils" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/serve_api/remote", "config"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] } , "launch": diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 46be0083..99fd9f62 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -15,15 +15,19 @@ #include "src/other_tools/just_mr/fetch.hpp" #include <filesystem> +#include <optional> #include <utility> // std::move #include "fmt/core.h" +#include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -398,31 +402,35 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, bool remote_compatible{common_args.compatible == true}; // setup the API for serving roots - auto serve_api_exists = JustMR::Utils::SetupServeApi( - common_args.remote_serve_address, auth_args); + std::optional<gsl::not_null<const ServeApi*>> serve; + if (JustMR::Utils::SetupServeApi(common_args.remote_serve_address, + auth_args)) { + serve = &ServeApi::Instance(); + } // check configuration of the serve endpoint provided - if (serve_api_exists) { + if (serve) { + // if we have a remote endpoint explicitly given by the user, it must + // match what the serve endpoint expects + if (remote_api and common_args.remote_execution_address and + not(*serve)->CheckServeRemoteExecution()) { + return kExitFetchError; // this check logs error on failure + } + // check the compatibility mode of the serve endpoint - auto compatible = ServeApi::Instance().IsCompatible(); + auto compatible = (*serve)->IsCompatible(); if (not compatible) { Logger::Log(LogLevel::Warning, "Checking compatibility configuration of the provided " "serve endpoint failed. Serve endpoint ignored."); - serve_api_exists = false; + serve = std::nullopt; } if (*compatible != remote_compatible) { Logger::Log( LogLevel::Warning, "Provided serve endpoint operates in a different compatibility " "mode than stated. Serve endpoint ignored."); - serve_api_exists = false; - } - // if we have a remote endpoint explicitly given by the user, it must - // match what the serve endpoint expects - if (remote_api and common_args.remote_execution_address and - not ServeApi::Instance().CheckServeRemoteExecution()) { - return kExitFetchError; // this check logs error on failure + serve = std::nullopt; } } @@ -435,7 +443,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, common_args.alternative_mirrors, common_args.ca_info, &critical_git_op_map, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) @@ -462,7 +470,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, &import_to_git_map, common_args.git_path->string(), *common_args.local_launcher, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 2d60a8da..b0d530f2 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -20,6 +20,7 @@ #include <exception> #include <thread> +#include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" @@ -27,6 +28,8 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -117,31 +120,35 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, bool remote_compatible{common_args.compatible == true}; // setup the API for serving roots - auto serve_api_exists = JustMR::Utils::SetupServeApi( - common_args.remote_serve_address, auth_args); + std::optional<gsl::not_null<const ServeApi*>> serve; + if (JustMR::Utils::SetupServeApi(common_args.remote_serve_address, + auth_args)) { + serve = &ServeApi::Instance(); + } // check configuration of the serve endpoint provided - if (serve_api_exists) { + if (serve) { + // if we have a remote endpoint explicitly given by the user, it must + // match what the serve endpoint expects + if (remote_api and common_args.remote_execution_address and + not(*serve)->CheckServeRemoteExecution()) { + return std::nullopt; // this check logs error on failure + } + // check the compatibility mode of the serve endpoint - auto compatible = ServeApi::Instance().IsCompatible(); + auto compatible = (*serve)->IsCompatible(); if (not compatible) { Logger::Log(LogLevel::Warning, "Checking compatibility configuration of the provided " "serve endpoint failed."); - serve_api_exists = false; + serve = std::nullopt; } if (*compatible != remote_compatible) { Logger::Log( LogLevel::Warning, "Provided serve endpoint operates in a different compatibility " "mode than stated. Serve endpoint ignored."); - serve_api_exists = false; - } - // if we have a remote endpoint explicitly given by the user, it must - // match what the serve endpoint expects - if (remote_api and common_args.remote_execution_address and - not ServeApi::Instance().CheckServeRemoteExecution()) { - return std::nullopt; // this check logs error on failure + serve = std::nullopt; } } @@ -154,7 +161,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.alternative_mirrors, common_args.ca_info, &critical_git_op_map, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) @@ -172,7 +179,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &import_to_git_map, common_args.git_path->string(), *common_args.local_launcher, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) @@ -189,7 +196,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.alternative_mirrors, common_args.git_path->string(), *common_args.local_launcher, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) @@ -205,7 +212,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.ca_info, &resolve_symlinks_map, &critical_git_op_map, - serve_api_exists, + serve, (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) : std::nullopt, @@ -215,7 +222,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, auto foreign_file_git_map = CreateForeignFileGitMap(&content_cas_map, &import_to_git_map, - serve_api_exists, + serve, common_args.fetch_absent, common_args.jobs); @@ -224,7 +231,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &critical_git_op_map, &import_to_git_map, &resolve_symlinks_map, - serve_api_exists, + serve, (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) : std::nullopt, @@ -237,7 +244,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, CreateDistdirGitMap(&content_cas_map, &import_to_git_map, &critical_git_op_map, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) @@ -249,7 +256,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &critical_git_op_map, &import_to_git_map, common_args.fetch_absent, - serve_api_exists, + serve, &(*local_api), (remote_api and not remote_compatible) ? std::make_optional(&(*remote_api)) diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 94894e60..dcf58392 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -66,6 +66,7 @@ , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/utils/cpp", "hash_combine"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": @@ -73,7 +74,6 @@ , ["src/other_tools/utils", "content"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] @@ -113,6 +113,7 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": @@ -122,7 +123,6 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/system", "system_command"] diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 20d85b8a..527bd7de 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -18,7 +18,6 @@ #include "fmt/core.h" #include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" @@ -109,7 +108,7 @@ auto CreateContentCASMap( MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> ContentCASMap { @@ -117,7 +116,7 @@ auto CreateContentCASMap( additional_mirrors, ca_info, critical_git_op_map, - serve_api_exists, + serve, local_api, remote_api](auto ts, auto setter, @@ -150,7 +149,7 @@ auto CreateContentCASMap( just_mr_paths, additional_mirrors, ca_info, - serve_api_exists, + serve, local_api, remote_api, setter, @@ -216,8 +215,8 @@ auto CreateContentCASMap( return; } // check if content is known to remote serve service - if (serve_api_exists and remote_api and - ServeApi::Instance().ContentInRemoteCAS(key.content)) { + if (serve and remote_api and + (*serve)->ContentInRemoteCAS(key.content)) { // try to get content from remote CAS if (remote_api.value()->RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp index 63934b24..9e8d9f46 100644 --- a/src/other_tools/ops_maps/content_cas_map.hpp +++ b/src/other_tools/ops_maps/content_cas_map.hpp @@ -25,6 +25,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -83,7 +84,7 @@ using ContentCASMap = AsyncMapConsumer<ArchiveContent, std::nullptr_t>; MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> ContentCASMap; diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp index fec72b27..10095045 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp @@ -24,7 +24,6 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/system/system_command.hpp" @@ -129,7 +128,7 @@ auto CreateGitTreeFetchMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, std::string const& git_bin, std::vector<std::string> const& launcher, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool backup_to_remote, @@ -138,7 +137,7 @@ auto CreateGitTreeFetchMap( import_to_git_map, git_bin, launcher, - serve_api_exists, + serve, local_api, remote_api, backup_to_remote](auto ts, @@ -164,7 +163,7 @@ auto CreateGitTreeFetchMap( import_to_git_map, git_bin, launcher, - serve_api_exists, + serve, local_api, remote_api, backup_to_remote, @@ -232,12 +231,11 @@ auto CreateGitTreeFetchMap( JustMRProgress::Instance().TaskTracker().Start(key.origin); // check if tree is known to remote serve service and can be // made available in remote CAS - if (serve_api_exists and remote_api) { + if (serve and remote_api) { // as we anyway interrogate the remote execution endpoint, // we're only interested here in the serve endpoint making // an attempt to upload the tree, if known, to remote CAS - std::ignore = - ServeApi::Instance().TreeInRemoteCAS(key.hash); + std::ignore = (*serve)->TreeInRemoteCAS(key.hash); } // check if tree is in remote CAS, if a remote is given if (remote_api and diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp index 05ad1c3c..3f6e6769 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp @@ -23,6 +23,7 @@ #include "gsl/gsl" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -59,7 +60,7 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>; gsl::not_null<ImportToGitMap*> const& import_to_git_map, std::string const& git_bin, std::vector<std::string> const& launcher, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool backup_to_remote, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 671a1d40..d88f09f8 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -8,6 +8,7 @@ , ["@", "json", "", "json"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/other_tools/ops_maps", "content_cas_map"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -22,7 +23,6 @@ , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] @@ -79,6 +79,7 @@ , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path_hash"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -109,6 +110,7 @@ , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -121,7 +123,6 @@ , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] @@ -142,12 +143,12 @@ , ["@", "json", "", "json"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/logging", "log_level"] - , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] @@ -169,6 +170,7 @@ , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -190,6 +192,7 @@ [ ["@", "gsl", "", "gsl"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] + , ["src/buildtool/serve_api/remote", "serve_api"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -198,7 +201,6 @@ , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/git", "git"] , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/serve_api/remote", "serve_api"] ] } } diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index b6dac07a..fc77e9e7 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -21,7 +21,6 @@ #include "fmt/core.h" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" @@ -63,24 +62,24 @@ void EnsureRootAsAbsent( std::string const& tree_id, std::filesystem::path const& repo_root, GitRepoInfo const& repo_info, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // this is an absent root - if (serve_api_exists) { + if (serve) { // check if the serve endpoint has this root - auto has_tree = CheckServeHasAbsentRoot(tree_id, logger); + auto has_tree = CheckServeHasAbsentRoot(**serve, tree_id, logger); if (not has_tree) { return; } if (not *has_tree) { // try to see if serve endpoint has the information to prepare the // root itself - auto serve_result = ServeApi::Instance().RetrieveTreeFromCommit( - repo_info.hash, - repo_info.subdir, - /*sync_tree = */ false); + auto serve_result = + (*serve)->RetrieveTreeFromCommit(repo_info.hash, + repo_info.subdir, + /*sync_tree = */ false); if (std::holds_alternative<std::string>(serve_result)) { // if serve has set up the tree, it must match what we expect auto const& served_tree_id = @@ -116,7 +115,8 @@ void EnsureRootAsAbsent( } // the tree is known locally, so we can upload it to remote CAS // for the serve endpoint to retrieve it and set up the root - if (not EnsureAbsentRootOnServe(tree_id, + if (not EnsureAbsentRootOnServe(**serve, + tree_id, repo_root, &(*remote_api.value()), logger, @@ -411,7 +411,7 @@ void EnsureCommit( gsl::not_null<ImportToGitMap*> const& import_to_git_map, std::string const& git_bin, std::vector<std::string> const& launcher, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool fetch_absent, @@ -487,7 +487,7 @@ void EnsureCommit( EnsureRootAsAbsent(*tree_id, StorageConfig::GitRoot(), repo_info, - serve_api_exists, + serve, remote_api, ws_setter, logger); @@ -510,13 +510,13 @@ void EnsureCommit( // no id file association exists JustMRProgress::Instance().TaskTracker().Start(repo_info.origin); // check if commit is known to remote serve service - if (serve_api_exists) { + if (serve) { // if root purely absent, request only the subdir tree if (repo_info.absent and not fetch_absent) { - auto serve_result = ServeApi::Instance().RetrieveTreeFromCommit( - repo_info.hash, - repo_info.subdir, - /*sync_tree = */ false); + auto serve_result = + (*serve)->RetrieveTreeFromCommit(repo_info.hash, + repo_info.subdir, + /*sync_tree = */ false); if (std::holds_alternative<std::string>(serve_result)) { // set the workspace root as absent JustMRProgress::Instance().TaskTracker().Stop( @@ -544,10 +544,10 @@ void EnsureCommit( // otherwise, request (and sync) the whole commit tree, to ensure // we maintain the id file association else { - auto serve_result = ServeApi::Instance().RetrieveTreeFromCommit( - repo_info.hash, - /*subdir = */ ".", - /*sync_tree = */ true); + auto serve_result = + (*serve)->RetrieveTreeFromCommit(repo_info.hash, + /*subdir = */ ".", + /*sync_tree = */ true); if (std::holds_alternative<std::string>(serve_result)) { auto const& root_tree_id = std::get<std::string>(serve_result); @@ -894,7 +894,7 @@ void EnsureCommit( EnsureRootAsAbsent(subtree, repo_root, repo_info, - serve_api_exists, + serve, remote_api, ws_setter, logger); @@ -923,7 +923,7 @@ auto CreateCommitGitMap( MirrorsPtr const& additional_mirrors, std::string const& git_bin, std::vector<std::string> const& launcher, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool fetch_absent, @@ -934,7 +934,7 @@ auto CreateCommitGitMap( additional_mirrors, git_bin, launcher, - serve_api_exists, + serve, local_api, remote_api, fetch_absent](auto ts, @@ -975,7 +975,7 @@ auto CreateCommitGitMap( import_to_git_map, git_bin, launcher, - serve_api_exists, + serve, local_api, remote_api, fetch_absent, @@ -1008,7 +1008,7 @@ auto CreateCommitGitMap( import_to_git_map, git_bin, launcher, - serve_api_exists, + serve, local_api, remote_api, fetch_absent, diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index 79fe873d..1f2ff707 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -25,6 +25,7 @@ #include "nlohmann/json.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -80,7 +81,7 @@ using CommitGitMap = MirrorsPtr const& additional_mirrors, std::string const& git_bin, std::vector<std::string> const& launcher, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool fetch_absent, diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 785c675b..6fccb871 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -20,7 +20,6 @@ #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" @@ -56,15 +55,15 @@ namespace { void EnsureRootAsAbsent( std::string const& tree_id, ArchiveRepoInfo const& key, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool is_cache_hit, ContentGitMap::SetterPtr const& ws_setter, ContentGitMap::LoggerPtr const& logger) { // this is an absent root - if (serve_api_exists) { + if (serve) { // check if the serve endpoint has this root - auto has_tree = CheckServeHasAbsentRoot(tree_id, logger); + auto has_tree = CheckServeHasAbsentRoot(**serve, tree_id, logger); if (not has_tree) { return; } @@ -73,12 +72,11 @@ void EnsureRootAsAbsent( // root itself; this is redundant if root is not already cached if (is_cache_hit) { auto serve_result = - ServeApi::Instance().RetrieveTreeFromArchive( - key.archive.content, - key.repo_type, - key.subdir, - key.pragma_special, - /*sync_tree=*/false); + (*serve)->RetrieveTreeFromArchive(key.archive.content, + key.repo_type, + key.subdir, + key.pragma_special, + /*sync_tree=*/false); if (std::holds_alternative<std::string>(serve_result)) { // if serve has set up the tree, it must match what we // expect @@ -119,6 +117,7 @@ void EnsureRootAsAbsent( // CAS for the serve endpoint to retrieve it and set up the // root if (not EnsureAbsentRootOnServe( + **serve, tree_id, StorageConfig::GitRoot(), &(*remote_api.value()), @@ -143,7 +142,8 @@ void EnsureRootAsAbsent( // the tree is known locally, so we can upload it to remote // CAS for the serve endpoint to retrieve it and set up the // root - if (not EnsureAbsentRootOnServe(tree_id, + if (not EnsureAbsentRootOnServe(**serve, + tree_id, StorageConfig::GitRoot(), &(*remote_api.value()), logger, @@ -175,7 +175,7 @@ void ResolveContentTree( GitCASPtr const& just_git_cas, bool is_cache_hit, bool is_absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -200,7 +200,7 @@ void ResolveContentTree( if (is_absent) { EnsureRootAsAbsent(*resolved_tree_id, key, - serve_api_exists, + serve, remote_api, is_cache_hit, ws_setter, @@ -231,7 +231,7 @@ void ResolveContentTree( is_cache_hit, key, is_absent, - serve_api_exists, + serve, remote_api, ts, ws_setter, @@ -273,7 +273,7 @@ void ResolveContentTree( key, tree_id_file, is_absent, - serve_api_exists, + serve, remote_api, is_cache_hit, ws_setter, @@ -299,7 +299,7 @@ void ResolveContentTree( if (is_absent) { EnsureRootAsAbsent(resolved_tree_id, key, - serve_api_exists, + serve, remote_api, is_cache_hit, ws_setter, @@ -339,7 +339,7 @@ void ResolveContentTree( if (is_absent) { EnsureRootAsAbsent(tree_hash, key, - serve_api_exists, + serve, remote_api, is_cache_hit, ws_setter, @@ -364,7 +364,7 @@ void WriteIdFileAndSetWSRoot( GitCASPtr const& just_git_cas, std::filesystem::path const& archive_tree_id_file, bool is_absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -408,7 +408,7 @@ void WriteIdFileAndSetWSRoot( just_git_cas, false, /*is_cache_hit*/ is_absent, - serve_api_exists, + serve, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -425,7 +425,7 @@ void ExtractAndImportToGit( std::filesystem::path const& content_cas_path, std::filesystem::path const& archive_tree_id_file, bool is_absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -461,7 +461,7 @@ void ExtractAndImportToGit( archive_tree_id_file, key, is_absent, - serve_api_exists, + serve, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -482,7 +482,7 @@ void ExtractAndImportToGit( values[0]->second, /*just_git_cas*/ archive_tree_id_file, is_absent, - serve_api_exists, + serve, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -509,7 +509,7 @@ auto CreateContentGitMap( CAInfoPtr const& ca_info, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool fetch_absent, std::size_t jobs) -> ContentGitMap { @@ -520,7 +520,7 @@ auto CreateContentGitMap( just_mr_paths, additional_mirrors, ca_info, - serve_api_exists, + serve, remote_api, fetch_absent](auto ts, auto setter, @@ -556,7 +556,7 @@ auto CreateContentGitMap( [archive_tree_id = *archive_tree_id, key, fetch_absent, - serve_api_exists, + serve, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -600,7 +600,7 @@ auto CreateContentGitMap( op_result.git_cas, /*is_cache_hit = */ true, /*is_absent = */ (key.absent and not fetch_absent), - serve_api_exists, + serve, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -623,14 +623,13 @@ auto CreateContentGitMap( if (key.absent and not fetch_absent) { // request the resolved subdir tree from the serve endpoint, if // given - if (serve_api_exists) { - auto serve_result = - ServeApi::Instance().RetrieveTreeFromArchive( - key.archive.content, - key.repo_type, - key.subdir, - key.pragma_special, - /*sync_tree = */ false); + if (serve) { + auto serve_result = (*serve)->RetrieveTreeFromArchive( + key.archive.content, + key.repo_type, + key.subdir, + key.pragma_special, + /*sync_tree = */ false); if (std::holds_alternative<std::string>(serve_result)) { // set the workspace root as absent JustMRProgress::Instance().TaskTracker().Stop( @@ -667,7 +666,7 @@ auto CreateContentGitMap( *content_cas_path, archive_tree_id_file, /*is_absent = */ true, - serve_api_exists, + serve, remote_api, critical_git_op_map, import_to_git_map, @@ -703,7 +702,7 @@ auto CreateContentGitMap( just_mr_paths, additional_mirrors, ca_info, - serve_api_exists, + serve, remote_api, ts, setter, @@ -758,7 +757,7 @@ auto CreateContentGitMap( *content_cas_path, archive_tree_id_file, /*is_absent=*/true, - serve_api_exists, + serve, remote_api, critical_git_op_map, import_to_git_map, @@ -797,7 +796,7 @@ auto CreateContentGitMap( *content_cas_path, archive_tree_id_file, /*is_absent=*/true, - serve_api_exists, + serve, remote_api, critical_git_op_map, import_to_git_map, @@ -849,7 +848,7 @@ auto CreateContentGitMap( content_cas_path, archive_tree_id_file, /*is_absent=*/false, - /*serve_api_exists=*/false, + /*serve=*/std::nullopt, /*remote_api=*/std::nullopt, critical_git_op_map, import_to_git_map, diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index d278b44a..06759ec8 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -24,6 +24,7 @@ #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -42,7 +43,7 @@ using ContentGitMap = CAInfoPtr const& ca_info, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, bool fetch_absent, std::size_t jobs) -> ContentGitMap; diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 390a5de4..2caa4b74 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -25,7 +25,6 @@ #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" @@ -133,14 +132,14 @@ auto CreateDistdirGitMap( gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> DistdirGitMap { auto distdir_to_git = [content_cas_map, import_to_git_map, critical_git_op_map, - serve_api_exists, + serve, local_api, remote_api](auto ts, auto setter, @@ -176,7 +175,7 @@ auto CreateDistdirGitMap( [distdir_tree_id = *distdir_tree_id, content_id = key.content_id, key, - serve_api_exists, + serve, remote_api, setter, logger](auto const& values) { @@ -190,10 +189,10 @@ auto CreateDistdirGitMap( // subdir is "." here, so no need to deal with the Git cache // and we can simply set the workspace root if (key.absent) { - if (serve_api_exists) { + if (serve) { // check if serve endpoint has this root auto has_tree = CheckServeHasAbsentRoot( - distdir_tree_id, logger); + **serve, distdir_tree_id, logger); if (not has_tree) { return; } @@ -201,10 +200,9 @@ auto CreateDistdirGitMap( // try to see if serve endpoint has the // information to prepare the root itself auto serve_result = - ServeApi::Instance() - .RetrieveTreeFromDistdir( - key.content_list, - /*sync_tree=*/false); + (*serve)->RetrieveTreeFromDistdir( + key.content_list, + /*sync_tree=*/false); if (std::holds_alternative<std::string>( serve_result)) { // if serve has set up the tree, it must @@ -253,6 +251,7 @@ auto CreateDistdirGitMap( // it to remote CAS for the serve endpoint // to retrieve it and set up the root if (not EnsureAbsentRootOnServe( + **serve, distdir_tree_id, StorageConfig::GitRoot(), &(*remote_api.value()), @@ -332,9 +331,10 @@ auto CreateDistdirGitMap( // up the absent root without actually checking the local status of // each content blob individually if (key.absent) { - if (serve_api_exists) { + if (serve) { // first check if serve endpoint has tree - auto has_tree = CheckServeHasAbsentRoot(tree_id, logger); + auto has_tree = + CheckServeHasAbsentRoot(**serve, tree_id, logger); if (not has_tree) { return; } @@ -349,9 +349,8 @@ auto CreateDistdirGitMap( // try to see if serve endpoint has the information to // prepare the root itself auto serve_result = - ServeApi::Instance().RetrieveTreeFromDistdir( - key.content_list, - /*sync_tree=*/false); + (*serve)->RetrieveTreeFromDistdir(key.content_list, + /*sync_tree=*/false); if (std::holds_alternative<std::string>(serve_result)) { // if serve has set up the tree, it must match what we // expect @@ -399,6 +398,7 @@ auto CreateDistdirGitMap( // tell serve to set up the root from the remote CAS // tree; upload can be skipped if (EnsureAbsentRootOnServe( + **serve, tree_id, /*repo_path=*/"", /*remote_api=*/std::nullopt, @@ -435,6 +435,7 @@ auto CreateDistdirGitMap( // tell serve to set up the root from the remote CAS // tree; upload can be skipped if (EnsureAbsentRootOnServe( + **serve, tree_id, /*repo_path=*/"", /*remote_api=*/std::nullopt, @@ -482,11 +483,10 @@ auto CreateDistdirGitMap( } // now ask serve endpoint if it can set up the root; as this is for // a present root, a corresponding remote endpoint is needed - if (serve_api_exists and remote_api) { + if (serve and remote_api) { auto serve_result = - ServeApi::Instance().RetrieveTreeFromDistdir( - key.content_list, - /*sync_tree=*/true); + (*serve)->RetrieveTreeFromDistdir(key.content_list, + /*sync_tree=*/true); if (std::holds_alternative<std::string>(serve_result)) { // if serve has set up the tree, it must match what we // expect diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp index abaa1e84..d26c80e9 100644 --- a/src/other_tools/root_maps/distdir_git_map.hpp +++ b/src/other_tools/root_maps/distdir_git_map.hpp @@ -25,6 +25,7 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -53,7 +54,7 @@ using DistdirGitMap = gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> DistdirGitMap; diff --git a/src/other_tools/root_maps/foreign_file_git_map.cpp b/src/other_tools/root_maps/foreign_file_git_map.cpp index 3119d84e..7a2d60dd 100644 --- a/src/other_tools/root_maps/foreign_file_git_map.cpp +++ b/src/other_tools/root_maps/foreign_file_git_map.cpp @@ -18,7 +18,6 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" @@ -113,10 +112,11 @@ void UseCacheHit(const std::string& tree_id, /*is_cache_hit=*/true)); } -void HandleAbsentForeignFile(ForeignFileInfo const& key, - bool serve_api_exists, - ForeignFileGitMap::SetterPtr const& setter, - ForeignFileGitMap::LoggerPtr const& logger) { +void HandleAbsentForeignFile( + ForeignFileInfo const& key, + std::optional<gsl::not_null<const ServeApi*>> const& serve, + ForeignFileGitMap::SetterPtr const& setter, + ForeignFileGitMap::LoggerPtr const& logger) { // Compute tree in memory GitRepo::tree_entries_t entries{}; auto raw_id = FromHexString(key.archive.content); @@ -138,8 +138,8 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, return; } auto tree_id = ToHexString(tree->first); - if (serve_api_exists) { - auto has_tree = CheckServeHasAbsentRoot(tree_id, logger); + if (serve) { + auto has_tree = CheckServeHasAbsentRoot(**serve, tree_id, logger); if (not has_tree) { return; } @@ -149,7 +149,7 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, /*is_cache_hit=*/false)); return; } - auto serve_result = ServeApi::Instance().RetrieveTreeFromForeignFile( + auto serve_result = (*serve)->RetrieveTreeFromForeignFile( key.archive.content, key.name, key.executable); if (std::holds_alternative<std::string>(serve_result)) { // if serve has set up the tree, it must match what we @@ -196,18 +196,18 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, [[nodiscard]] auto CreateForeignFileGitMap( gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap { auto setup_foreign_file = - [content_cas_map, import_to_git_map, fetch_absent, serve_api_exists]( + [content_cas_map, import_to_git_map, fetch_absent, serve]( auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { if (key.absent and not fetch_absent) { - HandleAbsentForeignFile(key, serve_api_exists, setter, logger); + HandleAbsentForeignFile(key, serve, setter, logger); return; } auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( diff --git a/src/other_tools/root_maps/foreign_file_git_map.hpp b/src/other_tools/root_maps/foreign_file_git_map.hpp index 363a80f1..bc5f3c01 100644 --- a/src/other_tools/root_maps/foreign_file_git_map.hpp +++ b/src/other_tools/root_maps/foreign_file_git_map.hpp @@ -21,10 +21,11 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -/// \brief Maps a oreign file to the resulting Git tree WS root, +/// \brief Maps a foreign file to the resulting Git tree WS root, /// together with the information whether it was a cache hit. using ForeignFileGitMap = AsyncMapConsumer<ForeignFileInfo, @@ -33,7 +34,7 @@ using ForeignFileGitMap = [[nodiscard]] auto CreateForeignFileGitMap( gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap; diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 186261b7..913c3db7 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -37,15 +37,15 @@ void CheckServeAndSetRoot( std::string const& tree_id, std::string const& repo_root, bool absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, FilePathGitMap::SetterPtr const& ws_setter, FilePathGitMap::LoggerPtr const& logger) { // if serve endpoint is given, try to ensure it has this tree available to // be able to build against it. If root is not absent, do not fail if we // don't have a suitable remote endpoint, but warn user nonetheless. - if (serve_api_exists) { - auto has_tree = CheckServeHasAbsentRoot(tree_id, logger); + if (serve) { + auto has_tree = CheckServeHasAbsentRoot(**serve, tree_id, logger); if (not has_tree) { return; // fatal } @@ -63,7 +63,8 @@ void CheckServeAndSetRoot( } } else { - if (not EnsureAbsentRootOnServe(tree_id, + if (not EnsureAbsentRootOnServe(**serve, + tree_id, repo_root, &(*remote_api.value()), logger, @@ -100,7 +101,7 @@ void ResolveFilePathTree( bool absent, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, gsl::not_null<TaskSystem*> const& ts, FilePathGitMap::SetterPtr const& ws_setter, @@ -125,7 +126,7 @@ void ResolveFilePathTree( CheckServeAndSetRoot(*resolved_tree_id, StorageConfig::GitRoot().string(), absent, - serve_api_exists, + serve, remote_api, ws_setter, logger); @@ -145,7 +146,7 @@ void ResolveFilePathTree( tree_hash, tree_id_file, absent, - serve_api_exists, + serve, remote_api, ts, ws_setter, @@ -186,7 +187,7 @@ void ResolveFilePathTree( [resolved_tree_id, tree_id_file, absent, - serve_api_exists, + serve, remote_api, ws_setter, logger](auto const& values) { @@ -214,7 +215,7 @@ void ResolveFilePathTree( resolved_tree_id, StorageConfig::GitRoot().string(), absent, - serve_api_exists, + serve, remote_api, ws_setter, logger); @@ -242,13 +243,8 @@ void ResolveFilePathTree( // tree needs no further processing; // if serve endpoint is given, try to ensure it has this tree available // to be able to build against it - CheckServeAndSetRoot(tree_hash, - repo_root, - absent, - serve_api_exists, - remote_api, - ws_setter, - logger); + CheckServeAndSetRoot( + tree_hash, repo_root, absent, serve, remote_api, ws_setter, logger); } } @@ -259,7 +255,7 @@ auto CreateFilePathGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs, std::string multi_repo_tool_name, @@ -268,7 +264,7 @@ auto CreateFilePathGitMap( critical_git_op_map, import_to_git_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, multi_repo_tool_name, build_tool_name](auto ts, @@ -307,7 +303,7 @@ auto CreateFilePathGitMap( repo_root = std::move(*repo_root), critical_git_op_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, ts, setter, @@ -367,7 +363,7 @@ auto CreateFilePathGitMap( absent, critical_git_op_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, ts, setter, @@ -389,7 +385,7 @@ auto CreateFilePathGitMap( absent, critical_git_op_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, ts, setter, @@ -456,7 +452,7 @@ auto CreateFilePathGitMap( absent = key.absent, critical_git_op_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, ts, setter, @@ -480,7 +476,7 @@ auto CreateFilePathGitMap( absent, critical_git_op_map, resolve_symlinks_map, - serve_api_exists, + serve, remote_api, ts, setter, diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 398e9c3c..362b5a4b 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -25,6 +25,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -53,7 +54,7 @@ using FilePathGitMap = AsyncMapConsumer<FpathInfo, nlohmann::json>; gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs, std::string multi_repo_tool_name, diff --git a/src/other_tools/root_maps/root_utils.cpp b/src/other_tools/root_maps/root_utils.cpp index f0269b33..38c1e205 100644 --- a/src/other_tools/root_maps/root_utils.cpp +++ b/src/other_tools/root_maps/root_utils.cpp @@ -20,12 +20,12 @@ #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/serve_api/remote/serve_api.hpp" -auto CheckServeHasAbsentRoot(std::string const& tree_id, +auto CheckServeHasAbsentRoot(ServeApi const& serve, + std::string const& tree_id, AsyncMapConsumerLoggerPtr const& logger) -> std::optional<bool> { - if (auto has_tree = ServeApi::Instance().CheckRootTree(tree_id)) { + if (auto has_tree = serve.CheckRootTree(tree_id)) { return *has_tree; } (*logger)(fmt::format("Checking that the serve endpoint knows tree " @@ -36,6 +36,7 @@ auto CheckServeHasAbsentRoot(std::string const& tree_id, } auto EnsureAbsentRootOnServe( + ServeApi const& serve, std::string const& tree_id, std::filesystem::path const& repo_path, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, @@ -64,7 +65,7 @@ auto EnsureAbsentRootOnServe( } } // ask serve endpoint to retrieve the uploaded tree - if (not ServeApi::Instance().GetTreeFromRemote(tree_id)) { + if (not serve.GetTreeFromRemote(tree_id)) { // respond based on no_sync_is_fatal flag (*logger)( fmt::format("Serve endpoint failed to sync root tree {}.", tree_id), diff --git a/src/other_tools/root_maps/root_utils.hpp b/src/other_tools/root_maps/root_utils.hpp index d154eb3e..aae9e84e 100644 --- a/src/other_tools/root_maps/root_utils.hpp +++ b/src/other_tools/root_maps/root_utils.hpp @@ -21,6 +21,7 @@ #include "gsl/gsl" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" /// \brief Calls the ServeApi to check whether the serve endpoint has the given /// tree available to build against. @@ -30,6 +31,7 @@ /// whether the serve endpoint knows the tree on ServeApi call success. The /// logger is called with fatal ONLY if this method returns nullopt. [[nodiscard]] auto CheckServeHasAbsentRoot( + ServeApi const& serve, std::string const& tree_id, AsyncMapConsumerLoggerPtr const& logger) -> std::optional<bool>; @@ -53,6 +55,7 @@ /// \returns Status flag, with false if state is deemed fatal, and true /// otherwise. Logger is only called with fatal if returning false. [[nodiscard]] auto EnsureAbsentRootOnServe( + ServeApi const& serve, std::string const& tree_id, std::filesystem::path const& repo_path, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 9e8b1360..3bdb56e9 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -26,7 +26,8 @@ namespace { /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. -void UploadToServeAndSetRoot(std::string const& tree_id, +void UploadToServeAndSetRoot(ServeApi const& serve, + std::string const& tree_id, ArtifactDigest const& digest, gsl::not_null<IExecutionApi*> const& remote_api, bool ignore_special, @@ -55,7 +56,8 @@ void UploadToServeAndSetRoot(std::string const& tree_id, } // tell serve to set up the root from the remote CAS tree; // upload can be skipped - if (EnsureAbsentRootOnServe(tree_id, + if (EnsureAbsentRootOnServe(serve, + tree_id, /*repo_path=*/"", /*remote_api=*/std::nullopt, logger, @@ -73,6 +75,7 @@ void UploadToServeAndSetRoot(std::string const& tree_id, /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. void MoveCASTreeToGitAndProcess( + ServeApi const& serve, std::string const& tree_id, ArtifactDigest const& digest, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -104,7 +107,8 @@ void MoveCASTreeToGitAndProcess( import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, - [tmp_dir, // keep tmp_dir alive + [&serve, + tmp_dir, // keep tmp_dir alive tree_id, digest, remote_api, @@ -118,8 +122,13 @@ void MoveCASTreeToGitAndProcess( } // upload tree from Git cache to remote CAS and tell serve to set up // the root from the remote CAS tree; set root as absent on success - UploadToServeAndSetRoot( - tree_id, digest, remote_api, ignore_special, setter, logger); + UploadToServeAndSetRoot(serve, + tree_id, + digest, + remote_api, + ignore_special, + setter, + logger); }, [logger, tmp_dir, tree_id](auto const& msg, bool fatal) { (*logger)(fmt::format( @@ -138,7 +147,7 @@ auto CreateTreeIdGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> TreeIdGitMap { @@ -146,7 +155,7 @@ auto CreateTreeIdGitMap( critical_git_op_map, import_to_git_map, fetch_absent, - serve_api_exists, + serve, local_api, remote_api](auto ts, auto setter, @@ -158,10 +167,10 @@ auto CreateTreeIdGitMap( // found on the serve endpoint or it can be made available to it; // otherwise, error out if (key.absent and not fetch_absent) { - if (serve_api_exists) { + if (serve) { // check serve endpoint - auto has_tree = - CheckServeHasAbsentRoot(key.tree_info.hash, logger); + auto has_tree = CheckServeHasAbsentRoot( + **serve, key.tree_info.hash, logger); if (not has_tree) { return; } @@ -191,7 +200,8 @@ auto CreateTreeIdGitMap( if (remote_api.value()->IsAvailable({digest})) { // tell serve to set up the root from the remote CAS tree; // upload can be skipped - if (EnsureAbsentRootOnServe(key.tree_info.hash, + if (EnsureAbsentRootOnServe(**serve, + key.tree_info.hash, /*repo_path=*/"", /*remote_api=*/std::nullopt, logger, @@ -228,7 +238,8 @@ auto CreateTreeIdGitMap( critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [digest, + [serve, + digest, import_to_git_map, local_api, remote_api, @@ -275,7 +286,8 @@ auto CreateTreeIdGitMap( // upload tree from Git cache to remote CAS and tell // serve to set up the root from the remote CAS // tree, then set root as absent - UploadToServeAndSetRoot(key.tree_info.hash, + UploadToServeAndSetRoot(**serve, + key.tree_info.hash, digest, *remote_api, key.ignore_special, @@ -289,7 +301,8 @@ auto CreateTreeIdGitMap( if (auto path = cas.TreePath(digest)) { // Move tree locally from CAS to Git cache, then // continue processing it by UploadToServeAndSetRoot - MoveCASTreeToGitAndProcess(key.tree_info.hash, + MoveCASTreeToGitAndProcess(**serve, + key.tree_info.hash, digest, import_to_git_map, local_api, diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp index 56cf783d..52b306a0 100644 --- a/src/other_tools/root_maps/tree_id_git_map.hpp +++ b/src/other_tools/root_maps/tree_id_git_map.hpp @@ -24,6 +24,7 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -68,7 +69,7 @@ using TreeIdGitMap = gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, - bool serve_api_exists, + std::optional<gsl::not_null<const ServeApi*>> const& serve, gsl::not_null<IExecutionApi*> const& local_api, std::optional<gsl::not_null<IExecutionApi*>> const& remote_api, std::size_t jobs) -> TreeIdGitMap; |