diff options
Diffstat (limited to 'src')
34 files changed, 568 insertions, 407 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 4954ecb7..d9f8cf6f 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -104,7 +104,12 @@ , "name": ["fetch"] , "hdrs": ["fetch.hpp"] , "srcs": ["fetch.cpp"] - , "deps": [["src/buildtool/build_engine/expression", "expression"], "cli"] + , "deps": + [ ["src/buildtool/build_engine/expression", "expression"] + , "cli" + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] + ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "fmt", "", "fmt"] @@ -126,8 +131,6 @@ , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] - , ["src/buildtool/storage", "config"] - , ["src/buildtool/storage", "storage"] ] } , "update": @@ -135,7 +138,11 @@ , "name": ["update"] , "hdrs": ["update.hpp"] , "srcs": ["update.cpp"] - , "deps": [["src/buildtool/build_engine/expression", "expression"], "cli"] + , "deps": + [ ["src/buildtool/build_engine/expression", "expression"] + , "cli" + , ["src/buildtool/storage", "config"] + ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "fmt", "", "fmt"] @@ -156,7 +163,12 @@ , "name": ["setup"] , "hdrs": ["setup.hpp"] , "srcs": ["setup.cpp"] - , "deps": [["src/buildtool/build_engine/expression", "expression"], "cli"] + , "deps": + [ ["src/buildtool/build_engine/expression", "expression"] + , "cli" + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] + ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "json", "", "json"] @@ -183,8 +195,6 @@ , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] - , ["src/buildtool/storage", "storage"] - , ["src/buildtool/storage", "config"] ] } , "launch": @@ -192,15 +202,17 @@ , "name": ["launch"] , "hdrs": ["launch.hpp"] , "srcs": ["launch.cpp"] - , "deps": ["cli"] + , "deps": + [ "cli" + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] + ] , "stage": ["src", "other_tools", "just_mr"] , "private-deps": [ ["@", "json", "", "json"] , ["src/buildtool/build_engine/expression", "expression"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "storage"] - , ["src/buildtool/storage", "config"] , "exit_codes" , "setup" , "utils" diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index edd38652..dd0dbbe3 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -28,8 +28,6 @@ #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" -#include "src/buildtool/storage/config.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -46,6 +44,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, MultiRepoSetupArguments const& setup_args, MultiRepoFetchArguments const& fetch_args, MultiRepoRemoteAuthArguments const& auth_args, + StorageConfig const& storage_config, + Storage const& storage, std::string multi_repository_tool_name) -> int { // provide report Logger::Log(LogLevel::Info, "Performing repositories fetch"); @@ -405,8 +405,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, return kExitConfigError; } - ApiBundle const apis{&StorageConfig::Instance(), - &Storage::Instance(), + ApiBundle const apis{&storage_config, + &storage, /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; @@ -421,7 +421,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, return kExitConfigError; } - auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); + auto serve = ServeApi::Create(*serve_config, &storage, &apis); // check configuration of the serve endpoint provided if (serve) { // if we have a remote endpoint explicitly given by the user, it must @@ -458,6 +458,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, common_args.ca_info, &critical_git_op_map, serve ? &*serve : nullptr, + &storage_config, + &storage, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); @@ -465,6 +467,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, auto archive_fetch_map = CreateArchiveFetchMap( &content_cas_map, *fetch_dir, + &storage, &(*apis.local), (fetch_args.backup_to_remote and has_remote_api) ? &*apis.remote : nullptr, @@ -474,6 +477,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, CreateImportToGitMap(&critical_git_op_map, common_args.git_path->string(), *common_args.local_launcher, + &storage_config, common_args.jobs); auto git_tree_fetch_map = @@ -482,6 +486,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, common_args.git_path->string(), *common_args.local_launcher, serve ? &*serve : nullptr, + &storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, fetch_args.backup_to_remote, diff --git a/src/other_tools/just_mr/fetch.hpp b/src/other_tools/just_mr/fetch.hpp index 56b526b4..c024aef0 100644 --- a/src/other_tools/just_mr/fetch.hpp +++ b/src/other_tools/just_mr/fetch.hpp @@ -19,6 +19,8 @@ #include <string> #include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Fetching of distfiles for a multi-repository build. @@ -27,6 +29,8 @@ MultiRepoSetupArguments const& setup_args, MultiRepoFetchArguments const& fetch_args, MultiRepoRemoteAuthArguments const& auth_args, + StorageConfig const& storage_config, + Storage const& storage, std::string multi_repository_tool_name) -> int; diff --git a/src/other_tools/just_mr/launch.cpp b/src/other_tools/just_mr/launch.cpp index cfa45406..9fa2c849 100644 --- a/src/other_tools/just_mr/launch.cpp +++ b/src/other_tools/just_mr/launch.cpp @@ -38,6 +38,8 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, ForwardOnlyArguments const& launch_fwd, + StorageConfig const& storage_config, + Storage const& storage, bool forward_build_root, std::string multi_repo_tool_name) -> int { // check if subcmd_name can be taken from additional args @@ -62,7 +64,7 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file, if (subcommand and kKnownJustSubcommands.contains(*subcommand)) { // Read the config file if needed if (kKnownJustSubcommands.at(*subcommand).config) { - lock = GarbageCollector::SharedLock(StorageConfig::Instance()); + lock = GarbageCollector::SharedLock(storage_config); if (not lock) { return kExitGenericFailure; } @@ -75,6 +77,8 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file, setup_args, just_cmd_args, auth_args, + storage_config, + storage, /*interactive=*/false, std::move(multi_repo_tool_name)); if (not mr_config_path) { diff --git a/src/other_tools/just_mr/launch.hpp b/src/other_tools/just_mr/launch.hpp index d4c7ba17..01c139b0 100644 --- a/src/other_tools/just_mr/launch.hpp +++ b/src/other_tools/just_mr/launch.hpp @@ -18,6 +18,8 @@ #include <filesystem> #include <optional> +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Runs execvp for configured command. Only returns if execvp fails. @@ -30,6 +32,8 @@ MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, ForwardOnlyArguments const& launch_fwd, + StorageConfig const& storage_config, + Storage const& storage, bool forward_build_root, std::string multi_repo_tool_name) -> int; diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index f35d92b4..815d6453 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -345,6 +345,8 @@ auto main(int argc, char* argv[]) -> int { arguments.auth, arguments.retry, arguments.launch_fwd, + StorageConfig::Instance(), + Storage::Instance(), forward_build_root, my_name); } @@ -373,6 +375,8 @@ auto main(int argc, char* argv[]) -> int { arguments.setup, arguments.just_cmd, arguments.auth, + StorageConfig::Instance(), + Storage::Instance(), /*interactive=*/(arguments.cmd == SubCommand::kSetupEnv), my_name); // dump resulting config to stdout @@ -388,8 +392,11 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `update` if (arguments.cmd == SubCommand::kUpdate) { - return MultiRepoUpdate( - config, arguments.common, arguments.update, my_name); + return MultiRepoUpdate(config, + arguments.common, + arguments.update, + StorageConfig::Instance(), + my_name); } // Run subcommand `fetch` @@ -416,6 +423,8 @@ auto main(int argc, char* argv[]) -> int { arguments.setup, arguments.fetch, arguments.auth, + StorageConfig::Instance(), + Storage::Instance(), my_name); } diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 5c24f6c8..729725d2 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -31,9 +31,7 @@ #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -54,6 +52,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoRemoteAuthArguments const& auth_args, + StorageConfig const& storage_config, + Storage const& storage, bool interactive, std::string multi_repo_tool_name) -> std::optional<std::filesystem::path> { @@ -124,8 +124,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } - ApiBundle const apis{&StorageConfig::Instance(), - &Storage::Instance(), + ApiBundle const apis{&storage_config, + &storage, /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; @@ -140,7 +140,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } - auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); + auto serve = ServeApi::Create(*serve_config, &storage, &apis); // check configuration of the serve endpoint provided if (serve) { @@ -178,6 +178,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.ca_info, &critical_git_op_map, serve ? &*serve : nullptr, + &storage_config, + &storage, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); @@ -186,6 +188,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, CreateImportToGitMap(&critical_git_op_map, common_args.git_path->string(), *common_args.local_launcher, + &storage_config, common_args.jobs); auto git_tree_fetch_map = @@ -194,6 +197,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.git_path->string(), *common_args.local_launcher, serve ? &*serve : nullptr, + &storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, false, /* backup_to_remote */ @@ -209,6 +213,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, common_args.git_path->string(), *common_args.local_launcher, serve ? &*serve : nullptr, + &storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.fetch_absent, @@ -223,6 +228,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &resolve_symlinks_map, &critical_git_op_map, serve ? &*serve : nullptr, + &storage_config, + &storage, has_remote_api ? &*apis.remote : nullptr, common_args.fetch_absent, common_args.jobs); @@ -231,6 +238,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, CreateForeignFileGitMap(&content_cas_map, &import_to_git_map, serve ? &*serve : nullptr, + &storage_config, + &storage, common_args.fetch_absent, common_args.jobs); @@ -240,6 +249,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &import_to_git_map, &resolve_symlinks_map, serve ? &*serve : nullptr, + &storage_config, has_remote_api ? &*apis.remote : nullptr, common_args.jobs, multi_repo_tool_name, @@ -251,6 +261,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &import_to_git_map, &critical_git_op_map, serve ? &*serve : nullptr, + &storage_config, + &storage, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); @@ -261,6 +273,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &import_to_git_map, common_args.fetch_absent, serve ? &*serve : nullptr, + &storage_config, &(*apis.local), has_remote_api ? &*apis.remote : nullptr, common_args.jobs); @@ -385,5 +398,5 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } // if successful, return the output config - return StorageUtils::AddToCAS(Storage::Instance(), mr_config.dump(2)); + return StorageUtils::AddToCAS(storage, mr_config.dump(2)); } diff --git a/src/other_tools/just_mr/setup.hpp b/src/other_tools/just_mr/setup.hpp index df08658f..b32118b8 100644 --- a/src/other_tools/just_mr/setup.hpp +++ b/src/other_tools/just_mr/setup.hpp @@ -21,6 +21,8 @@ #include <string> #include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Setup for a multi-repository build. @@ -30,6 +32,8 @@ MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoRemoteAuthArguments const& auth_args, + StorageConfig const& storage_config, + Storage const& storage, bool interactive, std::string multi_repo_tool_name) -> std::optional<std::filesystem::path>; diff --git a/src/other_tools/just_mr/update.cpp b/src/other_tools/just_mr/update.cpp index 7c6e9506..cb40766a 100644 --- a/src/other_tools/just_mr/update.cpp +++ b/src/other_tools/just_mr/update.cpp @@ -26,7 +26,6 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -37,6 +36,7 @@ auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config, MultiRepoCommonArguments const& common_args, MultiRepoUpdateArguments const& update_args, + StorageConfig const& storage_config, std::string multi_repo_tool_name) -> int { // provide report Logger::Log(LogLevel::Info, "Performing repositories update"); @@ -191,7 +191,7 @@ auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config, } } // Create fake repo for the anonymous remotes - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir("update"); + auto tmp_dir = storage_config.CreateTypedTmpDir("update"); if (not tmp_dir) { Logger::Log(LogLevel::Error, "Failed to create commit update tmp dir"); return kExitUpdateError; @@ -220,6 +220,7 @@ auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config, auto git_update_map = CreateGitUpdateMap(git_repo->GetGitCAS(), common_args.git_path->string(), *common_args.local_launcher, + &storage_config, common_args.jobs); // set up progress observer diff --git a/src/other_tools/just_mr/update.hpp b/src/other_tools/just_mr/update.hpp index de70c62a..26729935 100644 --- a/src/other_tools/just_mr/update.hpp +++ b/src/other_tools/just_mr/update.hpp @@ -18,12 +18,14 @@ #include <memory> #include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/cli.hpp" /// \brief Update of Git repos commit information for a multi-repository build. [[nodiscard]] auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config, MultiRepoCommonArguments const& common_args, MultiRepoUpdateArguments const& update_args, + StorageConfig const& storage_config, std::string multi_repo_tool_name) -> int; #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UPDATE_HPP diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index ab8c6664..d564dc41 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -22,13 +22,13 @@ , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/storage", "fs_utils"] - , ["src/buildtool/storage", "storage"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "config"] ] @@ -42,6 +42,7 @@ [ ["src/other_tools/git_operations", "git_repo_remote"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": @@ -50,7 +51,6 @@ , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] - , ["src/buildtool/storage", "config"] ] } , "content_cas_map": @@ -68,19 +68,18 @@ , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/utils/cpp", "hash_combine"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/other_tools/utils", "content"] , ["src/buildtool/file_system", "file_storage"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] - , ["src/buildtool/storage", "storage"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] - , ["src/buildtool/storage", "storage"] ] } , "archive_fetch_map": @@ -92,6 +91,7 @@ [ ["@", "gsl", "", "gsl"] , "content_cas_map" , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": @@ -114,6 +114,7 @@ , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": @@ -122,8 +123,6 @@ , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "config"] - , ["src/buildtool/storage", "storage"] , ["src/buildtool/system", "system_command"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "progress"] diff --git a/src/other_tools/ops_maps/archive_fetch_map.cpp b/src/other_tools/ops_maps/archive_fetch_map.cpp index e035b128..f04d43f4 100644 --- a/src/other_tools/ops_maps/archive_fetch_map.cpp +++ b/src/other_tools/ops_maps/archive_fetch_map.cpp @@ -19,7 +19,6 @@ #include "fmt/core.h" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/utils.hpp" @@ -69,15 +68,19 @@ void ProcessContent(std::filesystem::path const& content_path, auto CreateArchiveFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, std::filesystem::path const& fetch_dir, + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> ArchiveFetchMap { - auto fetch_archive = [content_cas_map, fetch_dir, local_api, remote_api]( - auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + auto fetch_archive = [content_cas_map, + fetch_dir, + storage, + local_api, + remote_api](auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { // get corresponding distfile auto distfile = (key.distfile @@ -89,13 +92,14 @@ auto CreateArchiveFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, ts, {key}, [target_name, + storage, local_api, remote_api, content = key.content, setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); auto content_path = cas.BlobPath(ArtifactDigest{content, 0, /*is_tree=*/false}, /*is_executable=*/false) diff --git a/src/other_tools/ops_maps/archive_fetch_map.hpp b/src/other_tools/ops_maps/archive_fetch_map.hpp index 6ec3e44c..772d59e4 100644 --- a/src/other_tools/ops_maps/archive_fetch_map.hpp +++ b/src/other_tools/ops_maps/archive_fetch_map.hpp @@ -21,6 +21,7 @@ #include "gsl/gsl" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" /// \brief Maps an archive content hash to a status flag. @@ -29,6 +30,7 @@ using ArchiveFetchMap = AsyncMapConsumer<ArchiveContent, bool>; [[nodiscard]] auto CreateArchiveFetchMap( gsl::not_null<ContentCASMap*> const& content_cas_map, std::filesystem::path const& fetch_dir, // should exist! + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> ArchiveFetchMap; diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 59a22327..cacee577 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -18,9 +18,7 @@ #include "fmt/core.h" #include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -32,6 +30,7 @@ namespace { void FetchFromNetwork(ArchiveContent const& key, MirrorsPtr const& additional_mirrors, CAInfoPtr const& ca_info, + Storage const& storage, ContentCASMap::SetterPtr const& setter, ContentCASMap::LoggerPtr const& logger) { // first, check that mandatory fields are provided @@ -75,7 +74,7 @@ void FetchFromNetwork(ArchiveContent const& key, } } // add the fetched data to CAS - auto path = StorageUtils::AddToCAS(Storage::Instance(), *data); + auto path = StorageUtils::AddToCAS(storage, *data); // check one last time if content is in CAS now if (not path) { (*logger)(fmt::format("Failed to store fetched content from {}", @@ -84,7 +83,7 @@ void FetchFromNetwork(ArchiveContent const& key, return; } // check that the data we stored actually produces the requested digest - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage.CAS(); if (not cas.BlobPath(ArtifactDigest{key.content, 0, /*is_tree=*/false}, /*is_executable=*/false)) { (*logger)( @@ -107,6 +106,8 @@ auto CreateContentCASMap( CAInfoPtr const& ca_info, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> ContentCASMap { @@ -115,6 +116,8 @@ auto CreateContentCASMap( ca_info, critical_git_op_map, serve, + storage, + storage_config, local_api, remote_api](auto ts, auto setter, @@ -123,23 +126,21 @@ auto CreateContentCASMap( auto const& key) { auto digest = ArtifactDigest(key.content, 0, false); // check local CAS - auto const& cas = Storage::Instance().CAS(); - if (cas.BlobPath(digest, /*is_executable=*/false)) { + if (local_api->IsAvailable(digest)) { (*setter)(nullptr); return; } // check if content is in Git cache; // ensure Git cache - GitOpKey op_key = { - .params = - { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = {.params = + { + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -149,6 +150,7 @@ auto CreateContentCASMap( additional_mirrors, ca_info, serve, + storage, local_api, remote_api, setter, @@ -183,7 +185,7 @@ auto CreateContentCASMap( // blob check failed return; } - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); if (res.second) { // blob found; add it to CAS if (not cas.StoreBlob(*res.second, @@ -207,7 +209,7 @@ auto CreateContentCASMap( .filename() .string()); StorageUtils::AddDistfileToCAS( - Storage::Instance(), repo_distfile, just_mr_paths); + *storage, repo_distfile, just_mr_paths); // check if content is in CAS now if (cas.BlobPath(digest, /*is_executable=*/false)) { JustMRProgress::Instance().TaskTracker().Stop(key.origin); @@ -240,10 +242,10 @@ auto CreateContentCASMap( } // revert to network fetch FetchFromNetwork( - key, additional_mirrors, ca_info, setter, logger); + key, additional_mirrors, ca_info, *storage, setter, logger); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( - auto const& msg, bool fatal) { + [logger, target_path = storage_config->GitRoot()](auto const& msg, + bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp index 049edc5f..27bb9429 100644 --- a/src/other_tools/ops_maps/content_cas_map.hpp +++ b/src/other_tools/ops_maps/content_cas_map.hpp @@ -26,6 +26,8 @@ #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -85,6 +87,8 @@ using ContentCASMap = AsyncMapConsumer<ArchiveContent, std::nullptr_t>; CAInfoPtr const& ca_info, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> ContentCASMap; diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp index 60ea67da..f88f5d46 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp @@ -25,8 +25,6 @@ #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -35,11 +33,12 @@ namespace { void BackupToRemote(std::string const& tree_id, + StorageConfig const& storage_config, IExecutionApi const& remote_api, GitTreeFetchMap::LoggerPtr const& logger) { // try to back up to remote CAS auto repo = RepositoryConfig{}; - if (repo.SetGitCAS(StorageConfig::Instance().GitRoot())) { + if (repo.SetGitCAS(storage_config.GitRoot())) { auto git_api = GitApi{&repo}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{ @@ -56,7 +55,7 @@ void BackupToRemote(std::string const& tree_id, else { // give a warning (*logger)(fmt::format("Failed to SetGitCAS at {}", - StorageConfig::Instance().GitRoot().string()), + storage_config.GitRoot().string()), /*fatal=*/false); } } @@ -66,6 +65,7 @@ void BackupToRemote(std::string const& tree_id, void MoveCASTreeToGit(std::string const& tree_id, ArtifactDigest const& digest, gsl::not_null<ImportToGitMap*> const& import_to_git_map, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, @@ -73,8 +73,7 @@ void MoveCASTreeToGit(std::string const& tree_id, GitTreeFetchMap::SetterPtr const& setter, GitTreeFetchMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage - auto tmp_dir = - StorageConfig::Instance().CreateTypedTmpDir("fetch-remote-git-tree"); + auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", @@ -97,6 +96,7 @@ void MoveCASTreeToGit(std::string const& tree_id, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive tree_id, + storage_config, remote_api, backup_to_remote, setter, @@ -108,7 +108,7 @@ void MoveCASTreeToGit(std::string const& tree_id, } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(tree_id, *remote_api, logger); + BackupToRemote(tree_id, *storage_config, *remote_api, logger); } (*setter)(false /*no cache hit*/); }, @@ -130,6 +130,7 @@ auto CreateGitTreeFetchMap( std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, @@ -139,6 +140,7 @@ auto CreateGitTreeFetchMap( git_bin, launcher, serve, + storage_config, local_api, remote_api, backup_to_remote](auto ts, @@ -148,16 +150,15 @@ auto CreateGitTreeFetchMap( auto const& key) { // check whether tree exists already in Git cache; // ensure Git cache exists - GitOpKey op_key = { - .params = - { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = {.params = + { + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -166,6 +167,7 @@ auto CreateGitTreeFetchMap( git_bin, launcher, serve, + storage_config, local_api, remote_api, backup_to_remote, @@ -184,9 +186,8 @@ auto CreateGitTreeFetchMap( auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { - (*logger)(fmt::format( - "Could not open repository {}", - StorageConfig::Instance().GitRoot().string()), + (*logger)(fmt::format("Could not open repository {}", + storage_config->GitRoot().string()), /*fatal=*/true); return; } @@ -208,7 +209,8 @@ auto CreateGitTreeFetchMap( if (*tree_found) { // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(key.hash, *remote_api, logger); + BackupToRemote( + key.hash, *storage_config, *remote_api, logger); } // success (*setter)(true /*cache hit*/); @@ -216,12 +218,12 @@ auto CreateGitTreeFetchMap( } // check if tree is known to local CAS auto digest = ArtifactDigest{key.hash, 0, /*is_tree=*/true}; - auto const& cas = Storage::Instance().CAS(); - if (auto path = cas.TreePath(digest)) { + if (local_api->IsAvailable(digest)) { // import tree to Git cache MoveCASTreeToGit(key.hash, digest, import_to_git_map, + storage_config, local_api, remote_api, backup_to_remote, @@ -251,6 +253,7 @@ auto CreateGitTreeFetchMap( key.hash, digest, import_to_git_map, + storage_config, local_api, remote_api, false, // tree already in remote, so ignore backing up @@ -261,8 +264,7 @@ auto CreateGitTreeFetchMap( return; } // create temporary location for command execution root - auto tmp_dir = - StorageConfig::Instance().CreateTypedTmpDir("git-tree"); + auto tmp_dir = storage_config->CreateTypedTmpDir("git-tree"); if (not tmp_dir) { (*logger)( "Failed to create execution root tmp directory for " @@ -271,8 +273,7 @@ auto CreateGitTreeFetchMap( return; } // create temporary location for storing command result files - auto out_dir = - StorageConfig::Instance().CreateTypedTmpDir("git-tree"); + auto out_dir = storage_config->CreateTypedTmpDir("git-tree"); if (not out_dir) { (*logger)( "Failed to create results tmp directory for tree id " @@ -322,6 +323,7 @@ auto CreateGitTreeFetchMap( key, git_bin, launcher, + storage_config, remote_api, backup_to_remote, ts, @@ -401,8 +403,7 @@ auto CreateGitTreeFetchMap( } // define temp repo path auto tmp_dir = - StorageConfig::Instance().CreateTypedTmpDir( - "git-tree"); + storage_config->CreateTypedTmpDir("git-tree"); ; if (not tmp_dir) { (*logger)(fmt::format("Could not create unique " @@ -423,7 +424,7 @@ auto CreateGitTreeFetchMap( fatal); }); if (not just_git_repo->FetchViaTmpRepo( - StorageConfig::Instance(), + *storage_config, target_path.string(), std::nullopt, key.inherit_env, @@ -449,8 +450,7 @@ auto CreateGitTreeFetchMap( GitOpKey op_key = { .params = { - StorageConfig::Instance() - .GitRoot(), // target_path + storage_config->GitRoot(), // target_path *op_result.result, // git_hash "", // branch "Keep referenced tree alive" // message @@ -459,8 +459,12 @@ auto CreateGitTreeFetchMap( critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [remote_api, backup_to_remote, key, setter, logger]( - auto const& values) { + [remote_api, + storage_config, + backup_to_remote, + key, + setter, + logger](auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { @@ -473,8 +477,10 @@ auto CreateGitTreeFetchMap( // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote( - key.hash, *remote_api, logger); + BackupToRemote(key.hash, + *storage_config, + *remote_api, + logger); } // success (*setter)(false /*no cache hit*/); @@ -496,8 +502,8 @@ auto CreateGitTreeFetchMap( fatal); }); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( - auto const& msg, bool fatal) { + [logger, target_path = storage_config->GitRoot()](auto const& msg, + bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT bare for target {}:\n{}", target_path.string(), diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp index 4a8ab527..e61e1264 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp @@ -24,6 +24,7 @@ #include "gsl/gsl" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -61,6 +62,7 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>; std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp index 143000bc..de3bfb25 100644 --- a/src/other_tools/ops_maps/git_update_map.cpp +++ b/src/other_tools/ops_maps/git_update_map.cpp @@ -16,20 +16,22 @@ #include "fmt/core.h" #include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -auto CreateGitUpdateMap(GitCASPtr const& git_cas, - std::string const& git_bin, - std::vector<std::string> const& launcher, - std::size_t jobs) -> GitUpdateMap { - auto update_commits = [git_cas, git_bin, launcher](auto /* unused */, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { +auto CreateGitUpdateMap( + GitCASPtr const& git_cas, + std::string const& git_bin, + std::vector<std::string> const& launcher, + gsl::not_null<StorageConfig const*> const& storage_config, + std::size_t jobs) -> GitUpdateMap { + auto update_commits = [git_cas, git_bin, launcher, storage_config]( + auto /* unused */, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { // perform git update commit auto git_repo = GitRepoRemote::Open(git_cas); // wrap the tmp odb if (not git_repo) { @@ -49,14 +51,13 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, // update commit auto id = fmt::format("{}:{}", key.repo, key.branch); JustMRProgress::Instance().TaskTracker().Start(id); - auto new_commit = - git_repo->UpdateCommitViaTmpRepo(StorageConfig::Instance(), - key.repo, - key.branch, - key.inherit_env, - git_bin, - launcher, - wrapped_logger); + auto new_commit = git_repo->UpdateCommitViaTmpRepo(*storage_config, + key.repo, + key.branch, + key.inherit_env, + git_bin, + launcher, + wrapped_logger); JustMRProgress::Instance().TaskTracker().Stop(id); if (not new_commit) { return; diff --git a/src/other_tools/ops_maps/git_update_map.hpp b/src/other_tools/ops_maps/git_update_map.hpp index bc021113..4b043284 100644 --- a/src/other_tools/ops_maps/git_update_map.hpp +++ b/src/other_tools/ops_maps/git_update_map.hpp @@ -21,6 +21,7 @@ #include <vector> #include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -51,9 +52,11 @@ struct hash<RepoDescriptionForUpdating> { }; } // namespace std -[[nodiscard]] auto CreateGitUpdateMap(GitCASPtr const& git_cas, - std::string const& git_bin, - std::vector<std::string> const& launcher, - std::size_t jobs) -> GitUpdateMap; +[[nodiscard]] auto CreateGitUpdateMap( + GitCASPtr const& git_cas, + std::string const& git_bin, + std::vector<std::string> const& launcher, + gsl::not_null<StorageConfig const*> const& storage_config, + std::size_t jobs) -> GitUpdateMap; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_UPDATE_MAP_HPP diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp index 7e75ab71..51a4548e 100644 --- a/src/other_tools/ops_maps/import_to_git_map.cpp +++ b/src/other_tools/ops_maps/import_to_git_map.cpp @@ -18,7 +18,6 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" namespace { @@ -28,19 +27,19 @@ void KeepCommitAndSetTree( std::string const& commit, std::filesystem::path const& target_path, GitCASPtr const& just_git_cas, + StorageConfig const& storage_config, gsl::not_null<TaskSystem*> const& ts, ImportToGitMap::SetterPtr const& setter, ImportToGitMap::LoggerPtr const& logger) { // Keep tag for commit - GitOpKey op_key = { - .params = - { - StorageConfig::Instance().GitRoot(), // target_path - commit, // git_hash - "", // branch - "Keep referenced tree alive" // message - }, - .op_type = GitOpType::KEEP_TAG}; + GitOpKey op_key = {.params = + { + storage_config.GitRoot(), // target_path + commit, // git_hash + "", // branch + "Keep referenced tree alive" // message + }, + .op_type = GitOpType::KEEP_TAG}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -93,13 +92,16 @@ auto CreateImportToGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, std::string const& git_bin, std::vector<std::string> const& launcher, + gsl::not_null<StorageConfig const*> const& storage_config, std::size_t jobs) -> ImportToGitMap { - auto import_to_git = [critical_git_op_map, git_bin, launcher]( - auto ts, - auto setter, - auto logger, - auto /*unused*/, - auto const& key) { + auto import_to_git = [critical_git_op_map, + git_bin, + launcher, + storage_config](auto ts, + auto setter, + auto logger, + auto /*unused*/, + auto const& key) { // Perform initial commit at import location: init + add . + commit GitOpKey op_key = {.params = { @@ -118,6 +120,7 @@ auto CreateImportToGitMap( target_path = key.target_path, git_bin, launcher, + storage_config, ts, setter, logger](auto const& values) { @@ -133,11 +136,11 @@ auto CreateImportToGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -148,6 +151,7 @@ auto CreateImportToGitMap( target_path, git_bin, launcher, + storage_config, ts, setter, logger](auto const& values) { @@ -162,12 +166,11 @@ auto CreateImportToGitMap( auto just_git_repo = GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { - (*logger)(fmt::format("Could not open Git cache " - "repository {}", - StorageConfig::Instance() - .GitRoot() - .string()), - /*fatal=*/true); + (*logger)( + fmt::format("Could not open Git cache " + "repository {}", + storage_config->GitRoot().string()), + /*fatal=*/true); return; } auto wrapped_logger = @@ -181,7 +184,7 @@ auto CreateImportToGitMap( fatal); }); if (not just_git_repo->FetchViaTmpRepo( - StorageConfig::Instance(), + *storage_config, target_path.string(), std::nullopt, std::vector<std::string>{} /* inherit_env */, @@ -206,11 +209,12 @@ auto CreateImportToGitMap( commit, target_path, op_result.git_cas, /*just_git_cas*/ + *storage_config, ts, setter, wrapped_logger); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT bare for " diff --git a/src/other_tools/ops_maps/import_to_git_map.hpp b/src/other_tools/ops_maps/import_to_git_map.hpp index ce17d781..cd6fdbef 100644 --- a/src/other_tools/ops_maps/import_to_git_map.hpp +++ b/src/other_tools/ops_maps/import_to_git_map.hpp @@ -21,6 +21,7 @@ #include <utility> #include "gsl/gsl" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/utils/cpp/path.hpp" @@ -65,6 +66,7 @@ using ImportToGitMap = gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, std::string const& git_bin, std::vector<std::string> const& launcher, + gsl::not_null<StorageConfig const*> const& storage_config, std::size_t jobs) -> ImportToGitMap; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP
\ No newline at end of file diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 77623d14..82d4ccb0 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -10,6 +10,8 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -22,9 +24,7 @@ , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] - , ["src/buildtool/storage", "storage"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] @@ -47,6 +47,7 @@ , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -54,7 +55,6 @@ , "root_utils" , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -79,6 +79,7 @@ , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path_hash"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -89,7 +90,6 @@ , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/utils/cpp", "tmp_dir"] @@ -110,6 +110,8 @@ , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "storage"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -121,8 +123,6 @@ , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/multithreading", "task_system"] - , ["src/buildtool/storage", "storage"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -142,14 +142,14 @@ , ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] ] , "private-deps": [ ["@", "fmt", "", "fmt"] , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/logging", "log_level"] - , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "fs_utils"] - , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "tmp_dir"] , "root_utils" ] @@ -169,6 +169,7 @@ , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -177,8 +178,6 @@ , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/git", "git"] , ["src/buildtool/file_system", "file_root"] - , ["src/buildtool/storage", "config"] - , ["src/buildtool/storage", "storage"] ] } , "root_utils": diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index 4a33d240..d4e5c4e4 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -21,7 +21,6 @@ #include "fmt/core.h" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -49,10 +48,10 @@ namespace { } [[nodiscard]] auto IsCacheGitRoot( + StorageConfig const& storage_config, std::filesystem::path const& repo_root) noexcept -> bool { return std::filesystem::absolute(ToNormalPath(repo_root)) == - std::filesystem::absolute( - ToNormalPath(StorageConfig::Instance().GitRoot())); + std::filesystem::absolute(ToNormalPath(storage_config.GitRoot())); } /// \brief Helper function for ensuring the serve endpoint, if given, has the @@ -146,6 +145,7 @@ void EnsureRootAsAbsent(std::string const& tree_id, void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, std::string const& subdir, bool ignore_special, + StorageConfig const& storage_config, GitCASPtr const& git_cas, std::filesystem::path const& tree_id_file, CommitGitMap::SetterPtr const& ws_setter, @@ -162,7 +162,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open cache object database {}", - StorageConfig::Instance().GitRoot().string()), + storage_config.GitRoot().string()), /*fatal=*/true); return; } @@ -185,7 +185,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, - StorageConfig::Instance().GitRoot().string()}), + storage_config.GitRoot().string()}), false)); } @@ -195,6 +195,7 @@ void NetworkFetchAndSetPresentRoot( std::string const& fetch_repo, MirrorsPtr const& additional_mirrors, GitCASPtr const& git_cas, + StorageConfig const& storage_config, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, std::string const& git_bin, std::vector<std::string> const& launcher, @@ -254,7 +255,7 @@ void NetworkFetchAndSetPresentRoot( err_messages += fmt::format( "While attempting fetch from URL {}:\n{}\n", mirror, msg); }); - if (git_repo->FetchViaTmpRepo(StorageConfig::Instance(), + if (git_repo->FetchViaTmpRepo(storage_config, mirror, repo_info.branch, repo_info.inherit_env, @@ -300,7 +301,7 @@ void NetworkFetchAndSetPresentRoot( return; } // if witnessing repository is the Git cache, then also tag the commit - if (IsCacheGitRoot(repo_root)) { + if (IsCacheGitRoot(storage_config, repo_root)) { GitOpKey op_key = {.params = { repo_root, // target_path @@ -410,6 +411,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool fetch_absent, @@ -436,8 +438,8 @@ void EnsureCommit(GitRepoInfo const& repo_info, return; } if (not is_commit_present.value()) { - auto tree_id_file = StorageUtils::GetCommitTreeIDFile( - StorageConfig::Instance(), repo_info.hash); + auto tree_id_file = + StorageUtils::GetCommitTreeIDFile(*storage_config, repo_info.hash); // Check if we have stored a file association between commit and tree; // if an association file exists, the respective tree MUST be in the // Git cache @@ -450,21 +452,18 @@ void EnsureCommit(GitRepoInfo const& repo_info, /*fatal=*/true); return; } - auto just_git_cas = - GitCAS::Open(StorageConfig::Instance().GitRoot()); + auto just_git_cas = GitCAS::Open(storage_config->GitRoot()); if (not just_git_cas) { - (*logger)( - fmt::format("Could not open Git cache database {}", - StorageConfig::Instance().GitRoot().string()), - /*fatal=*/true); + (*logger)(fmt::format("Could not open Git cache database {}", + storage_config->GitRoot().string()), + /*fatal=*/true); return; } auto just_git_repo = GitRepo::Open(just_git_cas); if (not just_git_repo) { - (*logger)( - fmt::format("Could not open Git cache repository {}", - StorageConfig::Instance().GitRoot().string()), - /*fatal=*/true); + (*logger)(fmt::format("Could not open Git cache repository {}", + storage_config->GitRoot().string()), + /*fatal=*/true); return; } // extract the subdir tree @@ -487,7 +486,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (repo_info.absent and not fetch_absent) { // try by all available means to generate & set the absent root EnsureRootAsAbsent(*tree_id, - StorageConfig::Instance().GitRoot(), + storage_config->GitRoot(), repo_info, serve, remote_api, @@ -496,14 +495,14 @@ void EnsureCommit(GitRepoInfo const& repo_info, } else { // this root is present - (*ws_setter)(std::pair( - nlohmann::json::array( - {repo_info.ignore_special - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - *tree_id, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/false)); + (*ws_setter)( + std::pair(nlohmann::json::array( + {repo_info.ignore_special + ? FileRoot::kGitTreeIgnoreSpecialMarker + : FileRoot::kGitTreeMarker, + *tree_id, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/false)); } // done! return; @@ -552,16 +551,16 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (serve_result) { auto const& root_tree_id = *serve_result; // verify if we know the tree already in the local Git cache - GitOpKey op_key = {.params = - { - StorageConfig::Instance() - .GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = { + .params = + { + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -571,6 +570,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, repo_root, fetch_repo, additional_mirrors, + storage_config, git_cas, critical_git_op_map, import_to_git_map, @@ -592,31 +592,29 @@ void EnsureCommit(GitRepoInfo const& repo_info, auto just_git_repo = GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { - (*logger)(fmt::format("Could not open Git " - "cache repository " - "{}", - StorageConfig::Instance() - .GitRoot() - .string()), - /*fatal=*/true); + (*logger)( + fmt::format( + "Could not open Git " + "cache repository " + "{}", + storage_config->GitRoot().string()), + /*fatal=*/true); return; } // check tree existence - auto wrapped_logger = - std::make_shared<AsyncMapConsumerLogger>( - [logger, tree = root_tree_id]( - auto const& msg, bool fatal) { - (*logger)( - fmt::format( - "While verifying presence of " - "tree {} in repository {}:\n{}", - tree, - StorageConfig::Instance() - .GitRoot() - .string(), - msg), - fatal); - }); + auto wrapped_logger = std::make_shared< + AsyncMapConsumerLogger>( + [logger, storage_config, tree = root_tree_id]( + auto const& msg, bool fatal) { + (*logger)( + fmt::format( + "While verifying presence of " + "tree {} in repository {}:\n{}", + tree, + storage_config->GitRoot().string(), + msg), + fatal); + }); auto tree_present = just_git_repo->CheckTreeExists( root_tree_id, wrapped_logger); if (not tree_present) { @@ -631,6 +629,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, root_tree_id, repo_info.subdir, repo_info.ignore_special, + *storage_config, op_result.git_cas, tree_id_file, ws_setter, @@ -641,7 +640,8 @@ void EnsureCommit(GitRepoInfo const& repo_info, // now check if the tree is in the local checkout, // if this checkout is not our Git cache; this can // save an unnecessary remote CAS call - if (not IsCacheGitRoot(repo_root)) { + if (not IsCacheGitRoot(*storage_config, + repo_root)) { auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { (*logger)(fmt::format("Could not open Git " @@ -730,7 +730,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, repo_info.origin); // Move tree from local CAS to local Git storage auto tmp_dir = - StorageConfig::Instance().CreateTypedTmpDir( + storage_config->CreateTypedTmpDir( "fetch-absent-root"); if (not tmp_dir) { (*logger)( @@ -763,6 +763,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive root_tree_id, + storage_config, subdir = repo_info.subdir, ignore_special = repo_info.ignore_special, just_git_cas = op_result.git_cas, @@ -794,6 +795,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, WriteIdFileAndSetWSRoot(root_tree_id, subdir, ignore_special, + *storage_config, just_git_cas, tree_id_file, ws_setter, @@ -826,6 +828,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, fetch_repo, additional_mirrors, git_cas, + *storage_config, critical_git_op_map, git_bin, launcher, @@ -834,8 +837,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, ws_setter, logger); }, - [logger, - target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT bare for " @@ -866,6 +868,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, fetch_repo, additional_mirrors, git_cas, + *storage_config, critical_git_op_map, git_bin, launcher, @@ -927,6 +930,7 @@ auto CreateCommitGitMap( std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool fetch_absent, @@ -938,6 +942,7 @@ auto CreateCommitGitMap( git_bin, launcher, serve, + storage_config, local_api, remote_api, fetch_absent](auto ts, @@ -953,7 +958,7 @@ auto CreateCommitGitMap( fetch_repo = std::filesystem::absolute(*fetch_repo_path).string(); } std::filesystem::path repo_root = StorageUtils::GetGitRoot( - StorageConfig::Instance(), just_mr_paths, fetch_repo); + *storage_config, just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { @@ -979,6 +984,7 @@ auto CreateCommitGitMap( git_bin, launcher, serve, + storage_config, local_api, remote_api, fetch_absent, @@ -1012,6 +1018,7 @@ auto CreateCommitGitMap( git_bin, launcher, serve, + storage_config, local_api, remote_api, fetch_absent, diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index 78c67800..b1894464 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -26,6 +26,7 @@ #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -82,6 +83,7 @@ using CommitGitMap = std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool fetch_absent, diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index c73d39a5..ef345ff0 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -20,9 +20,7 @@ #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/root_maps/root_utils.hpp" @@ -52,13 +50,15 @@ namespace { /// root if it was marked absent. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void EnsureRootAsAbsent(std::string const& tree_id, - ArchiveRepoInfo const& key, - ServeApi const* serve, - IExecutionApi const* remote_api, - bool is_cache_hit, - ContentGitMap::SetterPtr const& ws_setter, - ContentGitMap::LoggerPtr const& logger) { +void EnsureRootAsAbsent( + std::string const& tree_id, + ArchiveRepoInfo const& key, + ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + IExecutionApi const* remote_api, + bool is_cache_hit, + ContentGitMap::SetterPtr const& ws_setter, + ContentGitMap::LoggerPtr const& logger) { // this is an absent root if (serve != nullptr) { // check if the serve endpoint has this root @@ -116,7 +116,7 @@ void EnsureRootAsAbsent(std::string const& tree_id, if (not EnsureAbsentRootOnServe( *serve, tree_id, - StorageConfig::Instance().GitRoot(), + storage_config->GitRoot(), remote_api, logger, /*no_sync_is_fatal=*/true)) { @@ -139,13 +139,12 @@ void EnsureRootAsAbsent(std::string const& tree_id, // the tree is known locally, so we can upload it to remote // CAS for the serve endpoint to retrieve it and set up the // root - if (not EnsureAbsentRootOnServe( - *serve, - tree_id, - StorageConfig::Instance().GitRoot(), - remote_api, - logger, - /*no_sync_is_fatal=*/true)) { + if (not EnsureAbsentRootOnServe(*serve, + tree_id, + storage_config->GitRoot(), + remote_api, + logger, + /*no_sync_is_fatal=*/true)) { return; } } @@ -174,6 +173,7 @@ void ResolveContentTree( bool is_cache_hit, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -183,7 +183,7 @@ void ResolveContentTree( if (key.pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - StorageConfig::Instance(), tree_hash, *key.pragma_special); + *storage_config, tree_hash, *key.pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -199,6 +199,7 @@ void ResolveContentTree( EnsureRootAsAbsent(*resolved_tree_id, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, @@ -206,10 +207,9 @@ void ResolveContentTree( } else { (*ws_setter)(std::pair( - nlohmann::json::array( - {FileRoot::kGitTreeMarker, - *resolved_tree_id, - StorageConfig::Instance().GitRoot().string()}), + nlohmann::json::array({FileRoot::kGitTreeMarker, + *resolved_tree_id, + storage_config->GitRoot().string()}), /*is_cache_hit=*/is_cache_hit)); } } @@ -231,6 +231,7 @@ void ResolveContentTree( key, is_absent, serve, + storage_config, remote_api, ts, ws_setter, @@ -259,8 +260,7 @@ void ResolveContentTree( GitOpKey op_key = { .params = { - StorageConfig::Instance() - .GitRoot(), // target_path + storage_config->GitRoot(), // target_path resolved_tree_id, // git_hash "", // branch "Keep referenced tree alive" // message @@ -274,6 +274,7 @@ void ResolveContentTree( tree_id_file, is_absent, serve, + storage_config, remote_api, is_cache_hit, ws_setter, @@ -300,24 +301,22 @@ void ResolveContentTree( EnsureRootAsAbsent(resolved_tree_id, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - resolved_tree_id, - StorageConfig::Instance() - .GitRoot() - .string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)(std::pair( + nlohmann::json::array( + {FileRoot::kGitTreeMarker, + resolved_tree_id, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } }, - [logger, - target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -343,18 +342,18 @@ void ResolveContentTree( EnsureRootAsAbsent(tree_hash, key, serve, + storage_config, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - tree_hash, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)(std::pair( + nlohmann::json::array({FileRoot::kGitTreeMarker, + tree_hash, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } } } @@ -369,6 +368,7 @@ void WriteIdFileAndSetWSRoot( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -413,6 +413,7 @@ void WriteIdFileAndSetWSRoot( false, /*is_cache_hit*/ is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -430,6 +431,7 @@ void ExtractAndImportToGit( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -438,7 +440,7 @@ void ExtractAndImportToGit( ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // extract archive - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir(key.repo_type); + auto tmp_dir = storage_config->CreateTypedTmpDir(key.repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for {} target {}", key.repo_type, @@ -466,6 +468,7 @@ void ExtractAndImportToGit( key, is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -487,6 +490,7 @@ void ExtractAndImportToGit( archive_tree_id_file, is_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -514,6 +518,8 @@ auto CreateContentGitMap( gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, IExecutionApi const* remote_api, bool fetch_absent, std::size_t jobs) -> ContentGitMap { @@ -525,6 +531,8 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, + storage, + storage_config, remote_api, fetch_absent](auto ts, auto setter, @@ -532,7 +540,7 @@ auto CreateContentGitMap( auto /* unused */, auto const& key) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( - StorageConfig::Instance(), key.repo_type, key.archive.content); + *storage_config, key.repo_type, key.archive.content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file auto archive_tree_id = @@ -548,11 +556,11 @@ auto CreateContentGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -562,6 +570,7 @@ auto CreateContentGitMap( key, fetch_absent, serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -606,6 +615,7 @@ auto CreateContentGitMap( /*is_cache_hit = */ true, /*is_absent = */ (key.absent and not fetch_absent), serve, + storage_config, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -613,7 +623,7 @@ auto CreateContentGitMap( setter, logger); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT for " @@ -661,7 +671,7 @@ auto CreateContentGitMap( // a serve endpoint exists we can upload it the root ourselves; // check if content already in CAS - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); auto digest = ArtifactDigest(key.archive.content, 0, false); if (auto content_cas_path = cas.BlobPath(digest, /*is_executable=*/false)) { @@ -670,6 +680,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent = */ true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -686,11 +697,11 @@ auto CreateContentGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -706,6 +717,8 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, + storage, + storage_config, remote_api, ts, setter, @@ -743,7 +756,7 @@ auto CreateContentGitMap( // blob check failed return; } - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); if (res.second) { // blob found; add it to CAS if (not cas.StoreBlob(*res.second, @@ -761,6 +774,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -789,7 +803,7 @@ auto CreateContentGitMap( .filename() .string()); StorageUtils::AddDistfileToCAS( - Storage::Instance(), repo_distfile, just_mr_paths); + *storage, repo_distfile, just_mr_paths); // check if content is in CAS now if (auto content_cas_path = cas.BlobPath(digest, /*is_executable=*/false)) { @@ -800,6 +814,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/true, serve, + storage_config, remote_api, critical_git_op_map, import_to_git_map, @@ -816,7 +831,7 @@ auto CreateContentGitMap( key.archive.content), /*fatal=*/true); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -832,6 +847,8 @@ auto CreateContentGitMap( {key.archive}, [archive_tree_id_file, key, + storage, + storage_config, critical_git_op_map, import_to_git_map, resolve_symlinks_map, @@ -839,7 +856,7 @@ auto CreateContentGitMap( setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage->CAS(); auto content_cas_path = cas.BlobPath(ArtifactDigest( key.archive.content, 0, false), @@ -852,6 +869,7 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/false, /*serve=*/nullptr, + storage_config, /*remote_api=*/nullptr, critical_git_op_map, import_to_git_map, diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index 959edaa9..ddf922f4 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -25,6 +25,8 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -44,6 +46,8 @@ using ContentGitMap = gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, IExecutionApi const* remote_api, bool fetch_absent, std::size_t jobs) -> ContentGitMap; diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index b5c863e1..54abaae2 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -25,9 +25,7 @@ #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" @@ -39,10 +37,11 @@ namespace { /// \brief Create links from CAS content to distdir tmp directory [[nodiscard]] auto LinkToCAS( + Storage const& storage, std::shared_ptr<std::unordered_map<std::string, std::string>> const& content_list, std::filesystem::path const& tmp_dir) noexcept -> bool { - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage.CAS(); return std::all_of(content_list->begin(), content_list->end(), [&cas, tmp_dir](auto const& kv) { @@ -64,13 +63,15 @@ namespace { /// the setter on success. void ImportFromCASAndSetRoot( DistdirInfo const& key, + StorageConfig const& storage_config, + Storage const& storage, std::filesystem::path const& distdir_tree_id_file, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<TaskSystem*> const& ts, DistdirGitMap::SetterPtr const& setter, DistdirGitMap::LoggerPtr const& logger) { // create the links to CAS - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir("distdir"); + auto tmp_dir = storage_config.CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", @@ -79,7 +80,7 @@ void ImportFromCASAndSetRoot( return; } // link content from CAS into tmp dir - if (not LinkToCAS(key.content_list, tmp_dir->GetPath())) { + if (not LinkToCAS(storage, key.content_list, tmp_dir->GetPath())) { (*logger)(fmt::format("Failed to create links to CAS content!", key.content_id), /*fatal=*/true); @@ -92,6 +93,7 @@ void ImportFromCASAndSetRoot( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, + git_root = storage_config.GitRoot().string(), setter, logger](auto const& values) { // check for errors @@ -111,12 +113,10 @@ void ImportFromCASAndSetRoot( return; } // set the workspace root as present - (*setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - distdir_tree_id, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/false)); + (*setter)(std::pair( + nlohmann::json::array( + {FileRoot::kGitTreeMarker, distdir_tree_id, git_root}), + /*is_cache_hit=*/false)); }, [logger, target_path = tmp_dir->GetPath()](auto const& msg, bool fatal) { @@ -134,6 +134,8 @@ auto CreateDistdirGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap { @@ -141,14 +143,16 @@ auto CreateDistdirGitMap( import_to_git_map, critical_git_op_map, serve, + storage, + storage_config, local_api, remote_api](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { - auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile( - StorageConfig::Instance(), key.content_id); + auto distdir_tree_id_file = + StorageUtils::GetDistdirTreeIDFile(*storage_config, key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = @@ -164,11 +168,11 @@ auto CreateDistdirGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -178,6 +182,7 @@ auto CreateDistdirGitMap( content_id = key.content_id, key, serve, + storage_config, remote_api, setter, logger](auto const& values) { @@ -252,7 +257,7 @@ auto CreateDistdirGitMap( if (not EnsureAbsentRootOnServe( *serve, distdir_tree_id, - StorageConfig::Instance().GitRoot(), + storage_config->GitRoot(), remote_api, logger, true /*no_sync_is_fatal*/)) { @@ -277,15 +282,15 @@ auto CreateDistdirGitMap( } else { // set root as present - (*setter)(std::pair( - nlohmann::json::array( - {FileRoot::kGitTreeMarker, - distdir_tree_id, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/true)); + (*setter)( + std::pair(nlohmann::json::array( + {FileRoot::kGitTreeMarker, + distdir_tree_id, + storage_config->GitRoot().string()}), + /*is_cache_hit=*/true)); } }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -470,6 +475,8 @@ auto CreateDistdirGitMap( // first, look in the local CAS if (local_api->IsAvailable({digest})) { ImportFromCASAndSetRoot(key, + *storage_config, + *storage, distdir_tree_id_file, import_to_git_map, ts, @@ -523,10 +530,14 @@ auto CreateDistdirGitMap( key, import_to_git_map, ts, + storage, + storage_config, setter, logger]([[maybe_unused]] auto const& values) { // archive blobs are in CAS ImportFromCASAndSetRoot(key, + *storage_config, + *storage, distdir_tree_id_file, import_to_git_map, ts, diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp index c83b9512..9244f341 100644 --- a/src/other_tools/root_maps/distdir_git_map.hpp +++ b/src/other_tools/root_maps/distdir_git_map.hpp @@ -26,6 +26,8 @@ #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -55,6 +57,8 @@ using DistdirGitMap = gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap; diff --git a/src/other_tools/root_maps/foreign_file_git_map.cpp b/src/other_tools/root_maps/foreign_file_git_map.cpp index 2ce582ce..37ab10e1 100644 --- a/src/other_tools/root_maps/foreign_file_git_map.cpp +++ b/src/other_tools/root_maps/foreign_file_git_map.cpp @@ -18,15 +18,14 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/root_maps/root_utils.hpp" #include "src/utils/cpp/tmp_dir.hpp" namespace { void WithRootImportedToGit(ForeignFileInfo const& key, + StorageConfig const& storage_config, std::pair<std::string, GitCASPtr> const& result, ForeignFileGitMap::SetterPtr const& setter, ForeignFileGitMap::LoggerPtr const& logger) { @@ -34,11 +33,8 @@ void WithRootImportedToGit(ForeignFileInfo const& key, (*logger)("Importing to git failed", /*fatal=*/true); return; } - auto tree_id_file = - StorageUtils::GetForeignFileTreeIDFile(StorageConfig::Instance(), - key.archive.content, - key.name, - key.executable); + auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( + storage_config, key.archive.content, key.name, key.executable); auto cache_written = StorageUtils::WriteTreeIDFile(tree_id_file, result.first); if (not cache_written) { @@ -46,20 +42,22 @@ void WithRootImportedToGit(ForeignFileInfo const& key, fmt::format("Failed to write cache file {}", tree_id_file.string()), /*fatal=*/false); } - (*setter)(std::pair( - nlohmann::json::array({FileRoot::kGitTreeMarker, - result.first, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/false)); + (*setter)( + std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, + result.first, + storage_config.GitRoot().string()}), + /*is_cache_hit=*/false)); } void WithFetchedFile(ForeignFileInfo const& key, + gsl::not_null<StorageConfig const*> const& storage_config, + Storage const& storage, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<TaskSystem*> const& ts, ForeignFileGitMap::SetterPtr const& setter, ForeignFileGitMap::LoggerPtr const& logger) { - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir("foreign-file"); - auto const& cas = Storage::Instance().CAS(); + auto tmp_dir = storage_config->CreateTypedTmpDir("foreign-file"); + auto const& cas = storage.CAS(); auto digest = ArtifactDigest(key.archive.content, 0, key.executable); auto content_cas_path = cas.BlobPath(digest, key.executable); if (not content_cas_path) { @@ -90,9 +88,11 @@ void WithFetchedFile(ForeignFileInfo const& key, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive key, + storage_config, setter, logger](auto const& values) { - WithRootImportedToGit(key, *values[0], setter, logger); + WithRootImportedToGit( + key, *storage_config, *values[0], setter, logger); }, [logger, target_path = tmp_dir->GetPath()](auto const& msg, bool fatal) { @@ -103,16 +103,17 @@ void WithFetchedFile(ForeignFileInfo const& key, }); } -void UseCacheHit(const std::string& tree_id, +void UseCacheHit(StorageConfig const& storage_config, + const std::string& tree_id, ForeignFileGitMap::SetterPtr const& setter) { // We keep the invariant, that, whenever a cache entry is written, // the root is in our git root; in particular, the latter is present, // initialized, etc; so we can directly write the result. - (*setter)(std::pair( - nlohmann::json::array({FileRoot::kGitTreeMarker, - tree_id, - StorageConfig::Instance().GitRoot().string()}), - /*is_cache_hit=*/true)); + (*setter)( + std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, + tree_id, + storage_config.GitRoot().string()}), + /*is_cache_hit=*/true)); } void HandleAbsentForeignFile(ForeignFileInfo const& key, @@ -198,25 +199,26 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap { auto setup_foreign_file = [content_cas_map, import_to_git_map, fetch_absent, - serve](auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + serve, + storage, + storage_config](auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { if (key.absent and not fetch_absent) { HandleAbsentForeignFile(key, serve, setter, logger); return; } - auto tree_id_file = - StorageUtils::GetForeignFileTreeIDFile(StorageConfig::Instance(), - key.archive.content, - key.name, - key.executable); + auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( + *storage_config, key.archive.content, key.name, key.executable); if (FileSystemManager::Exists(tree_id_file)) { auto tree_id = FileSystemManager::ReadFile(tree_id_file); if (not tree_id) { @@ -225,15 +227,26 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, /*fatal=*/true); return; } - UseCacheHit(*tree_id, setter); + UseCacheHit(*storage_config, *tree_id, setter); return; } content_cas_map->ConsumeAfterKeysReady( ts, {key.archive}, - [key, import_to_git_map, setter, logger, ts]( - [[maybe_unused]] auto const& values) { - WithFetchedFile(key, import_to_git_map, ts, setter, logger); + [key, + import_to_git_map, + storage, + storage_config, + setter, + logger, + ts]([[maybe_unused]] auto const& values) { + WithFetchedFile(key, + storage_config, + *storage, + import_to_git_map, + ts, + setter, + logger); }, [logger, content = key.archive.content](auto const& msg, bool fatal) { diff --git a/src/other_tools/root_maps/foreign_file_git_map.hpp b/src/other_tools/root_maps/foreign_file_git_map.hpp index 941d28f7..2efd1ae6 100644 --- a/src/other_tools/root_maps/foreign_file_git_map.hpp +++ b/src/other_tools/root_maps/foreign_file_git_map.hpp @@ -22,6 +22,8 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -35,6 +37,8 @@ using ForeignFileGitMap = gsl::not_null<ContentCASMap*> const& content_cas_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap; diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index f15051d7..2de194d9 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -22,7 +22,6 @@ #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/multithreading/task_system.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/root_maps/root_utils.hpp" @@ -101,6 +100,7 @@ void ResolveFilePathTree( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, FilePathGitMap::SetterPtr const& ws_setter, @@ -108,7 +108,7 @@ void ResolveFilePathTree( if (pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - StorageConfig::Instance(), tree_hash, *pragma_special); + *storage_config, tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -123,7 +123,7 @@ void ResolveFilePathTree( // available to be able to build against it; the tree is resolved, // so it is in our Git cache CheckServeAndSetRoot(*resolved_tree_id, - StorageConfig::Instance().GitRoot().string(), + storage_config->GitRoot().string(), absent, serve, remote_api, @@ -146,6 +146,7 @@ void ResolveFilePathTree( tree_id_file, absent, serve, + storage_config, remote_api, ts, ws_setter, @@ -174,8 +175,7 @@ void ResolveFilePathTree( GitOpKey op_key = { .params = { - StorageConfig::Instance() - .GitRoot(), // target_path + storage_config->GitRoot(), // target_path resolved_tree_id, // git_hash "", // branch "Keep referenced tree alive" // message @@ -188,6 +188,7 @@ void ResolveFilePathTree( tree_id_file, absent, serve, + storage_config, remote_api, ws_setter, logger](auto const& values) { @@ -213,15 +214,14 @@ void ResolveFilePathTree( // it; the resolved tree is in the Git cache CheckServeAndSetRoot( resolved_tree_id, - StorageConfig::Instance().GitRoot().string(), + storage_config->GitRoot().string(), absent, serve, remote_api, ws_setter, logger); }, - [logger, - target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -257,6 +257,7 @@ auto CreateFilePathGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, std::size_t jobs, std::string multi_repo_tool_name, @@ -266,6 +267,7 @@ auto CreateFilePathGitMap( import_to_git_map, resolve_symlinks_map, serve, + storage_config, remote_api, multi_repo_tool_name, build_tool_name](auto ts, @@ -305,6 +307,7 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, + storage_config, remote_api, ts, setter, @@ -343,16 +346,16 @@ auto CreateFilePathGitMap( // resolve tree and set workspace root; tree gets resolved // from source repo into the Git cache, which we first need // to ensure is initialized - GitOpKey op_key = {.params = - { - StorageConfig::Instance() - .GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = { + .params = + { + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -365,6 +368,7 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, + storage_config, remote_api, ts, setter, @@ -387,13 +391,13 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, + storage_config, remote_api, ts, setter, logger); }, - [logger, - target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -426,7 +430,7 @@ auto CreateFilePathGitMap( /*fatal=*/false); } // it's not a git repo, so import it to git cache - auto tmp_dir = StorageConfig::Instance().CreateTypedTmpDir("file"); + auto tmp_dir = storage_config->CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); @@ -455,6 +459,7 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, + storage_config, remote_api, ts, setter, @@ -469,21 +474,21 @@ auto CreateFilePathGitMap( std::string tree = values[0]->first; // resolve tree and set workspace root; // we work on the Git CAS directly - ResolveFilePathTree( - StorageConfig::Instance().GitRoot().string(), - fpath.string(), - tree, - pragma_special, - values[0]->second, /*source_cas*/ - values[0]->second, /*target_cas*/ - absent, - critical_git_op_map, - resolve_symlinks_map, - serve, - remote_api, - ts, - setter, - logger); + ResolveFilePathTree(storage_config->GitRoot().string(), + fpath.string(), + tree, + pragma_special, + values[0]->second, /*source_cas*/ + values[0]->second, /*target_cas*/ + absent, + critical_git_op_map, + resolve_symlinks_map, + serve, + storage_config, + remote_api, + ts, + setter, + logger); }, [logger, target_path = key.fpath](auto const& msg, bool fatal) { (*logger)( diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 772cdc74..d116496f 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -26,6 +26,7 @@ #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -55,6 +56,7 @@ using FilePathGitMap = AsyncMapConsumer<FpathInfo, nlohmann::json>; gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, IExecutionApi const* remote_api, std::size_t jobs, std::string multi_repo_tool_name, diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 80f3d075..ad06b34d 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -18,8 +18,6 @@ #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/file_system/file_root.hpp" -#include "src/buildtool/storage/config.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/root_maps/root_utils.hpp" namespace { @@ -27,6 +25,7 @@ namespace { /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. void UploadToServeAndSetRoot(ServeApi const& serve, + StorageConfig const& storage_config, std::string const& tree_id, ArtifactDigest const& digest, IExecutionApi const& remote_api, @@ -35,7 +34,7 @@ void UploadToServeAndSetRoot(ServeApi const& serve, TreeIdGitMap::LoggerPtr const& logger) { // upload to remote CAS auto repo_config = RepositoryConfig{}; - if (repo_config.SetGitCAS(StorageConfig::Instance().GitRoot())) { + if (repo_config.SetGitCAS(storage_config.GitRoot())) { auto git_api = GitApi{&repo_config}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, @@ -50,7 +49,7 @@ void UploadToServeAndSetRoot(ServeApi const& serve, } else { (*logger)(fmt::format("Failed to SetGitCAS at {}", - StorageConfig::Instance().GitRoot().string()), + storage_config.GitRoot().string()), /*fatal=*/true); return; } @@ -76,6 +75,7 @@ void UploadToServeAndSetRoot(ServeApi const& serve, /// logger with fatal. void MoveCASTreeToGitAndProcess( ServeApi const& serve, + gsl::not_null<StorageConfig const*> const& storage_config, std::string const& tree_id, ArtifactDigest const& digest, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -86,8 +86,7 @@ void MoveCASTreeToGitAndProcess( TreeIdGitMap::SetterPtr const& setter, TreeIdGitMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage - auto tmp_dir = - StorageConfig::Instance().CreateTypedTmpDir("fetch-remote-git-tree"); + auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", @@ -109,6 +108,7 @@ void MoveCASTreeToGitAndProcess( ts, {std::move(c_info)}, [&serve, + storage_config, tmp_dir, // keep tmp_dir alive tree_id, digest, @@ -124,6 +124,7 @@ void MoveCASTreeToGitAndProcess( // upload tree from Git cache to remote CAS and tell serve to set up // the root from the remote CAS tree; set root as absent on success UploadToServeAndSetRoot(serve, + *storage_config, tree_id, digest, *remote_api, @@ -149,6 +150,7 @@ auto CreateTreeIdGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap { @@ -157,6 +159,7 @@ auto CreateTreeIdGitMap( import_to_git_map, fetch_absent, serve, + storage_config, local_api, remote_api](auto ts, auto setter, @@ -229,17 +232,18 @@ auto CreateTreeIdGitMap( GitOpKey op_key = { .params = { - StorageConfig::Instance().GitRoot(), // target_path - "", // git_hash - "", // branch - std::nullopt, // message - true // init_bare + storage_config->GitRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [serve, + storage_config, digest, import_to_git_map, local_api, @@ -262,9 +266,7 @@ auto CreateTreeIdGitMap( if (not git_repo) { (*logger)( fmt::format("Could not open repository {}", - StorageConfig::Instance() - .GitRoot() - .string()), + storage_config->GitRoot().string()), /*fatal=*/true); return; } @@ -290,6 +292,7 @@ auto CreateTreeIdGitMap( // serve to set up the root from the remote CAS // tree, then set root as absent UploadToServeAndSetRoot(*serve, + *storage_config, key.tree_info.hash, digest, *remote_api, @@ -300,11 +303,11 @@ auto CreateTreeIdGitMap( return; } // check if tree is known to local CAS - auto const& cas = Storage::Instance().CAS(); - if (auto path = cas.TreePath(digest)) { + if (local_api->IsAvailable(digest)) { // Move tree locally from CAS to Git cache, then // continue processing it by UploadToServeAndSetRoot MoveCASTreeToGitAndProcess(*serve, + storage_config, key.tree_info.hash, digest, import_to_git_map, @@ -326,7 +329,7 @@ auto CreateTreeIdGitMap( key.tree_info.hash), /*fatal=*/true); }, - [logger, target_path = StorageConfig::Instance().GitRoot()]( + [logger, target_path = storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -357,19 +360,19 @@ auto CreateTreeIdGitMap( git_tree_fetch_map->ConsumeAfterKeysReady( ts, {key.tree_info}, - [key, setter](auto const& values) { + [storage_config, key, setter](auto const& values) { // tree is now in Git cache; // get cache hit info auto is_cache_hit = *values[0]; // set the workspace root as present - (*setter)(std::pair( - nlohmann::json::array( - {key.ignore_special - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - key.tree_info.hash, - StorageConfig::Instance().GitRoot().string()}), - is_cache_hit)); + (*setter)( + std::pair(nlohmann::json::array( + {key.ignore_special + ? FileRoot::kGitTreeIgnoreSpecialMarker + : FileRoot::kGitTreeMarker, + key.tree_info.hash, + storage_config->GitRoot().string()}), + is_cache_hit)); }, [logger, tree_id = key.tree_info.hash](auto const& msg, bool fatal) { diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp index d2de005f..4276d143 100644 --- a/src/other_tools/root_maps/tree_id_git_map.hpp +++ b/src/other_tools/root_maps/tree_id_git_map.hpp @@ -25,6 +25,7 @@ #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -70,6 +71,7 @@ using TreeIdGitMap = gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap; |