diff options
20 files changed, 960 insertions, 525 deletions
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS index 6b5afe1c..954a487a 100644 --- a/src/buildtool/serve_api/serve_service/TARGETS +++ b/src/buildtool/serve_api/serve_service/TARGETS @@ -33,7 +33,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/crypto", "hash_function"] - , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/multithreading", "async_map_utils"] diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp index 3099de77..8bc06c39 100644 --- a/src/buildtool/serve_api/serve_service/source_tree.cpp +++ b/src/buildtool/serve_api/serve_service/source_tree.cpp @@ -26,7 +26,7 @@ #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/crypto/hash_function.hpp" -#include "src/buildtool/execution_api/git/git_api.hpp" +#include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -288,6 +288,23 @@ auto SourceTreeService::SyncGitEntryToCas( std::string const& object_hash, std::filesystem::path const& repo_path) const noexcept -> std::remove_cvref_t<decltype(TResponse::OK)> { + // get gc locks for the local storages + auto native_lock = + GarbageCollector::SharedLock(*native_context_->storage_config); + if (not native_lock) { + logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); + return TResponse::INTERNAL_ERROR; + } + std::optional<LockFile> compat_lock = std::nullopt; + if (compat_context_ != nullptr) { + compat_lock = + GarbageCollector::SharedLock(*compat_context_->storage_config); + if (not compat_lock) { + logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock"); + return TResponse::INTERNAL_ERROR; + } + } + auto const hash_type = native_context_->storage_config->hash_function.GetType(); if (IsTreeObject(kType) and not ProtocolTraits::IsTreeAllowed(hash_type)) { @@ -308,11 +325,17 @@ auto SourceTreeService::SyncGitEntryToCas( auto const digest = ArtifactDigestFactory::Create( hash_type, object_hash, 0, IsTreeObject(kType)); if (not digest) { - logger_->Emit(LogLevel::Error, "{}", digest.error()); + logger_->Emit(LogLevel::Error, "SyncGitEntryToCas: {}", digest.error()); return TResponse::INTERNAL_ERROR; } - auto git_api = GitApi{&repo}; + auto const is_compat = compat_context_ != nullptr; + auto git_api = + MRGitApi{&repo, + native_context_->storage_config, + is_compat ? &*compat_context_->storage_config : nullptr, + is_compat ? &*compat_context_->storage : nullptr, + is_compat ? &*apis_.local : nullptr}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = *digest, .type = kType}}, *apis_.remote)) { diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 04c6c013..bdca1331 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -491,18 +491,20 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, &native_storage_config, common_args.jobs); - auto git_tree_fetch_map = - CreateGitTreeFetchMap(&critical_git_op_map, - &import_to_git_map, - common_args.git_path->string(), - *common_args.local_launcher, - serve ? &*serve : nullptr, - &native_storage_config, - &(*apis.local), - has_remote_api ? &*apis.remote : nullptr, - fetch_args.backup_to_remote, - &progress, - common_args.jobs); + auto git_tree_fetch_map = CreateGitTreeFetchMap( + &critical_git_op_map, + &import_to_git_map, + common_args.git_path->string(), + *common_args.local_launcher, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + compat_storage != nullptr ? &*compat_storage : nullptr, + &(*apis.local), + has_remote_api ? &*apis.remote : nullptr, + fetch_args.backup_to_remote, + &progress, + common_args.jobs); // set up progress observer std::atomic<bool> done{false}; diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 012629c2..8b464fce 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -293,51 +293,58 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &native_storage_config, common_args.jobs); - auto git_tree_fetch_map = - CreateGitTreeFetchMap(&critical_git_op_map, - &import_to_git_map, - common_args.git_path->string(), - *common_args.local_launcher, - serve ? &*serve : nullptr, - &native_storage_config, - &(*apis.local), - has_remote_api ? &*apis.remote : nullptr, - false, /* backup_to_remote */ - &progress, - common_args.jobs); + auto git_tree_fetch_map = CreateGitTreeFetchMap( + &critical_git_op_map, + &import_to_git_map, + common_args.git_path->string(), + *common_args.local_launcher, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + compat_storage != nullptr ? &*compat_storage : nullptr, + &(*apis.local), + has_remote_api ? &*apis.remote : nullptr, + false, /* backup_to_remote */ + &progress, + common_args.jobs); auto resolve_symlinks_map = CreateResolveSymlinksMap(); - auto commit_git_map = - CreateCommitGitMap(&critical_git_op_map, - &import_to_git_map, - common_args.just_mr_paths, - common_args.alternative_mirrors, - common_args.git_path->string(), - *common_args.local_launcher, - serve ? &*serve : nullptr, - &native_storage_config, - &(*apis.local), - has_remote_api ? &*apis.remote : nullptr, - common_args.fetch_absent, - &progress, - common_args.jobs); - - auto content_git_map = - CreateContentGitMap(&content_cas_map, - &import_to_git_map, - common_args.just_mr_paths, - common_args.alternative_mirrors, - common_args.ca_info, - &resolve_symlinks_map, - &critical_git_op_map, - serve ? &*serve : nullptr, - &native_storage_config, - &native_storage, - has_remote_api ? &*apis.remote : nullptr, - common_args.fetch_absent, - &progress, - common_args.jobs); + auto commit_git_map = CreateCommitGitMap( + &critical_git_op_map, + &import_to_git_map, + common_args.just_mr_paths, + common_args.alternative_mirrors, + common_args.git_path->string(), + *common_args.local_launcher, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + compat_storage != nullptr ? &*compat_storage : nullptr, + &(*apis.local), + has_remote_api ? &*apis.remote : nullptr, + common_args.fetch_absent, + &progress, + common_args.jobs); + + auto content_git_map = CreateContentGitMap( + &content_cas_map, + &import_to_git_map, + common_args.just_mr_paths, + common_args.alternative_mirrors, + common_args.ca_info, + &resolve_symlinks_map, + &critical_git_op_map, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + &native_storage, + compat_storage != nullptr ? &*compat_storage : nullptr, + has_remote_api ? &*apis.local : nullptr, // only needed if remote given + has_remote_api ? &*apis.remote : nullptr, + common_args.fetch_absent, + &progress, + common_args.jobs); auto foreign_file_git_map = CreateForeignFileGitMap(&content_cas_map, @@ -355,33 +362,40 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, &resolve_symlinks_map, serve ? &*serve : nullptr, &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + compat_storage != nullptr ? &*compat_storage : nullptr, + has_remote_api ? &*apis.local : nullptr, // only needed if remote given has_remote_api ? &*apis.remote : nullptr, common_args.jobs, multi_repo_tool_name, common_args.just_path ? common_args.just_path->string() : kDefaultJustPath); - auto distdir_git_map = - CreateDistdirGitMap(&content_cas_map, - &import_to_git_map, - &critical_git_op_map, - serve ? &*serve : nullptr, - &native_storage_config, - &native_storage, - &(*apis.local), - has_remote_api ? &*apis.remote : nullptr, - common_args.jobs); + auto distdir_git_map = CreateDistdirGitMap( + &content_cas_map, + &import_to_git_map, + &critical_git_op_map, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + &native_storage, + compat_storage != nullptr ? &*compat_storage : nullptr, + &(*apis.local), + has_remote_api ? &*apis.remote : nullptr, + common_args.jobs); - auto tree_id_git_map = - CreateTreeIdGitMap(&git_tree_fetch_map, - &critical_git_op_map, - &import_to_git_map, - common_args.fetch_absent, - serve ? &*serve : nullptr, - &native_storage_config, - &(*apis.local), - has_remote_api ? &*apis.remote : nullptr, - common_args.jobs); + auto tree_id_git_map = CreateTreeIdGitMap( + &git_tree_fetch_map, + &critical_git_op_map, + &import_to_git_map, + common_args.fetch_absent, + serve ? &*serve : nullptr, + &native_storage_config, + compat_storage_config != nullptr ? &*compat_storage_config : nullptr, + compat_storage != nullptr ? &*compat_storage : nullptr, + &(*apis.local), + has_remote_api ? &*apis.remote : nullptr, + common_args.jobs); auto repos_to_setup_map = CreateReposToSetupMap(config, main, diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index da8be93f..96d9db57 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -115,6 +115,7 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] @@ -125,7 +126,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/common", "protocol_traits"] - , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/system", "system_command"] diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp index 1bfc25ad..6f1b5556 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp @@ -23,7 +23,7 @@ #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/git/git_api.hpp" +#include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/system/system_command.hpp" @@ -32,13 +32,21 @@ namespace { void BackupToRemote(ArtifactDigest const& digest, - StorageConfig const& storage_config, + StorageConfig const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const& remote_api, GitTreeFetchMap::LoggerPtr const& logger) { // try to back up to remote CAS auto repo = RepositoryConfig{}; - if (repo.SetGitCAS(storage_config.GitRoot())) { - auto git_api = GitApi{&repo}; + if (repo.SetGitCAS(native_storage_config.GitRoot())) { + auto git_api = + MRGitApi{&repo, + &native_storage_config, + compat_storage_config, + compat_storage, + compat_storage_config != nullptr ? &*local_api : nullptr}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, @@ -53,24 +61,28 @@ void BackupToRemote(ArtifactDigest const& digest, else { // give a warning (*logger)(fmt::format("Failed to SetGitCAS at {}", - storage_config.GitRoot().string()), + native_storage_config.GitRoot().string()), /*fatal=*/false); } } /// \brief Moves the root tree from local CAS to the Git cache and sets the /// root. -void MoveCASTreeToGit(ArtifactDigest const& digest, - gsl::not_null<ImportToGitMap*> const& import_to_git_map, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<IExecutionApi const*> const& local_api, - IExecutionApi const* remote_api, - bool backup_to_remote, - gsl::not_null<TaskSystem*> const& ts, - GitTreeFetchMap::SetterPtr const& setter, - GitTreeFetchMap::LoggerPtr const& logger) { +void MoveCASTreeToGit( + ArtifactDigest const& digest, + gsl::not_null<ImportToGitMap*> const& import_to_git_map, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + gsl::not_null<IExecutionApi const*> const& local_api, + IExecutionApi const* remote_api, + bool backup_to_remote, + gsl::not_null<TaskSystem*> const& ts, + GitTreeFetchMap::SetterPtr const& setter, + GitTreeFetchMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage - auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); + auto tmp_dir = + native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", @@ -93,7 +105,10 @@ void MoveCASTreeToGit(ArtifactDigest const& digest, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive digest, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, backup_to_remote, setter, @@ -105,7 +120,13 @@ void MoveCASTreeToGit(ArtifactDigest const& digest, } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(digest, *storage_config, *remote_api, logger); + BackupToRemote(digest, + *native_storage_config, + compat_storage_config, + compat_storage, + local_api, + *remote_api, + logger); } (*setter)(false /*no cache hit*/); }, @@ -119,15 +140,19 @@ void MoveCASTreeToGit(ArtifactDigest const& digest, }); } -void TagAndSetRoot(ArtifactDigest const& digest, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - IExecutionApi const* remote_api, - bool backup_to_remote, - gsl::not_null<TaskSystem*> const& ts, - GitTreeFetchMap::SetterPtr const& setter, - GitTreeFetchMap::LoggerPtr const& logger) { - auto repo = storage_config->GitRoot(); +void TagAndSetRoot( + ArtifactDigest const& digest, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + gsl::not_null<IExecutionApi const*> const& local_api, + IExecutionApi const* remote_api, + bool backup_to_remote, + gsl::not_null<TaskSystem*> const& ts, + GitTreeFetchMap::SetterPtr const& setter, + GitTreeFetchMap::LoggerPtr const& logger) { + auto repo = native_storage_config->GitRoot(); GitOpKey op_key = {.params = { repo, // target_path @@ -138,8 +163,15 @@ void TagAndSetRoot(ArtifactDigest const& digest, critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [digest, backup_to_remote, storage_config, remote_api, logger, setter]( - auto const& values) { + [digest, + backup_to_remote, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + logger, + setter](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Tree tagging failed", @@ -148,7 +180,13 @@ void TagAndSetRoot(ArtifactDigest const& digest, } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(digest, *storage_config, *remote_api, logger); + BackupToRemote(digest, + *native_storage_config, + compat_storage_config, + compat_storage, + local_api, + *remote_api, + logger); } (*setter)(false /*no cache hit*/); }, @@ -165,15 +203,18 @@ void TagAndSetRoot(ArtifactDigest const& digest, void TakeTreeFromOlderGeneration( std::size_t generation, ArtifactDigest const& digest, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, GitCASPtr const& git_cas, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, gsl::not_null<TaskSystem*> const& ts, GitTreeFetchMap::SetterPtr const& setter, GitTreeFetchMap::LoggerPtr const& logger) { - auto source = storage_config->GitGenerationRoot(generation); + auto source = native_storage_config->GitGenerationRoot(generation); GitOpKey op_key = {.params = { source, // target_path @@ -187,13 +228,16 @@ void TakeTreeFromOlderGeneration( [digest, git_cas, critical_git_op_map, + local_api, remote_api, backup_to_remote, ts, setter, logger, source, - storage_config](auto const& values) { + native_storage_config, + compat_storage_config, + compat_storage](auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { (*logger)("Tree tagging failed", /*fatal=*/true); @@ -214,12 +258,15 @@ void TakeTreeFromOlderGeneration( fatal); }); if (not git_repo->LocalFetchViaTmpRepo( - *storage_config, source, tag, fetch_logger)) { + *native_storage_config, source, tag, fetch_logger)) { return; } TagAndSetRoot(digest, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, critical_git_op_map, + local_api, remote_api, backup_to_remote, ts, @@ -244,7 +291,9 @@ auto CreateGitTreeFetchMap( std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, @@ -255,7 +304,9 @@ auto CreateGitTreeFetchMap( git_bin, launcher, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, backup_to_remote, @@ -266,15 +317,16 @@ auto CreateGitTreeFetchMap( auto const& key) { // check whether tree exists already in Git cache; // ensure Git cache exists - GitOpKey op_key = {.params = - { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = { + .params = + { + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -283,7 +335,9 @@ auto CreateGitTreeFetchMap( git_bin, launcher, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, backup_to_remote, @@ -303,9 +357,10 @@ auto CreateGitTreeFetchMap( auto git_repo = GitRepoRemote::Open( op_result.git_cas); // link fake repo to odb if (not git_repo) { - (*logger)(fmt::format("Could not open repository {}", - storage_config->GitRoot().string()), - /*fatal=*/true); + (*logger)( + fmt::format("Could not open repository {}", + native_storage_config->GitRoot().string()), + /*fatal=*/true); return; } // setup wrapped logger @@ -327,7 +382,10 @@ auto CreateGitTreeFetchMap( // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { BackupToRemote(ArtifactDigest{key.tree_hash, 0}, - *storage_config, + *native_storage_config, + compat_storage_config, + compat_storage, + local_api, *remote_api, logger); } @@ -338,9 +396,10 @@ auto CreateGitTreeFetchMap( // Check older generations for presence of the tree for (std::size_t generation = 1; - generation < storage_config->num_generations; + generation < native_storage_config->num_generations; generation++) { - auto old = storage_config->GitGenerationRoot(generation); + auto old = + native_storage_config->GitGenerationRoot(generation); if (FileSystemManager::IsDirectory(old)) { auto old_repo = GitRepo::Open(old); auto no_logging = @@ -353,9 +412,12 @@ auto CreateGitTreeFetchMap( TakeTreeFromOlderGeneration( generation, ArtifactDigest{key.tree_hash, 0}, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, op_result.git_cas, critical_git_op_map, + local_api, remote_api, backup_to_remote, ts, @@ -373,7 +435,9 @@ auto CreateGitTreeFetchMap( // import tree to Git cache MoveCASTreeToGit(digest, import_to_git_map, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, backup_to_remote, @@ -402,7 +466,9 @@ auto CreateGitTreeFetchMap( MoveCASTreeToGit( digest, import_to_git_map, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, false, // tree already in remote, so ignore backing up @@ -414,7 +480,7 @@ auto CreateGitTreeFetchMap( } // create temporary location for command execution root auto content_dir = - storage_config->CreateTypedTmpDir("git-tree"); + native_storage_config->CreateTypedTmpDir("git-tree"); if (not content_dir) { (*logger)( "Failed to create execution root tmp directory for " @@ -423,7 +489,8 @@ auto CreateGitTreeFetchMap( return; } // create temporary location for storing command result files - auto out_dir = storage_config->CreateTypedTmpDir("git-tree"); + auto out_dir = + native_storage_config->CreateTypedTmpDir("git-tree"); if (not out_dir) { (*logger)( "Failed to create results tmp directory for tree id " @@ -454,7 +521,7 @@ auto CreateGitTreeFetchMap( } // create temporary location for the import repository auto repo_dir = - storage_config->CreateTypedTmpDir("import-repo"); + native_storage_config->CreateTypedTmpDir("import-repo"); if (not repo_dir) { (*logger)( "Failed to create tmp directory for import repository", @@ -484,7 +551,10 @@ auto CreateGitTreeFetchMap( key, git_bin, launcher, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, backup_to_remote, progress, @@ -558,14 +628,15 @@ auto CreateGitTreeFetchMap( auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)( - fmt::format("Could not open Git repository {}", - storage_config->GitRoot().string()), + fmt::format( + "Could not open Git repository {}", + native_storage_config->GitRoot().string()), /*fatal=*/true); return; } // define temp repo path - auto tmp_dir = - storage_config->CreateTypedTmpDir("git-tree"); + auto tmp_dir = native_storage_config->CreateTypedTmpDir( + "git-tree"); ; if (not tmp_dir) { (*logger)(fmt::format("Could not create unique " @@ -586,7 +657,7 @@ auto CreateGitTreeFetchMap( fatal); }); if (not just_git_repo->FetchViaTmpRepo( - *storage_config, + *native_storage_config, target_path.string(), std::nullopt, key.inherit_env, @@ -612,7 +683,8 @@ auto CreateGitTreeFetchMap( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path + native_storage_config + ->GitRoot(), // target_path *op_result.result, // git_hash "Keep referenced tree alive" // message }, @@ -621,7 +693,10 @@ auto CreateGitTreeFetchMap( ts, {std::move(op_key)}, [remote_api, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, backup_to_remote, key, progress, @@ -640,7 +715,10 @@ auto CreateGitTreeFetchMap( remote_api != nullptr) { BackupToRemote( ArtifactDigest{key.tree_hash, 0}, - *storage_config, + *native_storage_config, + compat_storage_config, + compat_storage, + local_api, *remote_api, logger); } @@ -649,7 +727,7 @@ auto CreateGitTreeFetchMap( }, [logger, commit = *op_result.result, - target_path = storage_config->GitRoot()]( + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -671,8 +749,8 @@ auto CreateGitTreeFetchMap( fatal); }); }, - [logger, target_path = storage_config->GitRoot()](auto const& msg, - bool fatal) { + [logger, target_path = native_storage_config->GitRoot()]( + auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT bare for target {}:\n{}", target_path.string(), diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp index e5949099..7ff6b301 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp @@ -27,6 +27,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -65,7 +66,9 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>; std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool backup_to_remote, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 83ab83de..61e16cb7 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -45,6 +45,7 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr", "mirrors"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/ops_maps", "critical_git_op_map"] @@ -80,6 +81,7 @@ , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] , ["src/other_tools/just_mr", "utils"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] @@ -169,6 +171,7 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] @@ -181,7 +184,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/crypto", "hash_info"] - , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/file_system", "file_root"] ] } @@ -195,6 +198,8 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -203,7 +208,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/crypto", "hash_function"] - , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/execution_api/serve", "mr_git_api"] , ["src/buildtool/file_system", "object_type"] ] } diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index e6463d57..f221f63f 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -49,23 +49,29 @@ namespace { } [[nodiscard]] auto IsCacheGitRoot( - StorageConfig const& storage_config, + StorageConfig const& native_storage_config, std::filesystem::path const& repo_root) noexcept -> bool { return std::filesystem::absolute(ToNormalPath(repo_root)) == - std::filesystem::absolute(ToNormalPath(storage_config.GitRoot())); + std::filesystem::absolute( + ToNormalPath(native_storage_config.GitRoot())); } /// \brief Helper function for ensuring the serve endpoint, if given, has the /// root if it was marked absent. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void EnsureRootAsAbsent(std::string const& tree_id, - std::filesystem::path const& repo_root, - GitRepoInfo const& repo_info, - ServeApi const* serve, - IExecutionApi const* remote_api, - CommitGitMap::SetterPtr const& ws_setter, - CommitGitMap::LoggerPtr const& logger) { +void EnsureRootAsAbsent( + std::string const& tree_id, + std::filesystem::path const& repo_root, + GitRepoInfo const& repo_info, + ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + gsl::not_null<IExecutionApi const*> const& local_api, + IExecutionApi const* remote_api, + CommitGitMap::SetterPtr const& ws_setter, + CommitGitMap::LoggerPtr const& logger) { // this is an absent root if (serve != nullptr) { // check if the serve endpoint has this root @@ -116,6 +122,10 @@ void EnsureRootAsAbsent(std::string const& tree_id, if (not EnsureAbsentRootOnServe(*serve, tree_id, repo_root, + native_storage_config, + compat_storage_config, + compat_storage, + &*local_api, remote_api, logger, true /*no_sync_is_fatal*/)) { @@ -146,7 +156,7 @@ void EnsureRootAsAbsent(std::string const& tree_id, void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, std::string const& subdir, bool ignore_special, - StorageConfig const& storage_config, + StorageConfig const& native_storage_config, GitCASPtr const& git_cas, std::filesystem::path const& tree_id_file, CommitGitMap::SetterPtr const& ws_setter, @@ -163,7 +173,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb if (not git_repo) { (*logger)(fmt::format("Could not open cache object database {}", - storage_config.GitRoot().string()), + native_storage_config.GitRoot().string()), /*fatal=*/true); return; } @@ -186,7 +196,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, - storage_config.GitRoot().string()}), + native_storage_config.GitRoot().string()}), false)); } @@ -269,7 +279,7 @@ void TagAndSetRoot(std::filesystem::path const& repo_root, void TakeCommitFromOlderGeneration( std::filesystem::path const& source, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, std::filesystem::path const& repo_root, GitRepoInfo const& repo_info, GitCASPtr const& git_cas, @@ -291,7 +301,7 @@ void TakeCommitFromOlderGeneration( [logger, git_cas, repo_root, - storage_config, + native_storage_config, source, repo_info, critical_git_op_map, @@ -321,7 +331,7 @@ void TakeCommitFromOlderGeneration( fatal); }); if (not git_repo->LocalFetchViaTmpRepo( - *storage_config, source, tag, fetch_logger)) { + *native_storage_config, source, tag, fetch_logger)) { return; } TagAndSetRoot(repo_root, @@ -349,7 +359,7 @@ void NetworkFetchAndSetPresentRoot( std::string const& fetch_repo, MirrorsPtr const& additional_mirrors, GitCASPtr const& git_cas, - StorageConfig const& storage_config, + StorageConfig const& native_storage_config, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, std::string const& git_bin, std::vector<std::string> const& launcher, @@ -410,7 +420,7 @@ void NetworkFetchAndSetPresentRoot( err_messages += fmt::format( "While attempting fetch from URL {}:\n{}\n", mirror, msg); }); - if (git_repo->FetchViaTmpRepo(storage_config, + if (git_repo->FetchViaTmpRepo(native_storage_config, mirror, repo_info.branch, repo_info.inherit_env, @@ -456,7 +466,7 @@ void NetworkFetchAndSetPresentRoot( return; } // if witnessing repository is the Git cache, then also tag the commit - if (IsCacheGitRoot(storage_config, repo_root)) { + if (IsCacheGitRoot(native_storage_config, repo_root)) { TagAndSetRoot(repo_root, repo_info, true, @@ -505,24 +515,27 @@ void NetworkFetchAndSetPresentRoot( /// the root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void EnsureCommit(GitRepoInfo const& repo_info, - std::filesystem::path const& repo_root, - std::string const& fetch_repo, - MirrorsPtr const& additional_mirrors, - GitCASPtr const& git_cas, - gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, - gsl::not_null<ImportToGitMap*> const& import_to_git_map, - std::string const& git_bin, - std::vector<std::string> const& launcher, - ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<IExecutionApi const*> const& local_api, - IExecutionApi const* remote_api, - bool fetch_absent, - gsl::not_null<JustMRProgress*> const& progress, - gsl::not_null<TaskSystem*> const& ts, - CommitGitMap::SetterPtr const& ws_setter, - CommitGitMap::LoggerPtr const& logger) { +void EnsureCommit( + GitRepoInfo const& repo_info, + std::filesystem::path const& repo_root, + std::string const& fetch_repo, + MirrorsPtr const& additional_mirrors, + GitCASPtr const& git_cas, + gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + gsl::not_null<ImportToGitMap*> const& import_to_git_map, + std::string const& git_bin, + std::vector<std::string> const& launcher, + ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + gsl::not_null<IExecutionApi const*> const& local_api, + IExecutionApi const* remote_api, + bool fetch_absent, + gsl::not_null<JustMRProgress*> const& progress, + gsl::not_null<TaskSystem*> const& ts, + CommitGitMap::SetterPtr const& ws_setter, + CommitGitMap::LoggerPtr const& logger) { // link fake repo to odb auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { @@ -543,8 +556,8 @@ void EnsureCommit(GitRepoInfo const& repo_info, return; } if (not is_commit_present.value()) { - auto tree_id_file = - StorageUtils::GetCommitTreeIDFile(*storage_config, repo_info.hash); + auto tree_id_file = StorageUtils::GetCommitTreeIDFile( + *native_storage_config, repo_info.hash); // Check if we have stored a file association between commit and tree; // if an association file exists, the respective tree MUST be in the // Git cache @@ -557,18 +570,20 @@ void EnsureCommit(GitRepoInfo const& repo_info, /*fatal=*/true); return; } - auto just_git_cas = GitCAS::Open(storage_config->GitRoot()); + auto just_git_cas = GitCAS::Open(native_storage_config->GitRoot()); if (not just_git_cas) { - (*logger)(fmt::format("Could not open Git cache database {}", - storage_config->GitRoot().string()), - /*fatal=*/true); + (*logger)( + fmt::format("Could not open Git cache database {}", + native_storage_config->GitRoot().string()), + /*fatal=*/true); return; } auto just_git_repo = GitRepo::Open(just_git_cas); if (not just_git_repo) { - (*logger)(fmt::format("Could not open Git cache repository {}", - storage_config->GitRoot().string()), - /*fatal=*/true); + (*logger)( + fmt::format("Could not open Git cache repository {}", + native_storage_config->GitRoot().string()), + /*fatal=*/true); return; } // extract the subdir tree @@ -590,13 +605,18 @@ void EnsureCommit(GitRepoInfo const& repo_info, // set the workspace root if (repo_info.absent and not fetch_absent) { // try by all available means to generate & set the absent root - EnsureRootAsAbsent(*tree_id, - storage_config->GitRoot(), - repo_info, - serve, - remote_api, - ws_setter, - logger); + EnsureRootAsAbsent( + *tree_id, + native_storage_config->GitRoot(), /*repo_root*/ + repo_info, + serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + ws_setter, + logger); } else { // this root is present @@ -606,7 +626,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *tree_id, - storage_config->GitRoot().string()}), + native_storage_config->GitRoot().string()}), /*is_cache_hit=*/false)); } // done! @@ -615,9 +635,9 @@ void EnsureCommit(GitRepoInfo const& repo_info, // Check older generations for presence of the commit for (std::size_t generation = 1; - generation < storage_config->num_generations; + generation < native_storage_config->num_generations; generation++) { - auto old = storage_config->GitGenerationRoot(generation); + auto old = native_storage_config->GitGenerationRoot(generation); if (FileSystemManager::IsDirectory(old)) { auto old_repo = GitRepo::Open(old); auto no_logging = std::make_shared<AsyncMapConsumerLogger>( @@ -627,7 +647,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, old_repo->CheckCommitExists(repo_info.hash, no_logging); if (check_result and *check_result) { TakeCommitFromOlderGeneration(old, - storage_config, + native_storage_config, repo_root, repo_info, git_cas, @@ -687,16 +707,16 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (serve_result) { auto const& root_tree_id = *serve_result; // verify if we know the tree already in the local Git cache - GitOpKey op_key = { - .params = - { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = {.params = + { + native_storage_config + ->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -706,7 +726,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, repo_root, fetch_repo, additional_mirrors, - storage_config, + native_storage_config, git_cas, critical_git_op_map, import_to_git_map, @@ -730,28 +750,31 @@ void EnsureCommit(GitRepoInfo const& repo_info, GitRepoRemote::Open(op_result.git_cas); if (not just_git_repo) { (*logger)( - fmt::format( - "Could not open Git " - "cache repository " - "{}", - storage_config->GitRoot().string()), + fmt::format("Could not open Git " + "cache repository " + "{}", + native_storage_config->GitRoot() + .string()), /*fatal=*/true); return; } // check tree existence - auto wrapped_logger = std::make_shared< - AsyncMapConsumerLogger>( - [logger, storage_config, tree = root_tree_id]( - auto const& msg, bool fatal) { - (*logger)( - fmt::format( - "While verifying presence of " - "tree {} in repository {}:\n{}", - tree, - storage_config->GitRoot().string(), - msg), - fatal); - }); + auto wrapped_logger = + std::make_shared<AsyncMapConsumerLogger>( + [logger, + native_storage_config, + tree = root_tree_id](auto const& msg, + bool fatal) { + (*logger)( + fmt::format( + "While verifying presence of " + "tree {} in repository {}:\n{}", + tree, + native_storage_config->GitRoot() + .string(), + msg), + fatal); + }); auto tree_present = just_git_repo->CheckTreeExists( root_tree_id, wrapped_logger); if (not tree_present) { @@ -765,7 +788,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, root_tree_id, repo_info.subdir, repo_info.ignore_special, - *storage_config, + *native_storage_config, op_result.git_cas, tree_id_file, ws_setter, @@ -776,7 +799,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, // now check if the tree is in the local checkout, // if this checkout is not our Git cache; this can // save an unnecessary remote CAS call - if (not IsCacheGitRoot(*storage_config, + if (not IsCacheGitRoot(*native_storage_config, repo_root)) { auto git_repo = GitRepoRemote::Open(git_cas); if (not git_repo) { @@ -855,7 +878,8 @@ void EnsureCommit(GitRepoInfo const& repo_info, // try to get root tree from remote CAS auto const root_digest = ArtifactDigestFactory::Create( - storage_config->hash_function.GetType(), + native_storage_config->hash_function + .GetType(), root_tree_id, 0, /*is_tree=*/true); @@ -868,7 +892,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, progress->TaskTracker().Stop(repo_info.origin); // Move tree from local CAS to local Git storage auto tmp_dir = - storage_config->CreateTypedTmpDir( + native_storage_config->CreateTypedTmpDir( "fetch-absent-root"); if (not tmp_dir) { (*logger)( @@ -901,7 +925,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive root_tree_id, - storage_config, + native_storage_config, subdir = repo_info.subdir, ignore_special = repo_info.ignore_special, just_git_cas = op_result.git_cas, @@ -930,14 +954,15 @@ void EnsureCommit(GitRepoInfo const& repo_info, // write association to id file, get // subdir tree, and set the workspace // root as present - WriteIdFileAndSetWSRoot(root_tree_id, - subdir, - ignore_special, - *storage_config, - just_git_cas, - tree_id_file, - ws_setter, - logger); + WriteIdFileAndSetWSRoot( + root_tree_id, + subdir, + ignore_special, + *native_storage_config, + just_git_cas, + tree_id_file, + ws_setter, + logger); }, [logger, tmp_dir, root_tree_id]( auto const& msg, bool fatal) { @@ -961,22 +986,24 @@ void EnsureCommit(GitRepoInfo const& repo_info, root_tree_id), /*fatal=*/false); - NetworkFetchAndSetPresentRoot(repo_info, - repo_root, - fetch_repo, - additional_mirrors, - git_cas, - *storage_config, - critical_git_op_map, - git_bin, - launcher, - fetch_absent, - progress, - ts, - ws_setter, - logger); + NetworkFetchAndSetPresentRoot( + repo_info, + repo_root, + fetch_repo, + additional_mirrors, + git_cas, + *native_storage_config, + critical_git_op_map, + git_bin, + launcher, + fetch_absent, + progress, + ts, + ws_setter, + logger); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT bare for " @@ -1007,7 +1034,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, fetch_repo, additional_mirrors, git_cas, - *storage_config, + *native_storage_config, critical_git_op_map, git_bin, launcher, @@ -1041,6 +1068,10 @@ void EnsureCommit(GitRepoInfo const& repo_info, repo_root, repo_info, serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger); @@ -1070,7 +1101,9 @@ auto CreateCommitGitMap( std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool fetch_absent, @@ -1083,7 +1116,9 @@ auto CreateCommitGitMap( git_bin, launcher, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, fetch_absent, @@ -1100,7 +1135,7 @@ auto CreateCommitGitMap( fetch_repo = std::filesystem::absolute(*fetch_repo_path).string(); } std::filesystem::path repo_root = StorageUtils::GetGitRoot( - *storage_config, just_mr_paths, fetch_repo); + *native_storage_config, just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { @@ -1126,7 +1161,9 @@ auto CreateCommitGitMap( git_bin, launcher, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, fetch_absent, @@ -1161,7 +1198,9 @@ auto CreateCommitGitMap( git_bin, launcher, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api, fetch_absent, diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index 1c5e9f6b..fad04614 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -27,6 +27,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/mirrors.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" @@ -84,7 +85,9 @@ using CommitGitMap = std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, bool fetch_absent, diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 43222763..ecae60ac 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -52,7 +52,10 @@ void EnsureRootAsAbsent( std::string const& tree_id, ArchiveRepoInfo const& key, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, bool is_cache_hit, ContentGitMap::SetterPtr const& ws_setter, @@ -114,7 +117,11 @@ void EnsureRootAsAbsent( if (not EnsureAbsentRootOnServe( *serve, tree_id, - storage_config->GitRoot(), + native_storage_config->GitRoot(), /*repo_path*/ + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, logger, /*no_sync_is_fatal=*/true)) { @@ -137,12 +144,17 @@ void EnsureRootAsAbsent( // the tree is known locally, so we can upload it to remote // CAS for the serve endpoint to retrieve it and set up the // root - if (not EnsureAbsentRootOnServe(*serve, - tree_id, - storage_config->GitRoot(), - remote_api, - logger, - /*no_sync_is_fatal=*/true)) { + if (not EnsureAbsentRootOnServe( + *serve, + tree_id, + native_storage_config->GitRoot(), /*repo_root*/ + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + logger, + /*no_sync_is_fatal=*/true)) { return; } } @@ -171,7 +183,10 @@ void ResolveContentTree( bool is_cache_hit, bool is_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -181,7 +196,7 @@ void ResolveContentTree( if (key.pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - *storage_config, tree_hash, *key.pragma_special); + *native_storage_config, tree_hash, *key.pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -197,18 +212,22 @@ void ResolveContentTree( EnsureRootAsAbsent(*resolved_tree_id, key, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)(std::pair( - nlohmann::json::array({FileRoot::kGitTreeMarker, - *resolved_tree_id, - storage_config->GitRoot().string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)( + std::pair(nlohmann::json::array( + {FileRoot::kGitTreeMarker, + *resolved_tree_id, + native_storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } } else { @@ -229,7 +248,10 @@ void ResolveContentTree( key, is_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, ws_setter, @@ -239,7 +261,8 @@ void ResolveContentTree( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path + native_storage_config + ->GitRoot(), // target_path resolved_tree_id, // git_hash "Keep referenced tree alive" // message }, @@ -252,7 +275,10 @@ void ResolveContentTree( tree_id_file, is_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, is_cache_hit, ws_setter, @@ -279,7 +305,10 @@ void ResolveContentTree( EnsureRootAsAbsent(resolved_tree_id, key, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, is_cache_hit, ws_setter, @@ -290,11 +319,13 @@ void ResolveContentTree( nlohmann::json::array( {FileRoot::kGitTreeMarker, resolved_tree_id, - storage_config->GitRoot().string()}), + native_storage_config->GitRoot() + .string()}), /*is_cache_hit=*/is_cache_hit)); } }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -320,18 +351,22 @@ void ResolveContentTree( EnsureRootAsAbsent(tree_hash, key, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, is_cache_hit, ws_setter, logger); } else { - (*ws_setter)(std::pair( - nlohmann::json::array({FileRoot::kGitTreeMarker, - tree_hash, - storage_config->GitRoot().string()}), - /*is_cache_hit=*/is_cache_hit)); + (*ws_setter)( + std::pair(nlohmann::json::array( + {FileRoot::kGitTreeMarker, + tree_hash, + native_storage_config->GitRoot().string()}), + /*is_cache_hit=*/is_cache_hit)); } } } @@ -346,7 +381,10 @@ void WriteIdFileAndSetWSRoot( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, @@ -391,7 +429,10 @@ void WriteIdFileAndSetWSRoot( false, /*is_cache_hit*/ is_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -409,7 +450,10 @@ void ExtractAndImportToGit( std::filesystem::path const& archive_tree_id_file, bool is_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -418,7 +462,7 @@ void ExtractAndImportToGit( ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { // extract archive - auto tmp_dir = storage_config->CreateTypedTmpDir(key.repo_type); + auto tmp_dir = native_storage_config->CreateTypedTmpDir(key.repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for {} target {}", key.repo_type, @@ -447,7 +491,10 @@ void ExtractAndImportToGit( key, is_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -469,7 +516,10 @@ void ExtractAndImportToGit( archive_tree_id_file, is_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -487,13 +537,13 @@ void ExtractAndImportToGit( } auto IdFileExistsInOlderGeneration( - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, ArchiveRepoInfo const& key) -> std::optional<std::size_t> { for (std::size_t generation = 1; - generation < storage_config->num_generations; + generation < native_storage_config->num_generations; generation++) { auto archive_tree_id_file = - StorageUtils::GetArchiveTreeIDFile(*storage_config, + StorageUtils::GetArchiveTreeIDFile(*native_storage_config, key.repo_type, key.archive.content_hash.Hash(), generation); @@ -509,9 +559,12 @@ void HandleLocallyKnownTree( std::filesystem::path const& archive_tree_id_file, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, ContentGitMap::SetterPtr const& setter, @@ -528,11 +581,11 @@ void HandleLocallyKnownTree( // define Git operation to be done GitOpKey op_key = {.params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -542,7 +595,10 @@ void HandleLocallyKnownTree( key, fetch_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -585,7 +641,10 @@ void HandleLocallyKnownTree( /*is_cache_hit = */ true, /*is_absent = */ (key.absent and not fetch_absent), serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, resolve_symlinks_map, @@ -593,8 +652,8 @@ void HandleLocallyKnownTree( setter, logger); }, - [logger, target_path = storage_config->GitRoot()](auto const& msg, - bool fatal) { + [logger, target_path = native_storage_config->GitRoot()]( + auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git " "op ENSURE_INIT for " "target {}:\n{}", @@ -609,9 +668,12 @@ void HandleKnownInOlderGenerationAfterImport( const std::string& tree_id, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, ContentGitMap::SetterPtr const& setter, @@ -619,7 +681,7 @@ void HandleKnownInOlderGenerationAfterImport( // Now that we have the tree persisted in the git repository of the youngest // generation; hence we can write the map-entry. auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( - *storage_config, key.repo_type, key.archive.content_hash.Hash()); + *native_storage_config, key.repo_type, key.archive.content_hash.Hash()); if (not StorageUtils::WriteTreeIDFile(archive_tree_id_file, tree_id)) { (*logger)(fmt::format("Failed to write tree id to file {}", archive_tree_id_file.string()), @@ -632,9 +694,12 @@ void HandleKnownInOlderGenerationAfterImport( archive_tree_id_file, fetch_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -649,10 +714,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit( std::filesystem::path const& source, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, ContentGitMap::SetterPtr const& setter, @@ -672,14 +739,14 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit( fatal); }); if (not git_repo->LocalFetchViaTmpRepo( - *storage_config, source, tag, fetch_logger)) { + *native_storage_config, source, tag, fetch_logger)) { return; } GitOpKey op_key = {.params = { - storage_config->GitRoot(), // target_path - tree_id, // git_hash - "Keep referenced tree alive" // message + native_storage_config->GitRoot(), // target_path + tree_id, // git_hash + "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( @@ -690,10 +757,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit( git_cas, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -709,9 +778,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit( tree_id, fetch_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -732,21 +804,23 @@ void HandleKnownInOlderGenerationAfterTagging( std::filesystem::path const& source, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { GitOpKey op_key = {.params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -758,10 +832,12 @@ void HandleKnownInOlderGenerationAfterTagging( source, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -780,17 +856,19 @@ void HandleKnownInOlderGenerationAfterTagging( source, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, logger); }, - [logger, target_path = storage_config->GitRoot()](auto const& msg, - bool fatal) { + [logger, target_path = native_storage_config->GitRoot()]( + auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", target_path.string(), @@ -804,16 +882,18 @@ void HandleKnownInOlderGeneration( std::size_t generation, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, ContentGitMap::SetterPtr const& setter, ContentGitMap::LoggerPtr const& logger) { auto archive_tree_id_file = - StorageUtils::GetArchiveTreeIDFile(*storage_config, + StorageUtils::GetArchiveTreeIDFile(*native_storage_config, key.repo_type, key.archive.content_hash.Hash(), generation); @@ -824,7 +904,7 @@ void HandleKnownInOlderGeneration( /*fatal=*/true); return; } - auto source = storage_config->GitGenerationRoot(generation); + auto source = native_storage_config->GitGenerationRoot(generation); GitOpKey op_key = {.params = { @@ -841,10 +921,12 @@ void HandleKnownInOlderGeneration( source, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -861,10 +943,12 @@ void HandleKnownInOlderGeneration( source, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -891,8 +975,11 @@ auto CreateContentGitMap( gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + gsl::not_null<Storage const*> const& native_storage, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, bool fetch_absent, gsl::not_null<JustMRProgress*> const& progress, @@ -905,8 +992,11 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, - storage, - storage_config, + native_storage_config, + compat_storage_config, + native_storage, + compat_storage, + local_api, remote_api, fetch_absent, progress](auto ts, @@ -914,31 +1004,38 @@ auto CreateContentGitMap( auto logger, auto /* unused */, auto const& key) { - auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( - *storage_config, key.repo_type, key.archive.content_hash.Hash()); + auto archive_tree_id_file = + StorageUtils::GetArchiveTreeIDFile(*native_storage_config, + key.repo_type, + key.archive.content_hash.Hash()); if (FileSystemManager::Exists(archive_tree_id_file)) { HandleLocallyKnownTree(key, archive_tree_id_file, fetch_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, logger); } - else if (auto generation = - IdFileExistsInOlderGeneration(storage_config, key)) { + else if (auto generation = IdFileExistsInOlderGeneration( + native_storage_config, key)) { HandleKnownInOlderGeneration(key, *generation, fetch_absent, serve, - storage_config, - storage, + native_storage_config, + compat_storage_config, + compat_storage, resolve_symlinks_map, critical_git_op_map, + local_api, remote_api, ts, setter, @@ -981,16 +1078,19 @@ auto CreateContentGitMap( // a serve endpoint exists we can upload it the root ourselves; // check if content already in CAS - auto const& cas = storage->CAS(); + auto const& native_cas = native_storage->CAS(); auto const digest = ArtifactDigest{key.archive.content_hash, 0}; if (auto content_cas_path = - cas.BlobPath(digest, /*is_executable=*/false)) { + native_cas.BlobPath(digest, /*is_executable=*/false)) { ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent = */ true, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, import_to_git_map, @@ -1007,11 +1107,11 @@ auto CreateContentGitMap( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -1027,8 +1127,11 @@ auto CreateContentGitMap( additional_mirrors, ca_info, serve, - storage, - storage_config, + native_storage_config, + compat_storage_config, + native_storage, + compat_storage, + local_api, remote_api, progress, ts, @@ -1068,11 +1171,12 @@ auto CreateContentGitMap( // blob check failed return; } - auto const& cas = storage->CAS(); + auto const& native_cas = native_storage->CAS(); if (res.second) { // blob found; add it to CAS - if (not cas.StoreBlob(*res.second, - /*is_executable=*/false)) { + if (not native_cas.StoreBlob( + *res.second, + /*is_executable=*/false)) { (*logger)(fmt::format( "Failed to store content " "{} to local CAS", @@ -1080,14 +1184,17 @@ auto CreateContentGitMap( /*fatal=*/true); return; } - if (auto content_cas_path = cas.BlobPath( + if (auto content_cas_path = native_cas.BlobPath( digest, /*is_executable=*/false)) { ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent=*/true, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, import_to_git_map, @@ -1115,17 +1222,20 @@ auto CreateContentGitMap( .filename() .string()); StorageUtils::AddDistfileToCAS( - *storage, repo_distfile, just_mr_paths); + *native_storage, repo_distfile, just_mr_paths); // check if content is in CAS now - if (auto content_cas_path = - cas.BlobPath(digest, /*is_executable=*/false)) { + if (auto content_cas_path = native_cas.BlobPath( + digest, /*is_executable=*/false)) { progress->TaskTracker().Stop(key.archive.origin); ExtractAndImportToGit(key, *content_cas_path, archive_tree_id_file, /*is_absent=*/true, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, critical_git_op_map, import_to_git_map, @@ -1142,7 +1252,7 @@ auto CreateContentGitMap( key.archive.content_hash.Hash()), /*fatal=*/true); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -1158,8 +1268,8 @@ auto CreateContentGitMap( {key.archive}, [archive_tree_id_file, key, - storage, - storage_config, + native_storage_config, + native_storage, critical_git_op_map, import_to_git_map, resolve_symlinks_map, @@ -1167,11 +1277,12 @@ auto CreateContentGitMap( setter, logger]([[maybe_unused]] auto const& values) { // content is in local CAS now - auto const& cas = storage->CAS(); + auto const& native_cas = native_storage->CAS(); auto content_cas_path = - cas.BlobPath( - ArtifactDigest{key.archive.content_hash, 0}, - /*is_executable=*/false) + native_cas + .BlobPath( + ArtifactDigest{key.archive.content_hash, 0}, + /*is_executable=*/false) .value(); // root can only be present, so default all arguments // that refer to a serve endpoint @@ -1180,7 +1291,10 @@ auto CreateContentGitMap( archive_tree_id_file, /*is_absent=*/false, /*serve=*/nullptr, - storage_config, + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, critical_git_op_map, import_to_git_map, diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index e969f20a..52e6929e 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -47,8 +47,11 @@ using ContentGitMap = gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + gsl::not_null<Storage const*> const& native_storage, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, bool fetch_absent, gsl::not_null<JustMRProgress*> const& progress, diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index a19889b2..82c0c623 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -74,15 +74,15 @@ namespace { /// the setter on success. void ImportFromCASAndSetRoot( DistdirInfo const& key, - StorageConfig const& storage_config, - Storage const& storage, + StorageConfig const& native_storage_config, + Storage const& native_storage, std::filesystem::path const& distdir_tree_id_file, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<TaskSystem*> const& ts, DistdirGitMap::SetterPtr const& setter, DistdirGitMap::LoggerPtr const& logger) { // create the links to CAS - auto tmp_dir = storage_config.CreateTypedTmpDir("distdir"); + auto tmp_dir = native_storage_config.CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", @@ -91,7 +91,7 @@ void ImportFromCASAndSetRoot( return; } // link content from CAS into tmp dir - if (not LinkToCAS(storage, key.content_list, tmp_dir->GetPath())) { + if (not LinkToCAS(native_storage, key.content_list, tmp_dir->GetPath())) { (*logger)(fmt::format("Failed to create links to CAS content!", key.content_id), /*fatal=*/true); @@ -104,7 +104,7 @@ void ImportFromCASAndSetRoot( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, - git_root = storage_config.GitRoot().string(), + git_root = native_storage_config.GitRoot().string(), setter, logger](auto const& values) { // check for errors @@ -145,8 +145,10 @@ auto CreateDistdirGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + gsl::not_null<Storage const*> const& native_storage, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap { @@ -154,16 +156,18 @@ auto CreateDistdirGitMap( import_to_git_map, critical_git_op_map, serve, - storage, - storage_config, + native_storage_config, + compat_storage_config, + native_storage, + compat_storage, local_api, remote_api](auto ts, auto setter, auto logger, auto /* unused */, auto const& key) { - auto distdir_tree_id_file = - StorageUtils::GetDistdirTreeIDFile(*storage_config, key.content_id); + auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile( + *native_storage_config, key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = @@ -179,11 +183,11 @@ auto CreateDistdirGitMap( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -193,7 +197,10 @@ auto CreateDistdirGitMap( content_id = key.content_id, key, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, setter, logger](auto const& values) { @@ -268,7 +275,12 @@ auto CreateDistdirGitMap( if (not EnsureAbsentRootOnServe( *serve, distdir_tree_id, - storage_config->GitRoot(), + native_storage_config + ->GitRoot(), /*repo_root*/ + native_storage_config, + compat_storage_config, + compat_storage, + &*local_api, remote_api, logger, true /*no_sync_is_fatal*/)) { @@ -293,15 +305,15 @@ auto CreateDistdirGitMap( } else { // set root as present - (*setter)( - std::pair(nlohmann::json::array( - {FileRoot::kGitTreeMarker, - distdir_tree_id, - storage_config->GitRoot().string()}), - /*is_cache_hit=*/true)); + (*setter)(std::pair( + nlohmann::json::array( + {FileRoot::kGitTreeMarker, + distdir_tree_id, + native_storage_config->GitRoot().string()}), + /*is_cache_hit=*/true)); } }, - [logger, target_path = storage_config->GitRoot()]( + [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)(fmt::format("While running critical Git op " "ENSURE_INIT for target {}:\n{}", @@ -415,6 +427,10 @@ auto CreateDistdirGitMap( *serve, tree_id, /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, logger, /*no_sync_is_fatal=*/true)) { @@ -452,6 +468,10 @@ auto CreateDistdirGitMap( *serve, tree_id, /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, logger, /*no_sync_is_fatal=*/true)) { @@ -487,8 +507,8 @@ auto CreateDistdirGitMap( // first, look in the local CAS if (digest and local_api->IsAvailable({*digest})) { ImportFromCASAndSetRoot(key, - *storage_config, - *storage, + *native_storage_config, + *native_storage, distdir_tree_id_file, import_to_git_map, ts, @@ -541,15 +561,15 @@ auto CreateDistdirGitMap( [distdir_tree_id_file, key, import_to_git_map, + native_storage_config, + native_storage, ts, - storage, - storage_config, setter, logger]([[maybe_unused]] auto const& values) { // archive blobs are in CAS ImportFromCASAndSetRoot(key, - *storage_config, - *storage, + *native_storage_config, + *native_storage, distdir_tree_id_file, import_to_git_map, ts, diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp index 9244f341..2dc678f8 100644 --- a/src/other_tools/root_maps/distdir_git_map.hpp +++ b/src/other_tools/root_maps/distdir_git_map.hpp @@ -57,8 +57,10 @@ using DistdirGitMap = gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, - gsl::not_null<Storage const*> const& storage, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + gsl::not_null<Storage const*> const& native_storage, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> DistdirGitMap; diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 023dedf8..ca8bdd52 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -31,13 +31,18 @@ namespace { /// \brief Does the serve endpoint checks and sets the workspace root. /// It guarantees the logger is called exactly once with fatal on failure, and /// the setter on success. -void CheckServeAndSetRoot(std::string const& tree_id, - std::string const& repo_root, - bool absent, - ServeApi const* serve, - IExecutionApi const* remote_api, - FilePathGitMap::SetterPtr const& ws_setter, - FilePathGitMap::LoggerPtr const& logger) { +void CheckServeAndSetRoot( + std::string const& tree_id, + std::string const& repo_root, + bool absent, + ServeApi const* serve, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, + IExecutionApi const* remote_api, + FilePathGitMap::SetterPtr const& ws_setter, + FilePathGitMap::LoggerPtr const& logger) { // if serve endpoint is given, try to ensure it has this tree available to // be able to build against it. If root is not absent, do not fail if we // don't have a suitable remote endpoint, but warn user nonetheless. @@ -63,6 +68,10 @@ void CheckServeAndSetRoot(std::string const& tree_id, if (not EnsureAbsentRootOnServe(*serve, tree_id, repo_root, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, logger, /*no_sync_is_fatal=*/absent)) { @@ -99,7 +108,10 @@ void ResolveFilePathTree( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, gsl::not_null<TaskSystem*> const& ts, FilePathGitMap::SetterPtr const& ws_setter, @@ -107,7 +119,7 @@ void ResolveFilePathTree( if (pragma_special) { // get the resolved tree auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( - *storage_config, tree_hash, *pragma_special); + *native_storage_config, tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -122,9 +134,13 @@ void ResolveFilePathTree( // available to be able to build against it; the tree is resolved, // so it is in our Git cache CheckServeAndSetRoot(*resolved_tree_id, - storage_config->GitRoot().string(), + native_storage_config->GitRoot().string(), absent, serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger); @@ -145,7 +161,10 @@ void ResolveFilePathTree( tree_id_file, absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, ws_setter, @@ -155,7 +174,8 @@ void ResolveFilePathTree( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path + native_storage_config + ->GitRoot(), // target_path resolved_tree_id, // git_hash "Keep referenced tree alive" // message }, @@ -167,7 +187,10 @@ void ResolveFilePathTree( tree_id_file, absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger](auto const& values) { @@ -193,14 +216,19 @@ void ResolveFilePathTree( // it; the resolved tree is in the Git cache CheckServeAndSetRoot( resolved_tree_id, - storage_config->GitRoot().string(), + native_storage_config->GitRoot().string(), absent, serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ws_setter, logger); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -223,8 +251,17 @@ void ResolveFilePathTree( // tree needs no further processing; // if serve endpoint is given, try to ensure it has this tree available // to be able to build against it - CheckServeAndSetRoot( - tree_hash, repo_root, absent, serve, remote_api, ws_setter, logger); + CheckServeAndSetRoot(tree_hash, + repo_root, + absent, + serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + ws_setter, + logger); } } @@ -236,7 +273,10 @@ auto CreateFilePathGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, std::size_t jobs, std::string const& multi_repo_tool_name, @@ -246,7 +286,10 @@ auto CreateFilePathGitMap( import_to_git_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, multi_repo_tool_name, build_tool_name](auto ts, @@ -285,7 +328,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -324,16 +370,16 @@ auto CreateFilePathGitMap( // resolve tree and set workspace root; tree gets resolved // from source repo into the Git cache, which we first need // to ensure is initialized - GitOpKey op_key = { - .params = - { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare - }, - .op_type = GitOpType::ENSURE_INIT}; + GitOpKey op_key = {.params = + { + native_storage_config + ->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare + }, + .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, @@ -346,7 +392,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -369,13 +418,17 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, logger); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, + target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -408,7 +461,7 @@ auto CreateFilePathGitMap( /*fatal=*/false); } // it's not a git repo, so import it to git cache - auto tmp_dir = storage_config->CreateTypedTmpDir("file"); + auto tmp_dir = native_storage_config->CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); @@ -437,7 +490,10 @@ auto CreateFilePathGitMap( critical_git_op_map, resolve_symlinks_map, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, remote_api, ts, setter, @@ -452,21 +508,25 @@ auto CreateFilePathGitMap( std::string tree = values[0]->first; // resolve tree and set workspace root; // we work on the Git CAS directly - ResolveFilePathTree(storage_config->GitRoot().string(), - fpath.string(), - tree, - pragma_special, - values[0]->second, /*source_cas*/ - values[0]->second, /*target_cas*/ - absent, - critical_git_op_map, - resolve_symlinks_map, - serve, - storage_config, - remote_api, - ts, - setter, - logger); + ResolveFilePathTree( + native_storage_config->GitRoot().string(), + fpath.string(), + tree, + pragma_special, + values[0]->second, /*source_cas*/ + values[0]->second, /*target_cas*/ + absent, + critical_git_op_map, + resolve_symlinks_map, + serve, + native_storage_config, + compat_storage_config, + compat_storage, + local_api, + remote_api, + ts, + setter, + logger); }, [logger, target_path = key.fpath](auto const& msg, bool fatal) { (*logger)( diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index cb13da5c..ad2f4c39 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -27,6 +27,7 @@ #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -56,7 +57,10 @@ using FilePathGitMap = AsyncMapConsumer<FpathInfo, nlohmann::json>; gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, std::size_t jobs, std::string const& multi_repo_tool_name, diff --git a/src/other_tools/root_maps/root_utils.cpp b/src/other_tools/root_maps/root_utils.cpp index 9dd7badf..0e61c548 100644 --- a/src/other_tools/root_maps/root_utils.cpp +++ b/src/other_tools/root_maps/root_utils.cpp @@ -20,7 +20,7 @@ #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_function.hpp" -#include "src/buildtool/execution_api/git/git_api.hpp" +#include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/file_system/object_type.hpp" auto CheckServeHasAbsentRoot(ServeApi const& serve, @@ -37,12 +37,17 @@ auto CheckServeHasAbsentRoot(ServeApi const& serve, return std::nullopt; } -auto EnsureAbsentRootOnServe(ServeApi const& serve, - std::string const& tree_id, - std::filesystem::path const& repo_path, - IExecutionApi const* remote_api, - AsyncMapConsumerLoggerPtr const& logger, - bool no_sync_is_fatal) -> bool { +auto EnsureAbsentRootOnServe( + ServeApi const& serve, + std::string const& tree_id, + std::filesystem::path const& repo_path, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, + IExecutionApi const* remote_api, + AsyncMapConsumerLoggerPtr const& logger, + bool no_sync_is_fatal) -> bool { if (remote_api != nullptr) { // upload tree to remote CAS auto repo = RepositoryConfig{}; @@ -55,7 +60,11 @@ auto EnsureAbsentRootOnServe(ServeApi const& serve, auto const digest = ArtifactDigestFactory::Create( HashFunction::Type::GitSHA1, tree_id, 0, /*is_tree=*/true); - auto git_api = GitApi{&repo}; + auto git_api = MRGitApi{&repo, + native_storage_config, + compat_storage_config, + compat_storage, + local_api}; if (not digest or not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = *digest, .type = ObjectType::Tree}}, diff --git a/src/other_tools/root_maps/root_utils.hpp b/src/other_tools/root_maps/root_utils.hpp index c055633f..2d3ec6a7 100644 --- a/src/other_tools/root_maps/root_utils.hpp +++ b/src/other_tools/root_maps/root_utils.hpp @@ -22,6 +22,8 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief Calls the ServeApi to check whether the serve endpoint has the given /// tree available to build against. @@ -43,9 +45,16 @@ /// used by given remote execution endpoint! /// \param tree_id The Git-tree identifier. /// \param repo_path Local witnessing Git repository for the tree. -/// \param remote_api Optional API of the remote-execution endpoint. If nullopt, -/// skip the upload to the remote CAS; this assumes prior knowledge which -/// guarantees the tree given by tree_id exists in the remote CAS for the +/// \param native_storage_config Configuration of the native local storage. +/// \param compat_storage_config Optional configuration of the compatible local +/// storage, if it was set up. +/// \param compat_storage Optional compatible local storage, if it was set up. +/// \param local_api Optional API that knows how to communicate with the +/// remote-execution endpoint specified by parameter remote_api, if given. In +/// particular, it is expected to be provided if the remote is compatible. +/// \param remote_api Optional API of the remote-execution endpoint. +/// If nullopt, skip the upload to the remote CAS; this assumes prior knowledge +/// which guarantees the tree given by tree_id exists in the remote CAS for the /// duration of the subsequent serve API call; this option should be used /// carefully, but does result in less remote communication. /// \param logger An AsyncMapConsumer logger instance. @@ -58,6 +67,10 @@ ServeApi const& serve, std::string const& tree_id, std::filesystem::path const& repo_path, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + IExecutionApi const* local_api, IExecutionApi const* remote_api, AsyncMapConsumerLoggerPtr const& logger, bool no_sync_is_fatal) -> bool; diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 76b29045..7b3099ae 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -18,7 +18,7 @@ #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/crypto/hash_info.hpp" -#include "src/buildtool/execution_api/git/git_api.hpp" +#include "src/buildtool/execution_api/serve/mr_git_api.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/other_tools/root_maps/root_utils.hpp" @@ -26,18 +26,27 @@ namespace { /// \brief Guarantees it terminates by either calling the setter or calling the /// logger with fatal. -void UploadToServeAndSetRoot(ServeApi const& serve, - StorageConfig const& storage_config, - std::string const& tree_id, - ArtifactDigest const& digest, - IExecutionApi const& remote_api, - bool ignore_special, - TreeIdGitMap::SetterPtr const& setter, - TreeIdGitMap::LoggerPtr const& logger) { +void UploadToServeAndSetRoot( + ServeApi const& serve, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, + std::string const& tree_id, + ArtifactDigest const& digest, + gsl::not_null<IExecutionApi const*> const& local_api, + IExecutionApi const& remote_api, + bool ignore_special, + TreeIdGitMap::SetterPtr const& setter, + TreeIdGitMap::LoggerPtr const& logger) { // upload to remote CAS auto repo_config = RepositoryConfig{}; - if (repo_config.SetGitCAS(storage_config.GitRoot())) { - auto git_api = GitApi{&repo_config}; + if (repo_config.SetGitCAS(native_storage_config->GitRoot())) { + auto git_api = + MRGitApi{&repo_config, + native_storage_config, + compat_storage_config, + compat_storage, + compat_storage_config != nullptr ? &*local_api : nullptr}; if (not git_api.RetrieveToCas( {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, @@ -51,7 +60,7 @@ void UploadToServeAndSetRoot(ServeApi const& serve, } else { (*logger)(fmt::format("Failed to SetGitCAS at {}", - storage_config.GitRoot().string()), + native_storage_config->GitRoot().string()), /*fatal=*/true); return; } @@ -60,6 +69,10 @@ void UploadToServeAndSetRoot(ServeApi const& serve, if (EnsureAbsentRootOnServe(serve, tree_id, /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, /*remote_api=*/nullptr, logger, /*no_sync_is_fatal=*/true)) { @@ -77,7 +90,9 @@ void UploadToServeAndSetRoot(ServeApi const& serve, /// logger with fatal. void MoveCASTreeToGitAndProcess( ServeApi const& serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, std::string const& tree_id, ArtifactDigest const& digest, gsl::not_null<ImportToGitMap*> const& import_to_git_map, @@ -88,7 +103,8 @@ void MoveCASTreeToGitAndProcess( TreeIdGitMap::SetterPtr const& setter, TreeIdGitMap::LoggerPtr const& logger) { // Move tree from CAS to local Git storage - auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); + auto tmp_dir = + native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp directory for copying " "git-tree {} from remote CAS", @@ -110,10 +126,13 @@ void MoveCASTreeToGitAndProcess( ts, {std::move(c_info)}, [&serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, tmp_dir, // keep tmp_dir alive tree_id, digest, + local_api, remote_api, ignore_special, setter, @@ -126,9 +145,12 @@ void MoveCASTreeToGitAndProcess( // upload tree from Git cache to remote CAS and tell serve to set up // the root from the remote CAS tree; set root as absent on success UploadToServeAndSetRoot(serve, - *storage_config, + native_storage_config, + compat_storage_config, + compat_storage, tree_id, digest, + local_api, *remote_api, ignore_special, setter, @@ -152,7 +174,9 @@ auto CreateTreeIdGitMap( gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap { @@ -161,7 +185,9 @@ auto CreateTreeIdGitMap( import_to_git_map, fetch_absent, serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, local_api, remote_api](auto ts, auto setter, @@ -205,12 +231,17 @@ auto CreateTreeIdGitMap( if (remote_api->IsAvailable({digest})) { // tell serve to set up the root from the remote CAS tree; // upload can be skipped - if (EnsureAbsentRootOnServe(*serve, - key.tree_info.tree_hash.Hash(), - /*repo_path=*/"", - /*remote_api=*/nullptr, - logger, - /*no_sync_is_fatal=*/true)) { + if (EnsureAbsentRootOnServe( + *serve, + key.tree_info.tree_hash.Hash(), + /*repo_path=*/"", + native_storage_config, + /*compat_storage_config=*/nullptr, + /*compat_storage=*/nullptr, + /*local_api=*/nullptr, + /*remote_api=*/nullptr, + logger, + /*no_sync_is_fatal=*/true)) { // set workspace root as absent auto root = nlohmann::json::array( {key.ignore_special @@ -233,18 +264,20 @@ auto CreateTreeIdGitMap( GitOpKey op_key = { .params = { - storage_config->GitRoot(), // target_path - "", // git_hash - std::nullopt, // message - std::nullopt, // source_path - true // init_bare + native_storage_config->GitRoot(), // target_path + "", // git_hash + std::nullopt, // message + std::nullopt, // source_path + true // init_bare }, .op_type = GitOpType::ENSURE_INIT}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, [serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, digest, import_to_git_map, local_api, @@ -266,8 +299,9 @@ auto CreateTreeIdGitMap( op_result.git_cas); // link fake repo to odb if (not git_repo) { (*logger)( - fmt::format("Could not open repository {}", - storage_config->GitRoot().string()), + fmt::format( + "Could not open repository {}", + native_storage_config->GitRoot().string()), /*fatal=*/true); return; } @@ -294,9 +328,12 @@ auto CreateTreeIdGitMap( // tree, then set root as absent UploadToServeAndSetRoot( *serve, - *storage_config, + native_storage_config, + compat_storage_config, + compat_storage, key.tree_info.tree_hash.Hash(), digest, + local_api, *remote_api, key.ignore_special, setter, @@ -310,7 +347,9 @@ auto CreateTreeIdGitMap( // continue processing it by UploadToServeAndSetRoot MoveCASTreeToGitAndProcess( *serve, - storage_config, + native_storage_config, + compat_storage_config, + compat_storage, key.tree_info.tree_hash.Hash(), digest, import_to_git_map, @@ -332,7 +371,7 @@ auto CreateTreeIdGitMap( key.tree_info.tree_hash.Hash()), /*fatal=*/true); }, - [logger, target_path = storage_config->GitRoot()]( + [logger, target_path = native_storage_config->GitRoot()]( auto const& msg, bool fatal) { (*logger)( fmt::format("While running critical Git op " @@ -363,7 +402,7 @@ auto CreateTreeIdGitMap( git_tree_fetch_map->ConsumeAfterKeysReady( ts, {key.tree_info}, - [storage_config, key, setter](auto const& values) { + [native_storage_config, key, setter](auto const& values) { // tree is now in Git cache; // get cache hit info auto is_cache_hit = *values[0]; @@ -374,7 +413,7 @@ auto CreateTreeIdGitMap( ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, key.tree_info.tree_hash.Hash(), - storage_config->GitRoot().string()}), + native_storage_config->GitRoot().string()}), is_cache_hit)); }, [logger, hash = key.tree_info.tree_hash.Hash()](auto const& msg, diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp index faec4379..db7dfd68 100644 --- a/src/other_tools/root_maps/tree_id_git_map.hpp +++ b/src/other_tools/root_maps/tree_id_git_map.hpp @@ -26,6 +26,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/git_tree_fetch_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" @@ -71,7 +72,9 @@ using TreeIdGitMap = gsl::not_null<ImportToGitMap*> const& import_to_git_map, bool fetch_absent, ServeApi const* serve, - gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compat_storage_config, + Storage const* compat_storage, gsl::not_null<IExecutionApi const*> const& local_api, IExecutionApi const* remote_api, std::size_t jobs) -> TreeIdGitMap; |