diff options
author | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2023-08-24 14:02:37 +0200 |
---|---|---|
committer | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2023-08-25 18:11:23 +0200 |
commit | 089733c819066b801e28d6441cbff887e60aef51 (patch) | |
tree | b5d1528c0ce3e25f042c70684812711fa33fb9b8 /src | |
parent | 833974c986eb46f88c87b3a95b4b840a333b24da (diff) | |
download | justbuild-089733c819066b801e28d6441cbff887e60aef51.tar.gz |
just-mr fetch: Allow to back up fetched archives to a given remote CAS
Diffstat (limited to 'src')
-rw-r--r-- | src/other_tools/just_mr/cli.hpp | 5 | ||||
-rw-r--r-- | src/other_tools/just_mr/fetch.cpp | 8 | ||||
-rw-r--r-- | src/other_tools/just_mr/main.cpp | 9 | ||||
-rw-r--r-- | src/other_tools/ops_maps/TARGETS | 3 | ||||
-rw-r--r-- | src/other_tools/ops_maps/repo_fetch_map.cpp | 131 | ||||
-rw-r--r-- | src/other_tools/ops_maps/repo_fetch_map.hpp | 3 |
6 files changed, 103 insertions, 56 deletions
diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp index 60c8f745..484665a5 100644 --- a/src/other_tools/just_mr/cli.hpp +++ b/src/other_tools/just_mr/cli.hpp @@ -63,6 +63,7 @@ struct MultiRepoSetupArguments { struct MultiRepoFetchArguments { std::optional<std::filesystem::path> fetch_dir{std::nullopt}; + bool backup_to_remote{false}; }; struct MultiRepoUpdateArguments { @@ -243,6 +244,10 @@ static inline void SetupMultiRepoFetchArguments( }, "Directory to write distfiles when fetching.") ->type_name("PATH"); + app->add_flag("--backup-to-remote", + clargs->backup_to_remote, + "Backup fetched archives to a remote CAS, if a " + "remote-execution service is provided."); } static inline void SetupMultiRepoUpdateArguments( diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 49cca32e..4519a2cc 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -264,8 +264,12 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, local_api ? &(*local_api) : nullptr, remote_api ? &(*remote_api) : nullptr, common_args.jobs); - auto repo_fetch_map = - CreateRepoFetchMap(&content_cas_map, *fetch_dir, common_args.jobs); + auto repo_fetch_map = CreateRepoFetchMap( + &content_cas_map, + *fetch_dir, + (fetch_args.backup_to_remote and local_api) ? &(*local_api) : nullptr, + (fetch_args.backup_to_remote and remote_api) ? &(*remote_api) : nullptr, + common_args.jobs); // set up progress observer JustMRProgress::Instance().SetTotal(repos_to_fetch.size()); diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index 21cba2f7..29e53048 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -749,6 +749,15 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `fetch` if (arguments.cmd == SubCommand::kFetch) { + // check fetch configuration arguments for validity + if (arguments.common.remote_execution_address and + arguments.fetch.backup_to_remote and + Compatibility::IsCompatible()) { + Logger::Log(LogLevel::Error, + "Remote backup for fetched archives only available " + "in native mode!"); + return kExitConfigError; + } return MultiRepoFetch(config, arguments.common, arguments.setup, diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 8f88f8b0..a2121e33 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -77,7 +77,8 @@ , "name": ["repo_fetch_map"] , "hdrs": ["repo_fetch_map.hpp"] , "srcs": ["repo_fetch_map.cpp"] - , "deps": ["content_cas_map"] + , "deps": + ["content_cas_map", ["src/buildtool/execution_api/common", "common"]] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["src/other_tools/just_mr", "utils"] diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp index 1cc663d7..f8d28d8d 100644 --- a/src/other_tools/ops_maps/repo_fetch_map.cpp +++ b/src/other_tools/ops_maps/repo_fetch_map.cpp @@ -20,61 +20,96 @@ #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" #include "src/other_tools/just_mr/utils.hpp" +namespace { + +void ProcessContent(std::filesystem::path const& content_path, + std::filesystem::path const& target_name, + IExecutionApi* local_api, + IExecutionApi* remote_api, + std::string const& content, + ArtifactDigest const& digest, + RepoFetchMap::SetterPtr const& setter, + RepoFetchMap::LoggerPtr const& logger) { + // try to back up to remote CAS + if (local_api != nullptr and remote_api != nullptr) { + if (not local_api->RetrieveToCas( + {Artifact::ObjectInfo{.digest = digest, + .type = ObjectType::File}}, + remote_api)) { + // give a warning + (*logger)(fmt::format("Failed to back up content {} from local CAS " + "to remote", + content), + /*fatal=*/false); + } + } + // then, copy content into fetch_dir + if (FileSystemManager::Exists(target_name)) { + std::filesystem::permissions(target_name, + std::filesystem::perms::owner_write, + std::filesystem::perm_options::add); + } + if (not FileSystemManager::CopyFile(content_path, target_name)) { + (*logger)(fmt::format("Failed to copy content {} from CAS to {}", + content, + target_name.string()), + /*fatal=*/true); + return; + } + // success + JustMRStatistics::Instance().IncrementExecutedCounter(); + (*setter)(true); +} + +} // namespace + auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, std::filesystem::path const& fetch_dir, + IExecutionApi* local_api, + IExecutionApi* remote_api, std::size_t jobs) -> RepoFetchMap { - auto fetch_repo = [content_cas_map, fetch_dir](auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { + auto fetch_repo = [content_cas_map, fetch_dir, local_api, remote_api]( + auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { // get corresponding distfile auto distfile = (key.archive.distfile ? key.archive.distfile.value() : std::filesystem::path(key.archive.fetch_url) .filename() .string()); + auto target_name = fetch_dir / distfile; // check if content not already in CAS + auto digest = ArtifactDigest(key.archive.content, 0, false); auto const& cas = Storage::Instance().CAS(); - auto content_path = - cas.BlobPath(ArtifactDigest(key.archive.content, 0, false), - /*is_executable=*/false); + auto content_path = cas.BlobPath(digest, + /*is_executable=*/false); if (not content_path) { // make sure content is in CAS content_cas_map->ConsumeAfterKeysReady( ts, {key.archive}, - [fetch_dir, + [target_name, + local_api, + remote_api, content = key.archive.content, - distfile, + digest = std::move(digest), setter, logger]([[maybe_unused]] auto const& values) { - // content is now in CAS, so copy content into fetch_dir auto const& cas = Storage::Instance().CAS(); - auto content_path = - cas.BlobPath(ArtifactDigest(content, 0, false), - /*is_executable=*/false) - .value(); - auto target_name = fetch_dir / distfile; - if (FileSystemManager::Exists(target_name)) { - std::filesystem::permissions( - target_name, - std::filesystem::perms::owner_write, - std::filesystem::perm_options::add); - } - if (not FileSystemManager::CopyFile(content_path, - target_name)) { - (*logger)( - fmt::format("Failed to copy content {} from CAS " - "to {}", - content, - target_name.string()), - /*fatal=*/true); - return; - } - // success - JustMRStatistics::Instance().IncrementExecutedCounter(); - (*setter)(true); + auto content_path = cas.BlobPath(digest, + /*is_executable=*/false) + .value(); + ProcessContent(content_path, + target_name, + local_api, + remote_api, + content, + digest, + setter, + logger); }, [logger, content = key.archive.content](auto const& msg, bool fatal) { @@ -86,24 +121,14 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map, }); } else { - auto target_name = fetch_dir / distfile; - if (FileSystemManager::Exists(target_name)) { - std::filesystem::permissions( - target_name, - std::filesystem::perms::owner_write, - std::filesystem::perm_options::add); - } - if (not FileSystemManager::CopyFile(*content_path, target_name)) { - (*logger)(fmt::format("Failed to copy content {} from CAS " - "to {}", - key.archive.content, - target_name.string()), - /*fatal=*/true); - return; - } - // success - JustMRStatistics::Instance().IncrementCacheHitsCounter(); - (*setter)(true); + ProcessContent(*content_path, + target_name, + local_api, + remote_api, + key.archive.content, + digest, + setter, + logger); } }; return AsyncMapConsumer<ArchiveRepoInfo, bool>(fetch_repo, jobs); diff --git a/src/other_tools/ops_maps/repo_fetch_map.hpp b/src/other_tools/ops_maps/repo_fetch_map.hpp index 9952bce5..424c9122 100644 --- a/src/other_tools/ops_maps/repo_fetch_map.hpp +++ b/src/other_tools/ops_maps/repo_fetch_map.hpp @@ -15,6 +15,7 @@ #ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP #define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP +#include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" /// \brief Maps an archive content hash to a status flag. @@ -23,6 +24,8 @@ using RepoFetchMap = AsyncMapConsumer<ArchiveRepoInfo, bool>; [[nodiscard]] auto CreateRepoFetchMap( gsl::not_null<ContentCASMap*> const& content_cas_map, std::filesystem::path const& fetch_dir, // should exist! + IExecutionApi* local_api, + IExecutionApi* remote_api, std::size_t jobs) -> RepoFetchMap; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP
\ No newline at end of file |