From 51a3dadf88303e017c134e5bb821192fa0562546 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 4 Dec 2023 14:47:37 +0100 Subject: just-mr fetch: Clarify naming of current fetch async map As preparatory step for extending the types of repositories that are targeted by a just-mr fetch subcommand, names of certain variables and of the currently only fetch operation async map were changed to clarify that they only target archives. --- src/other_tools/just_mr/TARGETS | 2 +- src/other_tools/just_mr/fetch.cpp | 18 ++-- src/other_tools/ops_maps/TARGETS | 8 +- src/other_tools/ops_maps/archive_fetch_map.cpp | 135 +++++++++++++++++++++++++ src/other_tools/ops_maps/archive_fetch_map.hpp | 31 ++++++ src/other_tools/ops_maps/repo_fetch_map.cpp | 135 ------------------------- src/other_tools/ops_maps/repo_fetch_map.hpp | 31 ------ 7 files changed, 180 insertions(+), 180 deletions(-) create mode 100644 src/other_tools/ops_maps/archive_fetch_map.cpp create mode 100644 src/other_tools/ops_maps/archive_fetch_map.hpp delete mode 100644 src/other_tools/ops_maps/repo_fetch_map.cpp delete mode 100644 src/other_tools/ops_maps/repo_fetch_map.hpp (limited to 'src') diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index c4a75aef..391a8240 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -110,8 +110,8 @@ , "exit_codes" , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] + , ["src/other_tools/ops_maps", "archive_fetch_map"] , ["src/other_tools/ops_maps", "content_cas_map"] - , ["src/other_tools/ops_maps", "repo_fetch_map"] , "setup_utils" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "local"] diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 1f4a04d1..bfddf5ce 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -26,8 +26,8 @@ #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" +#include "src/other_tools/ops_maps/archive_fetch_map.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" -#include "src/other_tools/ops_maps/repo_fetch_map.hpp" auto MultiRepoFetch(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, @@ -120,8 +120,8 @@ auto MultiRepoFetch(std::shared_ptr const& config, Logger::Log(LogLevel::Info, "Fetching to {}", fetch_dir->string()); // gather all repos to be fetched - std::vector repos_to_fetch{}; - repos_to_fetch.reserve( + std::vector archives_to_fetch{}; + archives_to_fetch.reserve( fetch_repos->to_include.size()); // pre-reserve a maximum size for (auto const& repo_name : fetch_repos->to_include) { auto repo_desc = repos->At(repo_name); @@ -249,7 +249,7 @@ auto MultiRepoFetch(std::shared_ptr const& config, .absent = false // not used }; // add to list - repos_to_fetch.emplace_back(std::move(archive_info)); + archives_to_fetch.emplace_back(std::move(archive_info)); } } } @@ -262,7 +262,7 @@ auto MultiRepoFetch(std::shared_ptr const& config, } // report progress - auto nr = repos_to_fetch.size(); + auto nr = archives_to_fetch.size(); Logger::Log(LogLevel::Info, "Found {} {} to fetch", nr, @@ -287,7 +287,7 @@ auto MultiRepoFetch(std::shared_ptr const& config, local_api ? &(*local_api) : nullptr, remote_api ? &(*remote_api) : nullptr, common_args.jobs); - auto repo_fetch_map = CreateRepoFetchMap( + auto archive_fetch_map = CreateArchiveFetchMap( &content_cas_map, *fetch_dir, (fetch_args.backup_to_remote and local_api) ? &(*local_api) : nullptr, @@ -295,7 +295,7 @@ auto MultiRepoFetch(std::shared_ptr const& config, common_args.jobs); // set up progress observer - JustMRProgress::Instance().SetTotal(repos_to_fetch.size()); + JustMRProgress::Instance().SetTotal(archives_to_fetch.size()); std::atomic done{false}; std::condition_variable cv{}; auto reporter = JustMRProgressReporter::Reporter(); @@ -306,9 +306,9 @@ auto MultiRepoFetch(std::shared_ptr const& config, bool failed{false}; { TaskSystem ts{common_args.jobs}; - repo_fetch_map.ConsumeAfterKeysReady( + archive_fetch_map.ConsumeAfterKeysReady( &ts, - repos_to_fetch, + archives_to_fetch, []([[maybe_unused]] auto const& values) {}, [&failed](auto const& msg, bool fatal) { Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning, diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 5745f69c..c0f05d63 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -76,11 +76,11 @@ , ["src/other_tools/just_mr/progress_reporting", "progress"] ] } -, "repo_fetch_map": +, "archive_fetch_map": { "type": ["@", "rules", "CC", "library"] - , "name": ["repo_fetch_map"] - , "hdrs": ["repo_fetch_map.hpp"] - , "srcs": ["repo_fetch_map.cpp"] + , "name": ["archive_fetch_map"] + , "hdrs": ["archive_fetch_map.hpp"] + , "srcs": ["archive_fetch_map.cpp"] , "deps": ["content_cas_map", ["src/buildtool/execution_api/common", "common"]] , "stage": ["src", "other_tools", "ops_maps"] diff --git a/src/other_tools/ops_maps/archive_fetch_map.cpp b/src/other_tools/ops_maps/archive_fetch_map.cpp new file mode 100644 index 00000000..67b99e9a --- /dev/null +++ b/src/other_tools/ops_maps/archive_fetch_map.cpp @@ -0,0 +1,135 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/other_tools/ops_maps/archive_fetch_map.hpp" + +#include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/other_tools/just_mr/progress_reporting/progress.hpp" +#include "src/other_tools/just_mr/progress_reporting/statistics.hpp" +#include "src/other_tools/just_mr/utils.hpp" + +namespace { + +void ProcessContent(std::filesystem::path const& content_path, + std::filesystem::path const& target_name, + IExecutionApi* local_api, + IExecutionApi* remote_api, + std::string const& content, + ArtifactDigest const& digest, + ArchiveFetchMap::SetterPtr const& setter, + ArchiveFetchMap::LoggerPtr const& logger) { + // try to back up to remote CAS + if (local_api != nullptr and remote_api != nullptr) { + if (not local_api->RetrieveToCas( + {Artifact::ObjectInfo{.digest = digest, + .type = ObjectType::File}}, + remote_api)) { + // give a warning + (*logger)(fmt::format("Failed to back up content {} from local CAS " + "to remote", + content), + /*fatal=*/false); + } + } + // then, copy content into fetch_dir + if (FileSystemManager::Exists(target_name)) { + std::filesystem::permissions(target_name, + std::filesystem::perms::owner_write, + std::filesystem::perm_options::add); + } + if (not FileSystemManager::CopyFile(content_path, target_name)) { + (*logger)(fmt::format("Failed to copy content {} from CAS to {}", + content, + target_name.string()), + /*fatal=*/true); + return; + } + // success + JustMRStatistics::Instance().IncrementExecutedCounter(); + (*setter)(true); +} + +} // namespace + +auto CreateArchiveFetchMap(gsl::not_null const& content_cas_map, + std::filesystem::path const& fetch_dir, + IExecutionApi* local_api, + IExecutionApi* remote_api, + std::size_t jobs) -> ArchiveFetchMap { + auto fetch_archive = [content_cas_map, fetch_dir, local_api, remote_api]( + auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { + // get corresponding distfile + auto distfile = + (key.archive.distfile ? key.archive.distfile.value() + : std::filesystem::path(key.archive.fetch_url) + .filename() + .string()); + auto target_name = fetch_dir / distfile; + // check if content not already in CAS + auto digest = ArtifactDigest(key.archive.content, 0, false); + auto const& cas = Storage::Instance().CAS(); + auto content_path = cas.BlobPath(digest, + /*is_executable=*/false); + if (not content_path) { + // make sure content is in CAS + content_cas_map->ConsumeAfterKeysReady( + ts, + {key.archive}, + [target_name, + local_api, + remote_api, + content = key.archive.content, + digest = std::move(digest), + setter, + logger]([[maybe_unused]] auto const& values) { + auto const& cas = Storage::Instance().CAS(); + auto content_path = cas.BlobPath(digest, + /*is_executable=*/false) + .value(); + ProcessContent(content_path, + target_name, + local_api, + remote_api, + content, + digest, + setter, + logger); + }, + [logger, content = key.archive.content](auto const& msg, + bool fatal) { + (*logger)( + fmt::format("While ensuring content {} is in CAS:\n{}", + content, + msg), + fatal); + }); + } + else { + ProcessContent(*content_path, + target_name, + local_api, + remote_api, + key.archive.content, + digest, + setter, + logger); + } + }; + return AsyncMapConsumer(fetch_archive, jobs); +} diff --git a/src/other_tools/ops_maps/archive_fetch_map.hpp b/src/other_tools/ops_maps/archive_fetch_map.hpp new file mode 100644 index 00000000..a74b2428 --- /dev/null +++ b/src/other_tools/ops_maps/archive_fetch_map.hpp @@ -0,0 +1,31 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP +#define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP + +#include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/other_tools/ops_maps/content_cas_map.hpp" + +/// \brief Maps an archive content hash to a status flag. +using ArchiveFetchMap = AsyncMapConsumer; + +[[nodiscard]] auto CreateArchiveFetchMap( + gsl::not_null const& content_cas_map, + std::filesystem::path const& fetch_dir, // should exist! + IExecutionApi* local_api, + IExecutionApi* remote_api, + std::size_t jobs) -> ArchiveFetchMap; + +#endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_ARCHIVE_FETCH_MAP_HPP \ No newline at end of file diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp deleted file mode 100644 index f8d28d8d..00000000 --- a/src/other_tools/ops_maps/repo_fetch_map.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/other_tools/ops_maps/repo_fetch_map.hpp" - -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/storage/storage.hpp" -#include "src/other_tools/just_mr/progress_reporting/progress.hpp" -#include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -#include "src/other_tools/just_mr/utils.hpp" - -namespace { - -void ProcessContent(std::filesystem::path const& content_path, - std::filesystem::path const& target_name, - IExecutionApi* local_api, - IExecutionApi* remote_api, - std::string const& content, - ArtifactDigest const& digest, - RepoFetchMap::SetterPtr const& setter, - RepoFetchMap::LoggerPtr const& logger) { - // try to back up to remote CAS - if (local_api != nullptr and remote_api != nullptr) { - if (not local_api->RetrieveToCas( - {Artifact::ObjectInfo{.digest = digest, - .type = ObjectType::File}}, - remote_api)) { - // give a warning - (*logger)(fmt::format("Failed to back up content {} from local CAS " - "to remote", - content), - /*fatal=*/false); - } - } - // then, copy content into fetch_dir - if (FileSystemManager::Exists(target_name)) { - std::filesystem::permissions(target_name, - std::filesystem::perms::owner_write, - std::filesystem::perm_options::add); - } - if (not FileSystemManager::CopyFile(content_path, target_name)) { - (*logger)(fmt::format("Failed to copy content {} from CAS to {}", - content, - target_name.string()), - /*fatal=*/true); - return; - } - // success - JustMRStatistics::Instance().IncrementExecutedCounter(); - (*setter)(true); -} - -} // namespace - -auto CreateRepoFetchMap(gsl::not_null const& content_cas_map, - std::filesystem::path const& fetch_dir, - IExecutionApi* local_api, - IExecutionApi* remote_api, - std::size_t jobs) -> RepoFetchMap { - auto fetch_repo = [content_cas_map, fetch_dir, local_api, remote_api]( - auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { - // get corresponding distfile - auto distfile = - (key.archive.distfile ? key.archive.distfile.value() - : std::filesystem::path(key.archive.fetch_url) - .filename() - .string()); - auto target_name = fetch_dir / distfile; - // check if content not already in CAS - auto digest = ArtifactDigest(key.archive.content, 0, false); - auto const& cas = Storage::Instance().CAS(); - auto content_path = cas.BlobPath(digest, - /*is_executable=*/false); - if (not content_path) { - // make sure content is in CAS - content_cas_map->ConsumeAfterKeysReady( - ts, - {key.archive}, - [target_name, - local_api, - remote_api, - content = key.archive.content, - digest = std::move(digest), - setter, - logger]([[maybe_unused]] auto const& values) { - auto const& cas = Storage::Instance().CAS(); - auto content_path = cas.BlobPath(digest, - /*is_executable=*/false) - .value(); - ProcessContent(content_path, - target_name, - local_api, - remote_api, - content, - digest, - setter, - logger); - }, - [logger, content = key.archive.content](auto const& msg, - bool fatal) { - (*logger)( - fmt::format("While ensuring content {} is in CAS:\n{}", - content, - msg), - fatal); - }); - } - else { - ProcessContent(*content_path, - target_name, - local_api, - remote_api, - key.archive.content, - digest, - setter, - logger); - } - }; - return AsyncMapConsumer(fetch_repo, jobs); -} diff --git a/src/other_tools/ops_maps/repo_fetch_map.hpp b/src/other_tools/ops_maps/repo_fetch_map.hpp deleted file mode 100644 index 424c9122..00000000 --- a/src/other_tools/ops_maps/repo_fetch_map.hpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP -#define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP - -#include "src/buildtool/execution_api/common/execution_api.hpp" -#include "src/other_tools/ops_maps/content_cas_map.hpp" - -/// \brief Maps an archive content hash to a status flag. -using RepoFetchMap = AsyncMapConsumer; - -[[nodiscard]] auto CreateRepoFetchMap( - gsl::not_null const& content_cas_map, - std::filesystem::path const& fetch_dir, // should exist! - IExecutionApi* local_api, - IExecutionApi* remote_api, - std::size_t jobs) -> RepoFetchMap; - -#endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP \ No newline at end of file -- cgit v1.2.3