summaryrefslogtreecommitdiff
path: root/src/other_tools/ops_maps/repo_fetch_map.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/other_tools/ops_maps/repo_fetch_map.cpp')
-rw-r--r--src/other_tools/ops_maps/repo_fetch_map.cpp135
1 files changed, 0 insertions, 135 deletions
diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp
deleted file mode 100644
index f8d28d8d..00000000
--- a/src/other_tools/ops_maps/repo_fetch_map.cpp
+++ /dev/null
@@ -1,135 +0,0 @@
-// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "src/other_tools/ops_maps/repo_fetch_map.hpp"
-
-#include "src/buildtool/file_system/file_storage.hpp"
-#include "src/buildtool/storage/storage.hpp"
-#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
-#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
-#include "src/other_tools/just_mr/utils.hpp"
-
-namespace {
-
-void ProcessContent(std::filesystem::path const& content_path,
- std::filesystem::path const& target_name,
- IExecutionApi* local_api,
- IExecutionApi* remote_api,
- std::string const& content,
- ArtifactDigest const& digest,
- RepoFetchMap::SetterPtr const& setter,
- RepoFetchMap::LoggerPtr const& logger) {
- // try to back up to remote CAS
- if (local_api != nullptr and remote_api != nullptr) {
- if (not local_api->RetrieveToCas(
- {Artifact::ObjectInfo{.digest = digest,
- .type = ObjectType::File}},
- remote_api)) {
- // give a warning
- (*logger)(fmt::format("Failed to back up content {} from local CAS "
- "to remote",
- content),
- /*fatal=*/false);
- }
- }
- // then, copy content into fetch_dir
- if (FileSystemManager::Exists(target_name)) {
- std::filesystem::permissions(target_name,
- std::filesystem::perms::owner_write,
- std::filesystem::perm_options::add);
- }
- if (not FileSystemManager::CopyFile(content_path, target_name)) {
- (*logger)(fmt::format("Failed to copy content {} from CAS to {}",
- content,
- target_name.string()),
- /*fatal=*/true);
- return;
- }
- // success
- JustMRStatistics::Instance().IncrementExecutedCounter();
- (*setter)(true);
-}
-
-} // namespace
-
-auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
- std::filesystem::path const& fetch_dir,
- IExecutionApi* local_api,
- IExecutionApi* remote_api,
- std::size_t jobs) -> RepoFetchMap {
- auto fetch_repo = [content_cas_map, fetch_dir, local_api, remote_api](
- auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
- // get corresponding distfile
- auto distfile =
- (key.archive.distfile ? key.archive.distfile.value()
- : std::filesystem::path(key.archive.fetch_url)
- .filename()
- .string());
- auto target_name = fetch_dir / distfile;
- // check if content not already in CAS
- auto digest = ArtifactDigest(key.archive.content, 0, false);
- auto const& cas = Storage::Instance().CAS();
- auto content_path = cas.BlobPath(digest,
- /*is_executable=*/false);
- if (not content_path) {
- // make sure content is in CAS
- content_cas_map->ConsumeAfterKeysReady(
- ts,
- {key.archive},
- [target_name,
- local_api,
- remote_api,
- content = key.archive.content,
- digest = std::move(digest),
- setter,
- logger]([[maybe_unused]] auto const& values) {
- auto const& cas = Storage::Instance().CAS();
- auto content_path = cas.BlobPath(digest,
- /*is_executable=*/false)
- .value();
- ProcessContent(content_path,
- target_name,
- local_api,
- remote_api,
- content,
- digest,
- setter,
- logger);
- },
- [logger, content = key.archive.content](auto const& msg,
- bool fatal) {
- (*logger)(
- fmt::format("While ensuring content {} is in CAS:\n{}",
- content,
- msg),
- fatal);
- });
- }
- else {
- ProcessContent(*content_path,
- target_name,
- local_api,
- remote_api,
- key.archive.content,
- digest,
- setter,
- logger);
- }
- };
- return AsyncMapConsumer<ArchiveRepoInfo, bool>(fetch_repo, jobs);
-}