diff options
Diffstat (limited to 'src/other_tools/root_maps/distdir_git_map.cpp')
| -rw-r--r-- | src/other_tools/root_maps/distdir_git_map.cpp | 173 |
1 files changed, 173 insertions, 0 deletions
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp new file mode 100644 index 00000000..e1d15c2d --- /dev/null +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -0,0 +1,173 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/other_tools/root_maps/distdir_git_map.hpp" + +#include <algorithm> + +#include "src/buildtool/execution_api/common/execution_common.hpp" +#include "src/buildtool/execution_api/local/config.hpp" +#include "src/buildtool/execution_api/local/file_storage.hpp" +#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/other_tools/just_mr/utils.hpp" +#include "src/other_tools/ops_maps/critical_git_op_map.hpp" +#include "src/utils/cpp/tmp_dir.hpp" + +namespace { + +/// \brief Create links from CAS content to distdir tmp directory +[[nodiscard]] auto LinkToCAS( + std::shared_ptr<std::unordered_map<std::string, std::string>> const& + content_list, + std::filesystem::path const& tmp_dir) noexcept -> bool { + auto const& casf = LocalCAS<ObjectType::File>::Instance(); + return std::all_of(content_list->begin(), + content_list->end(), + [&casf, tmp_dir](auto const& kv) { + auto content_path = casf.BlobPath( + ArtifactDigest(kv.second, 0, false)); + if (content_path) { + return FileSystemManager::CreateFileHardlink( + *content_path, // from: cas_path/content_id + tmp_dir / kv.first); // to: tmp_dir/name + } + return false; + }); +} + +} // namespace + +auto CreateDistdirGitMap( + gsl::not_null<ImportToGitMap*> const& import_to_git_map, + gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, + std::size_t jobs) -> DistdirGitMap { + auto distdir_to_git = [import_to_git_map, critical_git_op_map]( + auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { + auto distdir_tree_id_file = + JustMR::Utils::GetDistdirTreeIDFile(key.content_id); + if (FileSystemManager::Exists(distdir_tree_id_file)) { + // read distdir_tree_id from file tree_id_file + auto distdir_tree_id = + FileSystemManager::ReadFile(distdir_tree_id_file); + if (not distdir_tree_id) { + (*logger)(fmt::format("Failed to read tree id from file {}", + distdir_tree_id_file.string()), + /*fatal=*/true); + return; + } + // ensure Git cache + // define Git operation to be done + GitOpKey op_key = { + { + JustMR::Utils::GetGitCacheRoot(), // target_path + "", // git_hash + "", // branch + std::nullopt, // message + true // init_bare + }, + GitOpType::ENSURE_INIT}; + critical_git_op_map->ConsumeAfterKeysReady( + ts, + {std::move(op_key)}, + [distdir_tree_id = *distdir_tree_id, setter, logger]( + auto const& values) { + GitOpValue op_result = *values[0]; + // check flag + if (not op_result.result) { + (*logger)("Git init failed", + /*fatal=*/true); + return; + } + // subdir is ".", so no need to deal with the Git cache + // set the workspace root + (*setter)(nlohmann::json::array( + {"git tree", + distdir_tree_id, + JustMR::Utils::GetGitCacheRoot().string()})); + }, + [logger, target_path = JustMR::Utils::GetGitCacheRoot()]( + auto const& msg, bool fatal) { + (*logger)(fmt::format("While running critical Git " + "op ENSURE_INIT for " + "target {}:\n{}", + target_path.string(), + msg), + fatal); + }); + } + else { + // create the links to CAS + auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir"); + if (not tmp_dir) { + (*logger)(fmt::format( + "Failed to create tmp path for distdir target {}", + key.content_id), + /*fatal=*/true); + return; + } + // link content from CAS into tmp dir + if (not LinkToCAS(key.content_list, tmp_dir->GetPath())) { + (*logger)(fmt::format("Failed to create links to CAS content!", + key.content_id), + /*fatal=*/true); + return; + } + // do import to git + CommitInfo c_info{tmp_dir->GetPath(), "distdir", key.content_id}; + import_to_git_map->ConsumeAfterKeysReady( + ts, + {std::move(c_info)}, + [tmp_dir, // keep tmp_dir alive + distdir_tree_id_file, + setter, + logger](auto const& values) { + // check for errors + if (not values[0]->second) { + (*logger)("Importing to git failed", + /*fatal=*/true); + return; + } + // only the tree is of interest + std::string distdir_tree_id = values[0]->first; + // write to tree id file + if (not JustMR::Utils::WriteTreeIDFile(distdir_tree_id_file, + distdir_tree_id)) { + (*logger)( + fmt::format("Failed to write tree id to file {}", + distdir_tree_id_file.string()), + /*fatal=*/true); + return; + } + // set the workspace root + (*setter)(nlohmann::json::array( + {"git tree", + distdir_tree_id, + JustMR::Utils::GetGitCacheRoot().string()})); + }, + [logger, target_path = tmp_dir->GetPath()](auto const& msg, + bool fatal) { + (*logger)( + fmt::format("While importing target {} to git:\n{}", + target_path.string(), + msg), + fatal); + }); + } + }; + return AsyncMapConsumer<DistdirInfo, nlohmann::json>(distdir_to_git, jobs); +}
\ No newline at end of file |