From c1a58d73d2fa21f9c692bbe3895443d3afc43d1d Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 27 Sep 2023 12:47:43 +0200 Subject: Decoupling symlinks map and CAS utilities from just-mr This is required in order to make them available to 'just serve' in a minimal just installation. --- src/buildtool/common/TARGETS | 11 + src/buildtool/common/user_structs.hpp | 72 +++++ src/buildtool/file_system/symlinks_map/TARGETS | 25 ++ .../file_system/symlinks_map/pragma_special.hpp | 42 +++ .../symlinks_map/resolve_symlinks_map.cpp | 335 +++++++++++++++++++++ .../symlinks_map/resolve_symlinks_map.hpp | 98 ++++++ src/buildtool/storage/TARGETS | 18 ++ src/buildtool/storage/fs_utils.cpp | 114 +++++++ src/buildtool/storage/fs_utils.hpp | 77 +++++ src/other_tools/just_mr/TARGETS | 14 +- src/other_tools/just_mr/cli.hpp | 5 +- src/other_tools/just_mr/setup.cpp | 5 +- src/other_tools/just_mr/update.cpp | 3 +- src/other_tools/just_mr/utils.cpp | 94 ------ src/other_tools/just_mr/utils.hpp | 121 +------- src/other_tools/ops_maps/TARGETS | 11 +- src/other_tools/ops_maps/content_cas_map.cpp | 13 +- src/other_tools/ops_maps/content_cas_map.hpp | 7 +- src/other_tools/ops_maps/git_update_map.cpp | 4 +- src/other_tools/ops_maps/import_to_git_map.cpp | 4 +- src/other_tools/repo_map/TARGETS | 1 + src/other_tools/repo_map/repos_to_setup_map.cpp | 1 + src/other_tools/root_maps/TARGETS | 16 +- src/other_tools/root_maps/commit_git_map.cpp | 14 +- src/other_tools/root_maps/commit_git_map.hpp | 4 +- src/other_tools/root_maps/content_git_map.cpp | 14 +- src/other_tools/root_maps/content_git_map.hpp | 2 +- src/other_tools/root_maps/distdir_git_map.cpp | 7 +- src/other_tools/root_maps/fpath_git_map.cpp | 9 +- src/other_tools/root_maps/fpath_git_map.hpp | 3 +- src/other_tools/root_maps/tree_id_git_map.cpp | 8 +- src/other_tools/symlinks_map/TARGETS | 18 -- .../symlinks_map/resolve_symlinks_map.cpp | 335 --------------------- .../symlinks_map/resolve_symlinks_map.hpp | 97 ------ 34 files changed, 883 insertions(+), 719 deletions(-) create mode 100644 src/buildtool/common/user_structs.hpp create mode 100644 src/buildtool/file_system/symlinks_map/TARGETS create mode 100644 src/buildtool/file_system/symlinks_map/pragma_special.hpp create mode 100644 src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp create mode 100644 src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp create mode 100644 src/buildtool/storage/fs_utils.cpp create mode 100644 src/buildtool/storage/fs_utils.hpp delete mode 100644 src/other_tools/symlinks_map/TARGETS delete mode 100644 src/other_tools/symlinks_map/resolve_symlinks_map.cpp delete mode 100644 src/other_tools/symlinks_map/resolve_symlinks_map.hpp (limited to 'src') diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index a05632c6..877d740a 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -116,4 +116,15 @@ , ["src/buildtool/storage", "storage"] ] } +, "user_structs": + { "type": ["@", "rules", "CC", "library"] + , "name": ["user_structs"] + , "hdrs": ["user_structs.hpp"] + , "stage": ["src", "buildtool", "common"] + , "deps": + [ ["@", "json", "", "json"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/main", "constants"] + ] + } } diff --git a/src/buildtool/common/user_structs.hpp b/src/buildtool/common/user_structs.hpp new file mode 100644 index 00000000..bebd8fab --- /dev/null +++ b/src/buildtool/common/user_structs.hpp @@ -0,0 +1,72 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP +#define INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP + +#include +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/main/constants.hpp" + +/* Structures populated exclusively from command line with user-defined data */ + +struct LocalPaths { + // user-defined locations + std::optional root{std::nullopt}; + std::filesystem::path setup_root{FileSystemManager::GetCurrentDirectory()}; + std::optional workspace_root{ + // find workspace root + []() -> std::optional { + std::function + is_workspace_root = [&](std::filesystem::path const& path) { + return std::any_of( + kRootMarkers.begin(), + kRootMarkers.end(), + [&path](auto const& marker) { + return FileSystemManager::Exists(path / marker); + }); + }; + // default path to current dir + auto path = FileSystemManager::GetCurrentDirectory(); + auto root_path = path.root_path(); + while (true) { + if (is_workspace_root(path)) { + return path; + } + if (path == root_path) { + return std::nullopt; + } + path = path.parent_path(); + } + }()}; + nlohmann::json git_checkout_locations{}; + std::vector distdirs{}; +}; + +struct CAInfo { + bool no_ssl_verify{false}; + std::optional ca_bundle{std::nullopt}; +}; + +using LocalPathsPtr = std::shared_ptr; +using CAInfoPtr = std::shared_ptr; + +#endif // INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP diff --git a/src/buildtool/file_system/symlinks_map/TARGETS b/src/buildtool/file_system/symlinks_map/TARGETS new file mode 100644 index 00000000..125a4f52 --- /dev/null +++ b/src/buildtool/file_system/symlinks_map/TARGETS @@ -0,0 +1,25 @@ +{ "resolve_symlinks_map": + { "type": ["@", "rules", "CC", "library"] + , "name": ["resolve_symlinks_map"] + , "hdrs": ["resolve_symlinks_map.hpp"] + , "srcs": ["resolve_symlinks_map.cpp"] + , "deps": + [ "pragma_special" + , ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/multithreading", "async_map_consumer"] + , ["src/utils/cpp", "hash_combine"] + , ["src/utils/cpp", "path"] + , ["src/utils/cpp", "path_hash"] + ] + , "stage": ["src", "buildtool", "file_system", "symlinks_map"] + , "private-deps": + [["@", "fmt", "", "fmt"], ["src/buildtool/storage", "config"]] + } +, "pragma_special": + { "type": ["@", "rules", "CC", "library"] + , "name": ["pragma_special"] + , "hdrs": ["pragma_special.hpp"] + , "stage": ["src", "buildtool", "file_system", "symlinks_map"] + } +} diff --git a/src/buildtool/file_system/symlinks_map/pragma_special.hpp b/src/buildtool/file_system/symlinks_map/pragma_special.hpp new file mode 100644 index 00000000..61935bdd --- /dev/null +++ b/src/buildtool/file_system/symlinks_map/pragma_special.hpp @@ -0,0 +1,42 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP + +#include +#include + +/* Enum used by the resolve_symlinks_map */ + +/// \brief Pragma "special" value enum +enum class PragmaSpecial : std::uint8_t { + Ignore, + ResolvePartially, + ResolveCompletely +}; + +/// \brief Pragma "special" value map, from string to enum +std::unordered_map const kPragmaSpecialMap = { + {"ignore", PragmaSpecial::Ignore}, + {"resolve-partially", PragmaSpecial::ResolvePartially}, + {"resolve-completely", PragmaSpecial::ResolveCompletely}}; + +/// \brief Pragma "special" value inverse map, from enum to string +std::unordered_map const kPragmaSpecialInverseMap = + {{PragmaSpecial::Ignore, "ignore"}, + {PragmaSpecial::ResolvePartially, "resolve-partially"}, + {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; + +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP diff --git a/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp new file mode 100644 index 00000000..1ae5db8b --- /dev/null +++ b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp @@ -0,0 +1,335 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" + +#include "fmt/core.h" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/storage/config.hpp" + +namespace { + +void ResolveKnownEntry(GitObjectToResolve const& obj, + GitRepo::TreeEntryInfo const& entry_info, + GitCASPtr const& just_git_cas, + ResolveSymlinksMap::SetterPtr const& setter, + ResolveSymlinksMap::LoggerPtr const& logger, + ResolveSymlinksMap::SubCallerPtr const& subcaller) { + // differentiated treatment based on object type + if (IsFileObject(entry_info.type)) { + // files are already resolved, so return the hash directly + (*setter)(ResolvedGitObject{.id = entry_info.id, + .type = entry_info.type, + .path = obj.rel_path}); + } + else if (IsTreeObject(entry_info.type)) { + // for tree types we resolve by rebuilding the tree from the + // resolved children + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)("ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto children = just_git_repo->ReadTree( + entry_info.id, + [](std::vector const& /*unused*/) { + return true; + }, + /*is_hex_id=*/true); + if (not children) { + (*logger)(fmt::format("ResolveSymlinks: failed to read entries of " + "subtree {} in root tree {}", + entry_info.id, + obj.root_tree_id), + /*fatal=*/true); + return; + } + // resolve children + std::vector children_info{}; + children_info.reserve(children->size()); + for (auto const& [raw_id, ev] : *children) { + for (auto const& e : ev) { + // must enforce ignore special at the tree level! + if (IsNonSpecialObject(e.type) or + obj.pragma_special != PragmaSpecial::Ignore) { + // children info is known, so pass this forward + if (IsSymlinkObject(e.type)) { + if (auto target = just_git_cas->ReadObject(raw_id)) { + children_info.emplace_back( + obj.root_tree_id, + obj.rel_path / e.name, + obj.pragma_special, + std::make_optional(GitRepo::TreeEntryInfo{ + .id = ToHexString(raw_id), + .type = e.type, + .symlink_content = *target})); + } + else { + (*logger)( + fmt::format("ResolveSymlinks: could not read " + "symlink {} in root tree {}", + (obj.rel_path / e.name).string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + } + else { + children_info.emplace_back( + obj.root_tree_id, + obj.rel_path / e.name, + obj.pragma_special, + GitRepo::TreeEntryInfo{ + .id = ToHexString(raw_id), + .type = e.type, + .symlink_content = std::nullopt}); + } + } + } + } + (*subcaller)( + children_info, + [children_info, parent = obj, just_git_cas, setter, logger]( + auto const& resolved_entries) { + // create the entries map of the children + GitRepo::tree_entries_t entries{}; + auto num = resolved_entries.size(); + entries.reserve(num); + for (auto i = 0; i < num; ++i) { + auto const& p = children_info[i].rel_path; + entries[*FromHexString(resolved_entries[i]->id)] + .emplace_back( + p.filename().string(), // we only need the name + resolved_entries[i]->type); + } + // create the tree inside our Git CAS, which is already + // existing by this point. Also, this operation is + // guarded internally, so no need for the + // critical_git_op map + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)( + "ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto tree_raw_id = just_git_repo->CreateTree(entries); + if (not tree_raw_id) { + (*logger)(fmt::format("ResolveSymlinks: failed to create " + "resolved tree {} in root tree {}", + parent.rel_path.string(), + parent.root_tree_id), + /*fatal=*/true); + return; + } + // set the resolved tree hash + (*setter)(ResolvedGitObject{.id = ToHexString(*tree_raw_id), + .type = ObjectType::Tree, + .path = parent.rel_path}); + }, + logger); + } + else { + // sanity check: cannot resolve a symlink called with ignore + // special, as that can only be handled by the parent tree + if (obj.pragma_special == PragmaSpecial::Ignore) { + (*logger)(fmt::format("ResolveSymlinks: asked to ignore symlink {} " + "in root tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // target should have already been read + if (not entry_info.symlink_content) { + (*logger)(fmt::format("ResolveSymlinks: missing target of symlink " + "{} in root tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // check if link target (unresolved) is confined to the tree + if (not PathIsConfined(*entry_info.symlink_content, obj.rel_path)) { + (*logger)(fmt::format("ResolveSymlinks: symlink {} is not confined " + "to tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // if partially resolved, return non-upwards symlinks as-is + if (obj.pragma_special == PragmaSpecial::ResolvePartially and + PathIsNonUpwards(*entry_info.symlink_content)) { + // return as symlink object + (*setter)(ResolvedGitObject{.id = entry_info.id, + .type = ObjectType::Symlink, + .path = obj.rel_path}); + return; + } + // resolve the target + auto n_target = ToNormalPath(obj.rel_path.parent_path() / + *entry_info.symlink_content); + (*subcaller)( + {GitObjectToResolve(obj.root_tree_id, + n_target, + obj.pragma_special, + /*known_info=*/std::nullopt)}, + [setter](auto const& values) { + (*setter)(ResolvedGitObject{*values[0]}); + }, + logger); + } +} + +} // namespace + +auto CreateResolveSymlinksMap() -> ResolveSymlinksMap { + auto resolve_symlinks = [](auto /*unused*/, + auto setter, + auto logger, + auto subcaller, + auto const& key) { + // look up entry by its relative path + auto just_git_cas = GitCAS::Open(StorageConfig::GitRoot()); + if (not just_git_cas) { + (*logger)("ResolveSymlinks: could not open Git cache database!", + /*fatal=*/true); + return; + } + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)("ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto entry_info = key.known_info + ? key.known_info + : just_git_repo->GetObjectByPathFromTree( + key.root_tree_id, key.rel_path); + + // differentiate between existing path and non-existing + if (entry_info) { + ResolveKnownEntry( + key, *entry_info, just_git_cas, setter, logger, subcaller); + } + else { + // non-existing paths come from symlinks, so treat accordingly + // sanity check: pragma ignore special should not be set if here + if (key.pragma_special == PragmaSpecial::Ignore) { + (*logger)( + fmt::format("ResolveSymlinks: asked to ignore indirect " + "symlink path {} in root tree {}", + key.rel_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + auto parent_path = key.rel_path.parent_path(); + if (parent_path == key.rel_path) { + (*logger)(fmt::format("ResolveSymlinks: found unresolved path " + "{} in root tree {}", + key.rel_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + // resolve parent + (*subcaller)( + {GitObjectToResolve(key.root_tree_id, + parent_path, + key.pragma_special, + /*known_info=*/std::nullopt)}, + [key, + parent_path, + filename = key.rel_path.filename(), + just_git_cas, + setter, + logger, + subcaller](auto const& values) { + auto resolved_parent = *values[0]; + // parent must be a tree + if (not IsTreeObject(resolved_parent.type)) { + (*logger)( + fmt::format("ResolveSymlinks: path {} in root tree " + "{} failed to resolve to a tree", + parent_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + // check if filename exists in resolved parent tree + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)( + "ResolveSymlinks: could not open Git cache " + "repository!", + /*fatal=*/true); + return; + } + auto entry_info = just_git_repo->GetObjectByPathFromTree( + resolved_parent.id, filename); + if (entry_info) { + ResolveKnownEntry( + GitObjectToResolve(key.root_tree_id, + resolved_parent.path / filename, + key.pragma_special, + /*known_info=*/std::nullopt), + std::move(*entry_info), + just_git_cas, + setter, + logger, + subcaller); + } + else { + // report unresolvable + (*logger)( + fmt::format( + "ResolveSymlinks: reached unresolvable " + "path {} in root tree {}", + (resolved_parent.path / filename).string(), + key.root_tree_id), + /*fatal=*/true); + } + }, + logger); + } + }; + return AsyncMapConsumer( + resolve_symlinks); +} + +auto DetectAndReportCycle(ResolveSymlinksMap const& map, + std::string const& root_tree_id) + -> std::optional { + using namespace std::string_literals; + auto cycle = map.DetectCycle(); + if (cycle) { + bool found{false}; + std::ostringstream oss{}; + oss << fmt::format("Cycle detected for Git tree {}:", root_tree_id) + << std::endl; + for (auto const& k : *cycle) { + auto match = (k == cycle->back()); + auto prefix{match ? found ? "`-- "s : ".-> "s + : found ? "| "s + : " "s}; + oss << prefix << k.rel_path << std::endl; + found = found or match; + } + return oss.str(); + } + return std::nullopt; +} diff --git a/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp new file mode 100644 index 00000000..55254a30 --- /dev/null +++ b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp @@ -0,0 +1,98 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP + +#include +#include +#include + +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/utils/cpp/hash_combine.hpp" +#include "src/utils/cpp/path.hpp" +#include "src/utils/cpp/path_hash.hpp" + +/// \brief Information needed to resolve an object (blob or tree) given its +/// path relative to the path of a root tree in a given CAS. +struct GitObjectToResolve { + // hash of the root tree + std::string root_tree_id{}; /* key */ + // path of this object relative to root tree, in normal form + std::filesystem::path rel_path{"."}; /* key */ + // how the tree should be resolved + PragmaSpecial pragma_special{}; /* key */ + // sometimes the info of the object at the required path is already known, + // so leverage this to avoid extra work + std::optional known_info{std::nullopt}; + + GitObjectToResolve() = default; // needed for cycle detection only! + + GitObjectToResolve(std::string root_tree_id_, + std::filesystem::path const& rel_path_, + PragmaSpecial const& pragma_special_, + std::optional known_info_) + : root_tree_id{std::move(root_tree_id_)}, + rel_path{ToNormalPath(rel_path_)}, + pragma_special{pragma_special_}, + known_info{std::move(known_info_)} {}; + + [[nodiscard]] auto operator==( + GitObjectToResolve const& other) const noexcept -> bool { + return root_tree_id == other.root_tree_id and + rel_path == other.rel_path and + pragma_special == other.pragma_special; + } +}; + +/// \brief For a possibly initially unresolved path by the end we should be able +/// to know its hash, its type, and its now resolved path. +struct ResolvedGitObject { + std::string id; + ObjectType type; + std::filesystem::path path; +}; + +/// \brief Maps information about a Git object to its Git ID, type, and path as +/// part of a Git tree where symlinks have been resolved according to the given +/// pragma value. +/// Returns a nullopt only if called on a symlink with pragma ignore special. +/// \note Call the map with type Tree and path "." to resolve a Git tree. +using ResolveSymlinksMap = + AsyncMapConsumer; + +[[nodiscard]] auto CreateResolveSymlinksMap() -> ResolveSymlinksMap; + +[[nodiscard]] auto DetectAndReportCycle(ResolveSymlinksMap const& map, + std::string const& root_tree_id) + -> std::optional; + +namespace std { +template <> +struct hash { + [[nodiscard]] auto operator()(const GitObjectToResolve& ct) const noexcept + -> std::size_t { + size_t seed{}; + hash_combine(&seed, ct.root_tree_id); + hash_combine(&seed, ct.rel_path); + hash_combine(&seed, ct.pragma_special); + return seed; + } +}; +} // namespace std + +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index ee2a2e13..38a118cf 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -58,4 +58,22 @@ , "stage": ["src", "buildtool", "storage"] , "private-deps": [["src/buildtool/execution_api/remote", "config"]] } +, "fs_utils": + { "type": ["@", "rules", "CC", "library"] + , "name": ["fs_utils"] + , "hdrs": ["fs_utils.hpp"] + , "srcs": ["fs_utils.cpp"] + , "deps": + [ ["src/buildtool/common", "user_structs"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] + , ["src/utils/cpp", "tmp_dir"] + ] + , "stage": ["src", "buildtool", "storage"] + , "private-deps": + [ ["src/buildtool/file_system", "file_system_manager"] + , ["src/utils/cpp", "path"] + , "config" + , "storage" + ] + } } diff --git a/src/buildtool/storage/fs_utils.cpp b/src/buildtool/storage/fs_utils.cpp new file mode 100644 index 00000000..f2241fc9 --- /dev/null +++ b/src/buildtool/storage/fs_utils.cpp @@ -0,0 +1,114 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/storage/fs_utils.hpp" + +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/utils/cpp/path.hpp" + +namespace StorageUtils { + +auto GetGitRoot(LocalPathsPtr const& just_mr_paths, + std::string const& repo_url) noexcept -> std::filesystem::path { + if (just_mr_paths->git_checkout_locations.contains(repo_url)) { + return std::filesystem::absolute(ToNormalPath(std::filesystem::path{ + just_mr_paths->git_checkout_locations[repo_url] + .get()})); + } + auto repo_url_as_path = std::filesystem::absolute( + ToNormalPath(std::filesystem::path(repo_url))); + if (not repo_url_as_path.empty() and + FileSystemManager::IsAbsolutePath(repo_url_as_path) and + FileSystemManager::IsDirectory(repo_url_as_path)) { + return repo_url_as_path; + } + return StorageConfig::GitRoot(); +} + +auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { + // try to create parent dir + auto parent_path = + StorageConfig::GenerationCacheRoot(0) / "tmp-workspaces" / type; + return TmpDir::Create(parent_path); +} + +auto GetCommitTreeIDFile(std::string const& commit) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "commit-tree-map" / commit; +} + +auto GetArchiveTreeIDFile(std::string const& repo_type, + std::string const& content) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; +} + +auto GetDistdirTreeIDFile(std::string const& content) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; +} + +auto GetResolvedTreeIDFile(std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "special-tree-map" / + kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; +} + +auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, + std::string const& tree_id) noexcept -> bool { + // needs to be done safely, so use the rename trick + auto tmp_dir = TmpDir::Create(tree_id_file.parent_path()); + if (not tmp_dir) { + Logger::Log(LogLevel::Error, + "could not create tmp dir for writing tree id file {}", + tree_id_file.string()); + return false; + } + std::filesystem::path tmp_file{tmp_dir->GetPath() / "tmp_file"}; + if (not FileSystemManager::WriteFile(tree_id, tmp_file)) { + Logger::Log(LogLevel::Error, "could not create tmp tree id file"); + return false; + } + return FileSystemManager::Rename(tmp_file.string(), tree_id_file); +} + +auto AddToCAS(std::string const& data) noexcept + -> std::optional { + // get file CAS instance + auto const& cas = Storage::Instance().CAS(); + // write to cas + auto digest = cas.StoreBlob(data); + if (digest) { + return cas.BlobPath(*digest, /*is_executable=*/false); + } + return std::nullopt; +} + +void AddDistfileToCAS(std::filesystem::path const& distfile, + LocalPathsPtr const& just_mr_paths) noexcept { + auto const& cas = Storage::Instance().CAS(); + for (auto const& dirpath : just_mr_paths->distdirs) { + auto candidate = dirpath / distfile; + if (FileSystemManager::Exists(candidate)) { + // try to add to CAS + [[maybe_unused]] auto digest = + cas.StoreBlob(candidate, /*is_executable=*/false); + } + } +} + +} // namespace StorageUtils diff --git a/src/buildtool/storage/fs_utils.hpp b/src/buildtool/storage/fs_utils.hpp new file mode 100644 index 00000000..ba66dac5 --- /dev/null +++ b/src/buildtool/storage/fs_utils.hpp @@ -0,0 +1,77 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP + +#include +#include +#include + +#include "src/buildtool/common/user_structs.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/utils/cpp/tmp_dir.hpp" + +/* Utilities related to CAS and paths therein */ + +namespace StorageUtils { + +/// \brief Get location of Git repository. Defaults to the Git cache root when +/// no better location is found. +[[nodiscard]] auto GetGitRoot(LocalPathsPtr const& just_mr_paths, + std::string const& repo_url) noexcept + -> std::filesystem::path; + +/// \brief Create a tmp directory with controlled lifetime for specific +/// operations (archive, zip, file, distdir checkouts; fetch; update). +[[nodiscard]] auto CreateTypedTmpDir(std::string const& type) noexcept + -> TmpDirPtr; + +/// \brief Get the path to the file storing the tree id associated with +/// a given commit. +[[nodiscard]] auto GetCommitTreeIDFile(std::string const& commit) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing the tree id of an archive +/// content. +[[nodiscard]] auto GetArchiveTreeIDFile(std::string const& repo_type, + std::string const& content) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing the tree id of a distdir list +/// content. +[[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing a resolved tree hash. +[[nodiscard]] auto GetResolvedTreeIDFile( + std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; + +/// \brief Write a tree id to file. The parent folder of the file must exist! +[[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, + std::string const& tree_id) noexcept -> bool; + +/// \brief Add data to file CAS. +/// Returns the path to the file added to CAS, or nullopt if not added. +[[nodiscard]] auto AddToCAS(std::string const& data) noexcept + -> std::optional; + +/// \brief Try to add distfile to CAS. +void AddDistfileToCAS(std::filesystem::path const& distfile, + LocalPathsPtr const& just_mr_paths) noexcept; + +} // namespace StorageUtils + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index a64f4c3c..a3ed87c9 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -41,17 +41,12 @@ , "hdrs": ["utils.hpp"] , "srcs": ["utils.cpp"] , "deps": - [ ["src/utils/cpp", "tmp_dir"] + [ ["@", "gsl", "", "gsl"] + , ["@", "json", "", "json"] , ["src/buildtool/storage", "config"] - , ["src/buildtool/main", "constants"] , ["src/buildtool/build_engine/expression", "expression"] ] , "stage": ["src", "other_tools", "just_mr"] - , "private-deps": - [ ["src/utils/cpp", "path"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "file_storage"] - ] } , "exit_codes": { "type": ["@", "rules", "CC", "library"] @@ -69,6 +64,7 @@ , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "clidefaults"] + , ["src/buildtool/common", "user_structs"] , ["src/buildtool/execution_api/local", "config"] , ["src/other_tools/just_mr", "utils"] , ["src/buildtool/logging", "log_level"] @@ -132,6 +128,7 @@ [ ["@", "json", "", "json"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_repo_remote"] , "exit_codes" , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -151,6 +148,7 @@ [ ["@", "json", "", "json"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] + , ["src/buildtool/storage", "fs_utils"] , "exit_codes" , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] @@ -162,7 +160,7 @@ , ["src/other_tools/root_maps", "distdir_git_map"] , ["src/other_tools/root_maps", "fpath_git_map"] , ["src/other_tools/root_maps", "tree_id_git_map"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] , "setup_utils" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "local"] diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp index 4c0c6e34..adc72578 100644 --- a/src/other_tools/just_mr/cli.hpp +++ b/src/other_tools/just_mr/cli.hpp @@ -26,6 +26,7 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/clidefaults.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/other_tools/just_mr/utils.hpp" @@ -36,9 +37,9 @@ struct MultiRepoCommonArguments { std::optional absent_repository_file{std::nullopt}; std::optional checkout_locations_file{std::nullopt}; std::vector explicit_distdirs{}; - JustMR::PathsPtr just_mr_paths = std::make_shared(); + LocalPathsPtr just_mr_paths = std::make_shared(); std::optional> local_launcher{std::nullopt}; - JustMR::CAInfoPtr ca_info = std::make_shared(); + CAInfoPtr ca_info = std::make_shared(); std::optional just_path{std::nullopt}; std::optional main{std::nullopt}; std::optional rc_path{std::nullopt}; diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 54295c6b..5fb5512f 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -19,9 +19,11 @@ #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -34,7 +36,6 @@ #include "src/other_tools/root_maps/distdir_git_map.hpp" #include "src/other_tools/root_maps/fpath_git_map.hpp" #include "src/other_tools/root_maps/tree_id_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" auto MultiRepoSetup(std::shared_ptr const& config, MultiRepoCommonArguments const& common_args, @@ -227,5 +228,5 @@ auto MultiRepoSetup(std::shared_ptr const& config, return std::nullopt; } // if successful, return the output config - return JustMR::Utils::AddToCAS(mr_config.dump(2)); + return StorageUtils::AddToCAS(mr_config.dump(2)); } diff --git a/src/other_tools/just_mr/update.cpp b/src/other_tools/just_mr/update.cpp index 435827d5..babfee5a 100644 --- a/src/other_tools/just_mr/update.cpp +++ b/src/other_tools/just_mr/update.cpp @@ -20,6 +20,7 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -147,7 +148,7 @@ auto MultiRepoUpdate(std::shared_ptr const& config, } } // Create fake repo for the anonymous remotes - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("update"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("update"); if (not tmp_dir) { Logger::Log(LogLevel::Error, "Failed to create commit update tmp dir"); return kExitUpdateError; diff --git a/src/other_tools/just_mr/utils.cpp b/src/other_tools/just_mr/utils.cpp index be0e841e..7e99eb43 100644 --- a/src/other_tools/just_mr/utils.cpp +++ b/src/other_tools/just_mr/utils.cpp @@ -14,102 +14,8 @@ #include "src/other_tools/just_mr/utils.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/storage/storage.hpp" -#include "src/utils/cpp/path.hpp" - namespace JustMR::Utils { -auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, - std::string const& repo_url) noexcept -> std::filesystem::path { - if (just_mr_paths->git_checkout_locations.contains(repo_url)) { - return std::filesystem::absolute(ToNormalPath(std::filesystem::path{ - just_mr_paths->git_checkout_locations[repo_url] - .get()})); - } - auto repo_url_as_path = std::filesystem::absolute( - ToNormalPath(std::filesystem::path(repo_url))); - if (not repo_url_as_path.empty() and - FileSystemManager::IsAbsolutePath(repo_url_as_path) and - FileSystemManager::IsDirectory(repo_url_as_path)) { - return repo_url_as_path; - } - return StorageConfig::GitRoot(); -} - -auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { - // try to create parent dir - auto parent_path = - StorageConfig::GenerationCacheRoot(0) / "tmp-workspaces" / type; - return TmpDir::Create(parent_path); -} - -auto GetCommitTreeIDFile(std::string const& commit) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "commit-tree-map" / commit; -} - -auto GetArchiveTreeIDFile(std::string const& repo_type, - std::string const& content) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; -} - -auto GetDistdirTreeIDFile(std::string const& content) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; -} - -auto GetResolvedTreeIDFile(std::string const& tree_hash, - PragmaSpecial const& pragma_special) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "special-tree-map" / - kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; -} - -auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, - std::string const& tree_id) noexcept -> bool { - // needs to be done safely, so use the rename trick - auto tmp_dir = TmpDir::Create(tree_id_file.parent_path()); - if (not tmp_dir) { - Logger::Log(LogLevel::Error, - "could not create tmp dir for writing tree id file {}", - tree_id_file.string()); - return false; - } - std::filesystem::path tmp_file{tmp_dir->GetPath() / "tmp_file"}; - if (not FileSystemManager::WriteFile(tree_id, tmp_file)) { - Logger::Log(LogLevel::Error, "could not create tmp tree id file"); - return false; - } - return FileSystemManager::Rename(tmp_file.string(), tree_id_file); -} - -auto AddToCAS(std::string const& data) noexcept - -> std::optional { - // get file CAS instance - auto const& cas = Storage::Instance().CAS(); - // write to cas - auto digest = cas.StoreBlob(data); - if (digest) { - return cas.BlobPath(*digest, /*is_executable=*/false); - } - return std::nullopt; -} - -void AddDistfileToCAS(std::filesystem::path const& distfile, - JustMR::PathsPtr const& just_mr_paths) noexcept { - auto const& cas = Storage::Instance().CAS(); - for (auto const& dirpath : just_mr_paths->distdirs) { - auto candidate = dirpath / distfile; - if (FileSystemManager::Exists(candidate)) { - // try to add to CAS - [[maybe_unused]] auto digest = - cas.StoreBlob(candidate, /*is_executable=*/false); - } - } -} - // NOLINTNEXTLINE(misc-no-recursion) auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos, diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 304cc839..3aff2348 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -15,13 +15,16 @@ #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP +#include +#include +#include #include +#include +#include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/main/constants.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/utils/cpp/tmp_dir.hpp" /* Paths and constants required by just-mr */ @@ -135,115 +138,7 @@ std::unordered_map const kCheckoutTypeMap = { {"file", CheckoutType::File}, {"distdir", CheckoutType::Distdir}, {"git tree", CheckoutType::GitTree}}; - -/// \brief Pragma "special" value enum -enum class PragmaSpecial : std::uint8_t { - Ignore, - ResolvePartially, - ResolveCompletely -}; - -/// \brief Pragma "special" value map -std::unordered_map const kPragmaSpecialMap = { - {"ignore", PragmaSpecial::Ignore}, - {"resolve-partially", PragmaSpecial::ResolvePartially}, - {"resolve-completely", PragmaSpecial::ResolveCompletely}}; - -/// \brief Pragma "special" value inverse map, from enum to string -std::unordered_map const kPragmaSpecialInverseMap = - {{PragmaSpecial::Ignore, "ignore"}, - {PragmaSpecial::ResolvePartially, "resolve-partially"}, - {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; - -namespace JustMR { - -struct Paths { - // user-defined locations - std::optional root{std::nullopt}; - std::filesystem::path setup_root{FileSystemManager::GetCurrentDirectory()}; - std::optional workspace_root{ - // find workspace root - []() -> std::optional { - std::function - is_workspace_root = [&](std::filesystem::path const& path) { - return std::any_of( - kRootMarkers.begin(), - kRootMarkers.end(), - [&path](auto const& marker) { - return FileSystemManager::Exists(path / marker); - }); - }; - // default path to current dir - auto path = FileSystemManager::GetCurrentDirectory(); - auto root_path = path.root_path(); - while (true) { - if (is_workspace_root(path)) { - return path; - } - if (path == root_path) { - return std::nullopt; - } - path = path.parent_path(); - } - }()}; - nlohmann::json git_checkout_locations{}; - std::vector distdirs{}; -}; - -struct CAInfo { - bool no_ssl_verify{false}; - std::optional ca_bundle{std::nullopt}; -}; - -using PathsPtr = std::shared_ptr; -using CAInfoPtr = std::shared_ptr; - -namespace Utils { - -/// \brief Get location of Git repository. Defaults to the Git cache root when -/// no better location is found. -[[nodiscard]] auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, - std::string const& repo_url) noexcept - -> std::filesystem::path; - -/// \brief Create a tmp directory with controlled lifetime for specific -/// operations (archive, zip, file, distdir checkouts; fetch; update). -[[nodiscard]] auto CreateTypedTmpDir(std::string const& type) noexcept - -> TmpDirPtr; - -/// \brief Get the path to the file storing the tree id associated with -/// a given commit. -[[nodiscard]] auto GetCommitTreeIDFile(std::string const& commit) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing the tree id of an archive -/// content. -[[nodiscard]] auto GetArchiveTreeIDFile(std::string const& repo_type, - std::string const& content) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing the tree id of a distdir list -/// content. -[[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing a resolved tree hash. -[[nodiscard]] auto GetResolvedTreeIDFile( - std::string const& tree_hash, - PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; - -/// \brief Write a tree id to file. The parent folder of the file must exist! -[[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, - std::string const& tree_id) noexcept -> bool; - -/// \brief Add data to file CAS. -/// Returns the path to the file added to CAS, or nullopt if not added. -[[nodiscard]] auto AddToCAS(std::string const& data) noexcept - -> std::optional; - -/// \brief Try to add distfile to CAS. -void AddDistfileToCAS(std::filesystem::path const& distfile, - JustMR::PathsPtr const& just_mr_paths) noexcept; +namespace JustMR::Utils { /// \brief Recursive part of the ResolveRepo function. /// Keeps track of repository names to detect any cyclic dependencies. @@ -260,8 +155,6 @@ auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos) noexcept -> std::optional; -} // namespace Utils - -} // namespace JustMR +} // namespace JustMR::Utils #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index a2121e33..9ca5e5f4 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -25,7 +25,8 @@ ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["src/buildtool/storage", "fs_utils"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "config"] ] @@ -39,12 +40,12 @@ [ ["src/other_tools/git_operations", "git_repo_remote"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] - , ["@", "fmt", "", "fmt"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["@", "fmt", "", "fmt"] , ["src/buildtool/execution_api/local", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/utils/cpp", "tmp_dir"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -56,7 +57,8 @@ , "hdrs": ["content_cas_map.hpp"] , "srcs": ["content_cas_map.cpp"] , "deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["src/buildtool/common", "user_structs"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] @@ -68,6 +70,7 @@ , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] ] diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index da37df4d..b796dc04 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -16,6 +16,7 @@ #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -26,7 +27,7 @@ namespace { /// \brief Fetches a file from the internet and stores its content in memory. /// Returns the content. [[nodiscard]] auto NetworkFetch(std::string const& fetch_url, - JustMR::CAInfoPtr const& ca_info) noexcept + CAInfoPtr const& ca_info) noexcept -> std::optional { auto curl_handle = CurlEasyHandle::Create(ca_info->no_ssl_verify, ca_info->ca_bundle); @@ -47,8 +48,8 @@ template } // namespace -auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, - JustMR::CAInfoPtr const& ca_info, +auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths, + CAInfoPtr const& ca_info, IExecutionApi* local_api, IExecutionApi* remote_api, std::size_t jobs) -> ContentCASMap { @@ -71,7 +72,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, (key.distfile ? key.distfile.value() : std::filesystem::path(key.fetch_url).filename().string()); - JustMR::Utils::AddDistfileToCAS(repo_distfile, just_mr_paths); + StorageUtils::AddDistfileToCAS(repo_distfile, just_mr_paths); // check if content is in CAS now if (cas.BlobPath(digest, /*is_executable=*/false)) { JustMRProgress::Instance().TaskTracker().Stop(key.origin); @@ -132,8 +133,8 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, } } // add the fetched data to CAS - auto path = JustMR::Utils::AddToCAS(*data); - // check that storing the fetched data succeeded + auto path = StorageUtils::AddToCAS(*data); + // check one last time if content is in CAS now if (not path) { (*logger)(fmt::format("Failed to store fetched content from {}", key.fetch_url), diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp index 859fbe6b..846b2762 100644 --- a/src/other_tools/ops_maps/content_cas_map.hpp +++ b/src/other_tools/ops_maps/content_cas_map.hpp @@ -19,9 +19,10 @@ #include #include "nlohmann/json.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/hash_combine.hpp" struct ArchiveContent { @@ -61,8 +62,8 @@ struct ArchiveRepoInfo { /// \brief Maps the content hash of an archive to an "exists" status flag. using ContentCASMap = AsyncMapConsumer; -[[nodiscard]] auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, - JustMR::CAInfoPtr const& ca_info, +[[nodiscard]] auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths, + CAInfoPtr const& ca_info, IExecutionApi* local_api, IExecutionApi* remote_api, std::size_t jobs) -> ContentCASMap; diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp index cc40aef6..157d13bb 100644 --- a/src/other_tools/ops_maps/git_update_map.cpp +++ b/src/other_tools/ops_maps/git_update_map.cpp @@ -16,9 +16,9 @@ #include "fmt/core.h" #include "src/buildtool/execution_api/local/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto CreateGitUpdateMap(GitCASPtr const& git_cas, @@ -39,7 +39,7 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, /*fatal=*/true); return; } - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("update"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("update"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create commit update tmp dir for " "remote {}", diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp index e8b7a89d..28365866 100644 --- a/src/other_tools/ops_maps/import_to_git_map.cpp +++ b/src/other_tools/ops_maps/import_to_git_map.cpp @@ -18,7 +18,7 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/other_tools/just_mr/utils.hpp" +#include "src/buildtool/storage/fs_utils.hpp" namespace { @@ -167,7 +167,7 @@ auto CreateImportToGitMap( } // create tmp directory auto tmp_dir = - JustMR::Utils::CreateTypedTmpDir("import-to-git"); + StorageUtils::CreateTypedTmpDir("import-to-git"); if (not tmp_dir) { (*logger)( fmt::format("Could not create unique path " diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS index b91730b9..c397d82c 100644 --- a/src/other_tools/repo_map/TARGETS +++ b/src/other_tools/repo_map/TARGETS @@ -16,6 +16,7 @@ [ ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] ] } } diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index f51e48ec..7de7e2ff 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -15,6 +15,7 @@ #include "src/other_tools/repo_map/repos_to_setup_map.hpp" #include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 2b5fbd1a..656e1fab 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -15,6 +15,7 @@ , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "tmp_dir"] , ["src/buildtool/file_system", "file_storage"] @@ -31,7 +32,7 @@ , "deps": [ ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/execution_api/common", "common"] - , ["src/other_tools/just_mr", "utils"] + , ["src/buildtool/common", "user_structs"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] @@ -44,6 +45,8 @@ , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] ] } , "fpath_git_map": @@ -57,7 +60,8 @@ , ["src/utils/cpp", "path_hash"] , ["src/utils/cpp", "hash_combine"] , ["src/other_tools/just_mr", "utils"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -67,6 +71,8 @@ , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_repo"] , ["src/other_tools/git_operations", "git_repo_remote"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] + , ["src/buildtool/storage", "fs_utils"] ] } , "content_git_map": @@ -77,15 +83,17 @@ , "deps": [ ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["src/utils/archive", "archive_ops"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] @@ -109,9 +117,9 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/system", "system_command"] , ["src/other_tools/git_operations", "git_repo_remote"] - , ["src/other_tools/just_mr", "utils"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/ops_maps", "content_cas_map"] diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index 9798eb46..f4ee1c17 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -17,6 +17,8 @@ #include #include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -44,7 +46,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // write association of the root tree in id file - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, root_tree_id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, root_tree_id)) { (*logger)(fmt::format("Failed to write tree id {} to file {}", root_tree_id, tree_id_file.string()), @@ -118,7 +120,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (not is_commit_present.value()) { if (repo_info.absent) { auto tree_id_file = - JustMR::Utils::GetCommitTreeIDFile(repo_info.hash); + StorageUtils::GetCommitTreeIDFile(repo_info.hash); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = @@ -223,7 +225,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, JustMRProgress::Instance().TaskTracker().Stop( repo_info.origin); // Move tree from CAS to local git storage - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir( + auto tmp_dir = StorageUtils::CreateTypedTmpDir( "fetch-absent-root"); if (not tmp_dir) { (*logger)( @@ -346,7 +348,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, } } // default to fetching it from network - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("fetch"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("fetch"); if (not tmp_dir) { (*logger)("Failed to create fetch tmp directory!", /*fatal=*/true); @@ -485,7 +487,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, auto CreateCommitGitMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, - JustMR::PathsPtr const& just_mr_paths, + LocalPathsPtr const& just_mr_paths, std::string const& git_bin, std::vector const& launcher, ServeApi* serve_api, @@ -513,7 +515,7 @@ auto CreateCommitGitMap( fetch_repo = std::filesystem::absolute(fetch_repo).string(); } std::filesystem::path repo_root = - JustMR::Utils::GetGitRoot(just_mr_paths, fetch_repo); + StorageUtils::GetGitRoot(just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index 4f59a56f..26ed018b 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -19,9 +19,9 @@ #include #include "nlohmann/json.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -69,7 +69,7 @@ using CommitGitMap = [[nodiscard]] auto CreateCommitGitMap( gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, - JustMR::PathsPtr const& just_mr_paths, + LocalPathsPtr const& just_mr_paths, std::string const& git_bin, std::vector const& launcher, ServeApi* serve_api, diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index c6e0809b..a82670ea 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -16,7 +16,9 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -54,7 +56,7 @@ void ResolveContentTree( if (pragma_special) { // get the resolved tree auto tree_id_file = - JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + StorageUtils::GetResolvedTreeIDFile(tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -108,8 +110,8 @@ void ResolveContentTree( } auto const& resolved_tree = *hashes[0]; // cache the resolved tree in the CAS map - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, - resolved_tree.id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { (*logger)(fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), @@ -160,7 +162,7 @@ auto CreateContentGitMap( auto logger, auto /* unused */, auto const& key) { - auto archive_tree_id_file = JustMR::Utils::GetArchiveTreeIDFile( + auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( key.repo_type, key.archive.content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file @@ -264,7 +266,7 @@ auto CreateContentGitMap( logger]([[maybe_unused]] auto const& values) { // content is in CAS // extract archive - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir(repo_type); + auto tmp_dir = StorageUtils::CreateTypedTmpDir(repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "{} target {}", @@ -314,7 +316,7 @@ auto CreateContentGitMap( // only tree id is needed std::string archive_tree_id = values[0]->first; // write to tree id file - if (not JustMR::Utils::WriteTreeIDFile( + if (not StorageUtils::WriteTreeIDFile( archive_tree_id_file, archive_tree_id)) { (*logger)( fmt::format("Failed to write tree id " diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index 6eed8448..db0950f0 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -17,9 +17,9 @@ #include +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" /// \brief Maps the content of an archive to the resulting Git tree WS root, /// together with the information whether it was a cache hit. diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index a132cf47..8f7bd98f 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -20,6 +20,7 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -65,7 +66,7 @@ auto CreateDistdirGitMap( auto /* unused */, auto const& key) { auto distdir_tree_id_file = - JustMR::Utils::GetDistdirTreeIDFile(key.content_id); + StorageUtils::GetDistdirTreeIDFile(key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = @@ -135,7 +136,7 @@ auto CreateDistdirGitMap( logger]([[maybe_unused]] auto const& values) mutable { // repos are in CAS // create the links to CAS - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", @@ -171,7 +172,7 @@ auto CreateDistdirGitMap( // only the tree is of interest std::string distdir_tree_id = values[0]->first; // write to tree id file - if (not JustMR::Utils::WriteTreeIDFile( + if (not StorageUtils::WriteTreeIDFile( distdir_tree_id_file, distdir_tree_id)) { (*logger)( fmt::format( diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 563876ff..9f545856 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -18,6 +18,7 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/tmp_dir.hpp" @@ -36,7 +37,7 @@ void ResolveFilePathTree( if (pragma_special) { // get the resolved tree auto tree_id_file = - JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + StorageUtils::GetResolvedTreeIDFile(tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -90,8 +91,8 @@ void ResolveFilePathTree( } auto const& resolved_tree = *hashes[0]; // cache the resolved tree in the CAS map - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, - resolved_tree.id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { (*logger)(fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), @@ -255,7 +256,7 @@ auto CreateFilePathGitMap( /*fatal=*/false); } // it's not a git repo, so import it to git cache - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("file"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 6063aa8d..139921b7 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -16,9 +16,10 @@ #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP #include "nlohmann/json.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 34da3121..a8db186a 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -19,11 +19,11 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -#include "src/other_tools/just_mr/utils.hpp" namespace { @@ -145,7 +145,7 @@ auto CreateTreeIdGitMap( if (not *tree_found) { JustMRProgress::Instance().TaskTracker().Start(key.origin); // create temporary location for command execution root - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("git-tree"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("git-tree"); if (not tmp_dir) { (*logger)( "Failed to create tmp directory for tree id map!", @@ -154,7 +154,7 @@ auto CreateTreeIdGitMap( } // create temporary location for storing command result // files - auto out_dir = JustMR::Utils::CreateTypedTmpDir("git-tree"); + auto out_dir = StorageUtils::CreateTypedTmpDir("git-tree"); if (not out_dir) { (*logger)( "Failed to create tmp directory for tree id map!", @@ -282,7 +282,7 @@ auto CreateTreeIdGitMap( } // define temp repo path auto tmp_dir = - JustMR::Utils::CreateTypedTmpDir("git-tree"); + StorageUtils::CreateTypedTmpDir("git-tree"); ; if (not tmp_dir) { (*logger)(fmt::format("Could not create unique " diff --git a/src/other_tools/symlinks_map/TARGETS b/src/other_tools/symlinks_map/TARGETS deleted file mode 100644 index 225000f5..00000000 --- a/src/other_tools/symlinks_map/TARGETS +++ /dev/null @@ -1,18 +0,0 @@ -{ "resolve_symlinks_map": - { "type": ["@", "rules", "CC", "library"] - , "name": ["resolve_symlinks_map"] - , "hdrs": ["resolve_symlinks_map.hpp"] - , "srcs": ["resolve_symlinks_map.cpp"] - , "deps": - [ ["src/buildtool/file_system", "git_repo"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/multithreading", "async_map_consumer"] - , ["src/other_tools/just_mr", "utils"] - , ["src/utils/cpp", "path"] - , ["src/utils/cpp", "path_hash"] - ] - , "stage": ["src", "other_tools", "symlinks_map"] - , "private-deps": - [["@", "fmt", "", "fmt"], ["src/buildtool/storage", "config"]] - } -} diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.cpp b/src/other_tools/symlinks_map/resolve_symlinks_map.cpp deleted file mode 100644 index 88d09aec..00000000 --- a/src/other_tools/symlinks_map/resolve_symlinks_map.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" - -#include "fmt/core.h" -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/storage/config.hpp" - -namespace { - -void ResolveKnownEntry(GitObjectToResolve const& obj, - GitRepo::TreeEntryInfo const& entry_info, - GitCASPtr const& just_git_cas, - ResolveSymlinksMap::SetterPtr const& setter, - ResolveSymlinksMap::LoggerPtr const& logger, - ResolveSymlinksMap::SubCallerPtr const& subcaller) { - // differentiated treatment based on object type - if (IsFileObject(entry_info.type)) { - // files are already resolved, so return the hash directly - (*setter)(ResolvedGitObject{.id = entry_info.id, - .type = entry_info.type, - .path = obj.rel_path}); - } - else if (IsTreeObject(entry_info.type)) { - // for tree types we resolve by rebuilding the tree from the - // resolved children - auto just_git_repo = GitRepo::Open(just_git_cas); - if (not just_git_repo) { - (*logger)("ResolveSymlinks: could not open Git cache repository!", - /*fatal=*/true); - return; - } - auto children = just_git_repo->ReadTree( - entry_info.id, - [](std::vector const& /*unused*/) { - return true; - }, - /*is_hex_id=*/true); - if (not children) { - (*logger)(fmt::format("ResolveSymlinks: failed to read entries of " - "subtree {} in root tree {}", - entry_info.id, - obj.root_tree_id), - /*fatal=*/true); - return; - } - // resolve children - std::vector children_info{}; - children_info.reserve(children->size()); - for (auto const& [raw_id, ev] : *children) { - for (auto const& e : ev) { - // must enforce ignore special at the tree level! - if (IsNonSpecialObject(e.type) or - obj.pragma_special != PragmaSpecial::Ignore) { - // children info is known, so pass this forward - if (IsSymlinkObject(e.type)) { - if (auto target = just_git_cas->ReadObject(raw_id)) { - children_info.emplace_back( - obj.root_tree_id, - obj.rel_path / e.name, - obj.pragma_special, - std::make_optional(GitRepo::TreeEntryInfo{ - .id = ToHexString(raw_id), - .type = e.type, - .symlink_content = *target})); - } - else { - (*logger)( - fmt::format("ResolveSymlinks: could not read " - "symlink {} in root tree {}", - (obj.rel_path / e.name).string(), - obj.root_tree_id), - /*fatal=*/true); - return; - } - } - else { - children_info.emplace_back( - obj.root_tree_id, - obj.rel_path / e.name, - obj.pragma_special, - GitRepo::TreeEntryInfo{ - .id = ToHexString(raw_id), - .type = e.type, - .symlink_content = std::nullopt}); - } - } - } - } - (*subcaller)( - children_info, - [children_info, parent = obj, just_git_cas, setter, logger]( - auto const& resolved_entries) { - // create the entries map of the children - GitRepo::tree_entries_t entries{}; - auto num = resolved_entries.size(); - entries.reserve(num); - for (auto i = 0; i < num; ++i) { - auto const& p = children_info[i].rel_path; - entries[*FromHexString(resolved_entries[i]->id)] - .emplace_back( - p.filename().string(), // we only need the name - resolved_entries[i]->type); - } - // create the tree inside our Git CAS, which is already - // existing by this point. Also, this operation is - // guarded internally, so no need for the - // critical_git_op map - auto just_git_repo = GitRepo::Open(just_git_cas); - if (not just_git_repo) { - (*logger)( - "ResolveSymlinks: could not open Git cache repository!", - /*fatal=*/true); - return; - } - auto tree_raw_id = just_git_repo->CreateTree(entries); - if (not tree_raw_id) { - (*logger)(fmt::format("ResolveSymlinks: failed to create " - "resolved tree {} in root tree {}", - parent.rel_path.string(), - parent.root_tree_id), - /*fatal=*/true); - return; - } - // set the resolved tree hash - (*setter)(ResolvedGitObject{.id = ToHexString(*tree_raw_id), - .type = ObjectType::Tree, - .path = parent.rel_path}); - }, - logger); - } - else { - // sanity check: cannot resolve a symlink called with ignore - // special, as that can only be handled by the parent tree - if (obj.pragma_special == PragmaSpecial::Ignore) { - (*logger)(fmt::format("ResolveSymlinks: asked to ignore symlink {} " - "in root tree {}", - obj.rel_path.string(), - obj.root_tree_id), - /*fatal=*/true); - return; - } - // target should have already been read - if (not entry_info.symlink_content) { - (*logger)(fmt::format("ResolveSymlinks: missing target of symlink " - "{} in root tree {}", - obj.rel_path.string(), - obj.root_tree_id), - /*fatal=*/true); - return; - } - // check if link target (unresolved) is confined to the tree - if (not PathIsConfined(*entry_info.symlink_content, obj.rel_path)) { - (*logger)(fmt::format("ResolveSymlinks: symlink {} is not confined " - "to tree {}", - obj.rel_path.string(), - obj.root_tree_id), - /*fatal=*/true); - return; - } - // if partially resolved, return non-upwards symlinks as-is - if (obj.pragma_special == PragmaSpecial::ResolvePartially and - PathIsNonUpwards(*entry_info.symlink_content)) { - // return as symlink object - (*setter)(ResolvedGitObject{.id = entry_info.id, - .type = ObjectType::Symlink, - .path = obj.rel_path}); - return; - } - // resolve the target - auto n_target = ToNormalPath(obj.rel_path.parent_path() / - *entry_info.symlink_content); - (*subcaller)( - {GitObjectToResolve(obj.root_tree_id, - n_target, - obj.pragma_special, - /*known_info=*/std::nullopt)}, - [setter](auto const& values) { - (*setter)(ResolvedGitObject{*values[0]}); - }, - logger); - } -} - -} // namespace - -auto CreateResolveSymlinksMap() -> ResolveSymlinksMap { - auto resolve_symlinks = [](auto /*unused*/, - auto setter, - auto logger, - auto subcaller, - auto const& key) { - // look up entry by its relative path - auto just_git_cas = GitCAS::Open(StorageConfig::GitRoot()); - if (not just_git_cas) { - (*logger)("ResolveSymlinks: could not open Git cache database!", - /*fatal=*/true); - return; - } - auto just_git_repo = GitRepo::Open(just_git_cas); - if (not just_git_repo) { - (*logger)("ResolveSymlinks: could not open Git cache repository!", - /*fatal=*/true); - return; - } - auto entry_info = key.known_info - ? key.known_info - : just_git_repo->GetObjectByPathFromTree( - key.root_tree_id, key.rel_path); - - // differentiate between existing path and non-existing - if (entry_info) { - ResolveKnownEntry( - key, *entry_info, just_git_cas, setter, logger, subcaller); - } - else { - // non-existing paths come from symlinks, so treat accordingly - // sanity check: pragma ignore special should not be set if here - if (key.pragma_special == PragmaSpecial::Ignore) { - (*logger)( - fmt::format("ResolveSymlinks: asked to ignore indirect " - "symlink path {} in root tree {}", - key.rel_path.string(), - key.root_tree_id), - /*fatal=*/true); - return; - } - auto parent_path = key.rel_path.parent_path(); - if (parent_path == key.rel_path) { - (*logger)(fmt::format("ResolveSymlinks: found unresolved path " - "{} in root tree {}", - key.rel_path.string(), - key.root_tree_id), - /*fatal=*/true); - return; - } - // resolve parent - (*subcaller)( - {GitObjectToResolve(key.root_tree_id, - parent_path, - key.pragma_special, - /*known_info=*/std::nullopt)}, - [key, - parent_path, - filename = key.rel_path.filename(), - just_git_cas, - setter, - logger, - subcaller](auto const& values) { - auto resolved_parent = *values[0]; - // parent must be a tree - if (not IsTreeObject(resolved_parent.type)) { - (*logger)( - fmt::format("ResolveSymlinks: path {} in root tree " - "{} failed to resolve to a tree", - parent_path.string(), - key.root_tree_id), - /*fatal=*/true); - return; - } - // check if filename exists in resolved parent tree - auto just_git_repo = GitRepo::Open(just_git_cas); - if (not just_git_repo) { - (*logger)( - "ResolveSymlinks: could not open Git cache " - "repository!", - /*fatal=*/true); - return; - } - auto entry_info = just_git_repo->GetObjectByPathFromTree( - resolved_parent.id, filename); - if (entry_info) { - ResolveKnownEntry( - GitObjectToResolve(key.root_tree_id, - resolved_parent.path / filename, - key.pragma_special, - /*known_info=*/std::nullopt), - std::move(*entry_info), - just_git_cas, - setter, - logger, - subcaller); - } - else { - // report unresolvable - (*logger)( - fmt::format( - "ResolveSymlinks: reached unresolvable " - "path {} in root tree {}", - (resolved_parent.path / filename).string(), - key.root_tree_id), - /*fatal=*/true); - } - }, - logger); - } - }; - return AsyncMapConsumer( - resolve_symlinks); -} - -auto DetectAndReportCycle(ResolveSymlinksMap const& map, - std::string const& root_tree_id) - -> std::optional { - using namespace std::string_literals; - auto cycle = map.DetectCycle(); - if (cycle) { - bool found{false}; - std::ostringstream oss{}; - oss << fmt::format("Cycle detected for Git tree {}:", root_tree_id) - << std::endl; - for (auto const& k : *cycle) { - auto match = (k == cycle->back()); - auto prefix{match ? found ? "`-- "s : ".-> "s - : found ? "| "s - : " "s}; - oss << prefix << k.rel_path << std::endl; - found = found or match; - } - return oss.str(); - } - return std::nullopt; -} diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.hpp b/src/other_tools/symlinks_map/resolve_symlinks_map.hpp deleted file mode 100644 index 8bae7583..00000000 --- a/src/other_tools/symlinks_map/resolve_symlinks_map.hpp +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP -#define INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP - -#include -#include -#include - -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/multithreading/async_map_consumer.hpp" -#include "src/other_tools/just_mr/utils.hpp" -#include "src/utils/cpp/path.hpp" -#include "src/utils/cpp/path_hash.hpp" - -/// \brief Information needed to resolve an object (blob or tree) given its -/// path relative to the path of a root tree in a given CAS. -struct GitObjectToResolve { - // hash of the root tree - std::string root_tree_id{}; /* key */ - // path of this object relative to root tree, in normal form - std::filesystem::path rel_path{"."}; /* key */ - // how the tree should be resolved - PragmaSpecial pragma_special{}; /* key */ - // sometimes the info of the object at the required path is already known, - // so leverage this to avoid extra work - std::optional known_info{std::nullopt}; - - GitObjectToResolve() = default; // needed for cycle detection only! - - GitObjectToResolve(std::string root_tree_id_, - std::filesystem::path const& rel_path_, - PragmaSpecial const& pragma_special_, - std::optional known_info_) - : root_tree_id{std::move(root_tree_id_)}, - rel_path{ToNormalPath(rel_path_)}, - pragma_special{pragma_special_}, - known_info{std::move(known_info_)} {}; - - [[nodiscard]] auto operator==( - GitObjectToResolve const& other) const noexcept -> bool { - return root_tree_id == other.root_tree_id and - rel_path == other.rel_path and - pragma_special == other.pragma_special; - } -}; - -/// \brief For a possibly initially unresolved path by the end we should be able -/// to know its hash, its type, and its now resolved path. -struct ResolvedGitObject { - std::string id; - ObjectType type; - std::filesystem::path path; -}; - -/// \brief Maps information about a Git object to its Git ID, type, and path as -/// part of a Git tree where symlinks have been resolved according to the given -/// pragma value. -/// Returns a nullopt only if called on a symlink with pragma ignore special. -/// \note Call the map with type Tree and path "." to resolve a Git tree. -using ResolveSymlinksMap = - AsyncMapConsumer; - -[[nodiscard]] auto CreateResolveSymlinksMap() -> ResolveSymlinksMap; - -[[nodiscard]] auto DetectAndReportCycle(ResolveSymlinksMap const& map, - std::string const& root_tree_id) - -> std::optional; - -namespace std { -template <> -struct hash { - [[nodiscard]] auto operator()(const GitObjectToResolve& ct) const noexcept - -> std::size_t { - size_t seed{}; - hash_combine(&seed, ct.root_tree_id); - hash_combine(&seed, ct.rel_path); - hash_combine(&seed, ct.pragma_special); - return seed; - } -}; -} // namespace std - -#endif // INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP -- cgit v1.2.3