diff options
Diffstat (limited to 'src')
31 files changed, 441 insertions, 277 deletions
diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index a05632c6..877d740a 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -116,4 +116,15 @@ , ["src/buildtool/storage", "storage"] ] } +, "user_structs": + { "type": ["@", "rules", "CC", "library"] + , "name": ["user_structs"] + , "hdrs": ["user_structs.hpp"] + , "stage": ["src", "buildtool", "common"] + , "deps": + [ ["@", "json", "", "json"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/main", "constants"] + ] + } } diff --git a/src/buildtool/common/user_structs.hpp b/src/buildtool/common/user_structs.hpp new file mode 100644 index 00000000..bebd8fab --- /dev/null +++ b/src/buildtool/common/user_structs.hpp @@ -0,0 +1,72 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP +#define INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP + +#include <filesystem> +#include <functional> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include "nlohmann/json.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/main/constants.hpp" + +/* Structures populated exclusively from command line with user-defined data */ + +struct LocalPaths { + // user-defined locations + std::optional<std::filesystem::path> root{std::nullopt}; + std::filesystem::path setup_root{FileSystemManager::GetCurrentDirectory()}; + std::optional<std::filesystem::path> workspace_root{ + // find workspace root + []() -> std::optional<std::filesystem::path> { + std::function<bool(std::filesystem::path const&)> + is_workspace_root = [&](std::filesystem::path const& path) { + return std::any_of( + kRootMarkers.begin(), + kRootMarkers.end(), + [&path](auto const& marker) { + return FileSystemManager::Exists(path / marker); + }); + }; + // default path to current dir + auto path = FileSystemManager::GetCurrentDirectory(); + auto root_path = path.root_path(); + while (true) { + if (is_workspace_root(path)) { + return path; + } + if (path == root_path) { + return std::nullopt; + } + path = path.parent_path(); + } + }()}; + nlohmann::json git_checkout_locations{}; + std::vector<std::filesystem::path> distdirs{}; +}; + +struct CAInfo { + bool no_ssl_verify{false}; + std::optional<std::filesystem::path> ca_bundle{std::nullopt}; +}; + +using LocalPathsPtr = std::shared_ptr<LocalPaths>; +using CAInfoPtr = std::shared_ptr<CAInfo>; + +#endif // INCLUDED_SRC_BUILDTOOL_COMMON_USER_STRUCTS_HPP diff --git a/src/other_tools/symlinks_map/TARGETS b/src/buildtool/file_system/symlinks_map/TARGETS index 225000f5..125a4f52 100644 --- a/src/other_tools/symlinks_map/TARGETS +++ b/src/buildtool/file_system/symlinks_map/TARGETS @@ -4,15 +4,22 @@ , "hdrs": ["resolve_symlinks_map.hpp"] , "srcs": ["resolve_symlinks_map.cpp"] , "deps": - [ ["src/buildtool/file_system", "git_repo"] + [ "pragma_special" + , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/multithreading", "async_map_consumer"] - , ["src/other_tools/just_mr", "utils"] + , ["src/utils/cpp", "hash_combine"] , ["src/utils/cpp", "path"] , ["src/utils/cpp", "path_hash"] ] - , "stage": ["src", "other_tools", "symlinks_map"] + , "stage": ["src", "buildtool", "file_system", "symlinks_map"] , "private-deps": [["@", "fmt", "", "fmt"], ["src/buildtool/storage", "config"]] } +, "pragma_special": + { "type": ["@", "rules", "CC", "library"] + , "name": ["pragma_special"] + , "hdrs": ["pragma_special.hpp"] + , "stage": ["src", "buildtool", "file_system", "symlinks_map"] + } } diff --git a/src/buildtool/file_system/symlinks_map/pragma_special.hpp b/src/buildtool/file_system/symlinks_map/pragma_special.hpp new file mode 100644 index 00000000..61935bdd --- /dev/null +++ b/src/buildtool/file_system/symlinks_map/pragma_special.hpp @@ -0,0 +1,42 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP + +#include <string> +#include <unordered_map> + +/* Enum used by the resolve_symlinks_map */ + +/// \brief Pragma "special" value enum +enum class PragmaSpecial : std::uint8_t { + Ignore, + ResolvePartially, + ResolveCompletely +}; + +/// \brief Pragma "special" value map, from string to enum +std::unordered_map<std::string, PragmaSpecial> const kPragmaSpecialMap = { + {"ignore", PragmaSpecial::Ignore}, + {"resolve-partially", PragmaSpecial::ResolvePartially}, + {"resolve-completely", PragmaSpecial::ResolveCompletely}}; + +/// \brief Pragma "special" value inverse map, from enum to string +std::unordered_map<PragmaSpecial, std::string> const kPragmaSpecialInverseMap = + {{PragmaSpecial::Ignore, "ignore"}, + {PragmaSpecial::ResolvePartially, "resolve-partially"}, + {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; + +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_PRAGMA_SPECIAL_HPP diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.cpp b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp index 88d09aec..1ae5db8b 100644 --- a/src/other_tools/symlinks_map/resolve_symlinks_map.cpp +++ b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "fmt/core.h" #include "src/buildtool/file_system/git_repo.hpp" diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.hpp b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp index 8bae7583..55254a30 100644 --- a/src/other_tools/symlinks_map/resolve_symlinks_map.hpp +++ b/src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP -#define INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP +#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP +#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP #include <filesystem> #include <optional> @@ -21,8 +21,9 @@ #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" -#include "src/other_tools/just_mr/utils.hpp" +#include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path.hpp" #include "src/utils/cpp/path_hash.hpp" @@ -94,4 +95,4 @@ struct hash<GitObjectToResolve> { }; } // namespace std -#endif // INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP +#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index ee2a2e13..38a118cf 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -58,4 +58,22 @@ , "stage": ["src", "buildtool", "storage"] , "private-deps": [["src/buildtool/execution_api/remote", "config"]] } +, "fs_utils": + { "type": ["@", "rules", "CC", "library"] + , "name": ["fs_utils"] + , "hdrs": ["fs_utils.hpp"] + , "srcs": ["fs_utils.cpp"] + , "deps": + [ ["src/buildtool/common", "user_structs"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] + , ["src/utils/cpp", "tmp_dir"] + ] + , "stage": ["src", "buildtool", "storage"] + , "private-deps": + [ ["src/buildtool/file_system", "file_system_manager"] + , ["src/utils/cpp", "path"] + , "config" + , "storage" + ] + } } diff --git a/src/buildtool/storage/fs_utils.cpp b/src/buildtool/storage/fs_utils.cpp new file mode 100644 index 00000000..f2241fc9 --- /dev/null +++ b/src/buildtool/storage/fs_utils.cpp @@ -0,0 +1,114 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/storage/fs_utils.hpp" + +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/utils/cpp/path.hpp" + +namespace StorageUtils { + +auto GetGitRoot(LocalPathsPtr const& just_mr_paths, + std::string const& repo_url) noexcept -> std::filesystem::path { + if (just_mr_paths->git_checkout_locations.contains(repo_url)) { + return std::filesystem::absolute(ToNormalPath(std::filesystem::path{ + just_mr_paths->git_checkout_locations[repo_url] + .get<std::string>()})); + } + auto repo_url_as_path = std::filesystem::absolute( + ToNormalPath(std::filesystem::path(repo_url))); + if (not repo_url_as_path.empty() and + FileSystemManager::IsAbsolutePath(repo_url_as_path) and + FileSystemManager::IsDirectory(repo_url_as_path)) { + return repo_url_as_path; + } + return StorageConfig::GitRoot(); +} + +auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { + // try to create parent dir + auto parent_path = + StorageConfig::GenerationCacheRoot(0) / "tmp-workspaces" / type; + return TmpDir::Create(parent_path); +} + +auto GetCommitTreeIDFile(std::string const& commit) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "commit-tree-map" / commit; +} + +auto GetArchiveTreeIDFile(std::string const& repo_type, + std::string const& content) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; +} + +auto GetDistdirTreeIDFile(std::string const& content) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; +} + +auto GetResolvedTreeIDFile(std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "special-tree-map" / + kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; +} + +auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, + std::string const& tree_id) noexcept -> bool { + // needs to be done safely, so use the rename trick + auto tmp_dir = TmpDir::Create(tree_id_file.parent_path()); + if (not tmp_dir) { + Logger::Log(LogLevel::Error, + "could not create tmp dir for writing tree id file {}", + tree_id_file.string()); + return false; + } + std::filesystem::path tmp_file{tmp_dir->GetPath() / "tmp_file"}; + if (not FileSystemManager::WriteFile(tree_id, tmp_file)) { + Logger::Log(LogLevel::Error, "could not create tmp tree id file"); + return false; + } + return FileSystemManager::Rename(tmp_file.string(), tree_id_file); +} + +auto AddToCAS(std::string const& data) noexcept + -> std::optional<std::filesystem::path> { + // get file CAS instance + auto const& cas = Storage::Instance().CAS(); + // write to cas + auto digest = cas.StoreBlob(data); + if (digest) { + return cas.BlobPath(*digest, /*is_executable=*/false); + } + return std::nullopt; +} + +void AddDistfileToCAS(std::filesystem::path const& distfile, + LocalPathsPtr const& just_mr_paths) noexcept { + auto const& cas = Storage::Instance().CAS(); + for (auto const& dirpath : just_mr_paths->distdirs) { + auto candidate = dirpath / distfile; + if (FileSystemManager::Exists(candidate)) { + // try to add to CAS + [[maybe_unused]] auto digest = + cas.StoreBlob(candidate, /*is_executable=*/false); + } + } +} + +} // namespace StorageUtils diff --git a/src/buildtool/storage/fs_utils.hpp b/src/buildtool/storage/fs_utils.hpp new file mode 100644 index 00000000..ba66dac5 --- /dev/null +++ b/src/buildtool/storage/fs_utils.hpp @@ -0,0 +1,77 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP + +#include <filesystem> +#include <optional> +#include <string> + +#include "src/buildtool/common/user_structs.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/utils/cpp/tmp_dir.hpp" + +/* Utilities related to CAS and paths therein */ + +namespace StorageUtils { + +/// \brief Get location of Git repository. Defaults to the Git cache root when +/// no better location is found. +[[nodiscard]] auto GetGitRoot(LocalPathsPtr const& just_mr_paths, + std::string const& repo_url) noexcept + -> std::filesystem::path; + +/// \brief Create a tmp directory with controlled lifetime for specific +/// operations (archive, zip, file, distdir checkouts; fetch; update). +[[nodiscard]] auto CreateTypedTmpDir(std::string const& type) noexcept + -> TmpDirPtr; + +/// \brief Get the path to the file storing the tree id associated with +/// a given commit. +[[nodiscard]] auto GetCommitTreeIDFile(std::string const& commit) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing the tree id of an archive +/// content. +[[nodiscard]] auto GetArchiveTreeIDFile(std::string const& repo_type, + std::string const& content) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing the tree id of a distdir list +/// content. +[[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept + -> std::filesystem::path; + +/// \brief Get the path to the file storing a resolved tree hash. +[[nodiscard]] auto GetResolvedTreeIDFile( + std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; + +/// \brief Write a tree id to file. The parent folder of the file must exist! +[[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, + std::string const& tree_id) noexcept -> bool; + +/// \brief Add data to file CAS. +/// Returns the path to the file added to CAS, or nullopt if not added. +[[nodiscard]] auto AddToCAS(std::string const& data) noexcept + -> std::optional<std::filesystem::path>; + +/// \brief Try to add distfile to CAS. +void AddDistfileToCAS(std::filesystem::path const& distfile, + LocalPathsPtr const& just_mr_paths) noexcept; + +} // namespace StorageUtils + +#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_FS_UTILS_HPP diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index a64f4c3c..a3ed87c9 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -41,17 +41,12 @@ , "hdrs": ["utils.hpp"] , "srcs": ["utils.cpp"] , "deps": - [ ["src/utils/cpp", "tmp_dir"] + [ ["@", "gsl", "", "gsl"] + , ["@", "json", "", "json"] , ["src/buildtool/storage", "config"] - , ["src/buildtool/main", "constants"] , ["src/buildtool/build_engine/expression", "expression"] ] , "stage": ["src", "other_tools", "just_mr"] - , "private-deps": - [ ["src/utils/cpp", "path"] - , ["src/buildtool/execution_api/local", "local"] - , ["src/buildtool/file_system", "file_storage"] - ] } , "exit_codes": { "type": ["@", "rules", "CC", "library"] @@ -69,6 +64,7 @@ , ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/common", "clidefaults"] + , ["src/buildtool/common", "user_structs"] , ["src/buildtool/execution_api/local", "config"] , ["src/other_tools/just_mr", "utils"] , ["src/buildtool/logging", "log_level"] @@ -132,6 +128,7 @@ [ ["@", "json", "", "json"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/git_operations", "git_repo_remote"] , "exit_codes" , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -151,6 +148,7 @@ [ ["@", "json", "", "json"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "task_system"] + , ["src/buildtool/storage", "fs_utils"] , "exit_codes" , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] @@ -162,7 +160,7 @@ , ["src/other_tools/root_maps", "distdir_git_map"] , ["src/other_tools/root_maps", "fpath_git_map"] , ["src/other_tools/root_maps", "tree_id_git_map"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] , "setup_utils" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "local"] diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp index 4c0c6e34..adc72578 100644 --- a/src/other_tools/just_mr/cli.hpp +++ b/src/other_tools/just_mr/cli.hpp @@ -26,6 +26,7 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/clidefaults.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/other_tools/just_mr/utils.hpp" @@ -36,9 +37,9 @@ struct MultiRepoCommonArguments { std::optional<std::filesystem::path> absent_repository_file{std::nullopt}; std::optional<std::filesystem::path> checkout_locations_file{std::nullopt}; std::vector<std::string> explicit_distdirs{}; - JustMR::PathsPtr just_mr_paths = std::make_shared<JustMR::Paths>(); + LocalPathsPtr just_mr_paths = std::make_shared<LocalPaths>(); std::optional<std::vector<std::string>> local_launcher{std::nullopt}; - JustMR::CAInfoPtr ca_info = std::make_shared<JustMR::CAInfo>(); + CAInfoPtr ca_info = std::make_shared<CAInfo>(); std::optional<std::filesystem::path> just_path{std::nullopt}; std::optional<std::string> main{std::nullopt}; std::optional<std::filesystem::path> rc_path{std::nullopt}; diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 54295c6b..5fb5512f 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -19,9 +19,11 @@ #include "nlohmann/json.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -34,7 +36,6 @@ #include "src/other_tools/root_maps/distdir_git_map.hpp" #include "src/other_tools/root_maps/fpath_git_map.hpp" #include "src/other_tools/root_maps/tree_id_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, MultiRepoCommonArguments const& common_args, @@ -227,5 +228,5 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } // if successful, return the output config - return JustMR::Utils::AddToCAS(mr_config.dump(2)); + return StorageUtils::AddToCAS(mr_config.dump(2)); } diff --git a/src/other_tools/just_mr/update.cpp b/src/other_tools/just_mr/update.cpp index 435827d5..babfee5a 100644 --- a/src/other_tools/just_mr/update.cpp +++ b/src/other_tools/just_mr/update.cpp @@ -20,6 +20,7 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -147,7 +148,7 @@ auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config, } } // Create fake repo for the anonymous remotes - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("update"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("update"); if (not tmp_dir) { Logger::Log(LogLevel::Error, "Failed to create commit update tmp dir"); return kExitUpdateError; diff --git a/src/other_tools/just_mr/utils.cpp b/src/other_tools/just_mr/utils.cpp index be0e841e..7e99eb43 100644 --- a/src/other_tools/just_mr/utils.cpp +++ b/src/other_tools/just_mr/utils.cpp @@ -14,102 +14,8 @@ #include "src/other_tools/just_mr/utils.hpp" -#include "src/buildtool/file_system/file_storage.hpp" -#include "src/buildtool/storage/storage.hpp" -#include "src/utils/cpp/path.hpp" - namespace JustMR::Utils { -auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, - std::string const& repo_url) noexcept -> std::filesystem::path { - if (just_mr_paths->git_checkout_locations.contains(repo_url)) { - return std::filesystem::absolute(ToNormalPath(std::filesystem::path{ - just_mr_paths->git_checkout_locations[repo_url] - .get<std::string>()})); - } - auto repo_url_as_path = std::filesystem::absolute( - ToNormalPath(std::filesystem::path(repo_url))); - if (not repo_url_as_path.empty() and - FileSystemManager::IsAbsolutePath(repo_url_as_path) and - FileSystemManager::IsDirectory(repo_url_as_path)) { - return repo_url_as_path; - } - return StorageConfig::GitRoot(); -} - -auto CreateTypedTmpDir(std::string const& type) noexcept -> TmpDirPtr { - // try to create parent dir - auto parent_path = - StorageConfig::GenerationCacheRoot(0) / "tmp-workspaces" / type; - return TmpDir::Create(parent_path); -} - -auto GetCommitTreeIDFile(std::string const& commit) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "commit-tree-map" / commit; -} - -auto GetArchiveTreeIDFile(std::string const& repo_type, - std::string const& content) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "tree-map" / repo_type / content; -} - -auto GetDistdirTreeIDFile(std::string const& content) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; -} - -auto GetResolvedTreeIDFile(std::string const& tree_hash, - PragmaSpecial const& pragma_special) noexcept - -> std::filesystem::path { - return StorageConfig::BuildRoot() / "special-tree-map" / - kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; -} - -auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, - std::string const& tree_id) noexcept -> bool { - // needs to be done safely, so use the rename trick - auto tmp_dir = TmpDir::Create(tree_id_file.parent_path()); - if (not tmp_dir) { - Logger::Log(LogLevel::Error, - "could not create tmp dir for writing tree id file {}", - tree_id_file.string()); - return false; - } - std::filesystem::path tmp_file{tmp_dir->GetPath() / "tmp_file"}; - if (not FileSystemManager::WriteFile(tree_id, tmp_file)) { - Logger::Log(LogLevel::Error, "could not create tmp tree id file"); - return false; - } - return FileSystemManager::Rename(tmp_file.string(), tree_id_file); -} - -auto AddToCAS(std::string const& data) noexcept - -> std::optional<std::filesystem::path> { - // get file CAS instance - auto const& cas = Storage::Instance().CAS(); - // write to cas - auto digest = cas.StoreBlob(data); - if (digest) { - return cas.BlobPath(*digest, /*is_executable=*/false); - } - return std::nullopt; -} - -void AddDistfileToCAS(std::filesystem::path const& distfile, - JustMR::PathsPtr const& just_mr_paths) noexcept { - auto const& cas = Storage::Instance().CAS(); - for (auto const& dirpath : just_mr_paths->distdirs) { - auto candidate = dirpath / distfile; - if (FileSystemManager::Exists(candidate)) { - // try to add to CAS - [[maybe_unused]] auto digest = - cas.StoreBlob(candidate, /*is_executable=*/false); - } - } -} - // NOLINTNEXTLINE(misc-no-recursion) auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos, diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 304cc839..3aff2348 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -15,13 +15,16 @@ #ifndef INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP #define INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP +#include <map> +#include <optional> +#include <string> #include <unordered_set> +#include <vector> +#include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" -#include "src/buildtool/main/constants.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/utils/cpp/tmp_dir.hpp" /* Paths and constants required by just-mr */ @@ -135,115 +138,7 @@ std::unordered_map<std::string, CheckoutType> const kCheckoutTypeMap = { {"file", CheckoutType::File}, {"distdir", CheckoutType::Distdir}, {"git tree", CheckoutType::GitTree}}; - -/// \brief Pragma "special" value enum -enum class PragmaSpecial : std::uint8_t { - Ignore, - ResolvePartially, - ResolveCompletely -}; - -/// \brief Pragma "special" value map -std::unordered_map<std::string, PragmaSpecial> const kPragmaSpecialMap = { - {"ignore", PragmaSpecial::Ignore}, - {"resolve-partially", PragmaSpecial::ResolvePartially}, - {"resolve-completely", PragmaSpecial::ResolveCompletely}}; - -/// \brief Pragma "special" value inverse map, from enum to string -std::unordered_map<PragmaSpecial, std::string> const kPragmaSpecialInverseMap = - {{PragmaSpecial::Ignore, "ignore"}, - {PragmaSpecial::ResolvePartially, "resolve-partially"}, - {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; - -namespace JustMR { - -struct Paths { - // user-defined locations - std::optional<std::filesystem::path> root{std::nullopt}; - std::filesystem::path setup_root{FileSystemManager::GetCurrentDirectory()}; - std::optional<std::filesystem::path> workspace_root{ - // find workspace root - []() -> std::optional<std::filesystem::path> { - std::function<bool(std::filesystem::path const&)> - is_workspace_root = [&](std::filesystem::path const& path) { - return std::any_of( - kRootMarkers.begin(), - kRootMarkers.end(), - [&path](auto const& marker) { - return FileSystemManager::Exists(path / marker); - }); - }; - // default path to current dir - auto path = FileSystemManager::GetCurrentDirectory(); - auto root_path = path.root_path(); - while (true) { - if (is_workspace_root(path)) { - return path; - } - if (path == root_path) { - return std::nullopt; - } - path = path.parent_path(); - } - }()}; - nlohmann::json git_checkout_locations{}; - std::vector<std::filesystem::path> distdirs{}; -}; - -struct CAInfo { - bool no_ssl_verify{false}; - std::optional<std::filesystem::path> ca_bundle{std::nullopt}; -}; - -using PathsPtr = std::shared_ptr<JustMR::Paths>; -using CAInfoPtr = std::shared_ptr<JustMR::CAInfo>; - -namespace Utils { - -/// \brief Get location of Git repository. Defaults to the Git cache root when -/// no better location is found. -[[nodiscard]] auto GetGitRoot(JustMR::PathsPtr const& just_mr_paths, - std::string const& repo_url) noexcept - -> std::filesystem::path; - -/// \brief Create a tmp directory with controlled lifetime for specific -/// operations (archive, zip, file, distdir checkouts; fetch; update). -[[nodiscard]] auto CreateTypedTmpDir(std::string const& type) noexcept - -> TmpDirPtr; - -/// \brief Get the path to the file storing the tree id associated with -/// a given commit. -[[nodiscard]] auto GetCommitTreeIDFile(std::string const& commit) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing the tree id of an archive -/// content. -[[nodiscard]] auto GetArchiveTreeIDFile(std::string const& repo_type, - std::string const& content) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing the tree id of a distdir list -/// content. -[[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept - -> std::filesystem::path; - -/// \brief Get the path to the file storing a resolved tree hash. -[[nodiscard]] auto GetResolvedTreeIDFile( - std::string const& tree_hash, - PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; - -/// \brief Write a tree id to file. The parent folder of the file must exist! -[[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, - std::string const& tree_id) noexcept -> bool; - -/// \brief Add data to file CAS. -/// Returns the path to the file added to CAS, or nullopt if not added. -[[nodiscard]] auto AddToCAS(std::string const& data) noexcept - -> std::optional<std::filesystem::path>; - -/// \brief Try to add distfile to CAS. -void AddDistfileToCAS(std::filesystem::path const& distfile, - JustMR::PathsPtr const& just_mr_paths) noexcept; +namespace JustMR::Utils { /// \brief Recursive part of the ResolveRepo function. /// Keeps track of repository names to detect any cyclic dependencies. @@ -260,8 +155,6 @@ auto ResolveRepo(ExpressionPtr const& repo_desc, ExpressionPtr const& repos) noexcept -> std::optional<ExpressionPtr>; -} // namespace Utils - -} // namespace JustMR +} // namespace JustMR::Utils #endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index a2121e33..9ca5e5f4 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -25,7 +25,8 @@ ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["src/buildtool/storage", "fs_utils"] + , ["src/buildtool/storage", "storage"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/local", "config"] ] @@ -39,12 +40,12 @@ [ ["src/other_tools/git_operations", "git_repo_remote"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] - , ["@", "fmt", "", "fmt"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["@", "fmt", "", "fmt"] , ["src/buildtool/execution_api/local", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/utils/cpp", "tmp_dir"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] @@ -56,7 +57,8 @@ , "hdrs": ["content_cas_map.hpp"] , "srcs": ["content_cas_map.cpp"] , "deps": - [ ["src/other_tools/just_mr", "utils"] + [ ["src/buildtool/common", "user_structs"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/multithreading", "async_map_consumer"] , ["src/utils/cpp", "hash_combine"] @@ -68,6 +70,7 @@ , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] ] diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index da37df4d..b796dc04 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -16,6 +16,7 @@ #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -26,7 +27,7 @@ namespace { /// \brief Fetches a file from the internet and stores its content in memory. /// Returns the content. [[nodiscard]] auto NetworkFetch(std::string const& fetch_url, - JustMR::CAInfoPtr const& ca_info) noexcept + CAInfoPtr const& ca_info) noexcept -> std::optional<std::string> { auto curl_handle = CurlEasyHandle::Create(ca_info->no_ssl_verify, ca_info->ca_bundle); @@ -47,8 +48,8 @@ template <Hasher::HashType type> } // namespace -auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, - JustMR::CAInfoPtr const& ca_info, +auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths, + CAInfoPtr const& ca_info, IExecutionApi* local_api, IExecutionApi* remote_api, std::size_t jobs) -> ContentCASMap { @@ -71,7 +72,7 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, (key.distfile ? key.distfile.value() : std::filesystem::path(key.fetch_url).filename().string()); - JustMR::Utils::AddDistfileToCAS(repo_distfile, just_mr_paths); + StorageUtils::AddDistfileToCAS(repo_distfile, just_mr_paths); // check if content is in CAS now if (cas.BlobPath(digest, /*is_executable=*/false)) { JustMRProgress::Instance().TaskTracker().Stop(key.origin); @@ -132,8 +133,8 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, } } // add the fetched data to CAS - auto path = JustMR::Utils::AddToCAS(*data); - // check that storing the fetched data succeeded + auto path = StorageUtils::AddToCAS(*data); + // check one last time if content is in CAS now if (not path) { (*logger)(fmt::format("Failed to store fetched content from {}", key.fetch_url), diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp index 859fbe6b..846b2762 100644 --- a/src/other_tools/ops_maps/content_cas_map.hpp +++ b/src/other_tools/ops_maps/content_cas_map.hpp @@ -19,9 +19,10 @@ #include <string> #include "nlohmann/json.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/multithreading/async_map_consumer.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/hash_combine.hpp" struct ArchiveContent { @@ -61,8 +62,8 @@ struct ArchiveRepoInfo { /// \brief Maps the content hash of an archive to an "exists" status flag. using ContentCASMap = AsyncMapConsumer<ArchiveContent, bool>; -[[nodiscard]] auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths, - JustMR::CAInfoPtr const& ca_info, +[[nodiscard]] auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths, + CAInfoPtr const& ca_info, IExecutionApi* local_api, IExecutionApi* remote_api, std::size_t jobs) -> ContentCASMap; diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp index cc40aef6..157d13bb 100644 --- a/src/other_tools/ops_maps/git_update_map.cpp +++ b/src/other_tools/ops_maps/git_update_map.cpp @@ -16,9 +16,9 @@ #include "fmt/core.h" #include "src/buildtool/execution_api/local/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto CreateGitUpdateMap(GitCASPtr const& git_cas, @@ -39,7 +39,7 @@ auto CreateGitUpdateMap(GitCASPtr const& git_cas, /*fatal=*/true); return; } - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("update"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("update"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create commit update tmp dir for " "remote {}", diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp index e8b7a89d..28365866 100644 --- a/src/other_tools/ops_maps/import_to_git_map.cpp +++ b/src/other_tools/ops_maps/import_to_git_map.cpp @@ -18,7 +18,7 @@ #include "src/buildtool/execution_api/common/execution_common.hpp" #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/other_tools/just_mr/utils.hpp" +#include "src/buildtool/storage/fs_utils.hpp" namespace { @@ -167,7 +167,7 @@ auto CreateImportToGitMap( } // create tmp directory auto tmp_dir = - JustMR::Utils::CreateTypedTmpDir("import-to-git"); + StorageUtils::CreateTypedTmpDir("import-to-git"); if (not tmp_dir) { (*logger)( fmt::format("Could not create unique path " diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS index b91730b9..c397d82c 100644 --- a/src/other_tools/repo_map/TARGETS +++ b/src/other_tools/repo_map/TARGETS @@ -16,6 +16,7 @@ [ ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] ] } } diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index f51e48ec..7de7e2ff 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -15,6 +15,7 @@ #include "src/other_tools/repo_map/repos_to_setup_map.hpp" #include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 2b5fbd1a..656e1fab 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -15,6 +15,7 @@ , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/storage", "storage"] , ["src/utils/cpp", "tmp_dir"] , ["src/buildtool/file_system", "file_storage"] @@ -31,7 +32,7 @@ , "deps": [ ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/execution_api/common", "common"] - , ["src/other_tools/just_mr", "utils"] + , ["src/buildtool/common", "user_structs"] , ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/utils/cpp", "hash_combine"] @@ -44,6 +45,8 @@ , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] ] } , "fpath_git_map": @@ -57,7 +60,8 @@ , ["src/utils/cpp", "path_hash"] , ["src/utils/cpp", "hash_combine"] , ["src/other_tools/just_mr", "utils"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -67,6 +71,8 @@ , ["src/buildtool/file_system", "file_root"] , ["src/buildtool/file_system", "git_repo"] , ["src/other_tools/git_operations", "git_repo_remote"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] + , ["src/buildtool/storage", "fs_utils"] ] } , "content_git_map": @@ -77,15 +83,17 @@ , "deps": [ ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] - , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] + , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": [ ["src/utils/archive", "archive_ops"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] + , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] @@ -109,9 +117,9 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "fs_utils"] , ["src/buildtool/system", "system_command"] , ["src/other_tools/git_operations", "git_repo_remote"] - , ["src/other_tools/just_mr", "utils"] , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/ops_maps", "content_cas_map"] diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index 9798eb46..f4ee1c17 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -17,6 +17,8 @@ #include <algorithm> #include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -44,7 +46,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id, CommitGitMap::SetterPtr const& ws_setter, CommitGitMap::LoggerPtr const& logger) { // write association of the root tree in id file - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, root_tree_id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, root_tree_id)) { (*logger)(fmt::format("Failed to write tree id {} to file {}", root_tree_id, tree_id_file.string()), @@ -118,7 +120,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, if (not is_commit_present.value()) { if (repo_info.absent) { auto tree_id_file = - JustMR::Utils::GetCommitTreeIDFile(repo_info.hash); + StorageUtils::GetCommitTreeIDFile(repo_info.hash); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = @@ -223,7 +225,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, JustMRProgress::Instance().TaskTracker().Stop( repo_info.origin); // Move tree from CAS to local git storage - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir( + auto tmp_dir = StorageUtils::CreateTypedTmpDir( "fetch-absent-root"); if (not tmp_dir) { (*logger)( @@ -346,7 +348,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, } } // default to fetching it from network - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("fetch"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("fetch"); if (not tmp_dir) { (*logger)("Failed to create fetch tmp directory!", /*fatal=*/true); @@ -485,7 +487,7 @@ void EnsureCommit(GitRepoInfo const& repo_info, auto CreateCommitGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, - JustMR::PathsPtr const& just_mr_paths, + LocalPathsPtr const& just_mr_paths, std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi* serve_api, @@ -513,7 +515,7 @@ auto CreateCommitGitMap( fetch_repo = std::filesystem::absolute(fetch_repo).string(); } std::filesystem::path repo_root = - JustMR::Utils::GetGitRoot(just_mr_paths, fetch_repo); + StorageUtils::GetGitRoot(just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp index 4f59a56f..26ed018b 100644 --- a/src/other_tools/root_maps/commit_git_map.hpp +++ b/src/other_tools/root_maps/commit_git_map.hpp @@ -19,9 +19,9 @@ #include <utility> #include "nlohmann/json.hpp" +#include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/critical_git_op_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" @@ -69,7 +69,7 @@ using CommitGitMap = [[nodiscard]] auto CreateCommitGitMap( gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, gsl::not_null<ImportToGitMap*> const& import_to_git_map, - JustMR::PathsPtr const& just_mr_paths, + LocalPathsPtr const& just_mr_paths, std::string const& git_bin, std::vector<std::string> const& launcher, ServeApi* serve_api, diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index c6e0809b..a82670ea 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -16,7 +16,9 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_storage.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -54,7 +56,7 @@ void ResolveContentTree( if (pragma_special) { // get the resolved tree auto tree_id_file = - JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + StorageUtils::GetResolvedTreeIDFile(tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -108,8 +110,8 @@ void ResolveContentTree( } auto const& resolved_tree = *hashes[0]; // cache the resolved tree in the CAS map - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, - resolved_tree.id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { (*logger)(fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), @@ -160,7 +162,7 @@ auto CreateContentGitMap( auto logger, auto /* unused */, auto const& key) { - auto archive_tree_id_file = JustMR::Utils::GetArchiveTreeIDFile( + auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( key.repo_type, key.archive.content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file @@ -264,7 +266,7 @@ auto CreateContentGitMap( logger]([[maybe_unused]] auto const& values) { // content is in CAS // extract archive - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir(repo_type); + auto tmp_dir = StorageUtils::CreateTypedTmpDir(repo_type); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "{} target {}", @@ -314,7 +316,7 @@ auto CreateContentGitMap( // only tree id is needed std::string archive_tree_id = values[0]->first; // write to tree id file - if (not JustMR::Utils::WriteTreeIDFile( + if (not StorageUtils::WriteTreeIDFile( archive_tree_id_file, archive_tree_id)) { (*logger)( fmt::format("Failed to write tree id " diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index 6eed8448..db0950f0 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -17,9 +17,9 @@ #include <utility> +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" /// \brief Maps the content of an archive to the resulting Git tree WS root, /// together with the information whether it was a cache hit. diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index a132cf47..8f7bd98f 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -20,6 +20,7 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" @@ -65,7 +66,7 @@ auto CreateDistdirGitMap( auto /* unused */, auto const& key) { auto distdir_tree_id_file = - JustMR::Utils::GetDistdirTreeIDFile(key.content_id); + StorageUtils::GetDistdirTreeIDFile(key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = @@ -135,7 +136,7 @@ auto CreateDistdirGitMap( logger]([[maybe_unused]] auto const& values) mutable { // repos are in CAS // create the links to CAS - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("distdir"); if (not tmp_dir) { (*logger)(fmt::format("Failed to create tmp path for " "distdir target {}", @@ -171,7 +172,7 @@ auto CreateDistdirGitMap( // only the tree is of interest std::string distdir_tree_id = values[0]->first; // write to tree id file - if (not JustMR::Utils::WriteTreeIDFile( + if (not StorageUtils::WriteTreeIDFile( distdir_tree_id_file, distdir_tree_id)) { (*logger)( fmt::format( diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 563876ff..9f545856 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -18,6 +18,7 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/tmp_dir.hpp" @@ -36,7 +37,7 @@ void ResolveFilePathTree( if (pragma_special) { // get the resolved tree auto tree_id_file = - JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + StorageUtils::GetResolvedTreeIDFile(tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -90,8 +91,8 @@ void ResolveFilePathTree( } auto const& resolved_tree = *hashes[0]; // cache the resolved tree in the CAS map - if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, - resolved_tree.id)) { + if (not StorageUtils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { (*logger)(fmt::format("Failed to write resolved tree " "id to file {}", tree_id_file.string()), @@ -255,7 +256,7 @@ auto CreateFilePathGitMap( /*fatal=*/false); } // it's not a git repo, so import it to git cache - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("file"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("file"); if (not tmp_dir) { (*logger)("Failed to create import-to-git tmp directory!", /*fatal=*/true); diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 6063aa8d..139921b7 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -16,9 +16,10 @@ #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP #include "nlohmann/json.hpp" +#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 34da3121..a8db186a 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -19,11 +19,11 @@ #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/statistics.hpp" -#include "src/other_tools/just_mr/utils.hpp" namespace { @@ -145,7 +145,7 @@ auto CreateTreeIdGitMap( if (not *tree_found) { JustMRProgress::Instance().TaskTracker().Start(key.origin); // create temporary location for command execution root - auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("git-tree"); + auto tmp_dir = StorageUtils::CreateTypedTmpDir("git-tree"); if (not tmp_dir) { (*logger)( "Failed to create tmp directory for tree id map!", @@ -154,7 +154,7 @@ auto CreateTreeIdGitMap( } // create temporary location for storing command result // files - auto out_dir = JustMR::Utils::CreateTypedTmpDir("git-tree"); + auto out_dir = StorageUtils::CreateTypedTmpDir("git-tree"); if (not out_dir) { (*logger)( "Failed to create tmp directory for tree id map!", @@ -282,7 +282,7 @@ auto CreateTreeIdGitMap( } // define temp repo path auto tmp_dir = - JustMR::Utils::CreateTypedTmpDir("git-tree"); + StorageUtils::CreateTypedTmpDir("git-tree"); ; if (not tmp_dir) { (*logger)(fmt::format("Could not create unique " |