diff options
Diffstat (limited to 'src/buildtool/file_system')
| -rw-r--r-- | src/buildtool/file_system/file_root.hpp | 50 | ||||
| -rw-r--r-- | src/buildtool/file_system/file_system_manager.hpp | 9 | ||||
| -rw-r--r-- | src/buildtool/file_system/git_repo.cpp | 44 | ||||
| -rw-r--r-- | src/buildtool/file_system/git_repo.hpp | 4 | ||||
| -rw-r--r-- | src/buildtool/file_system/git_tree.cpp | 52 | ||||
| -rw-r--r-- | src/buildtool/file_system/git_tree.hpp | 27 |
6 files changed, 144 insertions, 42 deletions
diff --git a/src/buildtool/file_system/file_root.hpp b/src/buildtool/file_system/file_root.hpp index e25a2455..af61c8fd 100644 --- a/src/buildtool/file_system/file_root.hpp +++ b/src/buildtool/file_system/file_root.hpp @@ -209,6 +209,9 @@ class FileRoot { return true; } + /// \brief Retrieve a root tree as a KNOWN artifact. + /// User should know whether this root tree is symlink free and only + /// call this function accordingly. [[nodiscard]] auto AsKnownTree(std::string const& repository) const noexcept -> std::optional<ArtifactDescription> { if (Compatibility::IsCompatible()) { @@ -270,20 +273,30 @@ class FileRoot { }; FileRoot() noexcept = default; + explicit FileRoot(bool ignore_special) noexcept + : ignore_special_(ignore_special) {} + // avoid type narrowing errors + explicit FileRoot(char const* root) noexcept : root_{fs_root_t{root}} {} explicit FileRoot(std::filesystem::path root) noexcept : root_{std::move(root)} {} + FileRoot(std::filesystem::path root, bool ignore_special) noexcept + : root_{std::move(root)}, ignore_special_{ignore_special} {} FileRoot(gsl::not_null<GitCASPtr> const& cas, - gsl::not_null<GitTreePtr> const& tree) noexcept - : root_{git_root_t{cas, tree}} {} + gsl::not_null<GitTreePtr> const& tree, + bool ignore_special = false) noexcept + : root_{git_root_t{cas, tree}}, ignore_special_{ignore_special} {} [[nodiscard]] static auto FromGit(std::filesystem::path const& repo_path, - std::string const& git_tree_id) noexcept + std::string const& git_tree_id, + bool ignore_special = false) noexcept -> std::optional<FileRoot> { if (auto cas = GitCAS::Open(repo_path)) { - if (auto tree = GitTree::Read(cas, git_tree_id)) { + if (auto tree = GitTree::Read(cas, git_tree_id, ignore_special)) { try { return FileRoot{ - cas, std::make_shared<GitTree const>(std::move(*tree))}; + cas, + std::make_shared<GitTree const>(std::move(*tree)), + ignore_special}; } catch (...) { } } @@ -291,13 +304,14 @@ class FileRoot { return std::nullopt; } - // Return a complete description of the content of this root, if that can be - // done without any file-system access. + // Return a complete description of the content of this root, if + // content-fixed. [[nodiscard]] auto ContentDescription() const noexcept -> std::optional<nlohmann::json> { try { if (std::holds_alternative<git_root_t>(root_)) { nlohmann::json j; + // ignore-special git-tree-based roots are still content-fixed j.push_back(kGitTreeMarker); j.push_back(std::get<git_root_t>(root_).tree->Hash()); return j; @@ -323,7 +337,12 @@ class FileRoot { return static_cast<bool>( std::get<git_root_t>(root_).tree->LookupEntryByPath(path)); } - return FileSystemManager::Exists(std::get<fs_root_t>(root_) / path); + // std::holds_alternative<fs_root_t>(root_) == true + auto root_path = std::get<fs_root_t>(root_) / path; + auto exists = FileSystemManager::Exists(root_path); + return (ignore_special_ ? exists and FileSystemManager::Type( + root_path) != std::nullopt + : exists); } [[nodiscard]] auto IsFile( @@ -380,7 +399,7 @@ class FileRoot { return DirectoryEntries{&(*tree)}; } if (auto entry = tree->LookupEntryByPath(dir_path)) { - if (auto const& found_tree = entry->Tree()) { + if (auto const& found_tree = entry->Tree(ignore_special_)) { return DirectoryEntries{&(*found_tree)}; } } @@ -392,7 +411,8 @@ class FileRoot { [&map](const auto& name, auto type) { map.emplace(name.string(), type); return true; - })) { + }, + ignore_special_)) { return DirectoryEntries{std::move(map)}; } } @@ -425,6 +445,7 @@ class FileRoot { return std::nullopt; } + /// \brief Read a blob from the root based on its ID. [[nodiscard]] auto ReadBlob(std::string const& blob_id) const noexcept -> std::optional<std::string> { if (std::holds_alternative<git_root_t>(root_)) { @@ -434,9 +455,12 @@ class FileRoot { return std::nullopt; } + /// \brief Read a root tree based on its ID. + /// User should know whether the desired tree is symlink free and only call + /// this function accordingly. [[nodiscard]] auto ReadTree(std::string const& tree_id) const noexcept -> std::optional<GitTree> { - if (std::holds_alternative<git_root_t>(root_)) { + if (std::holds_alternative<git_root_t>(root_) and not ignore_special_) { try { auto const& cas = std::get<git_root_t>(root_).cas; return GitTree::Read(cas, tree_id); @@ -481,6 +505,10 @@ class FileRoot { private: root_t root_; + // If set, forces lookups to ignore entries which are neither file nor + // directories instead of erroring out. This means implicitly also that + // there are no more fast tree lookups, i.e., tree traversal is a must. + bool ignore_special_{}; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp index c7a31b29..ec2ab398 100644 --- a/src/buildtool/file_system/file_system_manager.hpp +++ b/src/buildtool/file_system/file_system_manager.hpp @@ -514,9 +514,13 @@ class FileSystemManager { } } + /// \brief Read a filesystem directory tree. + /// \param ignore_special If true, do not error out when encountering + /// symlinks. [[nodiscard]] static auto ReadDirectory( std::filesystem::path const& dir, - ReadDirEntryFunc const& read_entry) noexcept -> bool { + ReadDirEntryFunc const& read_entry, + bool ignore_special = false) noexcept -> bool { try { for (auto const& entry : std::filesystem::directory_iterator{dir}) { ObjectType type{}; @@ -532,6 +536,9 @@ class FileSystemManager { else if (std::filesystem::is_directory(status)) { type = ObjectType::Tree; } + else if (ignore_special) { + continue; + } else { Logger::Log(LogLevel::Error, "unsupported type for dir entry {}", diff --git a/src/buildtool/file_system/git_repo.cpp b/src/buildtool/file_system/git_repo.cpp index 9fe63fb6..968978b5 100644 --- a/src/buildtool/file_system/git_repo.cpp +++ b/src/buildtool/file_system/git_repo.cpp @@ -15,6 +15,7 @@ #include "src/buildtool/file_system/git_repo.hpp" #include <thread> +#include <unordered_set> #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/gsl.hpp" @@ -29,6 +30,11 @@ extern "C" { #ifndef BOOTSTRAP_BUILD_TOOL namespace { +std::unordered_set<git_filemode_t> const kSupportedGitFileModes{ + GIT_FILEMODE_BLOB, + GIT_FILEMODE_BLOB_EXECUTABLE, + GIT_FILEMODE_TREE}; + [[nodiscard]] auto ToHexString(git_oid const& oid) noexcept -> std::optional<std::string> { std::string hex_id(GIT_OID_HEXSZ, '\0'); @@ -46,6 +52,12 @@ namespace { return std::nullopt; } +/// \brief Returns true if mode corresponds to a supported object type. +[[nodiscard]] auto GitFileModeIsSupported(git_filemode_t const& mode) noexcept + -> bool { + return kSupportedGitFileModes.contains(mode); +} + [[nodiscard]] auto GitFileModeToObjectType(git_filemode_t const& mode) noexcept -> std::optional<ObjectType> { switch (mode) { @@ -110,6 +122,29 @@ namespace { } #endif +[[nodiscard]] auto flat_tree_walker_ignore_special(const char* /*root*/, + const git_tree_entry* entry, + void* payload) noexcept + -> int { + auto* entries = + reinterpret_cast<GitRepo::tree_entries_t*>(payload); // NOLINT + + std::string name = git_tree_entry_name(entry); + auto const* oid = git_tree_entry_id(entry); + if (auto raw_id = ToRawString(*oid)) { + if (not GitFileModeIsSupported(git_tree_entry_filemode(entry))) { + return 0; // allow, but not store + } + if (auto type = + GitFileModeToObjectType(git_tree_entry_filemode(entry))) { + (*entries)[*raw_id].emplace_back(std::move(name), *type); + return 1; // return >=0 on success, 1 == skip subtrees (flat) + } + } + Logger::Log(LogLevel::Error, "failed walk for git tree entry: {}", name); + return -1; // fail +} + [[nodiscard]] auto flat_tree_walker(const char* /*root*/, const git_tree_entry* entry, void* payload) noexcept -> int { @@ -1110,7 +1145,9 @@ auto GitRepo::IsRepoFake() const noexcept -> bool { return is_repo_fake_; } -auto GitRepo::ReadTree(std::string const& id, bool is_hex_id) const noexcept +auto GitRepo::ReadTree(std::string const& id, + bool is_hex_id, + bool ignore_special) const noexcept -> std::optional<tree_entries_t> { #ifdef BOOTSTRAP_BUILD_TOOL return std::nullopt; @@ -1140,7 +1177,10 @@ auto GitRepo::ReadTree(std::string const& id, bool is_hex_id) const noexcept tree_entries_t entries{}; entries.reserve(git_tree_entrycount(tree.get())); if (git_tree_walk( - tree.get(), GIT_TREEWALK_PRE, flat_tree_walker, &entries) != 0) { + tree.get(), + GIT_TREEWALK_PRE, + ignore_special ? flat_tree_walker_ignore_special : flat_tree_walker, + &entries) != 0) { Logger::Log(LogLevel::Debug, "failed to walk Git tree {}", is_hex_id ? std::string{id} : ToHexString(id)); diff --git a/src/buildtool/file_system/git_repo.hpp b/src/buildtool/file_system/git_repo.hpp index c66a54db..43c5fa22 100644 --- a/src/buildtool/file_system/git_repo.hpp +++ b/src/buildtool/file_system/git_repo.hpp @@ -81,8 +81,10 @@ class GitRepo { /// repository is required. /// \param id The object id. /// \param is_hex_id Specify whether `id` is hex string or raw. + /// \param ignore_special If set, treat symlinks as absent. [[nodiscard]] auto ReadTree(std::string const& id, - bool is_hex_id = false) const noexcept + bool is_hex_id = false, + bool ignore_special = false) const noexcept -> std::optional<tree_entries_t>; /// \brief Create a flat tree from entries and store tree in CAS. diff --git a/src/buildtool/file_system/git_tree.cpp b/src/buildtool/file_system/git_tree.cpp index c3aa92e7..d594e638 100644 --- a/src/buildtool/file_system/git_tree.cpp +++ b/src/buildtool/file_system/git_tree.cpp @@ -36,15 +36,15 @@ namespace { [[nodiscard]] auto LookupEntryPyPath( GitTree const& tree, std::filesystem::path::const_iterator it, - std::filesystem::path::const_iterator const& end) noexcept - -> GitTreeEntryPtr { + std::filesystem::path::const_iterator const& end, + bool ignore_special = false) noexcept -> GitTreeEntryPtr { auto segment = *it; auto entry = tree.LookupEntryByName(segment); if (not entry) { return nullptr; } if (++it != end) { - auto const& subtree = entry->Tree(); + auto const& subtree = entry->Tree(ignore_special); if (not subtree) { return nullptr; } @@ -66,13 +66,19 @@ auto GitTree::Read(std::filesystem::path const& repo_path, } auto GitTree::Read(gsl::not_null<GitCASPtr> const& cas, - std::string const& tree_id) noexcept - -> std::optional<GitTree> { + std::string const& tree_id, + bool ignore_special) noexcept -> std::optional<GitTree> { if (auto raw_id = FromHexString(tree_id)) { auto repo = GitRepo::Open(cas); if (repo != std::nullopt) { - if (auto entries = repo->ReadTree(*raw_id)) { - return GitTree::FromEntries(cas, std::move(*entries), *raw_id); + if (auto entries = repo->ReadTree( + *raw_id, /*is_hex_id=*/false, ignore_special)) { + // the raw_id value is NOT recomputed when ignore_special==true, + // so we set it to empty to signal that it should not be used! + return GitTree::FromEntries(cas, + std::move(*entries), + ignore_special ? "" : *raw_id, + ignore_special); } } else { @@ -96,7 +102,8 @@ auto GitTree::LookupEntryByName(std::string const& name) const noexcept auto GitTree::LookupEntryByPath( std::filesystem::path const& path) const noexcept -> GitTreeEntryPtr { auto resolved = ResolveRelativePath(path); - return LookupEntryPyPath(*this, resolved.begin(), resolved.end()); + return LookupEntryPyPath( + *this, resolved.begin(), resolved.end(), ignore_special_); } auto GitTree::Size() const noexcept -> std::optional<std::size_t> { @@ -117,19 +124,24 @@ auto GitTreeEntry::Blob() const noexcept -> std::optional<std::string> { return cas_->ReadObject(raw_id_); } -auto GitTreeEntry::Tree() const& noexcept -> std::optional<GitTree> const& { - return tree_cached_.SetOnceAndGet([this]() -> std::optional<GitTree> { - if (IsTree()) { - auto repo = GitRepo::Open(cas_); - if (repo == std::nullopt) { - return std::nullopt; +auto GitTreeEntry::Tree(bool ignore_special) const& noexcept + -> std::optional<GitTree> const& { + return tree_cached_.SetOnceAndGet( + [this, ignore_special]() -> std::optional<GitTree> { + if (IsTree()) { + auto repo = GitRepo::Open(cas_); + if (repo == std::nullopt) { + return std::nullopt; + } + if (auto entries = repo->ReadTree( + raw_id_, /*is_hex_id=*/false, ignore_special)) { + // the raw_id value is not used when ignore_special==true + return GitTree::FromEntries( + cas_, std::move(*entries), raw_id_, ignore_special); + } } - if (auto entries = repo->ReadTree(raw_id_)) { - return GitTree::FromEntries(cas_, std::move(*entries), raw_id_); - } - } - return std::nullopt; - }); + return std::nullopt; + }); } auto GitTreeEntry::Size() const noexcept -> std::optional<std::size_t> { diff --git a/src/buildtool/file_system/git_tree.hpp b/src/buildtool/file_system/git_tree.hpp index 5007b106..b634ffb2 100644 --- a/src/buildtool/file_system/git_tree.hpp +++ b/src/buildtool/file_system/git_tree.hpp @@ -45,8 +45,13 @@ class GitTree { /// \brief Read tree with given id from CAS. /// \param cas Git CAS that contains the tree id. /// \param tree_id Tree id as as hex string. + /// \param ignore_special If set, treat symlinks as absent. + /// NOTE: If ignore_special==true, the stored entries might differ from the + /// actual tree, so the stored ID is set to empty to signal that it should + /// not be used. [[nodiscard]] static auto Read(gsl::not_null<GitCASPtr> const& cas, - std::string const& tree_id) noexcept + std::string const& tree_id, + bool ignore_special = false) noexcept -> std::optional<GitTree>; /// \brief Lookup by dir entry name. '.' and '..' are not allowed. @@ -68,15 +73,22 @@ class GitTree { gsl::not_null<GitCASPtr> cas_; entries_t entries_; std::string raw_id_; + // If set, ignore all fast tree lookups and always traverse + bool ignore_special_; GitTree(gsl::not_null<GitCASPtr> const& cas, entries_t&& entries, - std::string raw_id) noexcept - : cas_{cas}, entries_{std::move(entries)}, raw_id_{std::move(raw_id)} {} + std::string raw_id, + bool ignore_special = false) noexcept + : cas_{cas}, + entries_{std::move(entries)}, + raw_id_{std::move(raw_id)}, + ignore_special_{ignore_special} {} [[nodiscard]] static auto FromEntries(gsl::not_null<GitCASPtr> const& cas, GitRepo::tree_entries_t&& entries, - std::string raw_id) noexcept + std::string raw_id, + bool ignore_special = false) noexcept -> std::optional<GitTree> { entries_t e{}; e.reserve(entries.size()); @@ -91,7 +103,7 @@ class GitTree { } } } - return GitTree(cas, std::move(e), std::move(raw_id)); + return GitTree(cas, std::move(e), std::move(raw_id), ignore_special); } }; @@ -106,8 +118,9 @@ class GitTreeEntry { [[nodiscard]] auto IsTree() const noexcept { return IsTreeObject(type_); } [[nodiscard]] auto Blob() const noexcept -> std::optional<std::string>; - [[nodiscard]] auto Tree() && = delete; - [[nodiscard]] auto Tree() const& noexcept -> std::optional<GitTree> const&; + [[nodiscard]] auto Tree(bool) && = delete; + [[nodiscard]] auto Tree(bool ignore_special = false) const& noexcept + -> std::optional<GitTree> const&; [[nodiscard]] auto Hash() const noexcept { return ToHexString(raw_id_); } [[nodiscard]] auto Type() const noexcept { return type_; } |