// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP #define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP #include #include #include #include #include #include #include #include #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_tree.hpp" #include "src/utils/cpp/concepts.hpp" /// FilteredIterator is an helper class to allow for iteration over /// directory-only or file-only entries stored inside the class /// DirectoryEntries. Internally, DirectoryEntries holds a /// map or map. While iterating, we are /// just interested in the name of the entry (i.e., the string). /// To decide which entries retain, the FilteredIterator requires a predicate. template // I is a forward iterator // I::value_type is a std::pair class FilteredIterator { public: using value_type = std::string const; using pointer = value_type*; using reference = value_type&; using difference_type = std::ptrdiff_t; using iteratori_category = std::forward_iterator_tag; using predicate_t = std::function; FilteredIterator() noexcept = default; // [first, last) is a valid sequence FilteredIterator(I first, I last, predicate_t p) noexcept : iterator_{std::find_if(first, last, p)}, end_{std::move(last)}, p{std::move(p)} {} auto operator*() const noexcept -> reference { return iterator_->first; } auto operator++() noexcept -> FilteredIterator& { ++iterator_; iterator_ = std::find_if(iterator_, end_, p); return *this; } [[nodiscard]] auto begin() noexcept -> FilteredIterator& { return *this; } [[nodiscard]] auto end() const noexcept -> FilteredIterator { return FilteredIterator{end_, end_, p}; } [[nodiscard]] friend auto operator==(FilteredIterator const& x, FilteredIterator const& y) noexcept -> bool { return x.iterator_ == y.iterator_; } [[nodiscard]] friend auto operator!=(FilteredIterator const& x, FilteredIterator const& y) noexcept -> bool { return not(x == y); } private: I iterator_{}; const I end_{}; predicate_t p{}; }; class FileRoot { using fs_root_t = std::filesystem::path; struct git_root_t { gsl::not_null cas; gsl::not_null tree; }; using root_t = std::variant; public: static constexpr auto kGitTreeMarker = "git tree"; static constexpr auto kGitTreeIgnoreSpecialMarker = "git tree ignore-special"; static constexpr auto kFileIgnoreSpecialMarker = "file ignore-special"; class DirectoryEntries { friend class FileRoot; public: using pairs_t = std::unordered_map; using tree_t = gsl::not_null; using entries_t = std::variant; using fs_iterator_type = typename pairs_t::const_iterator; using fs_iterator = FilteredIterator; using git_iterator_type = GitTree::entries_t::const_iterator; using git_iterator = FilteredIterator; private: /// Iterator has two FilteredIterators, one for iterating over pairs_t /// and one for tree_t. Each FilteredIterator is constructed with a /// proper predicate, allowing for iteration on file-only or /// directory-only entries class Iterator { public: using value_type = std::string const; using pointer = value_type*; using reference = value_type&; using difference_type = std::ptrdiff_t; using iteratori_category = std::forward_iterator_tag; explicit Iterator(fs_iterator fs_it) : it_{std::move(fs_it)} {} explicit Iterator(git_iterator git_it) : it_{std::move(git_it)} {} auto operator*() const noexcept -> reference { if (std::holds_alternative(it_)) { return *std::get(it_); } return *std::get(it_); } [[nodiscard]] auto begin() noexcept -> Iterator& { return *this; } [[nodiscard]] auto end() const noexcept -> Iterator { if (std::holds_alternative(it_)) { return Iterator{std::get(it_).end()}; } return Iterator{std::get(it_).end()}; } auto operator++() noexcept -> Iterator& { if (std::holds_alternative(it_)) { ++(std::get(it_)); } else { ++(std::get(it_)); } return *this; } friend auto operator==(Iterator const& x, Iterator const& y) noexcept -> bool { return x.it_ == y.it_; } friend auto operator!=(Iterator const& x, Iterator const& y) noexcept -> bool { return not(x == y); } private: std::variant it_; }; public: explicit DirectoryEntries(pairs_t pairs) noexcept : data_{std::move(pairs)} {} explicit DirectoryEntries(tree_t const& git_tree) noexcept : data_{git_tree} {} [[nodiscard]] auto ContainsBlob(std::string const& name) const noexcept -> bool { try { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); auto ptr = data->LookupEntryByName(name); if (static_cast(ptr)) { return IsBlobObject(ptr->Type()); } return false; } if (std::holds_alternative(data_)) { auto const& data = std::get(data_); auto it = data.find(name); return (it != data.end() and IsBlobObject(it->second)); } } catch (...) { } return false; } [[nodiscard]] auto Empty() const noexcept -> bool { if (std::holds_alternative(data_)) { try { auto const& tree = std::get(data_); return tree->begin() == tree->end(); } catch (...) { return false; } } if (std::holds_alternative(data_)) { return std::get(data_).empty(); } return true; } /// \brief Retrieve a root tree as a KNOWN artifact. /// Only succeeds if no entries have to be ignored. [[nodiscard]] auto AsKnownTree(std::string const& repository) const noexcept -> std::optional { if (Compatibility::IsCompatible()) { return std::nullopt; } if (std::holds_alternative(data_)) { try { auto const& data = std::get(data_); // check if tree is ignore_special if (data->RawHash().empty()) { return std::nullopt; } auto const& id = data->Hash(); auto const& size = data->Size(); if (size) { return ArtifactDescription{ ArtifactDigest{id, *size, /*is_tree=*/true}, ObjectType::Tree, repository}; } } catch (...) { return std::nullopt; } } return std::nullopt; } [[nodiscard]] auto FilesIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsFileObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return IsFileObject(x.second->Type()); }}}; } [[nodiscard]] auto SymlinksIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsSymlinkObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return IsSymlinkObject(x.second->Type()); }}}; } [[nodiscard]] auto DirectoriesIterator() const -> Iterator { if (std::holds_alternative(data_)) { auto const& data = std::get(data_); return Iterator{FilteredIterator{ data.begin(), data.end(), [](auto const& x) { return IsTreeObject(x.second); }}}; } // std::holds_alternative(data_) == true auto const& data = std::get(data_); return Iterator{FilteredIterator{ data->begin(), data->end(), [](auto const& x) noexcept -> bool { return x.second->IsTree(); }}}; } private: entries_t data_; }; FileRoot() noexcept = default; explicit FileRoot(bool ignore_special) noexcept : ignore_special_(ignore_special) {} // avoid type narrowing errors explicit FileRoot(char const* root) noexcept : root_{fs_root_t{root}} {} explicit FileRoot(std::filesystem::path root) noexcept : root_{std::move(root)} {} FileRoot(std::filesystem::path root, bool ignore_special) noexcept : root_{std::move(root)}, ignore_special_{ignore_special} {} FileRoot(gsl::not_null const& cas, gsl::not_null const& tree, bool ignore_special = false) noexcept : root_{git_root_t{cas, tree}}, ignore_special_{ignore_special} {} [[nodiscard]] static auto FromGit(std::filesystem::path const& repo_path, std::string const& git_tree_id, bool ignore_special = false) noexcept -> std::optional { if (auto cas = GitCAS::Open(repo_path)) { if (auto tree = GitTree::Read(cas, git_tree_id, ignore_special)) { try { return FileRoot{ cas, std::make_shared(std::move(*tree)), ignore_special}; } catch (...) { } } } return std::nullopt; } // Return a complete description of the content of this root, if // content-fixed. [[nodiscard]] auto ContentDescription() const noexcept -> std::optional { try { if (std::holds_alternative(root_)) { nlohmann::json j; // ignore-special git-tree-based roots are still content-fixed j.push_back(kGitTreeMarker); j.push_back(std::get(root_).tree->Hash()); return j; } } catch (...) { } return std::nullopt; } // Indicates that subsequent calls to `Exists()`, `IsFile()`, // `IsDirectory()`, and `BlobType()` on contents of the same directory will // be served without any additional file system lookups. [[nodiscard]] auto HasFastDirectoryLookup() const noexcept -> bool { return std::holds_alternative(root_); } [[nodiscard]] auto Exists(std::filesystem::path const& path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (path == ".") { return true; } return static_cast( std::get(root_).tree->LookupEntryByPath(path)); } // std::holds_alternative(root_) == true auto root_path = std::get(root_) / path; auto exists = FileSystemManager::Exists(root_path); auto type = FileSystemManager::Type(root_path); return (ignore_special_ ? exists and type and IsNonSpecialObject(*type) : exists); } [[nodiscard]] auto IsFile( std::filesystem::path const& file_path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { return IsFileObject(entry->Type()); } return false; } return FileSystemManager::IsFile(std::get(root_) / file_path); } [[nodiscard]] auto IsSymlink( std::filesystem::path const& file_path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { return IsSymlinkObject(entry->Type()); } return false; } return FileSystemManager::IsNonUpwardsSymlink( std::get(root_) / file_path); } [[nodiscard]] auto IsBlob( std::filesystem::path const& file_path) const noexcept -> bool { return IsFile(file_path) or IsSymlink(file_path); } [[nodiscard]] auto IsDirectory( std::filesystem::path const& dir_path) const noexcept -> bool { if (std::holds_alternative(root_)) { if (dir_path == ".") { return true; } if (auto entry = std::get(root_).tree->LookupEntryByPath( dir_path)) { return entry->IsTree(); } return false; } return FileSystemManager::IsDirectory(std::get(root_) / dir_path); } /// \brief Read content of file or symlink. [[nodiscard]] auto ReadContent(std::filesystem::path const& file_path) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { if (IsBlobObject(entry->Type())) { return entry->Blob(); } } return std::nullopt; } auto full_path = std::get(root_) / file_path; if (auto type = FileSystemManager::Type(full_path)) { return IsSymlinkObject(*type) ? FileSystemManager::ReadSymlink(full_path) : FileSystemManager::ReadFile(full_path); } return std::nullopt; } [[nodiscard]] auto ReadDirectory(std::filesystem::path const& dir_path) const noexcept -> DirectoryEntries { try { if (std::holds_alternative(root_)) { auto const& tree = std::get(root_).tree; if (dir_path == ".") { return DirectoryEntries{&(*tree)}; } if (auto entry = tree->LookupEntryByPath(dir_path)) { if (auto const& found_tree = entry->Tree(ignore_special_)) { return DirectoryEntries{&(*found_tree)}; } } } else { DirectoryEntries::pairs_t map{}; if (FileSystemManager::ReadDirectory( std::get(root_) / dir_path, [&map](const auto& name, auto type) { map.emplace(name.string(), type); return true; }, ignore_special_)) { return DirectoryEntries{std::move(map)}; } } } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "reading directory {} failed with:\n{}", dir_path.string(), ex.what()); } return DirectoryEntries{DirectoryEntries::pairs_t{}}; } [[nodiscard]] auto BlobType(std::filesystem::path const& file_path) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { if (IsBlobObject(entry->Type())) { return entry->Type(); } } return std::nullopt; } auto type = FileSystemManager::Type(std::get(root_) / file_path); if (type and IsBlobObject(*type)) { return type; } return std::nullopt; } /// \brief Read a blob from the root based on its ID. [[nodiscard]] auto ReadBlob(std::string const& blob_id) const noexcept -> std::optional { if (std::holds_alternative(root_)) { return std::get(root_).cas->ReadObject( blob_id, /*is_hex_id=*/true); } return std::nullopt; } /// \brief Read a root tree based on its ID. /// This should include all valid entry types. [[nodiscard]] auto ReadTree(std::string const& tree_id) const noexcept -> std::optional { if (std::holds_alternative(root_)) { try { auto const& cas = std::get(root_).cas; return GitTree::Read(cas, tree_id); } catch (...) { return std::nullopt; } } return std::nullopt; } // Create LOCAL or KNOWN artifact. Does not check existence for LOCAL. // `file_path` must reference a blob. [[nodiscard]] auto ToArtifactDescription( std::filesystem::path const& file_path, std::string const& repository) const noexcept -> std::optional { if (std::holds_alternative(root_)) { if (auto entry = std::get(root_).tree->LookupEntryByPath( file_path)) { if (entry->IsBlob()) { if (Compatibility::IsCompatible()) { auto compatible_hash = Compatibility::RegisterGitEntry( entry->Hash(), *entry->Blob(), repository); return ArtifactDescription{ ArtifactDigest{compatible_hash, *entry->Size(), /*is_tree=*/false}, entry->Type()}; } return ArtifactDescription{ ArtifactDigest{ entry->Hash(), *entry->Size(), /*is_tree=*/false}, entry->Type(), repository}; } } return std::nullopt; } return ArtifactDescription{file_path, repository}; } private: root_t root_; // If set, forces lookups to ignore entries which are neither file nor // directories instead of erroring out. This means implicitly also that // there are no more fast tree lookups, i.e., tree traversal is a must. bool ignore_special_{}; }; #endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP