diff options
Diffstat (limited to 'src')
7 files changed, 210 insertions, 33 deletions
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp index 15845c08..ed82d92d 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp @@ -35,14 +35,15 @@ struct BazelBlob { bool is_exec{}; // optional: hint to put the blob in executable CAS }; -[[nodiscard]] static inline auto CreateBlobFromFile( - std::filesystem::path const& file_path) noexcept - -> std::optional<BazelBlob> { - auto const type = FileSystemManager::Type(file_path); +/// \brief Create a blob from the content found in file or symlink pointed to by +/// given path. +[[nodiscard]] static inline auto CreateBlobFromPath( + std::filesystem::path const& fpath) noexcept -> std::optional<BazelBlob> { + auto const type = FileSystemManager::Type(fpath); if (not type) { return std::nullopt; } - auto const content = FileSystemManager::ReadFile(file_path, *type); + auto const content = FileSystemManager::ReadContentAtPath(fpath, *type); if (not content.has_value()) { return std::nullopt; } diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp index ba52443b..12f7398f 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp @@ -241,6 +241,21 @@ template <class T> return node; } +/// \brief Create protobuf message 'SymlinkNode'. +[[nodiscard]] auto CreateSymlinkNode( + std::string const& link_name, + std::string const& target, + std::vector<bazel_re::NodeProperty> const& props) noexcept + -> bazel_re::SymlinkNode { + bazel_re::SymlinkNode node; + node.set_name(link_name); + node.set_target(target); + std::copy(props.cbegin(), + props.cend(), + pb::back_inserter(node.mutable_node_properties())); + return node; +} + /// \brief Create protobuf message FileNode from Artifact::ObjectInfo [[nodiscard]] auto CreateFileNodeFromObjectInfo( std::string const& name, @@ -266,6 +281,25 @@ template <class T> return dir_node; } +/// \brief Create protobuf message SymlinkNode from Digest for multiple +/// instances at once +[[nodiscard]] auto CreateSymlinkNodesFromDigests( + std::vector<std::string> const& symlink_names, + std::vector<bazel_re::Digest> const& symlink_digests, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) + -> std::vector<bazel_re::SymlinkNode> { + std::vector<std::string> symlink_targets; + resolve_links(symlink_digests, &symlink_targets); + auto it_name = symlink_names.begin(); + auto it_target = symlink_targets.begin(); + std::vector<bazel_re::SymlinkNode> symlink_nodes; + // both loops have same length + for (; it_name != symlink_names.end(); ++it_name, ++it_target) { + symlink_nodes.emplace_back(CreateSymlinkNode(*it_name, *it_target, {})); + } + return symlink_nodes; +} + /// \brief Create bundle for protobuf message DirectoryNode from Directory. [[nodiscard]] auto CreateDirectoryNodeBundle(std::string const& dir_name, bazel_re::Directory const& dir) @@ -361,27 +395,41 @@ template <class T> : ObjectType::File}; } +[[nodiscard]] auto CreateObjectInfo(bazel_re::SymlinkNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{ + ArtifactDigest::Create<ObjectType::File>(node.target()), + ObjectType::Symlink}; +} + /// \brief Convert `DirectoryTree` to `DirectoryNodeBundle`. /// NOLINTNEXTLINE(misc-no-recursion) [[nodiscard]] auto DirectoryTreeToBundle( std::string const& root_name, DirectoryTreePtr const& tree, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links, std::optional<BazelMsgFactory::BlobStoreFunc> const& store_blob, std::optional<BazelMsgFactory::InfoStoreFunc> const& store_info, std::filesystem::path const& parent = "") noexcept -> DirectoryNodeBundle::Ptr { - std::vector<bazel_re::FileNode> file_nodes; - std::vector<bazel_re::DirectoryNode> dir_nodes; + std::vector<bazel_re::FileNode> file_nodes{}; + std::vector<bazel_re::DirectoryNode> dir_nodes{}; + std::vector<std::string> symlink_names{}; + std::vector<bazel_re::Digest> symlink_digests{}; try { for (auto const& [name, node] : *tree) { if (std::holds_alternative<DirectoryTreePtr>(node)) { auto const& dir = std::get<DirectoryTreePtr>(node); - auto const dir_bundle = DirectoryTreeToBundle( - name, dir, store_blob, store_info, parent / name); + auto const dir_bundle = DirectoryTreeToBundle(name, + dir, + resolve_links, + store_blob, + store_info, + parent / name); if (not dir_bundle) { return nullptr; } - dir_nodes.push_back(dir_bundle->Message()); + dir_nodes.emplace_back(dir_bundle->Message()); if (store_blob) { (*store_blob)(dir_bundle->MakeBlob(/*is_exec=*/false)); } @@ -393,11 +441,17 @@ template <class T> return nullptr; } if (IsTreeObject(object_info->type)) { - dir_nodes.push_back( + dir_nodes.emplace_back( CreateDirectoryNodeFromObjectInfo(name, *object_info)); } + else if (IsSymlinkObject(object_info->type)) { + // for symlinks we need to retrieve the data from the + // digest, which we will handle in bulk + symlink_names.emplace_back(name); + symlink_digests.emplace_back(object_info->digest); + } else { - file_nodes.push_back( + file_nodes.emplace_back( CreateFileNodeFromObjectInfo(name, *object_info)); } if (store_info and @@ -407,7 +461,12 @@ template <class T> } } return CreateDirectoryNodeBundle( - root_name, CreateDirectory(file_nodes, dir_nodes, {}, {})); + root_name, + CreateDirectory(file_nodes, + dir_nodes, + CreateSymlinkNodesFromDigests( + symlink_names, symlink_digests, resolve_links), + {})); } catch (...) { return nullptr; } @@ -425,6 +484,11 @@ auto BazelMsgFactory::ReadObjectInfosFromDirectory( return false; } } + for (auto const& l : dir.symlinks()) { + if (not store_info(l.name(), CreateObjectInfo(l))) { + return false; + } + } for (auto const& d : dir.directories()) { if (not store_info(d.name(), CreateObjectInfo(d))) { return false; @@ -468,10 +532,12 @@ auto BazelMsgFactory::ReadObjectInfosFromGitTree( auto BazelMsgFactory::CreateDirectoryDigestFromTree( DirectoryTreePtr const& tree, + LinkDigestResolveFunc const& resolve_links, std::optional<BlobStoreFunc> const& store_blob, std::optional<InfoStoreFunc> const& store_info) noexcept -> std::optional<bazel_re::Digest> { - if (auto bundle = DirectoryTreeToBundle("", tree, store_blob, store_info)) { + if (auto bundle = DirectoryTreeToBundle( + "", tree, resolve_links, store_blob, store_info)) { if (store_blob) { try { (*store_blob)(bundle->MakeBlob(/*is_exec=*/false)); @@ -487,17 +553,25 @@ auto BazelMsgFactory::CreateDirectoryDigestFromTree( auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, - DirStoreFunc const& store_dir) noexcept -> std::optional<bazel_re::Digest> { + DirStoreFunc const& store_dir, + SymlinkStoreFunc const& store_symlink) noexcept + -> std::optional<bazel_re::Digest> { std::vector<bazel_re::FileNode> files{}; std::vector<bazel_re::DirectoryNode> dirs{}; - - auto dir_reader = [&files, &dirs, &root, &store_file, &store_dir]( - auto name, auto type) { + std::vector<bazel_re::SymlinkNode> symlinks{}; + + auto dir_reader = [&files, + &dirs, + &symlinks, + &root, + &store_file, + &store_dir, + &store_symlink](auto name, auto type) { const auto full_name = root / name; if (IsTreeObject(type)) { // create and store sub directory auto digest = CreateDirectoryDigestFromLocalTree( - root / name, store_file, store_dir); + root / name, store_file, store_dir, store_symlink); if (not digest) { Logger::Log(LogLevel::Error, "failed storing tree {}", @@ -512,8 +586,22 @@ auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( return true; } - // create and store file try { + if (IsSymlinkObject(type)) { + // create and store symlink + auto content = FileSystemManager::ReadSymlink(full_name); + if (content and PathIsNonUpwards(*content) and + store_symlink(*content)) { + symlinks.emplace_back( + CreateSymlinkNode(name.string(), *content, {})); + return true; + } + Logger::Log(LogLevel::Error, + "failed storing symlink {}", + full_name.string()); + return false; + } + // create and store file if (auto digest = store_file(full_name, IsExecutableObject(type))) { auto file = CreateFileNode(name.string(), type, {}); file.set_allocated_digest(gsl::owner<bazel_re::Digest*>{ @@ -531,7 +619,7 @@ auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( }; if (FileSystemManager::ReadDirectory(root, dir_reader)) { - auto dir = CreateDirectory(files, dirs, {}, {}); + auto dir = CreateDirectory(files, dirs, symlinks, {}); if (auto bytes = SerializeMessage(dir)) { try { if (auto digest = store_dir(*bytes, dir)) { @@ -551,16 +639,20 @@ auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( auto BazelMsgFactory::CreateGitTreeDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, - TreeStoreFunc const& store_tree) noexcept + TreeStoreFunc const& store_tree, + SymlinkStoreFunc const& store_symlink) noexcept -> std::optional<bazel_re::Digest> { GitRepo::tree_entries_t entries{}; - auto dir_reader = [&entries, &root, &store_file, &store_tree](auto name, - auto type) { + auto dir_reader = [&entries, + &root, + &store_file, + &store_tree, + &store_symlink](auto name, auto type) { const auto full_name = root / name; if (IsTreeObject(type)) { // create and store sub directory if (auto digest = CreateGitTreeDigestFromLocalTree( - full_name, store_file, store_tree)) { + full_name, store_file, store_tree, store_symlink)) { if (auto raw_id = FromHexString( NativeSupport::Unprefix(digest->hash()))) { entries[std::move(*raw_id)].emplace_back(name.string(), @@ -573,8 +665,30 @@ auto BazelMsgFactory::CreateGitTreeDigestFromLocalTree( return false; } - // create and store file try { + if (IsSymlinkObject(type)) { + auto content = FileSystemManager::ReadSymlink(full_name); + if (content and PathIsNonUpwards(*content)) { + if (auto digest = store_symlink(*content)) { + if (auto raw_id = FromHexString( + NativeSupport::Unprefix(digest->hash()))) { + entries[std::move(*raw_id)].emplace_back( + name.string(), type); + return true; + } + } + Logger::Log(LogLevel::Error, + "failed storing symlink {}", + full_name.string()); + } + else { + Logger::Log(LogLevel::Error, + "failed storing symlink {} -- not non-upwards", + full_name.string()); + } + return false; + } + // create and store file if (auto digest = store_file(full_name, IsExecutableObject(type))) { if (auto raw_id = FromHexString( NativeSupport::Unprefix(digest->hash()))) { diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp index faeab647..ca913c63 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp @@ -40,11 +40,16 @@ class BazelMsgFactory { using BlobStoreFunc = std::function<void(BazelBlob&&)>; using InfoStoreFunc = std::function<bool(std::filesystem::path const&, Artifact::ObjectInfo const&)>; + using LinkDigestResolveFunc = + std::function<void(std::vector<bazel_re::Digest> const&, + std::vector<std::string>*)>; using FileStoreFunc = std::function< std::optional<bazel_re::Digest>(std::filesystem::path const&, bool)>; using DirStoreFunc = std::function<std::optional<bazel_re::Digest>( std::string const&, bazel_re::Directory const&)>; + using SymlinkStoreFunc = + std::function<std::optional<bazel_re::Digest>(std::string const&)>; using TreeStoreFunc = std::function<std::optional<bazel_re::Digest>( std::string const&, GitRepo::tree_entries_t const&)>; @@ -63,12 +68,14 @@ class BazelMsgFactory { /// \brief Create Directory digest from artifact tree structure. /// Recursively traverse entire tree and create blobs for sub-directories. - /// \param tree Directory tree of artifacts. - /// \param store_blob Function for storing Directory blobs. - /// \param store_info Function for storing object infos. + /// \param tree Directory tree of artifacts. + /// \param resolve_links Function for resolving symlinks. + /// \param store_blob Function for storing Directory blobs. + /// \param store_info Function for storing object infos. /// \returns Digest representing the entire tree. [[nodiscard]] static auto CreateDirectoryDigestFromTree( DirectoryTreePtr const& tree, + LinkDigestResolveFunc const& resolve_links, std::optional<BlobStoreFunc> const& store_blob = std::nullopt, std::optional<InfoStoreFunc> const& store_info = std::nullopt) noexcept -> std::optional<bazel_re::Digest>; @@ -78,11 +85,13 @@ class BazelMsgFactory { /// \param root Path to local file root. /// \param store_file Function for storing local file via path. /// \param store_dir Function for storing Directory blobs. + /// \param store_symlink Function for storing symlink via content. /// \returns Digest representing the entire file root. [[nodiscard]] static auto CreateDirectoryDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, - DirStoreFunc const& store_dir) noexcept + DirStoreFunc const& store_dir, + SymlinkStoreFunc const& store_symlink) noexcept -> std::optional<bazel_re::Digest>; /// \brief Create Git tree digest from local file root. @@ -90,11 +99,13 @@ class BazelMsgFactory { /// \param root Path to local file root. /// \param store_file Function for storing local file via path. /// \param store_tree Function for storing git trees. + /// \param store_symlink Function for storing symlink via content. /// \returns Digest representing the entire file root. [[nodiscard]] static auto CreateGitTreeDigestFromLocalTree( std::filesystem::path const& root, FileStoreFunc const& store_file, - TreeStoreFunc const& store_tree) noexcept + TreeStoreFunc const& store_tree, + SymlinkStoreFunc const& store_symlink) noexcept -> std::optional<bazel_re::Digest>; /// \brief Creates Action digest from command line. diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index e2fe1199..e5457bcd 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -61,11 +61,14 @@ class BuildCleanupAnchor { auto /*dir*/) -> std::optional<bazel_re::Digest> { return cas.StoreTree(bytes); }; + auto store_symlink = [&cas](auto content) { + return cas.StoreBlob(content); + }; return Compatibility::IsCompatible() ? BazelMsgFactory::CreateDirectoryDigestFromLocalTree( - dir_path, store_blob, store_tree) + dir_path, store_blob, store_tree, store_symlink) : BazelMsgFactory::CreateGitTreeDigestFromLocalTree( - dir_path, store_blob, store_tree); + dir_path, store_blob, store_tree, store_symlink); } } // namespace diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 577d771b..87296414 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -309,8 +309,18 @@ class LocalApi final : public IExecutionApi { if (Compatibility::IsCompatible()) { BlobContainer blobs{}; + auto const& cas = storage_->CAS(); auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( *build_root, + [&cas](std::vector<bazel_re::Digest> const& digests, + std::vector<std::string>* targets) { + targets->reserve(digests.size()); + for (auto const& digest : digests) { + auto p = cas.BlobPath(digest, /*is_executable=*/false); + auto content = FileSystemManager::ReadFile(*p); + targets->emplace_back(*content); + } + }, [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); }); if (not digest) { Logger::Log(LogLevel::Debug, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index fcfb517d..f4fd3b1f 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -312,8 +312,21 @@ auto BazelApi::CreateAction( if (Compatibility::IsCompatible()) { BlobContainer blobs{}; + auto const& network = network_; auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( *build_root, + [&network](std::vector<bazel_re::Digest> const& digests, + std::vector<std::string>* targets) { + auto reader = network->ReadBlobs(digests); + auto blobs = reader.Next(); + targets->reserve(digests.size()); + while (not blobs.empty()) { + for (auto const& blob : blobs) { + targets->emplace_back(blob.data); + } + blobs = reader.Next(); + } + }, [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); }); if (not digest) { Logger::Log(LogLevel::Debug, diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp index 30a469af..a319cc0e 100644 --- a/src/buildtool/file_system/file_system_manager.hpp +++ b/src/buildtool/file_system/file_system_manager.hpp @@ -695,6 +695,31 @@ class FileSystemManager { return std::nullopt; } + /// \brief Read the content of given file or symlink. + [[nodiscard]] static auto ReadContentAtPath( + std::filesystem::path const& fpath, + ObjectType type) -> std::optional<std::string> { + try { + if (IsSymlinkObject(type)) { + return ReadSymlink(fpath); + } + if (IsFileObject(type)) { + return ReadFile(fpath, type); + } + Logger::Log( + LogLevel::Debug, + "{} can not be read because it is neither a file nor symlink.", + fpath.string()); + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "reading content at path {} failed:\n{}", + fpath.string(), + ex.what()); + } + + return std::nullopt; + } + /// \brief Write file /// If argument fd_less is given, the write will be performed in a child /// process to prevent polluting the parent with open writable file |