diff options
Diffstat (limited to 'src')
9 files changed, 216 insertions, 161 deletions
diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS index a0032c86..0e66a685 100644 --- a/src/buildtool/execution_api/bazel_msg/TARGETS +++ b/src/buildtool/execution_api/bazel_msg/TARGETS @@ -24,7 +24,6 @@ [ "bazel_msg" , "directory_tree" , ["src/buildtool/common", "common"] - , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/logging", "log_level"] @@ -35,6 +34,7 @@ , ["src/utils/cpp", "hex_string"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/file_system", "git_repo"] ] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] } diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp index d91fffa7..9ced1375 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp @@ -392,27 +392,6 @@ template <class T> return ActionBundle::Create(msg, content_creator, digest_creator); } -[[nodiscard]] auto CreateObjectInfo(bazel_re::DirectoryNode const& node) - -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, - .type = ObjectType::Tree}; -} - -[[nodiscard]] auto CreateObjectInfo(bazel_re::FileNode const& node) - -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, - .type = node.is_executable() - ? ObjectType::Executable - : ObjectType::File}; -} - -[[nodiscard]] auto CreateObjectInfo(bazel_re::SymlinkNode const& node) - -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{ - ArtifactDigest::Create<ObjectType::File>(node.target()), - ObjectType::Symlink}; -} - /// \brief Convert `DirectoryTree` to `DirectoryNodeBundle`. /// NOLINTNEXTLINE(misc-no-recursion) [[nodiscard]] auto DirectoryTreeToBundle( @@ -487,61 +466,6 @@ template <class T> } // namespace -auto BazelMsgFactory::ReadObjectInfosFromDirectory( - bazel_re::Directory const& dir, - InfoStoreFunc const& store_info) noexcept -> bool { - try { - for (auto const& f : dir.files()) { - if (not store_info(f.name(), CreateObjectInfo(f))) { - return false; - } - } - for (auto const& l : dir.symlinks()) { - if (not store_info(l.name(), CreateObjectInfo(l))) { - return false; - } - } - for (auto const& d : dir.directories()) { - if (not store_info(d.name(), CreateObjectInfo(d))) { - return false; - } - } - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "reading object infos from Directory failed with:\n{}", - ex.what()); - return false; - } - return true; -} - -auto BazelMsgFactory::ReadObjectInfosFromGitTree( - GitRepo::tree_entries_t const& entries, - InfoStoreFunc const& store_info) noexcept -> bool { - try { - for (auto const& [raw_id, es] : entries) { - auto const hex_id = ToHexString(raw_id); - for (auto const& entry : es) { - if (not store_info( - entry.name, - Artifact::ObjectInfo{ - .digest = ArtifactDigest{hex_id, - /*size is unknown*/ 0, - IsTreeObject(entry.type)}, - .type = entry.type})) { - return false; - } - } - } - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "reading object infos from Git tree failed with:\n{}", - ex.what()); - return false; - } - return true; -} - auto BazelMsgFactory::CreateDirectoryDigestFromTree( DirectoryTreePtr const& tree, LinkDigestResolveFunc const& resolve_links, @@ -763,48 +687,3 @@ auto BazelMsgFactory::CreateActionDigestFromCommandLine( return action->Digest(); } - -auto BazelMsgFactory::DirectoryToString(bazel_re::Directory const& dir) noexcept - -> std::optional<std::string> { - auto json = nlohmann::json::object(); - try { - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - dir, [&json](auto path, auto info) { - json[path.string()] = info.ToString(); - return true; - })) { - Logger::Log(LogLevel::Error, - "reading object infos from Directory failed"); - return std::nullopt; - } - return json.dump(2) + "\n"; - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "dumping Directory to string failed with:\n{}", - ex.what()); - return std::nullopt; - } -} - -auto BazelMsgFactory::GitTreeToString( - GitRepo::tree_entries_t const& entries) noexcept - -> std::optional<std::string> { - auto json = nlohmann::json::object(); - try { - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( - entries, [&json](auto path, auto info) { - json[path.string()] = info.ToString(/*size_unknown=*/true); - return true; - })) { - Logger::Log(LogLevel::Error, - "reading object infos from Directory failed"); - return std::nullopt; - } - return json.dump(2) + "\n"; - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "dumping Directory to string failed with:\n{}", - ex.what()); - return std::nullopt; - } -} diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp index 074daeb7..817925f9 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp @@ -30,7 +30,6 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" -#include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -55,18 +54,6 @@ class BazelMsgFactory { using TreeStoreFunc = std::function<std::optional<bazel_re::Digest>(std::string const&)>; - /// \brief Read object infos from directory. - /// \returns true on success. - [[nodiscard]] static auto ReadObjectInfosFromDirectory( - bazel_re::Directory const& dir, - InfoStoreFunc const& store_info) noexcept -> bool; - - /// \brief Read object infos from git tree. - /// \returns true on success. - [[nodiscard]] static auto ReadObjectInfosFromGitTree( - GitRepo::tree_entries_t const& entries, - InfoStoreFunc const& store_info) noexcept -> bool; - /// \brief Create Directory digest from artifact tree structure. /// Recursively traverse entire tree and create blobs for sub-directories. /// \param tree Directory tree of artifacts. @@ -135,15 +122,6 @@ class BazelMsgFactory { std::optional<BlobStoreFunc> const& store_blob = std::nullopt) -> bazel_re::Digest; - /// \brief Create descriptive string from Directory protobuf message. - [[nodiscard]] static auto DirectoryToString( - bazel_re::Directory const& dir) noexcept -> std::optional<std::string>; - - /// \brief Create descriptive string from Git tree entries. - [[nodiscard]] static auto GitTreeToString( - GitRepo::tree_entries_t const& entries) noexcept - -> std::optional<std::string>; - /// \brief Create message vector from std::map. /// \param[in] input map /// \tparam T protobuf message type. It must be a name-value diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 176acea5..579412d9 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -7,9 +7,11 @@ , "execution_action.hpp" , "execution_response.hpp" , "tree_reader.hpp" + , "tree_reader_utils.hpp" , "stream_dumper.hpp" , "artifact_blob_container.hpp" ] + , "srcs": ["tree_reader_utils.cpp"] , "deps": [ "content_blob_container" , ["@", "gsl", "", "gsl"] diff --git a/src/buildtool/execution_api/common/stream_dumper.hpp b/src/buildtool/execution_api/common/stream_dumper.hpp index 31f9b292..81d3bf35 100644 --- a/src/buildtool/execution_api/common/stream_dumper.hpp +++ b/src/buildtool/execution_api/common/stream_dumper.hpp @@ -23,7 +23,7 @@ #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" template <typename TImpl> @@ -68,7 +68,7 @@ class StreamDumper final { if (Compatibility::IsCompatible()) { auto directory = impl_.ReadDirectory(info.digest); auto data = directory - ? BazelMsgFactory::DirectoryToString(*directory) + ? TreeReaderUtils::DirectoryToString(*directory) : std::nullopt; if (data) { return DumpString(*data, stream); @@ -76,7 +76,7 @@ class StreamDumper final { } else { auto entries = impl_.ReadGitTree(info.digest); - auto data = entries ? BazelMsgFactory::GitTreeToString(*entries) + auto data = entries ? TreeReaderUtils::GitTreeToString(*entries) : std::nullopt; if (data) { return DumpString(*data, stream); diff --git a/src/buildtool/execution_api/common/tree_reader.hpp b/src/buildtool/execution_api/common/tree_reader.hpp index 0a6057fe..d9048e1f 100644 --- a/src/buildtool/execution_api/common/tree_reader.hpp +++ b/src/buildtool/execution_api/common/tree_reader.hpp @@ -23,7 +23,7 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" @@ -50,7 +50,7 @@ class TreeReader final { -> std::optional<ReadTreeResult> { ReadTreeResult result; - BazelMsgFactory::InfoStoreFunc store_info = + TreeReaderUtils::InfoStoreFunc store_info = [&result, &parent](std::filesystem::path const& path, Artifact::ObjectInfo const& info) { result.paths.emplace_back(parent / path); @@ -60,15 +60,15 @@ class TreeReader final { if (Compatibility::IsCompatible()) { auto tree = impl_.ReadDirectory(digest); - if (tree and not BazelMsgFactory::ReadObjectInfosFromDirectory( - *tree, store_info)) { + if (tree and + not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } else { auto tree = impl_.ReadGitTree(digest); - if (tree and not BazelMsgFactory::ReadObjectInfosFromGitTree( - *tree, store_info)) { + if (tree and + not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } @@ -117,11 +117,11 @@ class TreeReader final { } [[nodiscard]] auto ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store, + TreeReaderUtils::InfoStoreFunc const& store, std::filesystem::path const& parent, ArtifactDigest const& digest, bool const include_trees) const -> bool { - BazelMsgFactory::InfoStoreFunc internal_store = + TreeReaderUtils::InfoStoreFunc internal_store = [this, &store, &parent, include_trees]( std::filesystem::path const& path, Artifact::ObjectInfo const& info) -> bool { @@ -138,8 +138,7 @@ class TreeReader final { return false; } } - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *tree, internal_store); + return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } else { @@ -149,8 +148,7 @@ class TreeReader final { return false; } } - return BazelMsgFactory::ReadObjectInfosFromGitTree( - *tree, internal_store); + return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } return false; diff --git a/src/buildtool/execution_api/common/tree_reader_utils.cpp b/src/buildtool/execution_api/common/tree_reader_utils.cpp new file mode 100644 index 00000000..1ec7edc0 --- /dev/null +++ b/src/buildtool/execution_api/common/tree_reader_utils.cpp @@ -0,0 +1,143 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" + +#include <exception> +#include <type_traits> + +#include "nlohmann/json.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/hex_string.hpp" + +namespace { +[[nodiscard]] auto CreateObjectInfo(bazel_re::DirectoryNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, + .type = ObjectType::Tree}; +} + +[[nodiscard]] auto CreateObjectInfo(bazel_re::FileNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, + .type = node.is_executable() + ? ObjectType::Executable + : ObjectType::File}; +} + +[[nodiscard]] auto CreateObjectInfo(bazel_re::SymlinkNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{ + .digest = ArtifactDigest::Create<ObjectType::File>(node.target()), + .type = ObjectType::Symlink}; +} + +template <typename TTree> +[[nodiscard]] auto TreeToString(TTree const& entries) noexcept + -> std::optional<std::string> { + auto json = nlohmann::json::object(); + TreeReaderUtils::InfoStoreFunc store_infos = + [&json](std::filesystem::path const& path, + Artifact::ObjectInfo const& info) -> bool { + static constexpr bool kSizeUnknown = + std::is_same_v<TTree, GitRepo::tree_entries_t>; + + json[path.string()] = info.ToString(kSizeUnknown); + return true; + }; + + if (TreeReaderUtils::ReadObjectInfos(entries, store_infos)) { + try { + return json.dump(2) + "\n"; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "dumping Directory to string failed with:\n{}", + ex.what()); + return std::nullopt; + } + } + Logger::Log(LogLevel::Error, "reading object infos from Directory failed"); + return std::nullopt; +} + +} // namespace + +auto TreeReaderUtils::ReadObjectInfos(bazel_re::Directory const& dir, + InfoStoreFunc const& store_info) noexcept + -> bool { + try { + for (auto const& f : dir.files()) { + if (not store_info(f.name(), CreateObjectInfo(f))) { + return false; + } + } + for (auto const& l : dir.symlinks()) { + if (not store_info(l.name(), CreateObjectInfo(l))) { + return false; + } + } + for (auto const& d : dir.directories()) { + if (not store_info(d.name(), CreateObjectInfo(d))) { + return false; + } + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "reading object infos from Directory failed with:\n{}", + ex.what()); + return false; + } + return true; +} + +auto TreeReaderUtils::ReadObjectInfos(GitRepo::tree_entries_t const& entries, + InfoStoreFunc const& store_info) noexcept + -> bool { + try { + for (auto const& [raw_id, es] : entries) { + auto const hex_id = ToHexString(raw_id); + for (auto const& entry : es) { + if (not store_info( + entry.name, + Artifact::ObjectInfo{ + .digest = ArtifactDigest{hex_id, + /*size is unknown*/ 0, + IsTreeObject(entry.type)}, + .type = entry.type})) { + return false; + } + } + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "reading object infos from Git tree failed with:\n{}", + ex.what()); + return false; + } + return true; +} + +auto TreeReaderUtils::DirectoryToString(bazel_re::Directory const& dir) noexcept + -> std::optional<std::string> { + return TreeToString(dir); +} + +auto TreeReaderUtils::GitTreeToString( + GitRepo::tree_entries_t const& entries) noexcept + -> std::optional<std::string> { + return TreeToString(entries); +} diff --git a/src/buildtool/execution_api/common/tree_reader_utils.hpp b/src/buildtool/execution_api/common/tree_reader_utils.hpp new file mode 100644 index 00000000..ed151899 --- /dev/null +++ b/src/buildtool/execution_api/common/tree_reader_utils.hpp @@ -0,0 +1,54 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP + +#include <filesystem> +#include <functional> +#include <optional> +#include <string> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/file_system/git_repo.hpp" + +class TreeReaderUtils final { + public: + using InfoStoreFunc = std::function<bool(std::filesystem::path const&, + Artifact::ObjectInfo const&)>; + + /// \brief Read object infos from directory. + /// \returns true on success. + [[nodiscard]] static auto ReadObjectInfos( + bazel_re::Directory const& dir, + InfoStoreFunc const& store_info) noexcept -> bool; + + /// \brief Read object infos from git tree. + /// \returns true on success. + [[nodiscard]] static auto ReadObjectInfos( + GitRepo::tree_entries_t const& entries, + InfoStoreFunc const& store_info) noexcept -> bool; + + /// \brief Create descriptive string from Directory protobuf message. + [[nodiscard]] static auto DirectoryToString( + bazel_re::Directory const& dir) noexcept -> std::optional<std::string>; + + /// \brief Create descriptive string from Git tree entries. + [[nodiscard]] static auto GitTreeToString( + GitRepo::tree_entries_t const& entries) noexcept + -> std::optional<std::string>; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP diff --git a/src/buildtool/execution_api/utils/subobject.cpp b/src/buildtool/execution_api/utils/subobject.cpp index 5523d784..4217c910 100644 --- a/src/buildtool/execution_api/utils/subobject.cpp +++ b/src/buildtool/execution_api/utils/subobject.cpp @@ -17,6 +17,7 @@ #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -51,7 +52,7 @@ auto RetrieveSubPathId(Artifact::ObjectInfo object_info, break; } std::optional<Artifact::ObjectInfo> new_object_info{}; - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + if (not TreeReaderUtils::ReadObjectInfos( *directory, [&new_object_info, &segment](auto path, auto info) { if (path == segment) { @@ -87,7 +88,7 @@ auto RetrieveSubPathId(Artifact::ObjectInfo object_info, break; } std::optional<Artifact::ObjectInfo> new_object_info{}; - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( + if (not TreeReaderUtils::ReadObjectInfos( *entries, [&new_object_info, &segment](auto path, auto info) { if (path == segment) { |