diff options
Diffstat (limited to 'src/buildtool/execution_api/common')
5 files changed, 212 insertions, 15 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 176acea5..579412d9 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -7,9 +7,11 @@ , "execution_action.hpp" , "execution_response.hpp" , "tree_reader.hpp" + , "tree_reader_utils.hpp" , "stream_dumper.hpp" , "artifact_blob_container.hpp" ] + , "srcs": ["tree_reader_utils.cpp"] , "deps": [ "content_blob_container" , ["@", "gsl", "", "gsl"] diff --git a/src/buildtool/execution_api/common/stream_dumper.hpp b/src/buildtool/execution_api/common/stream_dumper.hpp index 31f9b292..81d3bf35 100644 --- a/src/buildtool/execution_api/common/stream_dumper.hpp +++ b/src/buildtool/execution_api/common/stream_dumper.hpp @@ -23,7 +23,7 @@ #include "gsl/gsl" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" template <typename TImpl> @@ -68,7 +68,7 @@ class StreamDumper final { if (Compatibility::IsCompatible()) { auto directory = impl_.ReadDirectory(info.digest); auto data = directory - ? BazelMsgFactory::DirectoryToString(*directory) + ? TreeReaderUtils::DirectoryToString(*directory) : std::nullopt; if (data) { return DumpString(*data, stream); @@ -76,7 +76,7 @@ class StreamDumper final { } else { auto entries = impl_.ReadGitTree(info.digest); - auto data = entries ? BazelMsgFactory::GitTreeToString(*entries) + auto data = entries ? TreeReaderUtils::GitTreeToString(*entries) : std::nullopt; if (data) { return DumpString(*data, stream); diff --git a/src/buildtool/execution_api/common/tree_reader.hpp b/src/buildtool/execution_api/common/tree_reader.hpp index 0a6057fe..d9048e1f 100644 --- a/src/buildtool/execution_api/common/tree_reader.hpp +++ b/src/buildtool/execution_api/common/tree_reader.hpp @@ -23,7 +23,7 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/file_system/object_type.hpp" @@ -50,7 +50,7 @@ class TreeReader final { -> std::optional<ReadTreeResult> { ReadTreeResult result; - BazelMsgFactory::InfoStoreFunc store_info = + TreeReaderUtils::InfoStoreFunc store_info = [&result, &parent](std::filesystem::path const& path, Artifact::ObjectInfo const& info) { result.paths.emplace_back(parent / path); @@ -60,15 +60,15 @@ class TreeReader final { if (Compatibility::IsCompatible()) { auto tree = impl_.ReadDirectory(digest); - if (tree and not BazelMsgFactory::ReadObjectInfosFromDirectory( - *tree, store_info)) { + if (tree and + not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } else { auto tree = impl_.ReadGitTree(digest); - if (tree and not BazelMsgFactory::ReadObjectInfosFromGitTree( - *tree, store_info)) { + if (tree and + not TreeReaderUtils::ReadObjectInfos(*tree, store_info)) { return std::nullopt; } } @@ -117,11 +117,11 @@ class TreeReader final { } [[nodiscard]] auto ReadObjectInfosRecursively( - BazelMsgFactory::InfoStoreFunc const& store, + TreeReaderUtils::InfoStoreFunc const& store, std::filesystem::path const& parent, ArtifactDigest const& digest, bool const include_trees) const -> bool { - BazelMsgFactory::InfoStoreFunc internal_store = + TreeReaderUtils::InfoStoreFunc internal_store = [this, &store, &parent, include_trees]( std::filesystem::path const& path, Artifact::ObjectInfo const& info) -> bool { @@ -138,8 +138,7 @@ class TreeReader final { return false; } } - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *tree, internal_store); + return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } else { @@ -149,8 +148,7 @@ class TreeReader final { return false; } } - return BazelMsgFactory::ReadObjectInfosFromGitTree( - *tree, internal_store); + return TreeReaderUtils::ReadObjectInfos(*tree, internal_store); } } return false; diff --git a/src/buildtool/execution_api/common/tree_reader_utils.cpp b/src/buildtool/execution_api/common/tree_reader_utils.cpp new file mode 100644 index 00000000..1ec7edc0 --- /dev/null +++ b/src/buildtool/execution_api/common/tree_reader_utils.cpp @@ -0,0 +1,143 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/common/tree_reader_utils.hpp" + +#include <exception> +#include <type_traits> + +#include "nlohmann/json.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/hex_string.hpp" + +namespace { +[[nodiscard]] auto CreateObjectInfo(bazel_re::DirectoryNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, + .type = ObjectType::Tree}; +} + +[[nodiscard]] auto CreateObjectInfo(bazel_re::FileNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, + .type = node.is_executable() + ? ObjectType::Executable + : ObjectType::File}; +} + +[[nodiscard]] auto CreateObjectInfo(bazel_re::SymlinkNode const& node) + -> Artifact::ObjectInfo { + return Artifact::ObjectInfo{ + .digest = ArtifactDigest::Create<ObjectType::File>(node.target()), + .type = ObjectType::Symlink}; +} + +template <typename TTree> +[[nodiscard]] auto TreeToString(TTree const& entries) noexcept + -> std::optional<std::string> { + auto json = nlohmann::json::object(); + TreeReaderUtils::InfoStoreFunc store_infos = + [&json](std::filesystem::path const& path, + Artifact::ObjectInfo const& info) -> bool { + static constexpr bool kSizeUnknown = + std::is_same_v<TTree, GitRepo::tree_entries_t>; + + json[path.string()] = info.ToString(kSizeUnknown); + return true; + }; + + if (TreeReaderUtils::ReadObjectInfos(entries, store_infos)) { + try { + return json.dump(2) + "\n"; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "dumping Directory to string failed with:\n{}", + ex.what()); + return std::nullopt; + } + } + Logger::Log(LogLevel::Error, "reading object infos from Directory failed"); + return std::nullopt; +} + +} // namespace + +auto TreeReaderUtils::ReadObjectInfos(bazel_re::Directory const& dir, + InfoStoreFunc const& store_info) noexcept + -> bool { + try { + for (auto const& f : dir.files()) { + if (not store_info(f.name(), CreateObjectInfo(f))) { + return false; + } + } + for (auto const& l : dir.symlinks()) { + if (not store_info(l.name(), CreateObjectInfo(l))) { + return false; + } + } + for (auto const& d : dir.directories()) { + if (not store_info(d.name(), CreateObjectInfo(d))) { + return false; + } + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "reading object infos from Directory failed with:\n{}", + ex.what()); + return false; + } + return true; +} + +auto TreeReaderUtils::ReadObjectInfos(GitRepo::tree_entries_t const& entries, + InfoStoreFunc const& store_info) noexcept + -> bool { + try { + for (auto const& [raw_id, es] : entries) { + auto const hex_id = ToHexString(raw_id); + for (auto const& entry : es) { + if (not store_info( + entry.name, + Artifact::ObjectInfo{ + .digest = ArtifactDigest{hex_id, + /*size is unknown*/ 0, + IsTreeObject(entry.type)}, + .type = entry.type})) { + return false; + } + } + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "reading object infos from Git tree failed with:\n{}", + ex.what()); + return false; + } + return true; +} + +auto TreeReaderUtils::DirectoryToString(bazel_re::Directory const& dir) noexcept + -> std::optional<std::string> { + return TreeToString(dir); +} + +auto TreeReaderUtils::GitTreeToString( + GitRepo::tree_entries_t const& entries) noexcept + -> std::optional<std::string> { + return TreeToString(entries); +} diff --git a/src/buildtool/execution_api/common/tree_reader_utils.hpp b/src/buildtool/execution_api/common/tree_reader_utils.hpp new file mode 100644 index 00000000..ed151899 --- /dev/null +++ b/src/buildtool/execution_api/common/tree_reader_utils.hpp @@ -0,0 +1,54 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP + +#include <filesystem> +#include <functional> +#include <optional> +#include <string> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/file_system/git_repo.hpp" + +class TreeReaderUtils final { + public: + using InfoStoreFunc = std::function<bool(std::filesystem::path const&, + Artifact::ObjectInfo const&)>; + + /// \brief Read object infos from directory. + /// \returns true on success. + [[nodiscard]] static auto ReadObjectInfos( + bazel_re::Directory const& dir, + InfoStoreFunc const& store_info) noexcept -> bool; + + /// \brief Read object infos from git tree. + /// \returns true on success. + [[nodiscard]] static auto ReadObjectInfos( + GitRepo::tree_entries_t const& entries, + InfoStoreFunc const& store_info) noexcept -> bool; + + /// \brief Create descriptive string from Directory protobuf message. + [[nodiscard]] static auto DirectoryToString( + bazel_re::Directory const& dir) noexcept -> std::optional<std::string>; + + /// \brief Create descriptive string from Git tree entries. + [[nodiscard]] static auto GitTreeToString( + GitRepo::tree_entries_t const& entries) noexcept + -> std::optional<std::string>; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_UTILS_HPP |