From 95df0b6d1bb7d65bdf02533cb7067067000776d3 Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Mon, 27 May 2024 11:05:56 +0200 Subject: Move blob_tree to the common stage since it is used only there. --- src/buildtool/execution_api/bazel_msg/TARGETS | 20 ------ .../execution_api/bazel_msg/blob_tree.cpp | 81 ---------------------- .../execution_api/bazel_msg/blob_tree.hpp | 59 ---------------- src/buildtool/execution_api/common/TARGETS | 21 +++++- src/buildtool/execution_api/common/blob_tree.cpp | 81 ++++++++++++++++++++++ src/buildtool/execution_api/common/blob_tree.hpp | 59 ++++++++++++++++ src/buildtool/execution_api/common/common_api.hpp | 2 +- src/buildtool/execution_api/local/TARGETS | 1 - src/buildtool/execution_api/local/local_api.hpp | 2 +- src/buildtool/execution_api/remote/TARGETS | 3 +- .../execution_api/remote/bazel/bazel_api.hpp | 2 +- 11 files changed, 164 insertions(+), 167 deletions(-) delete mode 100644 src/buildtool/execution_api/bazel_msg/blob_tree.cpp delete mode 100644 src/buildtool/execution_api/bazel_msg/blob_tree.hpp create mode 100644 src/buildtool/execution_api/common/blob_tree.cpp create mode 100644 src/buildtool/execution_api/common/blob_tree.hpp (limited to 'src/buildtool/execution_api') diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS index 2c17b873..a0032c86 100644 --- a/src/buildtool/execution_api/bazel_msg/TARGETS +++ b/src/buildtool/execution_api/bazel_msg/TARGETS @@ -55,24 +55,4 @@ ] , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] } -, "blob_tree": - { "type": ["@", "rules", "CC", "library"] - , "name": ["blob_tree"] - , "hdrs": ["blob_tree.hpp"] - , "srcs": ["blob_tree.cpp"] - , "deps": - [ "bazel_msg" - , "directory_tree" - , ["@", "gsl", "", "gsl"] - , ["src/buildtool/compatibility", "compatibility"] - ] - , "private-deps": - [ ["src/buildtool/common", "common"] - , ["src/buildtool/common", "bazel_types"] - , ["src/buildtool/file_system", "git_repo"] - , ["src/buildtool/file_system", "object_type"] - , ["src/utils/cpp", "hex_string"] - ] - , "stage": ["src", "buildtool", "execution_api", "bazel_msg"] - } } diff --git a/src/buildtool/execution_api/bazel_msg/blob_tree.cpp b/src/buildtool/execution_api/bazel_msg/blob_tree.cpp deleted file mode 100644 index 88797970..00000000 --- a/src/buildtool/execution_api/bazel_msg/blob_tree.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" - -#include -#include - -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/file_system/git_repo.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/utils/cpp/hex_string.hpp" - -/// NOLINTNEXTLINE(misc-no-recursion) -auto BlobTree::FromDirectoryTree(DirectoryTreePtr const& tree, - std::filesystem::path const& parent) noexcept - -> std::optional { - GitRepo::tree_entries_t entries; - std::vector nodes; - try { - entries.reserve(tree->size()); - for (auto const& [name, node] : *tree) { - if (std::holds_alternative(node)) { - auto const& dir = std::get(node); - auto blob_tree = FromDirectoryTree(dir, parent / name); - if (not blob_tree) { - return std::nullopt; - } - auto raw_id = FromHexString(NativeSupport::Unprefix( - (*blob_tree)->Blob().digest.hash())); - if (not raw_id) { - return std::nullopt; - } - entries[std::move(*raw_id)].emplace_back(name, - ObjectType::Tree); - // Only add tree objects to the blob tree to be uploaded. - nodes.emplace_back((*blob_tree)); - } - else { - auto const& artifact = std::get(node); - auto const& object_info = artifact->Info(); - if (not object_info) { - return std::nullopt; - } - auto raw_id = FromHexString(object_info->digest.hash()); - if (not raw_id) { - return std::nullopt; - } - entries[std::move(*raw_id)].emplace_back(name, - object_info->type); - } - } - if (auto git_tree = GitRepo::CreateShallowTree(entries)) { - bazel_re::Digest digest{}; - digest.set_hash(NativeSupport::Prefix(ToHexString(git_tree->first), - /*is_tree=*/true)); - digest.set_size_bytes( - gsl::narrow(git_tree->second.size())); - return std::make_shared(BazelBlob{digest, - git_tree->second, - /*is_exec=*/false}, - nodes); - } - } catch (...) { - return std::nullopt; - } - return std::nullopt; -} diff --git a/src/buildtool/execution_api/bazel_msg/blob_tree.hpp b/src/buildtool/execution_api/bazel_msg/blob_tree.hpp deleted file mode 100644 index b407e8ee..00000000 --- a/src/buildtool/execution_api/bazel_msg/blob_tree.hpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP - -#include -#include -#include -#include -#include - -#include "gsl/gsl" -#include "src/buildtool/compatibility/native_support.hpp" -#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" -#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" - -class BlobTree; -using BlobTreePtr = gsl::not_null>; - -/// \brief Tree-like blob container to enable tree-invariant satisfying blob -/// upload. -class BlobTree { - public: - BlobTree(BazelBlob blob, std::vector nodes) - : blob_{std::move(blob)}, nodes_{std::move(nodes)} {} - - [[nodiscard]] auto Blob() const noexcept -> BazelBlob { return blob_; } - [[nodiscard]] auto IsTree() const noexcept -> bool { - return NativeSupport::IsTree(blob_.digest.hash()); - } - - /// \brief Create a `BlobTree` from a `DirectoryTree`. - [[nodiscard]] static auto FromDirectoryTree( - DirectoryTreePtr const& tree, - std::filesystem::path const& parent = "") noexcept - -> std::optional; - - [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); } - [[nodiscard]] auto end() const noexcept { return nodes_.end(); } - [[nodiscard]] auto size() const noexcept { return nodes_.size(); } - - private: - BazelBlob blob_; - std::vector nodes_; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 4cb4e4b2..0eac0c15 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -58,10 +58,10 @@ , "srcs": ["common_api.cpp"] , "deps": [ "common" + , "blob_tree" , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] - , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] ] , "stage": ["src", "buildtool", "execution_api", "common"] @@ -71,6 +71,25 @@ , ["src/buildtool/logging", "log_level"] ] } +, "blob_tree": + { "type": ["@", "rules", "CC", "library"] + , "name": ["blob_tree"] + , "hdrs": ["blob_tree.hpp"] + , "srcs": ["blob_tree.cpp"] + , "deps": + [ "common" + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] + , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] + , ["@", "gsl", "", "gsl"] + , ["src/buildtool/compatibility", "compatibility"] + ] + , "private-deps": + [ ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/file_system", "object_type"] + , ["src/utils/cpp", "hex_string"] + ] + , "stage": ["src", "buildtool", "execution_api", "common"] + } , "content_blob_container": { "type": ["@", "rules", "CC", "library"] , "name": ["content_blob_container"] diff --git a/src/buildtool/execution_api/common/blob_tree.cpp b/src/buildtool/execution_api/common/blob_tree.cpp new file mode 100644 index 00000000..0954220e --- /dev/null +++ b/src/buildtool/execution_api/common/blob_tree.cpp @@ -0,0 +1,81 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/common/blob_tree.hpp" + +#include +#include + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/utils/cpp/hex_string.hpp" + +/// NOLINTNEXTLINE(misc-no-recursion) +auto BlobTree::FromDirectoryTree(DirectoryTreePtr const& tree, + std::filesystem::path const& parent) noexcept + -> std::optional { + GitRepo::tree_entries_t entries; + std::vector nodes; + try { + entries.reserve(tree->size()); + for (auto const& [name, node] : *tree) { + if (std::holds_alternative(node)) { + auto const& dir = std::get(node); + auto blob_tree = FromDirectoryTree(dir, parent / name); + if (not blob_tree) { + return std::nullopt; + } + auto raw_id = FromHexString(NativeSupport::Unprefix( + (*blob_tree)->Blob().digest.hash())); + if (not raw_id) { + return std::nullopt; + } + entries[std::move(*raw_id)].emplace_back(name, + ObjectType::Tree); + // Only add tree objects to the blob tree to be uploaded. + nodes.emplace_back((*blob_tree)); + } + else { + auto const& artifact = std::get(node); + auto const& object_info = artifact->Info(); + if (not object_info) { + return std::nullopt; + } + auto raw_id = FromHexString(object_info->digest.hash()); + if (not raw_id) { + return std::nullopt; + } + entries[std::move(*raw_id)].emplace_back(name, + object_info->type); + } + } + if (auto git_tree = GitRepo::CreateShallowTree(entries)) { + bazel_re::Digest digest{}; + digest.set_hash(NativeSupport::Prefix(ToHexString(git_tree->first), + /*is_tree=*/true)); + digest.set_size_bytes( + gsl::narrow(git_tree->second.size())); + return std::make_shared(BazelBlob{digest, + git_tree->second, + /*is_exec=*/false}, + nodes); + } + } catch (...) { + return std::nullopt; + } + return std::nullopt; +} diff --git a/src/buildtool/execution_api/common/blob_tree.hpp b/src/buildtool/execution_api/common/blob_tree.hpp new file mode 100644 index 00000000..59e58410 --- /dev/null +++ b/src/buildtool/execution_api/common/blob_tree.hpp @@ -0,0 +1,59 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP + +#include +#include +#include +#include +#include + +#include "gsl/gsl" +#include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" +#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" + +class BlobTree; +using BlobTreePtr = gsl::not_null>; + +/// \brief Tree-like blob container to enable tree-invariant satisfying blob +/// upload. +class BlobTree { + public: + BlobTree(BazelBlob blob, std::vector nodes) + : blob_{std::move(blob)}, nodes_{std::move(nodes)} {} + + [[nodiscard]] auto Blob() const noexcept -> BazelBlob { return blob_; } + [[nodiscard]] auto IsTree() const noexcept -> bool { + return NativeSupport::IsTree(blob_.digest.hash()); + } + + /// \brief Create a `BlobTree` from a `DirectoryTree`. + [[nodiscard]] static auto FromDirectoryTree( + DirectoryTreePtr const& tree, + std::filesystem::path const& parent = "") noexcept + -> std::optional; + + [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); } + [[nodiscard]] auto end() const noexcept { return nodes_.end(); } + [[nodiscard]] auto size() const noexcept { return nodes_.size(); } + + private: + BazelBlob blob_; + std::vector nodes_; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BLOB_TREE_HPP diff --git a/src/buildtool/execution_api/common/common_api.hpp b/src/buildtool/execution_api/common/common_api.hpp index 312f97db..1a2914fb 100644 --- a/src/buildtool/execution_api/common/common_api.hpp +++ b/src/buildtool/execution_api/common/common_api.hpp @@ -27,8 +27,8 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" -#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" #include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" +#include "src/buildtool/execution_api/common/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" /// \brief Stores a list of missing artifact digests, as well as a back-mapping diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index b192b402..294311e7 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -35,7 +35,6 @@ , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] - , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] , ["src/buildtool/execution_api/git", "git"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/compatibility", "compatibility"] diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 91b86de2..978d3968 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -34,7 +34,7 @@ #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" -#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" +#include "src/buildtool/execution_api/common/blob_tree.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/stream_dumper.hpp" diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index c919b7bf..661df625 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -66,10 +66,10 @@ [ "config" , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] - , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] , ["@", "gsl", "", "gsl"] , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "port"] + , ["src/buildtool/execution_api/common", "common_api"] ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": @@ -79,7 +79,6 @@ , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "fs_utils"] ] diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index 49827922..b96148b1 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -28,7 +28,7 @@ #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp" -#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" +#include "src/buildtool/execution_api/common/blob_tree.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" -- cgit v1.2.3