summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_api/bazel_msg/TARGETS37
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp169
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp7
-rw-r--r--src/buildtool/execution_api/bazel_msg/blob_tree.cpp79
-rw-r--r--src/buildtool/execution_api/bazel_msg/blob_tree.hpp64
-rw-r--r--src/buildtool/execution_api/bazel_msg/directory_tree.cpp69
-rw-r--r--src/buildtool/execution_api/bazel_msg/directory_tree.hpp62
-rw-r--r--src/buildtool/execution_api/local/TARGETS7
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp110
-rw-r--r--src/buildtool/execution_api/remote/TARGETS7
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.cpp113
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.hpp4
12 files changed, 555 insertions, 173 deletions
diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS
index 64cbddd3..26d5ef22 100644
--- a/src/buildtool/execution_api/bazel_msg/TARGETS
+++ b/src/buildtool/execution_api/bazel_msg/TARGETS
@@ -21,6 +21,7 @@
, "srcs": ["bazel_msg_factory.cpp"]
, "deps":
[ "bazel_msg"
+ , "directory_tree"
, ["src/buildtool/common", "common"]
, ["src/buildtool/file_system", "git_cas"]
, ["src/buildtool/execution_engine/dag", "dag"]
@@ -34,4 +35,40 @@
]
, "stage": ["src", "buildtool", "execution_api", "bazel_msg"]
}
+, "directory_tree":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["directory_tree"]
+ , "hdrs": ["directory_tree.hpp"]
+ , "srcs": ["directory_tree.cpp"]
+ , "deps":
+ [ ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/execution_engine/dag", "dag"]
+ ]
+ , "private-deps":
+ [ ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/logging", "logging"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "bazel_msg"]
+ }
+, "blob_tree":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["blob_tree"]
+ , "hdrs": ["blob_tree.hpp"]
+ , "srcs": ["blob_tree.cpp"]
+ , "deps":
+ [ "bazel_msg"
+ , "directory_tree"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/compatibility", "compatibility"]
+ ]
+ , "private-deps":
+ [ ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "bazel_types"]
+ , ["src/buildtool/file_system", "git_cas"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/utils/cpp", "hex_string"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "bazel_msg"]
+ }
}
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp
index a9022e55..7dcc359a 100644
--- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp
+++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp
@@ -19,9 +19,9 @@
#include <filesystem>
#include <functional>
#include <memory>
+#include <optional>
#include <sstream>
#include <string>
-#include <variant>
#include <vector>
#include "gsl-lite/gsl-lite.hpp"
@@ -360,39 +360,23 @@ template <class T>
node.is_executable() ? ObjectType::Executable : ObjectType::File};
}
-class DirectoryTree;
-using DirectoryTreePtr = std::unique_ptr<DirectoryTree>;
-
-/// \brief Tree of `Artifact*` that can be converted to `DirectoryNodeBundle`.
-class DirectoryTree {
- public:
- /// \brief Add `Artifact*` to tree.
- [[nodiscard]] auto AddArtifact(std::filesystem::path const& path,
- Artifact const* artifact) -> bool {
- auto const norm_path = path.lexically_normal();
- if (norm_path.empty() or
- not FileSystemManager::IsRelativePath(norm_path)) {
- return false;
- }
- auto it = norm_path.begin();
- return AddArtifact(&it, norm_path.end(), artifact);
- }
-
- /// \brief Convert tree to `DirectoryNodeBundle`.
- /// NOLINTNEXTLINE(misc-no-recursion)
- [[nodiscard]] auto ToBundle(
- std::string const& root_name,
- std::optional<BazelMsgFactory::BlobStoreFunc> const& store_blob,
- std::optional<BazelMsgFactory::InfoStoreFunc> const& store_info,
- std::filesystem::path const& parent = "") const
- -> DirectoryNodeBundle::Ptr {
- std::vector<bazel_re::FileNode> file_nodes;
- std::vector<bazel_re::DirectoryNode> dir_nodes;
- for (auto const& [name, node] : nodes_) {
+/// \brief Convert `DirectoryTree` to `DirectoryNodeBundle`.
+/// NOLINTNEXTLINE(misc-no-recursion)
+[[nodiscard]] auto DirectoryTreeToBundle(
+ std::string const& root_name,
+ DirectoryTreePtr const& tree,
+ std::optional<BazelMsgFactory::BlobStoreFunc> const& store_blob,
+ std::optional<BazelMsgFactory::InfoStoreFunc> const& store_info,
+ std::filesystem::path const& parent = "") noexcept
+ -> DirectoryNodeBundle::Ptr {
+ std::vector<bazel_re::FileNode> file_nodes;
+ std::vector<bazel_re::DirectoryNode> dir_nodes;
+ try {
+ for (auto const& [name, node] : *tree) {
if (std::holds_alternative<DirectoryTreePtr>(node)) {
auto const& dir = std::get<DirectoryTreePtr>(node);
- auto const dir_bundle =
- dir->ToBundle(name, store_blob, store_info, parent / name);
+ auto const dir_bundle = DirectoryTreeToBundle(
+ name, dir, store_blob, store_info, parent / name);
if (not dir_bundle) {
return nullptr;
}
@@ -423,89 +407,11 @@ class DirectoryTree {
}
return CreateDirectoryNodeBundle(
root_name, CreateDirectory(file_nodes, dir_nodes, {}, {}));
+ } catch (...) {
+ return nullptr;
}
-
- /// \brief Convert tree to `BazelBlob`.
- /// NOLINTNEXTLINE(misc-no-recursion)
- [[nodiscard]] auto ToBlob(
- std::optional<BazelMsgFactory::BlobStoreFunc> const& store_blob,
- std::optional<BazelMsgFactory::InfoStoreFunc> const& store_info,
- std::filesystem::path const& parent = "") const
- -> std::optional<BazelBlob> {
- GitCAS::tree_entries_t entries;
- entries.reserve(nodes_.size());
- for (auto const& [name, node] : nodes_) {
- if (std::holds_alternative<DirectoryTreePtr>(node)) {
- auto const& dir = std::get<DirectoryTreePtr>(node);
- auto blob = dir->ToBlob(store_blob, store_info, parent / name);
- if (not blob) {
- return std::nullopt;
- }
- auto raw_id =
- FromHexString(NativeSupport::Unprefix(blob->digest.hash()));
- if (not raw_id) {
- return std::nullopt;
- }
- entries[std::move(*raw_id)].emplace_back(name,
- ObjectType::Tree);
- if (store_blob) {
- (*store_blob)(std::move(*blob));
- }
- }
- else {
- auto const& artifact = std::get<Artifact const*>(node);
- auto const& object_info = artifact->Info();
- if (not object_info) {
- return std::nullopt;
- }
- auto raw_id = FromHexString(object_info->digest.hash());
- if (not raw_id) {
- return std::nullopt;
- }
- entries[std::move(*raw_id)].emplace_back(name,
- object_info->type);
- if (store_info and
- not(*store_info)(parent / name, *object_info)) {
- return std::nullopt;
- }
- }
- }
-
- if (auto git_tree = GitCAS::CreateShallowTree(entries)) {
- bazel_re::Digest digest{};
- digest.set_hash(NativeSupport::Prefix(ToHexString(git_tree->first),
- /*is_tree=*/true));
- digest.set_size_bytes(
- gsl::narrow<google::protobuf::int64>(git_tree->second.size()));
- return BazelBlob{digest, std::move(git_tree->second)};
- }
-
- return std::nullopt;
- }
-
- private:
- using Node = std::variant<DirectoryTreePtr, Artifact const*>;
- std::unordered_map<std::string, Node> nodes_;
-
- // NOLINTNEXTLINE(misc-no-recursion)
- [[nodiscard]] auto AddArtifact(std::filesystem::path::iterator* begin,
- std::filesystem::path::iterator const& end,
- Artifact const* artifact) -> bool {
- auto segment = *((*begin)++);
- if (segment == "." or segment == "..") { // fail on "." and ".."
- return false;
- }
- if (*begin == end) {
- return nodes_.emplace(segment, artifact).second;
- }
- auto const [it, success] =
- nodes_.emplace(segment, std::make_unique<DirectoryTree>());
- return (success or
- std::holds_alternative<DirectoryTreePtr>(it->second)) and
- std::get<DirectoryTreePtr>(it->second)
- ->AddArtifact(begin, end, artifact);
- }
-};
+ return nullptr;
+}
} // namespace
@@ -559,38 +465,19 @@ auto BazelMsgFactory::ReadObjectInfosFromGitTree(
}
auto BazelMsgFactory::CreateDirectoryDigestFromTree(
- std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts,
+ DirectoryTreePtr const& tree,
std::optional<BlobStoreFunc> const& store_blob,
- std::optional<InfoStoreFunc> const& store_info)
+ std::optional<InfoStoreFunc> const& store_info) noexcept
-> std::optional<bazel_re::Digest> {
- DirectoryTree build_root{};
- for (auto const& [local_path, node] : artifacts) {
- auto const* artifact = &node->Content();
- if (not build_root.AddArtifact(local_path, artifact)) {
- Logger::Log(LogLevel::Error,
- "failed to add artifact {} ({}) to build root",
- local_path,
- artifact->Digest().value_or(ArtifactDigest{}).hash());
- return std::nullopt;
- }
- }
-
- if (Compatibility::IsCompatible()) {
- if (auto bundle = build_root.ToBundle("", store_blob, store_info)) {
- if (store_blob) {
+ if (auto bundle = DirectoryTreeToBundle("", tree, store_blob, store_info)) {
+ if (store_blob) {
+ try {
(*store_blob)(bundle->MakeBlob());
+ } catch (...) {
+ return std::nullopt;
}
- return bundle->Digest();
- }
- }
- else {
- if (auto blob = build_root.ToBlob(store_blob, store_info)) {
- auto digest = blob->digest;
- if (store_blob) {
- (*store_blob)(std::move(*blob));
- }
- return digest;
}
+ return bundle->Digest();
}
return std::nullopt;
}
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp
index adb9d26e..ec4ded35 100644
--- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp
+++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp
@@ -28,6 +28,7 @@
#include "src/buildtool/common/bazel_types.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
#include "src/buildtool/file_system/git_cas.hpp"
@@ -62,14 +63,14 @@ class BazelMsgFactory {
/// \brief Create Directory digest from artifact tree structure.
/// Recursively traverse entire tree and create blobs for sub-directories.
- /// \param artifacts Artifact tree structure.
+ /// \param tree Directory tree of artifacts.
/// \param store_blob Function for storing Directory blobs.
/// \param store_info Function for storing object infos.
/// \returns Digest representing the entire tree.
[[nodiscard]] static auto CreateDirectoryDigestFromTree(
- std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts,
+ DirectoryTreePtr const& tree,
std::optional<BlobStoreFunc> const& store_blob = std::nullopt,
- std::optional<InfoStoreFunc> const& store_info = std::nullopt)
+ std::optional<InfoStoreFunc> const& store_info = std::nullopt) noexcept
-> std::optional<bazel_re::Digest>;
/// \brief Create Directory digest from local file root.
diff --git a/src/buildtool/execution_api/bazel_msg/blob_tree.cpp b/src/buildtool/execution_api/bazel_msg/blob_tree.cpp
new file mode 100644
index 00000000..4b06fd7a
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/blob_tree.cpp
@@ -0,0 +1,79 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp"
+
+#include <utility>
+#include <variant>
+
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/compatibility/native_support.hpp"
+#include "src/buildtool/file_system/git_cas.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+/// NOLINTNEXTLINE(misc-no-recursion)
+auto BlobTree::FromDirectoryTree(DirectoryTreePtr const& tree,
+ std::filesystem::path const& parent) noexcept
+ -> std::optional<BlobTreePtr> {
+ GitCAS::tree_entries_t entries;
+ std::vector<BlobTreePtr> nodes;
+ try {
+ entries.reserve(tree->size());
+ for (auto const& [name, node] : *tree) {
+ if (std::holds_alternative<DirectoryTreePtr>(node)) {
+ auto const& dir = std::get<DirectoryTreePtr>(node);
+ auto blob_tree = FromDirectoryTree(dir, parent / name);
+ if (not blob_tree) {
+ return std::nullopt;
+ }
+ auto raw_id = FromHexString(NativeSupport::Unprefix(
+ (*blob_tree)->Blob().digest.hash()));
+ if (not raw_id) {
+ return std::nullopt;
+ }
+ entries[std::move(*raw_id)].emplace_back(name,
+ ObjectType::Tree);
+ // Only add tree objects to the blob tree to be uploaded.
+ nodes.emplace_back((*blob_tree));
+ }
+ else {
+ auto const& artifact = std::get<Artifact const*>(node);
+ auto const& object_info = artifact->Info();
+ if (not object_info) {
+ return std::nullopt;
+ }
+ auto raw_id = FromHexString(object_info->digest.hash());
+ if (not raw_id) {
+ return std::nullopt;
+ }
+ entries[std::move(*raw_id)].emplace_back(name,
+ object_info->type);
+ }
+ }
+ if (auto git_tree = GitCAS::CreateShallowTree(entries)) {
+ bazel_re::Digest digest{};
+ digest.set_hash(NativeSupport::Prefix(ToHexString(git_tree->first),
+ /*is_tree=*/true));
+ digest.set_size_bytes(
+ gsl::narrow<google::protobuf::int64>(git_tree->second.size()));
+ return std::make_shared<BlobTree>(
+ BazelBlob{digest, git_tree->second}, nodes);
+ }
+ } catch (...) {
+ return std::nullopt;
+ }
+ return std::nullopt;
+}
diff --git a/src/buildtool/execution_api/bazel_msg/blob_tree.hpp b/src/buildtool/execution_api/bazel_msg/blob_tree.hpp
new file mode 100644
index 00000000..786e3c55
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/blob_tree.hpp
@@ -0,0 +1,64 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP
+
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/compatibility/native_support.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp"
+
+class BlobTree;
+using BlobTreePtr = gsl::not_null<std::shared_ptr<BlobTree>>;
+
+/// \brief Tree-like blob container to enable tree-invariant satisfying blob
+/// upload.
+class BlobTree {
+ public:
+ BlobTree(BazelBlob blob, std::vector<BlobTreePtr> nodes)
+ : blob_{std::move(blob)} {
+ nodes_.reserve(nodes.size());
+ for (auto& node : nodes) {
+ nodes_.emplace_back(std::move(node));
+ }
+ }
+
+ [[nodiscard]] auto Blob() const noexcept -> BazelBlob { return blob_; }
+ [[nodiscard]] auto IsTree() const noexcept -> bool {
+ return NativeSupport::IsTree(blob_.digest.hash());
+ }
+
+ /// \brief Create a `BlobTree` from a `DirectoryTree`.
+ [[nodiscard]] static auto FromDirectoryTree(
+ DirectoryTreePtr const& tree,
+ std::filesystem::path const& parent = "") noexcept
+ -> std::optional<BlobTreePtr>;
+
+ [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); }
+ [[nodiscard]] auto end() const noexcept { return nodes_.end(); }
+ [[nodiscard]] auto size() const noexcept { return nodes_.size(); }
+
+ private:
+ BazelBlob blob_;
+ std::vector<BlobTreePtr> nodes_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BLOB_TREE_HPP
diff --git a/src/buildtool/execution_api/bazel_msg/directory_tree.cpp b/src/buildtool/execution_api/bazel_msg/directory_tree.cpp
new file mode 100644
index 00000000..a7951f90
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/directory_tree.cpp
@@ -0,0 +1,69 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "directory_tree.hpp"
+
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto DirectoryTree::AddArtifact(std::filesystem::path const& path,
+ Artifact const* artifact) noexcept -> bool {
+ auto const norm_path = path.lexically_normal();
+ if (norm_path.empty() or not FileSystemManager::IsRelativePath(norm_path)) {
+ return false;
+ }
+ auto it = norm_path.begin();
+ try {
+ return AddArtifact(&it, norm_path.end(), artifact);
+ } catch (...) {
+ return false;
+ }
+}
+
+auto DirectoryTree::FromNamedArtifacts(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<DirectoryTreePtr> {
+ auto dir_tree = std::make_unique<DirectoryTree>();
+ for (auto const& [local_path, node] : artifacts) {
+ auto const* artifact = &node->Content();
+ if (not dir_tree->AddArtifact(local_path, artifact)) {
+ Logger::Log(LogLevel::Error,
+ "failed to add artifact {} ({}) to directory tree",
+ local_path,
+ artifact->Digest().value_or(ArtifactDigest{}).hash());
+ return std::nullopt;
+ }
+ }
+ return dir_tree;
+}
+
+// NOLINTNEXTLINE(misc-no-recursion)
+auto DirectoryTree::AddArtifact(std::filesystem::path::iterator* begin,
+ std::filesystem::path::iterator const& end,
+ Artifact const* artifact) -> bool {
+ gsl_ExpectsAudit(std::distance(*begin, end) > 0);
+ auto segment = *((*begin)++);
+ if (segment == "." or segment == "..") { // fail on "." and ".."
+ return false;
+ }
+ if (*begin == end) {
+ return nodes_.emplace(segment, artifact).second;
+ }
+ auto const [it, success] =
+ nodes_.emplace(segment, std::make_unique<DirectoryTree>());
+ return (success or std::holds_alternative<DirectoryTreePtr>(it->second)) and
+ std::get<DirectoryTreePtr>(it->second)
+ ->AddArtifact(begin, end, artifact);
+}
diff --git a/src/buildtool/execution_api/bazel_msg/directory_tree.hpp b/src/buildtool/execution_api/bazel_msg/directory_tree.hpp
new file mode 100644
index 00000000..968c8899
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/directory_tree.hpp
@@ -0,0 +1,62 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP
+
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <variant>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+
+class DirectoryTree;
+using DirectoryTreePtr = gsl::not_null<std::unique_ptr<DirectoryTree>>;
+
+/// \brief Tree of named artifacts. The path through the tree until a leaf node
+/// where an artifact is stored represents the file path of that artifact. The
+/// tree can be traversed and converted to, e.g., `BlobTree` or
+/// `DirectoryNodeBundle`.
+class DirectoryTree {
+ public:
+ using Node = std::variant<DirectoryTreePtr, Artifact const*>;
+
+ /// \brief Add `Artifact*` to tree.
+ [[nodiscard]] auto AddArtifact(std::filesystem::path const& path,
+ Artifact const* artifact) noexcept -> bool;
+
+ /// \brief Create a DirectoryTree from a list of named artifacts.
+ [[nodiscard]] static auto FromNamedArtifacts(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<DirectoryTreePtr>;
+
+ [[nodiscard]] auto begin() const noexcept { return nodes_.begin(); }
+ [[nodiscard]] auto end() const noexcept { return nodes_.end(); }
+ [[nodiscard]] auto size() const noexcept { return nodes_.size(); }
+
+ private:
+ std::unordered_map<std::string, Node> nodes_;
+
+ [[nodiscard]] auto AddArtifact(std::filesystem::path::iterator* begin,
+ std::filesystem::path::iterator const& end,
+ Artifact const* artifact) -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_DIRECTORY_TREE_HPP
diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS
index 7b839530..d6257f88 100644
--- a/src/buildtool/execution_api/local/TARGETS
+++ b/src/buildtool/execution_api/local/TARGETS
@@ -23,16 +23,19 @@
]
, "srcs": ["local_action.cpp", "local_storage.cpp"]
, "deps":
- [ ["@", "gsl-lite", "", "gsl-lite"]
- , "config"
+ [ "config"
+ , ["@", "fmt", "", "fmt"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
+ , ["src/buildtool/execution_api/bazel_msg", "blob_tree"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "bazel_types"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/logging", "logging"]
]
, "stage": ["src", "buildtool", "execution_api", "local"]
, "private-deps":
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 296bc27d..fb64f6a1 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -17,15 +17,21 @@
#include <map>
#include <memory>
+#include <optional>
#include <string>
+#include <unordered_map>
#include <vector>
+#include "fmt/core.h"
#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/compatibility/native_support.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_api/local/local_action.hpp"
#include "src/buildtool/execution_api/local/local_storage.hpp"
+#include "src/buildtool/logging/logger.hpp"
/// \brief API for local execution.
class LocalApi final : public IExecutionApi {
@@ -210,24 +216,108 @@ class LocalApi final : public IExecutionApi {
return true;
}
+ /// NOLINTNEXTLINE(misc-no-recursion)
+ [[nodiscard]] auto UploadBlobTree(BlobTreePtr const& blob_tree) noexcept
+ -> bool {
+
+ // Create digest list from blobs for batch availability check.
+ std::vector<ArtifactDigest> digests;
+ digests.reserve(blob_tree->size());
+ std::unordered_map<ArtifactDigest, BlobTreePtr> tree_map;
+ for (auto const& node : *blob_tree) {
+ auto digest = ArtifactDigest{node->Blob().digest};
+ digests.emplace_back(digest);
+ try {
+ tree_map.emplace(std::move(digest), node);
+ } catch (...) {
+ return false;
+ }
+ }
+
+ // Find missing digests.
+ auto missing_digests = IsAvailable(digests);
+
+ // Process missing blobs.
+ BlobContainer container;
+ for (auto const& digest : missing_digests) {
+ if (auto it = tree_map.find(digest); it != tree_map.end()) {
+ auto const& node = it->second;
+ // Process trees.
+ if (node->IsTree()) {
+ if (not UploadBlobTree(node)) {
+ return false;
+ }
+ }
+ // Store blob.
+ try {
+ container.Emplace(node->Blob());
+ } catch (...) {
+ return false;
+ }
+ }
+ }
+
+ return Upload(container, /*skip_find_missing=*/true);
+ }
+
[[nodiscard]] auto UploadTree(
std::vector<DependencyGraph::NamedArtifactNodePtr> const&
artifacts) noexcept -> std::optional<ArtifactDigest> final {
- BlobContainer blobs{};
- auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
- artifacts,
- [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); });
- if (not digest) {
- Logger::Log(LogLevel::Debug, "failed to create digest for tree.");
+ auto build_root = DirectoryTree::FromNamedArtifacts(artifacts);
+ if (not build_root) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create build root from artifacts.");
return std::nullopt;
}
- if (not Upload(blobs, /*skip_find_missing=*/false)) {
- Logger::Log(LogLevel::Debug, "failed to upload blobs for tree.");
- return std::nullopt;
+ if (Compatibility::IsCompatible()) {
+ BlobContainer blobs{};
+ auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
+ *build_root,
+ [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); });
+ if (not digest) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create digest for build root.");
+ return std::nullopt;
+ }
+ Logger::Log(LogLevel::Trace, [&digest]() {
+ std::ostringstream oss{};
+ oss << "upload root directory" << std::endl;
+ oss << fmt::format(" - root digest: {}", digest->hash())
+ << std::endl;
+ return oss.str();
+ });
+ if (not Upload(blobs, /*skip_find_missing=*/false)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload blobs for build root.");
+ return std::nullopt;
+ }
+ return ArtifactDigest{*digest};
}
- return ArtifactDigest{*digest};
+ auto blob_tree = BlobTree::FromDirectoryTree(*build_root);
+ if (not blob_tree) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create blob tree for build root.");
+ return std::nullopt;
+ }
+ auto tree_blob = (*blob_tree)->Blob();
+ // Upload blob tree if tree is not available at the remote side (content
+ // first).
+ if (not IsAvailable(ArtifactDigest{tree_blob.digest})) {
+ if (not UploadBlobTree(*blob_tree)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload blob tree for build root.");
+ return std::nullopt;
+ }
+ if (not Upload(BlobContainer{{tree_blob}},
+ /*skip_find_missing=*/true)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload tree blob for build root.");
+ return std::nullopt;
+ }
+ }
+ return ArtifactDigest{tree_blob.digest};
}
[[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS
index 8e15dcfd..4bf96ea0 100644
--- a/src/buildtool/execution_api/remote/TARGETS
+++ b/src/buildtool/execution_api/remote/TARGETS
@@ -50,11 +50,16 @@
[ "config"
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/execution_api/bazel_msg", "blob_tree"]
, ["@", "gsl-lite", "", "gsl-lite"]
, ["src/buildtool/common", "common"]
]
, "stage": ["src", "buildtool", "execution_api", "remote"]
- , "private-deps": ["bazel_network"]
+ , "private-deps":
+ [ "bazel_network"
+ , ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/compatibility", "compatibility"]
+ ]
}
, "config":
{ "type": ["@", "rules", "CC", "library"]
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
index 93e1be2a..f1442111 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
@@ -17,10 +17,14 @@
#include <algorithm>
#include <map>
#include <memory>
+#include <optional>
#include <string>
+#include <unordered_map>
#include <vector>
+#include "fmt/core.h"
#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
@@ -235,31 +239,108 @@ auto BazelApi::CreateAction(
return network_->UploadBlobs(blobs, skip_find_missing);
}
+/// NOLINTNEXTLINE(misc-no-recursion)
+[[nodiscard]] auto BazelApi::UploadBlobTree(
+ BlobTreePtr const& blob_tree) noexcept -> bool {
+
+ // Create digest list from blobs for batch availability check.
+ std::vector<bazel_re::Digest> digests;
+ digests.reserve(blob_tree->size());
+ std::unordered_map<bazel_re::Digest, BlobTreePtr> tree_map;
+ for (auto const& node : *blob_tree) {
+ auto digest = node->Blob().digest;
+ digests.emplace_back(digest);
+ try {
+ tree_map.emplace(std::move(digest), node);
+ } catch (...) {
+ return false;
+ }
+ }
+
+ // Find missing digests.
+ auto missing_digests = network_->IsAvailable(digests);
+
+ // Process missing blobs.
+ BlobContainer container;
+ for (auto const& digest : missing_digests) {
+ if (auto it = tree_map.find(digest); it != tree_map.end()) {
+ auto const& node = it->second;
+ // Process trees.
+ if (node->IsTree()) {
+ if (not UploadBlobTree(node)) {
+ return false;
+ }
+ }
+ // Store blob.
+ try {
+ container.Emplace(node->Blob());
+ } catch (...) {
+ return false;
+ }
+ }
+ }
+
+ return network_->UploadBlobs(container, /*skip_find_missing=*/true);
+}
+
[[nodiscard]] auto BazelApi::UploadTree(
std::vector<DependencyGraph::NamedArtifactNodePtr> const&
artifacts) noexcept -> std::optional<ArtifactDigest> {
- BlobContainer blobs{};
- auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
- artifacts,
- [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); });
- if (not digest) {
- Logger::Log(LogLevel::Debug, "failed to create digest for tree.");
+ auto build_root = DirectoryTree::FromNamedArtifacts(artifacts);
+ if (not build_root) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create build root from artifacts.");
return std::nullopt;
}
- Logger::Log(LogLevel::Trace, [&digest]() {
- std::ostringstream oss{};
- oss << "upload root directory" << std::endl;
- oss << fmt::format(" - root digest: {}", digest->hash()) << std::endl;
- return oss.str();
- });
+ if (Compatibility::IsCompatible()) {
+ BlobContainer blobs{};
+ auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
+ *build_root,
+ [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); });
+ if (not digest) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create digest for build root.");
+ return std::nullopt;
+ }
+ Logger::Log(LogLevel::Trace, [&digest]() {
+ std::ostringstream oss{};
+ oss << "upload root directory" << std::endl;
+ oss << fmt::format(" - root digest: {}", digest->hash())
+ << std::endl;
+ return oss.str();
+ });
+ if (not Upload(blobs, /*skip_find_missing=*/false)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload blobs for build root.");
+ return std::nullopt;
+ }
+ return ArtifactDigest{*digest};
+ }
- if (not Upload(blobs, /*skip_find_missing=*/false)) {
- Logger::Log(LogLevel::Debug, "failed to upload blobs for tree.");
+ auto blob_tree = BlobTree::FromDirectoryTree(*build_root);
+ if (not blob_tree) {
+ Logger::Log(LogLevel::Debug,
+ "failed to create blob tree for build root.");
return std::nullopt;
}
-
- return ArtifactDigest{*digest};
+ auto tree_blob = (*blob_tree)->Blob();
+ // Upload blob tree if tree is not available at the remote side (content
+ // first).
+ if (not network_->IsAvailable(tree_blob.digest)) {
+ if (not UploadBlobTree(*blob_tree)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload blob tree for build root.");
+ return std::nullopt;
+ }
+ if (not Upload(BlobContainer{{tree_blob}},
+ /*skip_find_missing=*/true)) {
+ Logger::Log(LogLevel::Debug,
+ "failed to upload tree blob for build root.");
+ return std::nullopt;
+ }
+ }
+ return ArtifactDigest{tree_blob.digest};
}
[[nodiscard]] auto BazelApi::IsAvailable(
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
index 7f7a0045..a435824e 100644
--- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
@@ -23,6 +23,7 @@
#include "gsl-lite/gsl-lite.hpp"
#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
@@ -81,6 +82,9 @@ class BazelApi final : public IExecutionApi {
private:
std::shared_ptr<BazelNetwork> network_;
+
+ [[nodiscard]] auto UploadBlobTree(BlobTreePtr const& blob_tree) noexcept
+ -> bool;
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 20:54:24 +0000