diff options
Diffstat (limited to 'src/buildtool/execution_api')
| -rw-r--r-- | src/buildtool/execution_api/common/local_tree_map.hpp | 4 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/TARGETS | 1 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/local_action.cpp | 71 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/local_api.hpp | 14 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/local_storage.cpp | 113 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/local_storage.hpp | 17 |
6 files changed, 168 insertions, 52 deletions
diff --git a/src/buildtool/execution_api/common/local_tree_map.hpp b/src/buildtool/execution_api/common/local_tree_map.hpp index 77de2d53..2758c339 100644 --- a/src/buildtool/execution_api/common/local_tree_map.hpp +++ b/src/buildtool/execution_api/common/local_tree_map.hpp @@ -11,7 +11,9 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/logging/logger.hpp" -/// \brief Maps digest of `bazel_re::Directory` to `LocalTree`. +/// \brief Maps `bazel_re::Digest` to `LocalTree`. +/// Digest may refer to `bazel_re::Directory` or Git tree object, depending on +/// mode being compatible or native, respectively. class LocalTreeMap { /// \brief Thread-safe pool of unique object infos. class ObjectInfoPool { diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index e395f7d2..4af36d70 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -29,6 +29,7 @@ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/system", "system_command"] , ["src/buildtool/logging", "logging"] ] diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index 3ccc58eb..331aaaa5 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -235,26 +235,59 @@ auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path, Logger::Log(LogLevel::Error, "expected directory at {}", local_path); return std::nullopt; } - auto digest = BazelMsgFactory::CreateDirectoryDigestFromLocalTree( - dir_path, - [this](auto path, auto is_exec) { - return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); - }, - [this](auto bytes, auto dir) -> std::optional<bazel_re::Digest> { - auto digest = storage_->StoreBlob(bytes); - if (digest and not tree_map_->HasTree(*digest)) { - auto tree = tree_map_->CreateTree(); - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - dir, - [&tree](auto path, auto info) { - return tree.AddInfo(path, info); - }) or - not tree_map_->AddTree(*digest, std::move(tree))) { - return std::nullopt; + std::optional<bazel_re::Digest> digest{std::nullopt}; + if (Compatibility::IsCompatible()) { + digest = BazelMsgFactory::CreateDirectoryDigestFromLocalTree( + dir_path, + [this](auto path, auto is_exec) { + return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); + }, + [this](auto bytes, auto dir) -> std::optional<bazel_re::Digest> { + auto digest = storage_->StoreBlob(bytes); + if (digest and not tree_map_->HasTree(*digest)) { + auto tree = tree_map_->CreateTree(); + if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + dir, + [&tree](auto path, auto info) { + return tree.AddInfo(path, info); + }) or + not tree_map_->AddTree(*digest, std::move(tree))) { + return std::nullopt; + } } - } - return digest; - }); + return digest; + }); + } + else { + digest = BazelMsgFactory::CreateGitTreeDigestFromLocalTree( + dir_path, + [this](auto path, auto is_exec) { + return storage_->StoreBlob</*kOwner=*/true>(path, is_exec); + }, + [this](auto bytes, + auto entries) -> std::optional<bazel_re::Digest> { + auto digest = storage_->StoreTree(bytes); + if (digest and not tree_map_->HasTree(*digest)) { + auto tree = tree_map_->CreateTree(); + for (auto const& [raw_id, es] : entries) { + auto id = ToHexString(raw_id); + for (auto const& entry : es) { + auto info = Artifact::ObjectInfo{ + ArtifactDigest{ + id, 0, entry.type == ObjectType::Tree}, + entry.type}; + if (not tree.AddInfo(entry.name, info)) { + return std::nullopt; + } + } + } + if (not tree_map_->AddTree(*digest, std::move(tree))) { + return std::nullopt; + } + } + return digest; + }); + } if (digest) { auto out_dir = bazel_re::OutputDirectory{}; out_dir.set_path(local_path); diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index c1cb1c57..1a43fe4b 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -7,6 +7,7 @@ #include <vector> #include "gsl-lite/gsl-lite.hpp" +#include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/common/local_tree_map.hpp" @@ -75,7 +76,7 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto RetrieveToFds( std::vector<Artifact::ObjectInfo> const& artifacts_info, std::vector<int> const& fds, - bool /*raw_tree*/) noexcept -> bool final { + bool raw_tree) noexcept -> bool final { if (artifacts_info.size() != fds.size()) { Logger::Log(LogLevel::Error, "different number of digests and file descriptors."); @@ -87,7 +88,8 @@ class LocalApi final : public IExecutionApi { auto const& info = artifacts_info[i]; if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT - auto const success = storage_->DumpToStream(info, out); + auto const success = + storage_->DumpToStream(info, out, raw_tree); std::fclose(out); if (not success) { Logger::Log(LogLevel::Error, @@ -112,7 +114,9 @@ class LocalApi final : public IExecutionApi { bool /*skip_find_missing*/) noexcept -> bool final { for (auto const& blob : blobs) { - auto cas_digest = storage_->StoreBlob(blob.data); + auto const is_tree = NativeSupport::IsTree(blob.digest.hash()); + auto cas_digest = is_tree ? storage_->StoreTree(blob.data) + : storage_->StoreBlob(blob.data); if (not cas_digest or not std::equal_to<bazel_re::Digest>{}( *cas_digest, blob.digest)) { return false; @@ -148,7 +152,9 @@ class LocalApi final : public IExecutionApi { [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept -> bool final { - return storage_->BlobPath(digest, false).has_value(); + return static_cast<bool>(digest.is_tree() + ? storage_->TreePath(digest) + : storage_->BlobPath(digest, false)); } [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests) diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp index 5e4115df..e6d9571c 100644 --- a/src/buildtool/execution_api/local/local_storage.cpp +++ b/src/buildtool/execution_api/local/local_storage.cpp @@ -17,15 +17,46 @@ namespace { return std::nullopt; } +[[nodiscard]] auto ReadGitTree( + gsl::not_null<LocalStorage const*> const& storage, + bazel_re::Digest const& digest) noexcept + -> std::optional<GitCAS::tree_entries_t> { + if (auto const path = storage->TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return GitCAS::ReadTreeData( + *content, + HashFunction::ComputeTreeHash(*content).Bytes(), + /*is_hex_id=*/false); + } + } + Logger::Log(LogLevel::Error, "Tree {} not found in CAS", digest.hash()); + return std::nullopt; +} + +[[nodiscard]] auto DumpToStream(gsl::not_null<FILE*> const& stream, + std::optional<std::string> const& data) noexcept + -> bool { + if (data) { + std::fwrite(data->data(), 1, data->size(), stream); + return true; + } + return false; +} + [[nodiscard]] auto TreeToStream( gsl::not_null<LocalStorage const*> const& storage, bazel_re::Digest const& tree_digest, gsl::not_null<FILE*> const& stream) noexcept -> bool { - if (auto dir = ReadDirectory(storage, tree_digest)) { - if (auto data = BazelMsgFactory::DirectoryToString(*dir)) { - auto const& str = *data; - std::fwrite(str.data(), 1, str.size(), stream); - return true; + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(storage, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::DirectoryToString(*dir)); + } + } + else { + if (auto entries = ReadGitTree(storage, tree_digest)) { + return DumpToStream(stream, + BazelMsgFactory::GitTreeToString(*entries)); } } return false; @@ -36,8 +67,13 @@ namespace { Artifact::ObjectInfo const& blob_info, gsl::not_null<FILE*> const& stream) noexcept -> bool { constexpr std::size_t kChunkSize{512}; - if (auto const path = storage->BlobPath( - blob_info.digest, IsExecutableObject(blob_info.type))) { + auto path = + storage->BlobPath(blob_info.digest, IsExecutableObject(blob_info.type)); + if (not path and not Compatibility::IsCompatible()) { + // in native mode, lookup object in tree cas to dump tree as blob + path = storage->TreePath(blob_info.digest); + } + if (path) { std::string data(kChunkSize, '\0'); if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) { while (auto size = std::fread(data.data(), 1, kChunkSize, in)) { @@ -103,26 +139,53 @@ auto LocalStorage::ReadObjectInfosRecursively( } // fallback read from CAS and cache it in in-memory tree map - if (auto dir = ReadDirectory(this, digest)) { - auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree()) - : std::nullopt; - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, - [this, &store_info, &parent, &tree](auto path, auto info) { - return (not tree or tree->AddInfo(path, info)) and - (IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, parent / path, info.digest) - : store_info(parent / path, info)); - }) and - (not tree_map_ or tree_map_->AddTree(digest, std::move(*tree))); + if (Compatibility::IsCompatible()) { + if (auto dir = ReadDirectory(this, digest)) { + auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree()) + : std::nullopt; + return BazelMsgFactory::ReadObjectInfosFromDirectory( + *dir, + [this, &store_info, &parent, &tree](auto path, + auto info) { + return (not tree or tree->AddInfo(path, info)) and + (IsTreeObject(info.type) + ? ReadObjectInfosRecursively( + store_info, + parent / path, + info.digest) + : store_info(parent / path, info)); + }) and + (not tree_map_ or + tree_map_->AddTree(digest, std::move(*tree))); + } + } + else { + if (auto entries = ReadGitTree(this, digest)) { + auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree()) + : std::nullopt; + return BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, + [this, &store_info, &parent, &tree](auto path, + auto info) { + return (not tree or tree->AddInfo(path, info)) and + (IsTreeObject(info.type) + ? ReadObjectInfosRecursively( + store_info, + parent / path, + info.digest) + : store_info(parent / path, info)); + }) and + (not tree_map_ or + tree_map_->AddTree(digest, std::move(*tree))); + } } return false; } -auto LocalStorage::DumpToStream( - Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream) const noexcept -> bool { - return IsTreeObject(info.type) ? TreeToStream(this, info.digest, stream) - : BlobToStream(this, info, stream); +auto LocalStorage::DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept -> bool { + return IsTreeObject(info.type) and not raw_tree + ? TreeToStream(this, info.digest, stream) + : BlobToStream(this, info, stream); } diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp index 4560a859..4ffe2881 100644 --- a/src/buildtool/execution_api/local/local_storage.hpp +++ b/src/buildtool/execution_api/local/local_storage.hpp @@ -43,6 +43,11 @@ class LocalStorage { : cas_file_.StoreBlobFromBytes(bytes); } + [[nodiscard]] auto StoreTree(std::string const& bytes) const noexcept + -> std::optional<bazel_re::Digest> { + return cas_tree_.StoreBlobFromBytes(bytes); + } + /// \brief Obtain blob path from digest with x-bit. /// NOLINTNEXTLINE(misc-no-recursion) [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest, @@ -53,6 +58,11 @@ class LocalStorage { return path ? path : TrySyncBlob(digest, is_executable); } + [[nodiscard]] auto TreePath(bazel_re::Digest const& digest) const noexcept + -> std::optional<std::filesystem::path> { + return cas_tree_.BlobPath(digest); + } + [[nodiscard]] auto StoreActionResult( bazel_re::Digest const& action_id, bazel_re::ActionResult const& result) const noexcept -> bool { @@ -70,13 +80,14 @@ class LocalStorage { -> std::optional<std::pair<std::vector<std::filesystem::path>, std::vector<Artifact::ObjectInfo>>>; - [[nodiscard]] auto DumpToStream( - Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream) const noexcept -> bool; + [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& stream, + bool raw_tree) const noexcept -> bool; private: LocalCAS<ObjectType::File> cas_file_{}; LocalCAS<ObjectType::Executable> cas_exec_{}; + LocalCAS<ObjectType::Tree> cas_tree_{}; LocalAC ac_{&cas_file_}; std::shared_ptr<LocalTreeMap> tree_map_; |