diff options
Diffstat (limited to 'src/buildtool/serve_api')
| -rw-r--r-- | src/buildtool/serve_api/remote/TARGETS | 6 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/serve_api.hpp | 26 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/source_tree_client.cpp | 46 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/source_tree_client.hpp | 39 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/just_serve.proto | 15 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/source_tree.cpp | 106 |
6 files changed, 202 insertions, 36 deletions
diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS index 5f946e79..9b4e3a4e 100644 --- a/src/buildtool/serve_api/remote/TARGETS +++ b/src/buildtool/serve_api/remote/TARGETS @@ -17,16 +17,20 @@ , "srcs": ["source_tree_client.cpp"] , "deps": [ ["@", "gsl", "", "gsl"] + , ["src/buildtool/common", "common"] , ["src/buildtool/common/remote", "remote_common"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_api/remote", "context"] , ["src/buildtool/file_system", "git_types"] , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , ["src/buildtool/logging", "logging"] + , ["src/utils/cpp", "expected"] ] , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] , "stage": ["src", "buildtool", "serve_api", "remote"] , "private-deps": - [ ["src/buildtool/common/remote", "client_common"] + [ ["src/buildtool/common", "artifact_digest_factory"] + , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/logging", "log_level"] ] } diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp index 23c87982..ee60f4f1 100644 --- a/src/buildtool/serve_api/remote/serve_api.hpp +++ b/src/buildtool/serve_api/remote/serve_api.hpp @@ -46,7 +46,9 @@ class ServeApi final { gsl::not_null<LocalContext const*> const& local_context, gsl::not_null<RemoteContext const*> const& remote_context, gsl::not_null<ApiBundle const*> const& apis) noexcept - : stc_{address, remote_context}, + : stc_{address, + &local_context->storage_config->hash_function, + remote_context}, tc_{address, local_context->storage, remote_context, apis}, cc_{address, remote_context} {} @@ -71,10 +73,10 @@ class ServeApi final { return std::nullopt; } - [[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit, - std::string const& subdir = ".", - bool sync_tree = false) - const noexcept -> expected<std::string, GitLookupError> { + [[nodiscard]] auto RetrieveTreeFromCommit( + std::string const& commit, + std::string const& subdir = ".", + bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeCommitTree(commit, subdir, sync_tree); } @@ -83,8 +85,7 @@ class ServeApi final { std::string const& archive_type = "archive", std::string const& subdir = ".", std::optional<PragmaSpecial> const& resolve_symlinks = std::nullopt, - bool sync_tree = false) const noexcept - -> expected<std::string, GitLookupError> { + bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeArchiveTree( content, archive_type, subdir, resolve_symlinks, sync_tree); } @@ -92,15 +93,14 @@ class ServeApi final { [[nodiscard]] auto RetrieveTreeFromDistdir( std::shared_ptr<std::unordered_map<std::string, std::string>> const& distfiles, - bool sync_tree = false) const noexcept - -> expected<std::string, GitLookupError> { + bool sync_tree = false) const noexcept -> SourceTreeClient::result_t { return stc_.ServeDistdirTree(distfiles, sync_tree); } - [[nodiscard]] auto RetrieveTreeFromForeignFile(const std::string& content, - const std::string& name, - bool executable) - const noexcept -> expected<std::string, GitLookupError> { + [[nodiscard]] auto RetrieveTreeFromForeignFile( + const std::string& content, + const std::string& name, + bool executable) const noexcept -> SourceTreeClient::result_t { return stc_.ServeForeignFileTree(content, name, executable); } diff --git a/src/buildtool/serve_api/remote/source_tree_client.cpp b/src/buildtool/serve_api/remote/source_tree_client.cpp index 994fcc21..f3538ca3 100644 --- a/src/buildtool/serve_api/remote/source_tree_client.cpp +++ b/src/buildtool/serve_api/remote/source_tree_client.cpp @@ -16,6 +16,7 @@ #include "src/buildtool/serve_api/remote/source_tree_client.hpp" +#include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -61,7 +62,9 @@ auto PragmaSpecialToSymlinksResolve( SourceTreeClient::SourceTreeClient( ServerAddress const& address, - gsl::not_null<RemoteContext const*> const& remote_context) noexcept { + gsl::not_null<HashFunction const*> const& hash_function, + gsl::not_null<RemoteContext const*> const& remote_context) noexcept + : hash_function_{*hash_function} { stub_ = justbuild::just_serve::SourceTree::NewStub(CreateChannelWithCredentials( address.host, address.port, remote_context->auth)); @@ -95,7 +98,18 @@ auto SourceTreeClient::ServeCommitTree(std::string const& commit_id, ? GitLookupError::Fatal : GitLookupError::NotFound}; } - return response.tree(); // success + TreeResult result = {response.tree(), std::nullopt}; + // if asked to sync, get digest from response + if (sync_tree) { + auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), + response.digest()); + if (not digest) { + logger_.Emit(LogLevel::Debug, std::move(digest).error()); + return unexpected{GitLookupError::Fatal}; + } + result.digest = *std::move(digest); + } + return result; // success } auto SourceTreeClient::ServeArchiveTree( @@ -131,7 +145,18 @@ auto SourceTreeClient::ServeArchiveTree( ? GitLookupError::Fatal : GitLookupError::NotFound}; } - return response.tree(); // success + TreeResult result = {response.tree(), std::nullopt}; + // if asked to sync, get digest from response + if (sync_tree) { + auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), + response.digest()); + if (not digest) { + logger_.Emit(LogLevel::Debug, std::move(digest).error()); + return unexpected{GitLookupError::Fatal}; + } + result.digest = *std::move(digest); + } + return result; // success } auto SourceTreeClient::ServeDistdirTree( @@ -166,7 +191,18 @@ auto SourceTreeClient::ServeDistdirTree( ? GitLookupError::Fatal : GitLookupError::NotFound}; } - return response.tree(); // success + TreeResult result = {response.tree(), std::nullopt}; + // if asked to sync, get digest from response + if (sync_tree) { + auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(), + response.digest()); + if (not digest) { + logger_.Emit(LogLevel::Debug, std::move(digest).error()); + return unexpected{GitLookupError::Fatal}; + } + result.digest = *std::move(digest); + } + return result; // success } auto SourceTreeClient::ServeForeignFileTree(const std::string& content, @@ -199,7 +235,7 @@ auto SourceTreeClient::ServeForeignFileTree(const std::string& content, ? GitLookupError::Fatal : GitLookupError::NotFound}; } - return response.tree(); // success + return TreeResult{response.tree(), std::nullopt}; // success } auto SourceTreeClient::ServeContent(std::string const& content) const noexcept diff --git a/src/buildtool/serve_api/remote/source_tree_client.hpp b/src/buildtool/serve_api/remote/source_tree_client.hpp index c849eeb3..39e0e721 100644 --- a/src/buildtool/serve_api/remote/source_tree_client.hpp +++ b/src/buildtool/serve_api/remote/source_tree_client.hpp @@ -16,12 +16,15 @@ #define INCLUDED_SRC_BUILDTOOL_SERVE_API_SOURCE_TREE_CLIENT_HPP #include <memory> +#include <optional> #include <string> #include <unordered_map> #include "gsl/gsl" #include "justbuild/just_serve/just_serve.grpc.pb.h" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/remote/remote_common.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/remote/context.hpp" #include "src/buildtool/file_system/git_types.hpp" #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" @@ -34,19 +37,23 @@ class SourceTreeClient { public: explicit SourceTreeClient( ServerAddress const& address, + gsl::not_null<HashFunction const*> const& hash_function, gsl::not_null<RemoteContext const*> const& remote_context) noexcept; - // An error + data union type - using result_t = expected<std::string, GitLookupError>; + struct TreeResult { + std::string tree; + std::optional<ArtifactDigest> digest; + }; + using result_t = expected<TreeResult, GitLookupError>; /// \brief Retrieve the Git tree of a given commit, if known by the /// endpoint. It is a fatal error if the commit is known to the endpoint but - /// no tree is able to be returned. + /// no result is able to be returned. /// \param[in] commit_id Hash of the Git commit to look up. /// \param[in] subdir Relative path of the tree inside commit. /// \param[in] sync_tree Sync tree to the remote-execution endpoint. - /// \returns The tree identifier on success or an unexpected error (fatal or - /// commit or subtree not found). + /// \returns The optional tree digest on success or an unexpected error + /// (fatal or commit or subtree not found). [[nodiscard]] auto ServeCommitTree(std::string const& commit_id, std::string const& subdir, bool sync_tree) const noexcept @@ -54,15 +61,15 @@ class SourceTreeClient { /// \brief Retrieve the Git tree of an archive content, if known by the /// endpoint. It is a fatal error if the content blob is known to the - /// endpoint but no tree is able to be returned. + /// endpoint but no result is able to be returned. /// \param[in] content Hash of the archive content to look up. /// \param[in] archive_type Type of archive ("archive"|"zip"). /// \param[in] subdir Relative path of the tree inside archive. /// \param[in] resolve_symlinks Optional enum to state how symlinks in the /// archive should be handled if the tree has to be actually computed. /// \param[in] sync_tree Sync tree to the remote-execution endpoint. - /// \returns The tree identifier on success or an unexpected error (fatal or - /// content blob not found). + /// \returns The optional tree digest on success or an unexpected error + /// (fatal or content blob not found). [[nodiscard]] auto ServeArchiveTree( std::string const& content, std::string const& archive_type, @@ -72,20 +79,25 @@ class SourceTreeClient { /// \brief Retrieve the Git tree of a directory of distfiles, if all the /// content blobs are known by the endpoint. It is a fatal error if all - /// content blobs are known but no tree is able to be returned. + /// content blobs are known but no result is able to be returned. /// \param[in] distfiles Mapping from distfile names to content blob ids. - /// \param[in] sync_tree Sync tree and all ditfile blobs to the + /// \param[in] sync_tree Sync tree and all distfile blobs to the /// remote-execution endpoint. - /// \returns The tree identifier on success or an unexpected error (fatal or - /// at least one distfile blob missing). + /// \returns The optional tree digest on success or an unexpected error + /// (fatal or at least one distfile blob missing). [[nodiscard]] auto ServeDistdirTree( std::shared_ptr<std::unordered_map<std::string, std::string>> const& distfiles, bool sync_tree) const noexcept -> result_t; /// \brief Retrieve the Git tree of a foreign-file directory, if all content - /// blobs are known to the end point and, as a side effect, make that tree + /// blobs are known to the endpoint and, as a side-effect, make that tree /// known to the serve endpoint. + /// \param[in] content Hash of the foreign-file content. + /// \param[in] name Name of the foreign-file. + /// \param[in] executable Executable flag of foreign-file. + /// \returns The optional tree digest on success or an unexpected error + /// (fatal or content not found). [[nodiscard]] auto ServeForeignFileTree(const std::string& content, const std::string& name, bool executable) const noexcept @@ -122,6 +134,7 @@ class SourceTreeClient { -> bool; private: + HashFunction const& hash_function_; // hash function of the remote std::unique_ptr<justbuild::just_serve::SourceTree::Stub> stub_; Logger logger_{"RemoteSourceTreeClient"}; }; diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto index 4894fec3..a0eba0ef 100644 --- a/src/buildtool/serve_api/serve_service/just_serve.proto +++ b/src/buildtool/serve_api/serve_service/just_serve.proto @@ -56,6 +56,11 @@ message ServeCommitTreeResponse { // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeCommitTreeStatus status = 2; + + // The digest of the requested tree, which can be used to retrieve it from + // the associated remote-execution endpoint CAS, if tree was uploaded to the + // remote-execution endpoint. + build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for @@ -126,6 +131,11 @@ message ServeArchiveTreeResponse { // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeArchiveTreeStatus status = 2; + + // The digest of the requested tree, which can be used to retrieve it from + // the associated remote-execution endpoint CAS, if tree was uploaded to the + // remote-execution endpoint. + build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for @@ -176,6 +186,11 @@ message ServeDistdirTreeResponse { // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct. // For any other value, the `tree` field is not set. ServeDistdirTreeStatus status = 2; + + // The digest of the requested tree, which can be used to retrieve it from + // the associated remote-execution endpoint CAS, if tree was uploaded to the + // remote-execution endpoint. + build.bazel.remote.execution.v2.Digest digest = 3; } // A request message for diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp index 8bc06c39..a6514c46 100644 --- a/src/buildtool/serve_api/serve_service/source_tree.cpp +++ b/src/buildtool/serve_api/serve_service/source_tree.cpp @@ -214,14 +214,30 @@ auto SourceTreeService::ServeCommitTree( auto res = GetSubtreeFromCommit( native_context_->storage_config->GitRoot(), commit, subdir, logger_); if (res) { - auto tree_id = *std::move(res); + auto const tree_id = *std::move(res); auto status = ServeCommitTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas<ObjectType::Tree, ServeCommitTreeResponse>( tree_id, native_context_->storage_config->GitRoot()); + if (status == ServeCommitTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function.GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, std::move(digest).error()); + response->set_status( + ServeCommitTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } - *(response->mutable_tree()) = std::move(tree_id); + *(response->mutable_tree()) = tree_id; response->set_status(status); return ::grpc::Status::OK; } @@ -240,14 +256,32 @@ auto SourceTreeService::ServeCommitTree( for (auto const& path : serve_config_.known_repositories) { auto res = GetSubtreeFromCommit(path, commit, subdir, logger_); if (res) { - auto tree_id = *std::move(res); + auto const tree_id = *std::move(res); auto status = ServeCommitTreeResponse::OK; if (request->sync_tree()) { status = SyncGitEntryToCas<ObjectType::Tree, ServeCommitTreeResponse>(tree_id, path); + if (status == ServeCommitTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function + .GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, + std::move(digest).error()); + response->set_status( + ServeCommitTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } - *(response->mutable_tree()) = std::move(tree_id); + *(response->mutable_tree()) = tree_id; response->set_status(status); return ::grpc::Status::OK; } @@ -277,6 +311,21 @@ auto SourceTreeService::SyncArchive(std::string const& tree_id, if (sync_tree) { status = SyncGitEntryToCas<ObjectType::Tree, ServeArchiveTreeResponse>( tree_id, repo_path); + if (status == ServeArchiveTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function.GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, std::move(digest).error()); + response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } *(response->mutable_tree()) = tree_id; response->set_status(status); @@ -992,6 +1041,21 @@ auto SourceTreeService::DistdirImportToGit( if (sync_tree) { status = SyncGitEntryToCas<ObjectType::Tree, ServeDistdirTreeResponse>( tree_id, native_context_->storage_config->GitRoot()); + if (status == ServeDistdirTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function.GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, std::move(digest).error()); + response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } // set response on success *(response->mutable_tree()) = std::move(tree_id); @@ -1200,6 +1264,22 @@ auto SourceTreeService::ServeDistdirTree( status = SyncGitEntryToCas<ObjectType::Tree, ServeDistdirTreeResponse>( tree_id, native_context_->storage_config->GitRoot()); + if (status == ServeDistdirTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function.GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, std::move(digest).error()); + response->set_status( + ServeDistdirTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } // set response on success *(response->mutable_tree()) = std::move(tree_id); @@ -1224,6 +1304,24 @@ auto SourceTreeService::ServeDistdirTree( status = SyncGitEntryToCas<ObjectType::Tree, ServeDistdirTreeResponse>(tree_id, path); + if (status == ServeDistdirTreeResponse::OK) { + // set digest in response + auto digest = ArtifactDigestFactory::Create( + native_context_->storage_config->hash_function + .GetType(), + tree_id, + /*size is unknown*/ 0, + /*is_tree=*/true); + if (not digest) { + logger_->Emit(LogLevel::Error, + std::move(digest).error()); + response->set_status( + ServeDistdirTreeResponse::INTERNAL_ERROR); + return ::grpc::Status::OK; + } + *(response->mutable_digest()) = + ArtifactDigestFactory::ToBazel(*std::move(digest)); + } } // set response on success *(response->mutable_tree()) = std::move(tree_id); |