summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/buildtool/serve_api/remote/TARGETS6
-rw-r--r--src/buildtool/serve_api/remote/serve_api.hpp26
-rw-r--r--src/buildtool/serve_api/remote/source_tree_client.cpp46
-rw-r--r--src/buildtool/serve_api/remote/source_tree_client.hpp39
-rw-r--r--src/buildtool/serve_api/serve_service/just_serve.proto15
-rw-r--r--src/buildtool/serve_api/serve_service/source_tree.cpp106
-rw-r--r--src/other_tools/just_mr/fetch.cpp11
-rw-r--r--src/other_tools/just_mr/setup.cpp11
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp31
-rw-r--r--src/other_tools/root_maps/content_git_map.cpp11
-rw-r--r--src/other_tools/root_maps/distdir_git_map.cpp21
-rw-r--r--src/other_tools/root_maps/foreign_file_git_map.cpp10
-rw-r--r--test/buildtool/serve_api/TARGETS11
-rw-r--r--test/buildtool/serve_api/source_tree_client.test.cpp34
14 files changed, 284 insertions, 94 deletions
diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS
index 5f946e79..9b4e3a4e 100644
--- a/src/buildtool/serve_api/remote/TARGETS
+++ b/src/buildtool/serve_api/remote/TARGETS
@@ -17,16 +17,20 @@
, "srcs": ["source_tree_client.cpp"]
, "deps":
[ ["@", "gsl", "", "gsl"]
+ , ["src/buildtool/common", "common"]
, ["src/buildtool/common/remote", "remote_common"]
+ , ["src/buildtool/crypto", "hash_function"]
, ["src/buildtool/execution_api/remote", "context"]
, ["src/buildtool/file_system", "git_types"]
, ["src/buildtool/file_system/symlinks_map", "pragma_special"]
, ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "expected"]
]
, "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]]
, "stage": ["src", "buildtool", "serve_api", "remote"]
, "private-deps":
- [ ["src/buildtool/common/remote", "client_common"]
+ [ ["src/buildtool/common", "artifact_digest_factory"]
+ , ["src/buildtool/common/remote", "client_common"]
, ["src/buildtool/logging", "log_level"]
]
}
diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp
index 23c87982..ee60f4f1 100644
--- a/src/buildtool/serve_api/remote/serve_api.hpp
+++ b/src/buildtool/serve_api/remote/serve_api.hpp
@@ -46,7 +46,9 @@ class ServeApi final {
gsl::not_null<LocalContext const*> const& local_context,
gsl::not_null<RemoteContext const*> const& remote_context,
gsl::not_null<ApiBundle const*> const& apis) noexcept
- : stc_{address, remote_context},
+ : stc_{address,
+ &local_context->storage_config->hash_function,
+ remote_context},
tc_{address, local_context->storage, remote_context, apis},
cc_{address, remote_context} {}
@@ -71,10 +73,10 @@ class ServeApi final {
return std::nullopt;
}
- [[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit,
- std::string const& subdir = ".",
- bool sync_tree = false)
- const noexcept -> expected<std::string, GitLookupError> {
+ [[nodiscard]] auto RetrieveTreeFromCommit(
+ std::string const& commit,
+ std::string const& subdir = ".",
+ bool sync_tree = false) const noexcept -> SourceTreeClient::result_t {
return stc_.ServeCommitTree(commit, subdir, sync_tree);
}
@@ -83,8 +85,7 @@ class ServeApi final {
std::string const& archive_type = "archive",
std::string const& subdir = ".",
std::optional<PragmaSpecial> const& resolve_symlinks = std::nullopt,
- bool sync_tree = false) const noexcept
- -> expected<std::string, GitLookupError> {
+ bool sync_tree = false) const noexcept -> SourceTreeClient::result_t {
return stc_.ServeArchiveTree(
content, archive_type, subdir, resolve_symlinks, sync_tree);
}
@@ -92,15 +93,14 @@ class ServeApi final {
[[nodiscard]] auto RetrieveTreeFromDistdir(
std::shared_ptr<std::unordered_map<std::string, std::string>> const&
distfiles,
- bool sync_tree = false) const noexcept
- -> expected<std::string, GitLookupError> {
+ bool sync_tree = false) const noexcept -> SourceTreeClient::result_t {
return stc_.ServeDistdirTree(distfiles, sync_tree);
}
- [[nodiscard]] auto RetrieveTreeFromForeignFile(const std::string& content,
- const std::string& name,
- bool executable)
- const noexcept -> expected<std::string, GitLookupError> {
+ [[nodiscard]] auto RetrieveTreeFromForeignFile(
+ const std::string& content,
+ const std::string& name,
+ bool executable) const noexcept -> SourceTreeClient::result_t {
return stc_.ServeForeignFileTree(content, name, executable);
}
diff --git a/src/buildtool/serve_api/remote/source_tree_client.cpp b/src/buildtool/serve_api/remote/source_tree_client.cpp
index 994fcc21..f3538ca3 100644
--- a/src/buildtool/serve_api/remote/source_tree_client.cpp
+++ b/src/buildtool/serve_api/remote/source_tree_client.cpp
@@ -16,6 +16,7 @@
#include "src/buildtool/serve_api/remote/source_tree_client.hpp"
+#include "src/buildtool/common/artifact_digest_factory.hpp"
#include "src/buildtool/common/remote/client_common.hpp"
#include "src/buildtool/logging/log_level.hpp"
@@ -61,7 +62,9 @@ auto PragmaSpecialToSymlinksResolve(
SourceTreeClient::SourceTreeClient(
ServerAddress const& address,
- gsl::not_null<RemoteContext const*> const& remote_context) noexcept {
+ gsl::not_null<HashFunction const*> const& hash_function,
+ gsl::not_null<RemoteContext const*> const& remote_context) noexcept
+ : hash_function_{*hash_function} {
stub_ =
justbuild::just_serve::SourceTree::NewStub(CreateChannelWithCredentials(
address.host, address.port, remote_context->auth));
@@ -95,7 +98,18 @@ auto SourceTreeClient::ServeCommitTree(std::string const& commit_id,
? GitLookupError::Fatal
: GitLookupError::NotFound};
}
- return response.tree(); // success
+ TreeResult result = {response.tree(), std::nullopt};
+ // if asked to sync, get digest from response
+ if (sync_tree) {
+ auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(),
+ response.digest());
+ if (not digest) {
+ logger_.Emit(LogLevel::Debug, std::move(digest).error());
+ return unexpected{GitLookupError::Fatal};
+ }
+ result.digest = *std::move(digest);
+ }
+ return result; // success
}
auto SourceTreeClient::ServeArchiveTree(
@@ -131,7 +145,18 @@ auto SourceTreeClient::ServeArchiveTree(
? GitLookupError::Fatal
: GitLookupError::NotFound};
}
- return response.tree(); // success
+ TreeResult result = {response.tree(), std::nullopt};
+ // if asked to sync, get digest from response
+ if (sync_tree) {
+ auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(),
+ response.digest());
+ if (not digest) {
+ logger_.Emit(LogLevel::Debug, std::move(digest).error());
+ return unexpected{GitLookupError::Fatal};
+ }
+ result.digest = *std::move(digest);
+ }
+ return result; // success
}
auto SourceTreeClient::ServeDistdirTree(
@@ -166,7 +191,18 @@ auto SourceTreeClient::ServeDistdirTree(
? GitLookupError::Fatal
: GitLookupError::NotFound};
}
- return response.tree(); // success
+ TreeResult result = {response.tree(), std::nullopt};
+ // if asked to sync, get digest from response
+ if (sync_tree) {
+ auto digest = ArtifactDigestFactory::FromBazel(hash_function_.GetType(),
+ response.digest());
+ if (not digest) {
+ logger_.Emit(LogLevel::Debug, std::move(digest).error());
+ return unexpected{GitLookupError::Fatal};
+ }
+ result.digest = *std::move(digest);
+ }
+ return result; // success
}
auto SourceTreeClient::ServeForeignFileTree(const std::string& content,
@@ -199,7 +235,7 @@ auto SourceTreeClient::ServeForeignFileTree(const std::string& content,
? GitLookupError::Fatal
: GitLookupError::NotFound};
}
- return response.tree(); // success
+ return TreeResult{response.tree(), std::nullopt}; // success
}
auto SourceTreeClient::ServeContent(std::string const& content) const noexcept
diff --git a/src/buildtool/serve_api/remote/source_tree_client.hpp b/src/buildtool/serve_api/remote/source_tree_client.hpp
index c849eeb3..39e0e721 100644
--- a/src/buildtool/serve_api/remote/source_tree_client.hpp
+++ b/src/buildtool/serve_api/remote/source_tree_client.hpp
@@ -16,12 +16,15 @@
#define INCLUDED_SRC_BUILDTOOL_SERVE_API_SOURCE_TREE_CLIENT_HPP
#include <memory>
+#include <optional>
#include <string>
#include <unordered_map>
#include "gsl/gsl"
#include "justbuild/just_serve/just_serve.grpc.pb.h"
+#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/common/remote/remote_common.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/remote/context.hpp"
#include "src/buildtool/file_system/git_types.hpp"
#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp"
@@ -34,19 +37,23 @@ class SourceTreeClient {
public:
explicit SourceTreeClient(
ServerAddress const& address,
+ gsl::not_null<HashFunction const*> const& hash_function,
gsl::not_null<RemoteContext const*> const& remote_context) noexcept;
- // An error + data union type
- using result_t = expected<std::string, GitLookupError>;
+ struct TreeResult {
+ std::string tree;
+ std::optional<ArtifactDigest> digest;
+ };
+ using result_t = expected<TreeResult, GitLookupError>;
/// \brief Retrieve the Git tree of a given commit, if known by the
/// endpoint. It is a fatal error if the commit is known to the endpoint but
- /// no tree is able to be returned.
+ /// no result is able to be returned.
/// \param[in] commit_id Hash of the Git commit to look up.
/// \param[in] subdir Relative path of the tree inside commit.
/// \param[in] sync_tree Sync tree to the remote-execution endpoint.
- /// \returns The tree identifier on success or an unexpected error (fatal or
- /// commit or subtree not found).
+ /// \returns The optional tree digest on success or an unexpected error
+ /// (fatal or commit or subtree not found).
[[nodiscard]] auto ServeCommitTree(std::string const& commit_id,
std::string const& subdir,
bool sync_tree) const noexcept
@@ -54,15 +61,15 @@ class SourceTreeClient {
/// \brief Retrieve the Git tree of an archive content, if known by the
/// endpoint. It is a fatal error if the content blob is known to the
- /// endpoint but no tree is able to be returned.
+ /// endpoint but no result is able to be returned.
/// \param[in] content Hash of the archive content to look up.
/// \param[in] archive_type Type of archive ("archive"|"zip").
/// \param[in] subdir Relative path of the tree inside archive.
/// \param[in] resolve_symlinks Optional enum to state how symlinks in the
/// archive should be handled if the tree has to be actually computed.
/// \param[in] sync_tree Sync tree to the remote-execution endpoint.
- /// \returns The tree identifier on success or an unexpected error (fatal or
- /// content blob not found).
+ /// \returns The optional tree digest on success or an unexpected error
+ /// (fatal or content blob not found).
[[nodiscard]] auto ServeArchiveTree(
std::string const& content,
std::string const& archive_type,
@@ -72,20 +79,25 @@ class SourceTreeClient {
/// \brief Retrieve the Git tree of a directory of distfiles, if all the
/// content blobs are known by the endpoint. It is a fatal error if all
- /// content blobs are known but no tree is able to be returned.
+ /// content blobs are known but no result is able to be returned.
/// \param[in] distfiles Mapping from distfile names to content blob ids.
- /// \param[in] sync_tree Sync tree and all ditfile blobs to the
+ /// \param[in] sync_tree Sync tree and all distfile blobs to the
/// remote-execution endpoint.
- /// \returns The tree identifier on success or an unexpected error (fatal or
- /// at least one distfile blob missing).
+ /// \returns The optional tree digest on success or an unexpected error
+ /// (fatal or at least one distfile blob missing).
[[nodiscard]] auto ServeDistdirTree(
std::shared_ptr<std::unordered_map<std::string, std::string>> const&
distfiles,
bool sync_tree) const noexcept -> result_t;
/// \brief Retrieve the Git tree of a foreign-file directory, if all content
- /// blobs are known to the end point and, as a side effect, make that tree
+ /// blobs are known to the endpoint and, as a side-effect, make that tree
/// known to the serve endpoint.
+ /// \param[in] content Hash of the foreign-file content.
+ /// \param[in] name Name of the foreign-file.
+ /// \param[in] executable Executable flag of foreign-file.
+ /// \returns The optional tree digest on success or an unexpected error
+ /// (fatal or content not found).
[[nodiscard]] auto ServeForeignFileTree(const std::string& content,
const std::string& name,
bool executable) const noexcept
@@ -122,6 +134,7 @@ class SourceTreeClient {
-> bool;
private:
+ HashFunction const& hash_function_; // hash function of the remote
std::unique_ptr<justbuild::just_serve::SourceTree::Stub> stub_;
Logger logger_{"RemoteSourceTreeClient"};
};
diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto
index 4894fec3..a0eba0ef 100644
--- a/src/buildtool/serve_api/serve_service/just_serve.proto
+++ b/src/buildtool/serve_api/serve_service/just_serve.proto
@@ -56,6 +56,11 @@ message ServeCommitTreeResponse {
// If the status has a code `OK` or `SYNC_ERROR`, the tree is correct.
// For any other value, the `tree` field is not set.
ServeCommitTreeStatus status = 2;
+
+ // The digest of the requested tree, which can be used to retrieve it from
+ // the associated remote-execution endpoint CAS, if tree was uploaded to the
+ // remote-execution endpoint.
+ build.bazel.remote.execution.v2.Digest digest = 3;
}
// A request message for
@@ -126,6 +131,11 @@ message ServeArchiveTreeResponse {
// If the status has a code `OK` or `SYNC_ERROR`, the tree is correct.
// For any other value, the `tree` field is not set.
ServeArchiveTreeStatus status = 2;
+
+ // The digest of the requested tree, which can be used to retrieve it from
+ // the associated remote-execution endpoint CAS, if tree was uploaded to the
+ // remote-execution endpoint.
+ build.bazel.remote.execution.v2.Digest digest = 3;
}
// A request message for
@@ -176,6 +186,11 @@ message ServeDistdirTreeResponse {
// If the status has a code `OK` or `SYNC_ERROR`, the tree is correct.
// For any other value, the `tree` field is not set.
ServeDistdirTreeStatus status = 2;
+
+ // The digest of the requested tree, which can be used to retrieve it from
+ // the associated remote-execution endpoint CAS, if tree was uploaded to the
+ // remote-execution endpoint.
+ build.bazel.remote.execution.v2.Digest digest = 3;
}
// A request message for
diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp
index 8bc06c39..a6514c46 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.cpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.cpp
@@ -214,14 +214,30 @@ auto SourceTreeService::ServeCommitTree(
auto res = GetSubtreeFromCommit(
native_context_->storage_config->GitRoot(), commit, subdir, logger_);
if (res) {
- auto tree_id = *std::move(res);
+ auto const tree_id = *std::move(res);
auto status = ServeCommitTreeResponse::OK;
if (request->sync_tree()) {
status =
SyncGitEntryToCas<ObjectType::Tree, ServeCommitTreeResponse>(
tree_id, native_context_->storage_config->GitRoot());
+ if (status == ServeCommitTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function.GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error, std::move(digest).error());
+ response->set_status(
+ ServeCommitTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
- *(response->mutable_tree()) = std::move(tree_id);
+ *(response->mutable_tree()) = tree_id;
response->set_status(status);
return ::grpc::Status::OK;
}
@@ -240,14 +256,32 @@ auto SourceTreeService::ServeCommitTree(
for (auto const& path : serve_config_.known_repositories) {
auto res = GetSubtreeFromCommit(path, commit, subdir, logger_);
if (res) {
- auto tree_id = *std::move(res);
+ auto const tree_id = *std::move(res);
auto status = ServeCommitTreeResponse::OK;
if (request->sync_tree()) {
status =
SyncGitEntryToCas<ObjectType::Tree,
ServeCommitTreeResponse>(tree_id, path);
+ if (status == ServeCommitTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function
+ .GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error,
+ std::move(digest).error());
+ response->set_status(
+ ServeCommitTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
- *(response->mutable_tree()) = std::move(tree_id);
+ *(response->mutable_tree()) = tree_id;
response->set_status(status);
return ::grpc::Status::OK;
}
@@ -277,6 +311,21 @@ auto SourceTreeService::SyncArchive(std::string const& tree_id,
if (sync_tree) {
status = SyncGitEntryToCas<ObjectType::Tree, ServeArchiveTreeResponse>(
tree_id, repo_path);
+ if (status == ServeArchiveTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function.GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error, std::move(digest).error());
+ response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
*(response->mutable_tree()) = tree_id;
response->set_status(status);
@@ -992,6 +1041,21 @@ auto SourceTreeService::DistdirImportToGit(
if (sync_tree) {
status = SyncGitEntryToCas<ObjectType::Tree, ServeDistdirTreeResponse>(
tree_id, native_context_->storage_config->GitRoot());
+ if (status == ServeDistdirTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function.GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error, std::move(digest).error());
+ response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
// set response on success
*(response->mutable_tree()) = std::move(tree_id);
@@ -1200,6 +1264,22 @@ auto SourceTreeService::ServeDistdirTree(
status =
SyncGitEntryToCas<ObjectType::Tree, ServeDistdirTreeResponse>(
tree_id, native_context_->storage_config->GitRoot());
+ if (status == ServeDistdirTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function.GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error, std::move(digest).error());
+ response->set_status(
+ ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
// set response on success
*(response->mutable_tree()) = std::move(tree_id);
@@ -1224,6 +1304,24 @@ auto SourceTreeService::ServeDistdirTree(
status =
SyncGitEntryToCas<ObjectType::Tree,
ServeDistdirTreeResponse>(tree_id, path);
+ if (status == ServeDistdirTreeResponse::OK) {
+ // set digest in response
+ auto digest = ArtifactDigestFactory::Create(
+ native_context_->storage_config->hash_function
+ .GetType(),
+ tree_id,
+ /*size is unknown*/ 0,
+ /*is_tree=*/true);
+ if (not digest) {
+ logger_->Emit(LogLevel::Error,
+ std::move(digest).error());
+ response->set_status(
+ ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ *(response->mutable_digest()) =
+ ArtifactDigestFactory::ToBazel(*std::move(digest));
+ }
}
// set response on success
*(response->mutable_tree()) = std::move(tree_id);
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index bdca1331..ace8cbda 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -422,10 +422,13 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
ApiBundle{.hash_function = hash_fct,
.local = native_local_api,
.remote = has_remote_api ? remote_api : native_local_api};
- auto serve = ServeApi::Create(*serve_config,
- &native_local_context, /*unused*/
- &remote_context,
- &apis /*unused*/);
+ auto serve = ServeApi::Create(
+ *serve_config,
+ compat_local_context != nullptr
+ ? &*compat_local_context
+ : &native_local_context, // defines the client's hash_function
+ &remote_context,
+ &apis /*unused*/);
// check configuration of the serve endpoint provided
if (serve) {
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 8b464fce..b29e9a5c 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -234,10 +234,13 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
ApiBundle{.hash_function = hash_fct,
.local = native_local_api,
.remote = has_remote_api ? remote_api : native_local_api};
- auto serve = ServeApi::Create(*serve_config,
- &native_local_context, /*unused*/
- &remote_context,
- &apis /*unused*/);
+ auto serve = ServeApi::Create(
+ *serve_config,
+ compat_local_context != nullptr
+ ? &*compat_local_context
+ : &native_local_context, // defines the client's hash_function
+ &remote_context,
+ &apis /*unused*/);
// check configuration of the serve endpoint provided
if (serve) {
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index f221f63f..8f96eb82 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -82,13 +82,13 @@ void EnsureRootAsAbsent(
if (not *has_tree) {
// try to see if serve endpoint has the information to prepare the
// root itself
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromCommit(repo_info.hash,
repo_info.subdir,
/*sync_tree = */ false);
if (serve_result) {
// if serve has set up the tree, it must match what we expect
- auto const& served_tree_id = *serve_result;
+ auto const& served_tree_id = serve_result->tree;
if (tree_id != served_tree_id) {
(*logger)(fmt::format("Mismatch in served root tree "
"id:\nexpected {}, but got {}",
@@ -671,7 +671,7 @@ void EnsureCommit(
if (serve != nullptr) {
// if root purely absent, request only the subdir tree
if (repo_info.absent and not fetch_absent) {
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromCommit(repo_info.hash,
repo_info.subdir,
/*sync_tree = */ false);
@@ -683,7 +683,7 @@ void EnsureCommit(
{repo_info.ignore_special
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
- *std::move(serve_result)}),
+ serve_result->tree}),
/*is_cache_hit=*/false));
return;
}
@@ -700,12 +700,13 @@ void EnsureCommit(
// otherwise, request (and sync) the whole commit tree, to ensure
// we maintain the id file association
else {
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromCommit(repo_info.hash,
/*subdir = */ ".",
/*sync_tree = */ true);
if (serve_result) {
- auto const& root_tree_id = *serve_result;
+ auto const root_tree_id = serve_result->tree;
+ auto const remote_digest = serve_result->digest;
// verify if we know the tree already in the local Git cache
GitOpKey op_key = {.params =
{
@@ -721,6 +722,7 @@ void EnsureCommit(
ts,
{std::move(op_key)},
[root_tree_id,
+ remote_digest,
tree_id_file,
repo_info,
repo_root,
@@ -875,18 +877,13 @@ void EnsureCommit(
}
}
- // try to get root tree from remote CAS
- auto const root_digest =
- ArtifactDigestFactory::Create(
- native_storage_config->hash_function
- .GetType(),
- root_tree_id,
- 0,
- /*is_tree=*/true);
- if (remote_api != nullptr and root_digest and
+ // try to get root tree from remote CAS; use the
+ // digest received from serve; whether native or
+ // compatible, it will either way be imported to Git
+ if (remote_api != nullptr and remote_digest and
remote_api->RetrieveToCas(
{Artifact::ObjectInfo{
- .digest = *root_digest,
+ .digest = *remote_digest,
.type = ObjectType::Tree}},
*local_api)) {
progress->TaskTracker().Stop(repo_info.origin);
@@ -907,7 +904,7 @@ void EnsureCommit(
}
if (not local_api->RetrieveToPaths(
{Artifact::ObjectInfo{
- .digest = *root_digest,
+ .digest = *remote_digest,
.type = ObjectType::Tree}},
{tmp_dir->GetPath()})) {
(*logger)(fmt::format(
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index ecae60ac..2338c066 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -71,7 +71,7 @@ void EnsureRootAsAbsent(
// try to see if serve endpoint has the information to prepare the
// root itself; this is redundant if root is not already cached
if (is_cache_hit) {
- auto serve_result = serve->RetrieveTreeFromArchive(
+ auto const serve_result = serve->RetrieveTreeFromArchive(
key.archive.content_hash.Hash(),
key.repo_type,
key.subdir,
@@ -80,12 +80,11 @@ void EnsureRootAsAbsent(
if (serve_result) {
// if serve has set up the tree, it must match what we
// expect
- auto const& served_tree_id = *serve_result;
- if (tree_id != served_tree_id) {
+ if (tree_id != serve_result->tree) {
(*logger)(fmt::format("Mismatch in served root tree "
"id:\nexpected {}, but got {}",
tree_id,
- served_tree_id),
+ serve_result->tree),
/*fatal=*/true);
return;
}
@@ -1047,7 +1046,7 @@ auto CreateContentGitMap(
// request the resolved subdir tree from the serve endpoint, if
// given
if (serve != nullptr) {
- auto serve_result = serve->RetrieveTreeFromArchive(
+ auto const serve_result = serve->RetrieveTreeFromArchive(
key.archive.content_hash.Hash(),
key.repo_type,
key.subdir,
@@ -1058,7 +1057,7 @@ auto CreateContentGitMap(
progress->TaskTracker().Stop(key.archive.origin);
(*setter)(std::pair(
nlohmann::json::array(
- {FileRoot::kGitTreeMarker, *serve_result}),
+ {FileRoot::kGitTreeMarker, serve_result->tree}),
/*is_cache_hit = */ false));
return;
}
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index 82c0c623..ef460949 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -224,21 +224,20 @@ auto CreateDistdirGitMap(
if (not *has_tree) {
// try to see if serve endpoint has the
// information to prepare the root itself
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromDistdir(
key.content_list,
/*sync_tree=*/false);
if (serve_result) {
// if serve has set up the tree, it must
// match what we expect
- auto const& served_tree_id = *serve_result;
- if (distdir_tree_id != served_tree_id) {
+ if (distdir_tree_id != serve_result->tree) {
(*logger)(
fmt::format(
"Mismatch in served root tree "
"id:\nexpected {}, but got {}",
distdir_tree_id,
- served_tree_id),
+ serve_result->tree),
/*fatal=*/true);
return;
}
@@ -376,19 +375,18 @@ auto CreateDistdirGitMap(
}
// try to see if serve endpoint has the information to
// prepare the root itself
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromDistdir(key.content_list,
/*sync_tree=*/false);
if (serve_result) {
// if serve has set up the tree, it must match what we
// expect
- auto const& served_tree_id = *serve_result;
- if (tree_id != served_tree_id) {
+ if (tree_id != serve_result->tree) {
(*logger)(
fmt::format("Mismatch in served root tree "
"id:\nexpected {}, but got {}",
tree_id,
- served_tree_id),
+ serve_result->tree),
/*fatal=*/true);
return;
}
@@ -520,18 +518,17 @@ auto CreateDistdirGitMap(
// now ask serve endpoint if it can set up the root; as this is for
// a present root, a corresponding remote endpoint is needed
if (serve != nullptr and remote_api != nullptr) {
- auto serve_result =
+ auto const serve_result =
serve->RetrieveTreeFromDistdir(key.content_list,
/*sync_tree=*/true);
if (serve_result) {
// if serve has set up the tree, it must match what we
// expect
- auto const& served_tree_id = *serve_result;
- if (tree_id != served_tree_id) {
+ if (tree_id != serve_result->tree) {
(*logger)(fmt::format("Mismatch in served root tree "
"id:\nexpected {}, but got {}",
tree_id,
- served_tree_id),
+ serve_result->tree),
/*fatal=*/true);
return;
}
diff --git a/src/other_tools/root_maps/foreign_file_git_map.cpp b/src/other_tools/root_maps/foreign_file_git_map.cpp
index 8bc09b86..25c77947 100644
--- a/src/other_tools/root_maps/foreign_file_git_map.cpp
+++ b/src/other_tools/root_maps/foreign_file_git_map.cpp
@@ -156,17 +156,15 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key,
/*is_cache_hit=*/false));
return;
}
- auto serve_result = serve->RetrieveTreeFromForeignFile(
+ auto const serve_result = serve->RetrieveTreeFromForeignFile(
key.archive.content_hash.Hash(), key.name, key.executable);
if (serve_result) {
- // if serve has set up the tree, it must match what we
- // expect
- auto const& served_tree_id = *serve_result;
- if (tree_id != served_tree_id) {
+ // if serve has set up the tree, it must match what we expect
+ if (tree_id != serve_result->tree) {
(*logger)(fmt::format("Mismatch in served root tree "
"id: expected {}, but got {}",
tree_id,
- served_tree_id),
+ serve_result->tree),
/*fatal=*/true);
return;
}
diff --git a/test/buildtool/serve_api/TARGETS b/test/buildtool/serve_api/TARGETS
index 656a3ae7..69455203 100644
--- a/test/buildtool/serve_api/TARGETS
+++ b/test/buildtool/serve_api/TARGETS
@@ -6,19 +6,28 @@
, "private-deps":
[ ["@", "catch2", "", "catch2"]
, ["@", "src", "src/buildtool/auth", "auth"]
+ , ["@", "src", "src/buildtool/common", "protocol_traits"]
, ["@", "src", "src/buildtool/common/remote", "retry_config"]
+ , ["@", "src", "src/buildtool/crypto", "hash_function"]
, ["@", "src", "src/buildtool/execution_api/remote", "config"]
, ["@", "src", "src/buildtool/execution_api/remote", "context"]
, ["@", "src", "src/buildtool/serve_api/remote", "config"]
, ["@", "src", "src/buildtool/serve_api/remote", "source_tree_client"]
, ["utils", "catch-main-serve"]
+ , ["utils", "test_hash_function_type"]
, ["utils", "test_serve_config"]
]
, "stage": ["test", "buildtool", "serve_api"]
}
, "TESTS":
{ "type": ["@", "rules", "test", "suite"]
+ , "arguments_config": ["TEST_COMPATIBLE_REMOTE"]
, "stage": ["serve_api"]
- , "deps": ["source_tree_client"]
+ , "deps":
+ { "type": "if"
+ , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"}
+ , "then": []
+ , "else": ["source_tree_client"]
+ }
}
}
diff --git a/test/buildtool/serve_api/source_tree_client.test.cpp b/test/buildtool/serve_api/source_tree_client.test.cpp
index 7c9634df..8ada1317 100644
--- a/test/buildtool/serve_api/source_tree_client.test.cpp
+++ b/test/buildtool/serve_api/source_tree_client.test.cpp
@@ -19,10 +19,13 @@
#include "catch2/catch_test_macros.hpp"
#include "src/buildtool/auth/authentication.hpp"
+#include "src/buildtool/common/protocol_traits.hpp"
#include "src/buildtool/common/remote/retry_config.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/execution_api/remote/context.hpp"
#include "src/buildtool/serve_api/remote/config.hpp"
+#include "test/utils/hermeticity/test_hash_function_type.hpp"
#include "test/utils/serve_service/test_serve_config.hpp"
auto const kRootCommit =
@@ -36,9 +39,11 @@ auto const kRootSymId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"};
auto const kBazSymId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"};
TEST_CASE("Serve service client: tree-of-commit request", "[serve_api]") {
- auto config = TestServeConfig::ReadFromEnvironment();
+ auto const config = TestServeConfig::ReadFromEnvironment();
REQUIRE(config);
REQUIRE(config->remote_address);
+ auto const hash_function =
+ HashFunction{TestHashType::ReadFromEnvironment()};
// Create TLC client
Auth auth{};
@@ -48,26 +53,39 @@ TEST_CASE("Serve service client: tree-of-commit request", "[serve_api]") {
.retry_config = &retry_config,
.exec_config = &exec_config};
- SourceTreeClient st_client(*config->remote_address, &remote_context);
+ SourceTreeClient st_client(
+ *config->remote_address, &hash_function, &remote_context);
SECTION("Commit in bare checkout") {
auto root_id = st_client.ServeCommitTree(kRootCommit, ".", false);
REQUIRE(root_id);
- CHECK(*root_id == kRootId);
+ CHECK_FALSE(root_id->digest); // digest is not provided if not syncing
+ if (ProtocolTraits::IsNative(hash_function.GetType())) {
+ CHECK(root_id->tree == kRootId);
+ }
auto baz_id = st_client.ServeCommitTree(kRootCommit, "baz", false);
REQUIRE(baz_id);
- CHECK(*baz_id == kBazId);
+ CHECK_FALSE(baz_id->digest); // digest is not provided if not syncing
+ if (ProtocolTraits::IsNative(hash_function.GetType())) {
+ CHECK(baz_id->tree == kBazId);
+ }
}
SECTION("Commit in non-bare checkout") {
auto root_id = st_client.ServeCommitTree(kRootSymCommit, ".", false);
REQUIRE(root_id);
- CHECK(*root_id == kRootSymId);
+ CHECK_FALSE(root_id->digest); // digest is not provided if not syncing
+ if (ProtocolTraits::IsNative(hash_function.GetType())) {
+ CHECK(root_id->tree == kRootSymId);
+ }
auto baz_id = st_client.ServeCommitTree(kRootSymCommit, "baz", false);
REQUIRE(baz_id);
- CHECK(*baz_id == kBazSymId);
+ CHECK_FALSE(baz_id->digest); // digest is not provided if not syncing
+ if (ProtocolTraits::IsNative(hash_function.GetType())) {
+ CHECK(baz_id->tree == kBazSymId);
+ }
}
SECTION("Subdir not found") {
@@ -81,7 +99,7 @@ TEST_CASE("Serve service client: tree-of-commit request", "[serve_api]") {
auto root_id = st_client.ServeCommitTree(
"0123456789abcdef0123456789abcdef01234567", ".", false);
REQUIRE_FALSE(root_id);
- CHECK_FALSE(root_id.error() ==
- GitLookupError::Fatal); // non-fatal failure
+ CHECK(root_id.error() ==
+ GitLookupError::NotFound); // non-fatal failure
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 20:52:02 +0000