summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/file_system/git_repo.cpp24
-rw-r--r--src/buildtool/file_system/git_repo.hpp13
-rw-r--r--src/buildtool/serve_api/serve_service/just_serve.proto4
-rw-r--r--src/buildtool/serve_api/serve_service/source_tree.cpp366
-rw-r--r--src/buildtool/serve_api/serve_service/source_tree.hpp29
-rw-r--r--src/other_tools/ops_maps/import_to_git_map.cpp7
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp15
7 files changed, 332 insertions, 126 deletions
diff --git a/src/buildtool/file_system/git_repo.cpp b/src/buildtool/file_system/git_repo.cpp
index f679647e..72f85f6a 100644
--- a/src/buildtool/file_system/git_repo.cpp
+++ b/src/buildtool/file_system/git_repo.cpp
@@ -932,9 +932,9 @@ auto GitRepo::FetchFromPath(std::shared_ptr<git_config> cfg,
auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
std::string const& subdir,
anon_logger_ptr const& logger) noexcept
- -> std::optional<std::string> {
+ -> std::variant<bool, std::string> {
#ifdef BOOTSTRAP_BUILD_TOOL
- return std::nullopt;
+ return true;
#else
try {
// preferably with a "fake" repository!
@@ -955,7 +955,7 @@ auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
GetGitCAS()->git_path_.string(),
GitLastError()),
true /*fatal*/);
- return std::nullopt;
+ return true; // fatal failure
}
git_commit* commit_ptr{nullptr};
@@ -968,7 +968,7 @@ auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
true /*fatal*/);
// cleanup resources
git_commit_free(commit_ptr);
- return std::nullopt;
+ return false; // non-fatal failure
}
auto commit_obj = std::unique_ptr<git_commit, decltype(&commit_closer)>(
commit_ptr, commit_closer);
@@ -985,7 +985,7 @@ auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
true /*fatal*/);
// cleanup resources
git_tree_free(tree_ptr);
- return std::nullopt;
+ return true; // fatal failure
}
auto tree = std::unique_ptr<git_tree, decltype(&tree_closer)>(
tree_ptr, tree_closer);
@@ -1004,7 +1004,7 @@ auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
true /*fatal*/);
// cleanup resources
git_tree_entry_free(subtree_entry_ptr);
- return std::nullopt;
+ return true; // fatal failure
}
auto subtree_entry =
std::unique_ptr<git_tree_entry, decltype(&tree_entry_closer)>(
@@ -1012,16 +1012,16 @@ auto GitRepo::GetSubtreeFromCommit(std::string const& commit,
std::string subtree_hash{
git_oid_tostr_s(git_tree_entry_id(subtree_entry.get()))};
- return subtree_hash;
+ return subtree_hash; // success
}
// if no subdir, get hash from tree
std::string tree_hash{git_oid_tostr_s(git_tree_id(tree.get()))};
- return tree_hash;
+ return tree_hash; // success
} catch (std::exception const& ex) {
Logger::Log(LogLevel::Error,
"get subtree from commit failed with:\n{}",
ex.what());
- return std::nullopt;
+ return true; // fatal failure
}
#endif // BOOTSTRAP_BUILD_TOOL
}
@@ -1145,7 +1145,11 @@ auto GitRepo::GetSubtreeFromPath(std::filesystem::path const& fpath,
// find relative path from root to given path
auto subdir = std::filesystem::relative(fpath, *root).string();
// get subtree from head commit and subdir
- return GetSubtreeFromCommit(head_commit, subdir, wrapped_logger);
+ auto res = GetSubtreeFromCommit(head_commit, subdir, wrapped_logger);
+ if (std::holds_alternative<bool>(res)) {
+ return std::nullopt;
+ }
+ return std::get<std::string>(res);
} catch (std::exception const& ex) {
Logger::Log(LogLevel::Error,
"get subtree from path failed with:\n{}",
diff --git a/src/buildtool/file_system/git_repo.hpp b/src/buildtool/file_system/git_repo.hpp
index 05bcfaf3..09e65402 100644
--- a/src/buildtool/file_system/git_repo.hpp
+++ b/src/buildtool/file_system/git_repo.hpp
@@ -15,7 +15,13 @@
#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_REPO_HPP
#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_REPO_HPP
+#include <filesystem>
#include <functional>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <variant>
+#include <vector>
#include "src/buildtool/common/bazel_types.hpp"
#include "src/buildtool/file_system/git_cas.hpp"
@@ -178,12 +184,15 @@ class GitRepo {
/// \brief Get the tree id of a subtree given the root commit
/// Calling it from a fake repository allows thread-safe use.
- /// Returns the subtree hash, as a string, or nullopt if failure.
+ /// Returns an error + data union, where at index 0 is a flag stating the
+ /// nature of the error on failure (true is fatal, false is non-fatal, i.e.,
+ /// commit not found), and at index 1 is the subtree hash on success.
/// It guarantees the logger is called exactly once with fatal if failure.
[[nodiscard]] auto GetSubtreeFromCommit(
std::string const& commit,
std::string const& subdir,
- anon_logger_ptr const& logger) noexcept -> std::optional<std::string>;
+ anon_logger_ptr const& logger) noexcept
+ -> std::variant<bool, std::string>;
/// \brief Get the tree id of a subtree given the root tree hash
/// Calling it from a fake repository allows thread-safe use.
diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto
index ac68f829..985dfecd 100644
--- a/src/buildtool/serve_api/serve_service/just_serve.proto
+++ b/src/buildtool/serve_api/serve_service/just_serve.proto
@@ -46,7 +46,7 @@ message ServeCommitTreeResponse {
// Failed to upload tree remotely
SYNC_ERROR = 1;
- // Tree not found
+ // Commit not found
NOT_FOUND = 2;
// Internally, something is very broken
@@ -116,7 +116,7 @@ message ServeArchiveTreeResponse {
// Failed to resolve symlinks as requested
RESOLVE_ERROR = 3;
- // Tree not found
+ // Content blob not known
NOT_FOUND = 4;
// Internally, something is very broken
diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp
index a92c53b1..609775ab 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.cpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.cpp
@@ -96,45 +96,46 @@ auto SourceTreeService::GetSubtreeFromCommit(
std::filesystem::path const& repo_path,
std::string const& commit,
std::string const& subdir,
- std::shared_ptr<Logger> const& logger) -> std::optional<std::string> {
+ std::shared_ptr<Logger> const& logger) -> std::variant<bool, std::string> {
if (auto git_cas = GitCAS::Open(repo_path)) {
if (auto repo = GitRepo::Open(git_cas)) {
// wrap logger for GitRepo call
auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>(
- [logger, repo_path, commit](auto const& msg, bool fatal) {
+ [logger, repo_path, commit, subdir](auto const& msg,
+ bool fatal) {
if (fatal) {
auto err = fmt::format(
- "ServeCommitTree: While retrieving subtree of "
- "commit {} from repository {}:\n{}",
+ "While retrieving subtree {} of commit {} from "
+ "repository {}:\n{}",
+ subdir,
commit,
repo_path.string(),
msg);
logger->Emit(LogLevel::Info, err);
}
});
- if (auto tree_id = repo->GetSubtreeFromCommit(
- commit, subdir, wrapped_logger)) {
- return tree_id;
- }
+ return repo->GetSubtreeFromCommit(commit, subdir, wrapped_logger);
}
}
- return std::nullopt;
+ return true; // fatal failure
}
auto SourceTreeService::GetSubtreeFromTree(
std::filesystem::path const& repo_path,
std::string const& tree_id,
std::string const& subdir,
- std::shared_ptr<Logger> const& logger) -> std::optional<std::string> {
+ std::shared_ptr<Logger> const& logger) -> std::variant<bool, std::string> {
if (auto git_cas = GitCAS::Open(repo_path)) {
if (auto repo = GitRepo::Open(git_cas)) {
// wrap logger for GitRepo call
auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>(
- [logger, repo_path, tree_id](auto const& msg, bool fatal) {
+ [logger, repo_path, tree_id, subdir](auto const& msg,
+ bool fatal) {
if (fatal) {
auto err = fmt::format(
- "ServeCommitTree: While retrieving subtree of tree "
- "{} from repository {}:\n{}",
+ "While retrieving subtree {} of tree {} from "
+ "repository {}:\n{}",
+ subdir,
tree_id,
repo_path.string(),
msg);
@@ -143,17 +144,18 @@ auto SourceTreeService::GetSubtreeFromTree(
});
if (auto subtree_id =
repo->GetSubtreeFromTree(tree_id, subdir, wrapped_logger)) {
- return subtree_id;
+ return *subtree_id;
}
+ return false; // non-fatal failure
}
}
- return std::nullopt;
+ return true; // fatal failure
}
auto SourceTreeService::GetBlobFromRepo(std::filesystem::path const& repo_path,
std::string const& blob_id,
std::shared_ptr<Logger> const& logger)
- -> std::optional<std::string> {
+ -> std::variant<bool, std::string> {
if (auto git_cas = GitCAS::Open(repo_path)) {
if (auto repo = GitRepo::Open(git_cas)) {
// wrap logger for GitRepo call
@@ -161,8 +163,8 @@ auto SourceTreeService::GetBlobFromRepo(std::filesystem::path const& repo_path,
[logger, repo_path, blob_id](auto const& msg, bool fatal) {
if (fatal) {
auto err = fmt::format(
- "ServeCommitTree: While checking existence of blob "
- "{} in repository {}:\n{}",
+ "While checking existence of blob {} in repository "
+ "{}:\n{}",
blob_id,
repo_path.string(),
msg);
@@ -171,18 +173,23 @@ auto SourceTreeService::GetBlobFromRepo(std::filesystem::path const& repo_path,
});
auto res = repo->TryReadBlob(blob_id, wrapped_logger);
if (not res.first) {
- return std::nullopt;
+ return true; // fatal failure
}
if (not res.second) {
auto str = fmt::format("Blob {} not found in repository {}",
blob_id,
repo_path.string());
logger->Emit(LogLevel::Info, str);
+ return false; // non-fatal failure
}
- return res.second;
+ return res.second.value();
}
}
- return std::nullopt;
+ // failed to open repository
+ logger->Emit(
+ LogLevel::Info,
+ fmt::format("Failed to open repository {}", repo_path.string()));
+ return true; // fatal failure
}
auto SourceTreeService::ServeCommitTree(
@@ -192,11 +199,13 @@ auto SourceTreeService::ServeCommitTree(
auto const& commit{request->commit()};
auto const& subdir{request->subdir()};
// try in local build root Git cache
- if (auto tree_id = GetSubtreeFromCommit(
- StorageConfig::GitRoot(), commit, subdir, logger_)) {
+ auto res =
+ GetSubtreeFromCommit(StorageConfig::GitRoot(), commit, subdir, logger_);
+ if (std::holds_alternative<std::string>(res)) {
+ auto tree_id = std::get<std::string>(res);
if (request->sync_tree()) {
// sync tree with remote CAS
- auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+ auto digest = ArtifactDigest{tree_id, 0, /*is_tree=*/true};
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(StorageConfig::GitRoot())) {
auto str = fmt::format("Failed to SetGitCAS at {}",
@@ -211,25 +220,38 @@ auto SourceTreeService::ServeCommitTree(
.type = ObjectType::Tree}},
&(*remote_api_))) {
auto str = fmt::format(
- "Failed to sync tree {} from local Git cache", *tree_id);
+ "Failed to sync tree {} from local Git cache", tree_id);
logger_->Emit(LogLevel::Error, str);
- *(response->mutable_tree()) = std::move(*tree_id);
+ *(response->mutable_tree()) = std::move(tree_id);
response->set_status(ServeCommitTreeResponse::SYNC_ERROR);
return ::grpc::Status::OK;
}
}
// set response
- *(response->mutable_tree()) = std::move(*tree_id);
+ *(response->mutable_tree()) = std::move(tree_id);
response->set_status(ServeCommitTreeResponse::OK);
return ::grpc::Status::OK;
}
+ // report fatal failure
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while retrieving subtree {} of commit {} from repository "
+ "{}",
+ subdir,
+ commit,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
// try given extra repositories, in order
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (auto tree_id =
- GetSubtreeFromCommit(path, commit, subdir, logger_)) {
+ auto res = GetSubtreeFromCommit(path, commit, subdir, logger_);
+ if (std::holds_alternative<std::string>(res)) {
+ auto tree_id = std::get<std::string>(res);
if (request->sync_tree()) {
// sync tree with remote CAS
- auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+ auto digest = ArtifactDigest{tree_id, 0, /*is_tree=*/true};
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(path)) {
auto str =
@@ -246,18 +268,31 @@ auto SourceTreeService::ServeCommitTree(
&(*remote_api_))) {
auto str = fmt::format(
"Failed to sync tree {} from known repository {}",
- *tree_id,
+ tree_id,
path.string());
logger_->Emit(LogLevel::Error, str);
- *(response->mutable_tree()) = std::move(*tree_id);
+ *(response->mutable_tree()) = std::move(tree_id);
response->set_status(ServeCommitTreeResponse::SYNC_ERROR);
+ return ::grpc::Status::OK;
}
}
// set response
- *(response->mutable_tree()) = std::move(*tree_id);
+ *(response->mutable_tree()) = std::move(tree_id);
response->set_status(ServeCommitTreeResponse::OK);
return ::grpc::Status::OK;
}
+ // report fatal failure
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while retrieving subtree {} of commit {} from "
+ "repository {}",
+ subdir,
+ commit,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeCommitTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
}
// commit not found
response->set_status(ServeCommitTreeResponse::NOT_FOUND);
@@ -403,11 +438,7 @@ auto SourceTreeService::CommonImportToGit(
auto commit_hash =
git_repo->StageAndCommitAllAnonymous(commit_message, wrapped_logger);
if (not commit_hash) {
- auto str =
- fmt::format("Failed to create initial commit in repository {}\n{}",
- root_path.string(),
- err);
- return result_t(std::in_place_index<0>, str);
+ return result_t(std::in_place_index<0>, err);
}
// create a tmp directory for the fetch to Git CAS
auto tmp_dir = StorageUtils::CreateTypedTmpDir("import-to-git");
@@ -445,9 +476,7 @@ auto SourceTreeService::CommonImportToGit(
root_path.string(),
/*branch=*/std::nullopt,
wrapped_logger)) {
- auto str = fmt::format(
- "Failed to fetch commit {} into Git CAS\n{}", *commit_hash, err);
- return result_t(std::in_place_index<0>, str);
+ return result_t(std::in_place_index<0>, err);
}
// wrap logger for GitRepo call
err.clear();
@@ -476,9 +505,7 @@ auto SourceTreeService::CommonImportToGit(
if (not git_repo->KeepTag(*commit_hash,
"Keep referenced tree alive", // message
wrapped_logger)) {
- auto str = fmt::format(
- "Failed to tag and keep commit {}\n{}", *commit_hash, err);
- return result_t(std::in_place_index<0>, str);
+ return result_t(std::in_place_index<0>, err);
}
}
// wrap logger for GitRepo call
@@ -492,15 +519,13 @@ auto SourceTreeService::CommonImportToGit(
}
});
// get the root tree of this commit; this is thread-safe
- auto tree_id =
+ auto res =
just_git_repo->GetSubtreeFromCommit(*commit_hash, ".", wrapped_logger);
- if (not tree_id) {
- auto str = fmt::format(
- "Failed to retrieve tree id of commit {}\n{}", *commit_hash, err);
- return result_t(std::in_place_index<0>, str);
+ if (not std::holds_alternative<std::string>(res)) {
+ return result_t(std::in_place_index<0>, err);
}
// return the root tree id
- return result_t(std::in_place_index<1>, *tree_id);
+ return result_t(std::in_place_index<1>, std::get<std::string>(res));
}
auto SourceTreeService::ArchiveImportToGit(
@@ -549,24 +574,21 @@ auto SourceTreeService::ArchiveImportToGit(
return ::grpc::Status::OK;
}
// wrap logger for GitRepo call
+ std::string err;
auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>(
- [logger = logger_, subdir, tree_id](auto const& msg, bool fatal) {
+ [&err, subdir, tree_id](auto const& msg, bool fatal) {
if (fatal) {
- auto err =
- fmt::format("While retrieving subtree {} of tree {}:\n{}",
- subdir,
- tree_id,
- msg);
- logger->Emit(LogLevel::Error, err);
+ err = fmt::format("While retrieving subtree {} of tree {}:\n{}",
+ subdir,
+ tree_id,
+ msg);
}
});
// get the subtree id; this is thread-safe
auto subtree_id =
just_git_repo->GetSubtreeFromTree(tree_id, subdir, wrapped_logger);
if (not subtree_id) {
- auto str = fmt::format(
- "Failed to retrieve subtree {} of tree {}", subdir, tree_id);
- logger_->Emit(LogLevel::Error, str);
+ logger_->Emit(LogLevel::Error, err);
response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
return ::grpc::Status::OK;
}
@@ -580,7 +602,7 @@ auto SourceTreeService::ArchiveImportToGit(
auto SourceTreeService::IsTreeInRepo(std::string const& tree_id,
std::filesystem::path const& repo_path,
std::shared_ptr<Logger> const& logger)
- -> bool {
+ -> std::optional<bool> {
if (auto git_cas = GitCAS::Open(repo_path)) {
if (auto repo = GitRepo::Open(git_cas)) {
// wrap logger for GitRepo call
@@ -588,20 +610,22 @@ auto SourceTreeService::IsTreeInRepo(std::string const& tree_id,
[logger, repo_path, tree_id](auto const& msg, bool fatal) {
if (fatal) {
auto err = fmt::format(
- "SourceTreeService: While checking existence of "
- "tree {} in repository {}:\n{}",
+ "While checking existence of tree {} in repository "
+ "{}:\n{}",
tree_id,
repo_path.string(),
msg);
logger->Emit(LogLevel::Info, err);
}
});
- if (auto res = repo->CheckTreeExists(tree_id, wrapped_logger)) {
- return *res;
- }
+ return repo->CheckTreeExists(tree_id, wrapped_logger);
}
}
- return false; // tree not found
+ // failed to open repository
+ logger->Emit(
+ LogLevel::Info,
+ fmt::format("Failed to open repository {}", repo_path.string()));
+ return std::nullopt;
}
auto SourceTreeService::ServeArchiveTree(
@@ -629,28 +653,49 @@ auto SourceTreeService::ServeArchiveTree(
return ::grpc::Status::OK;
}
// check local build root Git cache
- if (auto subtree_id = GetSubtreeFromTree(
- StorageConfig::GitRoot(), *archive_tree_id, subdir, logger_)) {
- return ResolveContentTree(*subtree_id,
+ auto res = GetSubtreeFromTree(
+ StorageConfig::GitRoot(), *archive_tree_id, subdir, logger_);
+ if (std::holds_alternative<std::string>(res)) {
+ return ResolveContentTree(std::get<std::string>(res), // tree_id
StorageConfig::GitRoot(),
resolve_special,
request->sync_tree(),
response);
}
+ // check for fatal error
+ if (std::get<bool>(res)) {
+ auto str = fmt::format("Failed to open repository {}",
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
// check known repositories
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (auto subtree_id = GetSubtreeFromTree(
- path, *archive_tree_id, subdir, logger_)) {
- return ResolveContentTree(*subtree_id,
- path,
- resolve_special,
- request->sync_tree(),
- response);
+ auto res =
+ GetSubtreeFromTree(path, *archive_tree_id, subdir, logger_);
+ if (std::holds_alternative<std::string>(res)) {
+ return ResolveContentTree(
+ std::get<std::string>(res), // tree_id
+ path,
+ resolve_special,
+ request->sync_tree(),
+ response);
+ }
+ // check for fatal error
+ if (std::get<bool>(res)) {
+ auto str =
+ fmt::format("Failed to open repository {}", path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
}
}
// report error for missing tree specified in id file
auto str =
- fmt::format("Tree {} is known, but missing!", *archive_tree_id);
+ fmt::format("Failed while retrieving subtree {} of known tree {}",
+ subdir,
+ *archive_tree_id);
logger_->Emit(LogLevel::Error, str);
response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
return ::grpc::Status::OK;
@@ -670,22 +715,44 @@ auto SourceTreeService::ServeArchiveTree(
if (content_cas_path = cas.BlobPath(digest, /*is_executable=*/false);
not content_cas_path) {
// check if content blob is in Git cache
- if (auto data =
- GetBlobFromRepo(StorageConfig::GitRoot(), content, logger_)) {
+ auto res = GetBlobFromRepo(StorageConfig::GitRoot(), content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// add to CAS
- content_cas_path = StorageUtils::AddToCAS(*data);
+ content_cas_path =
+ StorageUtils::AddToCAS(std::get<std::string>(res));
+ }
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while trying to retrieve content {} from repository {}",
+ content,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
}
}
if (not content_cas_path) {
// check if content blob is in a known repository
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (auto data = GetBlobFromRepo(path, content, logger_)) {
+ auto res = GetBlobFromRepo(path, content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// add to CAS
- content_cas_path = StorageUtils::AddToCAS(*data);
+ content_cas_path =
+ StorageUtils::AddToCAS(std::get<std::string>(res));
if (content_cas_path) {
break;
}
}
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while trying to retrieve content {} from "
+ "repository {}",
+ content,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeArchiveTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
}
}
if (not content_cas_path) {
@@ -853,10 +920,12 @@ auto SourceTreeService::ServeDistdirTree(
}
else {
// check local Git cache
- if (auto data = GetBlobFromRepo(
- StorageConfig::GitRoot(), content, logger_)) {
+ auto res =
+ GetBlobFromRepo(StorageConfig::GitRoot(), content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// add content to local CAS
- if (not cas.StoreBlob(*data, /*is_executable=*/false)) {
+ if (not cas.StoreBlob(std::get<std::string>(res),
+ /*is_executable=*/false)) {
auto str = fmt::format(
"Failed to store content {} from local Git cache to "
"local CAS",
@@ -869,12 +938,25 @@ auto SourceTreeService::ServeDistdirTree(
blob_found = true;
}
else {
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while trying to retrieve content {} from "
+ "repository {}",
+ content,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(
+ ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
// check known repositories
for (auto const& path :
RemoteServeConfig::KnownRepositories()) {
- if (auto data = GetBlobFromRepo(path, content, logger_)) {
+ auto res = GetBlobFromRepo(path, content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// add content to local CAS
- if (not cas.StoreBlob(*data, /*is_executable=*/false)) {
+ if (not cas.StoreBlob(std::get<std::string>(res),
+ /*is_executable=*/false)) {
auto str = fmt::format(
"Failed to store content {} from known "
"repository {} to local CAS",
@@ -888,6 +970,17 @@ auto SourceTreeService::ServeDistdirTree(
blob_found = true;
break;
}
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while trying to retrieve content {} from "
+ "repository {}",
+ content,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(
+ ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
}
if (not blob_found) {
// Explanation: clang-tidy gets confused by the break in the
@@ -967,7 +1060,17 @@ auto SourceTreeService::ServeDistdirTree(
return ::grpc::Status::OK;
}
// check if tree is already in Git cache
- if (IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_);
+ if (not has_tree) {
+ auto str =
+ fmt::format("Failed while checking for tree {} in repository {}",
+ tree_id,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// if asked, sync tree and all blobs with remote CAS
if (request->sync_tree()) {
if (not local_api_->RetrieveToCas(
@@ -989,7 +1092,17 @@ auto SourceTreeService::ServeDistdirTree(
}
// check if tree is in a known repository
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (IsTreeInRepo(tree_id, path, logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, path, logger_);
+ if (not has_tree) {
+ auto str = fmt::format(
+ "Failed while checking for tree {} in repository {}",
+ tree_id,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeDistdirTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// if asked, sync tree and all blobs with remote CAS
if (request->sync_tree()) {
if (not local_api_->RetrieveToCas(
@@ -1031,8 +1144,8 @@ auto SourceTreeService::ServeContent(
}
auto const digest = ArtifactDigest{content, 0, /*is_tree=*/false};
// check if content blob is in Git cache
- if (auto data =
- GetBlobFromRepo(StorageConfig::GitRoot(), content, logger_)) {
+ auto res = GetBlobFromRepo(StorageConfig::GitRoot(), content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// upload blob to remote CAS
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(StorageConfig::GitRoot())) {
@@ -1057,9 +1170,19 @@ auto SourceTreeService::ServeContent(
response->set_status(ServeContentResponse::OK);
return ::grpc::Status::OK;
}
+ if (std::get<bool>(res)) {
+ auto str =
+ fmt::format("Failed while checking for content {} in repository {}",
+ content,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeContentResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
// check if content blob is in a known repository
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (auto data = GetBlobFromRepo(path, content, logger_)) {
+ auto res = GetBlobFromRepo(path, content, logger_);
+ if (std::holds_alternative<std::string>(res)) {
// upload blob to remote CAS
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(path)) {
@@ -1086,6 +1209,15 @@ auto SourceTreeService::ServeContent(
response->set_status(ServeContentResponse::OK);
return ::grpc::Status::OK;
}
+ if (std::get<bool>(res)) {
+ auto str = fmt::format(
+ "Failed while checking for content {} in repository {}",
+ content,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeContentResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
}
// check also in the local CAS
if (local_api_->IsAvailable(digest) and
@@ -1118,7 +1250,17 @@ auto SourceTreeService::ServeTree(
}
auto digest = ArtifactDigest{tree_id, 0, /*is_tree=*/true};
// check if tree is in Git cache
- if (IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_);
+ if (not has_tree) {
+ auto str =
+ fmt::format("Failed while checking for tree {} in repository {}",
+ tree_id,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// upload tree to remote CAS
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(StorageConfig::GitRoot())) {
@@ -1145,7 +1287,17 @@ auto SourceTreeService::ServeTree(
}
// check if tree is in a known repository
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (IsTreeInRepo(tree_id, path, logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, path, logger_);
+ if (not has_tree) {
+ auto str = fmt::format(
+ "Failed while checking for tree {} in repository {}",
+ tree_id,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(ServeTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// upload tree to remote CAS
auto repo = RepositoryConfig{};
if (not repo.SetGitCAS(path)) {
@@ -1203,14 +1355,34 @@ auto SourceTreeService::CheckRootTree(
return ::grpc::Status::OK;
}
// check first in the Git cache
- if (IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger_);
+ if (not has_tree) {
+ auto str =
+ fmt::format("Failed while checking for tree {} in repository {}",
+ tree_id,
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(CheckRootTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// success!
response->set_status(CheckRootTreeResponse::OK);
return ::grpc::Status::OK;
}
// check if tree is in a known repository
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
- if (IsTreeInRepo(tree_id, path, logger_)) {
+ auto has_tree = IsTreeInRepo(tree_id, path, logger_);
+ if (not has_tree) {
+ auto str = fmt::format(
+ "Failed while checking for tree {} in repository {}",
+ tree_id,
+ path.string());
+ logger_->Emit(LogLevel::Error, str);
+ response->set_status(CheckRootTreeResponse::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ if (*has_tree) {
// success!
response->set_status(CheckRootTreeResponse::OK);
return ::grpc::Status::OK;
diff --git a/src/buildtool/serve_api/serve_service/source_tree.hpp b/src/buildtool/serve_api/serve_service/source_tree.hpp
index b17bdcf9..bd89bf78 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.hpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.hpp
@@ -132,22 +132,38 @@ class SourceTreeService final
// symlinks resolver map
ResolveSymlinksMap resolve_symlinks_map_{CreateResolveSymlinksMap()};
+ /// \brief Check if commit exists and tries to get the subtree if found.
+ /// \returns An error + data union, where at index 0 is a failure type flag
+ /// and at index 1 is the subtree hash on success. The failure flag is false
+ /// if commit was not found and true if failed otherwise.
[[nodiscard]] static auto GetSubtreeFromCommit(
std::filesystem::path const& repo_path,
std::string const& commit,
std::string const& subdir,
- std::shared_ptr<Logger> const& logger) -> std::optional<std::string>;
-
+ std::shared_ptr<Logger> const& logger)
+ -> std::variant<bool, std::string>;
+
+ /// \brief Check if tree exists and tries to get the subtree if found.
+ /// \returns An error + data union, where at index 0 is a failure type flag
+ /// and at index 1 is the subtree hash on success. The failure flag is true
+ /// if repository could not be opened (i.e., fatal failure), false if the
+ /// check for the subtree itself failed.
[[nodiscard]] static auto GetSubtreeFromTree(
std::filesystem::path const& repo_path,
std::string const& tree_id,
std::string const& subdir,
- std::shared_ptr<Logger> const& logger) -> std::optional<std::string>;
+ std::shared_ptr<Logger> const& logger)
+ -> std::variant<bool, std::string>;
+ /// \brief Tries to retrieve the blob from a repository.
+ /// \returns An error + data union, where at index 0 is a failure type flag
+ /// and at index 1 is the subtree hash on success. The failure flag is false
+ /// if blob was not found and true if failed otherwise.
[[nodiscard]] static auto GetBlobFromRepo(
std::filesystem::path const& repo_path,
std::string const& blob_id,
- std::shared_ptr<Logger> const& logger) -> std::optional<std::string>;
+ std::shared_ptr<Logger> const& logger)
+ -> std::variant<bool, std::string>;
[[nodiscard]] auto SyncArchive(std::string const& tree_id,
std::filesystem::path const& repo_path,
@@ -180,10 +196,13 @@ class SourceTreeService final
bool sync_tree,
ServeArchiveTreeResponse* response) -> ::grpc::Status;
+ /// \brief Checks if a given tree is a repository.
+ /// \returns A status of tree presence, or nullopt if non-check-related
+ /// failure.
[[nodiscard]] static auto IsTreeInRepo(
std::string const& tree_id,
std::filesystem::path const& repo_path,
- std::shared_ptr<Logger> const& logger) -> bool;
+ std::shared_ptr<Logger> const& logger) -> std::optional<bool>;
[[nodiscard]] auto DistdirImportToGit(
std::string const& tree_id,
diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp
index ec3e1c1d..1de1ef15 100644
--- a/src/other_tools/ops_maps/import_to_git_map.cpp
+++ b/src/other_tools/ops_maps/import_to_git_map.cpp
@@ -67,12 +67,13 @@ void KeepCommitAndSetTree(
msg),
fatal);
});
- auto tree_hash =
+ auto res =
git_repo->GetSubtreeFromCommit(commit, ".", wrapped_logger);
- if (not tree_hash) {
+ if (not std::holds_alternative<std::string>(res)) {
return;
}
- (*setter)(std::pair<std::string, GitCASPtr>(*tree_hash, git_cas));
+ (*setter)(std::pair<std::string, GitCASPtr>(
+ std::get<std::string>(res), git_cas));
},
[logger, commit, target_path](auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op KEEP_TAG for "
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index c39df472..63cfa02a 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -689,9 +689,9 @@ void EnsureCommit(
fatal);
});
// get tree id and return workspace root
- auto subtree = git_repo->GetSubtreeFromCommit(
+ auto res = git_repo->GetSubtreeFromCommit(
repo_info.hash, repo_info.subdir, wrapped_logger);
- if (not subtree) {
+ if (not std::holds_alternative<std::string>(res)) {
return;
}
// set the workspace root as present
@@ -701,7 +701,7 @@ void EnsureCommit(
{repo_info.ignore_special
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
- *subtree,
+ std::get<std::string>(res), // subtree id
repo_root}),
/*is_cache_hit=*/false));
},
@@ -724,15 +724,16 @@ void EnsureCommit(
fatal);
});
// get tree id and return workspace root
- auto subtree = git_repo->GetSubtreeFromCommit(
+ auto res = git_repo->GetSubtreeFromCommit(
repo_info.hash, repo_info.subdir, wrapped_logger);
- if (not subtree) {
+ if (not std::holds_alternative<std::string>(res)) {
return;
}
+ auto subtree = std::get<std::string>(res);
// set the workspace root
if (repo_info.absent and not fetch_absent) {
// try by all available means to generate and set the absent root
- EnsureRootAsAbsent(*subtree,
+ EnsureRootAsAbsent(subtree,
repo_root,
repo_info,
serve_api_exists,
@@ -747,7 +748,7 @@ void EnsureCommit(
{repo_info.ignore_special
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
- *subtree,
+ subtree,
repo_root.string()}),
/*is_cache_hit=*/true));
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 14:45:14 +0000