summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/computed_roots/TARGETS4
-rw-r--r--src/buildtool/computed_roots/evaluate.cpp82
-rw-r--r--src/buildtool/tree_structure/TARGETS3
-rw-r--r--src/buildtool/tree_structure/tree_structure_utils.cpp75
-rw-r--r--src/buildtool/tree_structure/tree_structure_utils.hpp17
5 files changed, 115 insertions, 66 deletions
diff --git a/src/buildtool/computed_roots/TARGETS b/src/buildtool/computed_roots/TARGETS
index 4d620db3..3d16e831 100644
--- a/src/buildtool/computed_roots/TARGETS
+++ b/src/buildtool/computed_roots/TARGETS
@@ -78,9 +78,6 @@
, ["src/buildtool/crypto", "hash_function"]
, ["src/buildtool/execution_api/common", "api_bundle"]
, ["src/buildtool/execution_api/common", "common"]
- , ["src/buildtool/execution_api/local", "config"]
- , ["src/buildtool/execution_api/local", "context"]
- , ["src/buildtool/execution_api/local", "local"]
, ["src/buildtool/execution_api/utils", "rehash_utils"]
, ["src/buildtool/file_system", "file_root"]
, ["src/buildtool/file_system", "file_system_manager"]
@@ -96,7 +93,6 @@
, ["src/buildtool/progress_reporting", "base_progress_reporter"]
, ["src/buildtool/progress_reporting", "progress"]
, ["src/buildtool/storage", "storage"]
- , ["src/buildtool/tree_structure", "tree_structure_cache"]
, ["src/buildtool/tree_structure", "tree_structure_utils"]
, ["src/utils/cpp", "expected"]
, ["src/utils/cpp", "tmp_dir"]
diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp
index ffac5e04..a7898413 100644
--- a/src/buildtool/computed_roots/evaluate.cpp
+++ b/src/buildtool/computed_roots/evaluate.cpp
@@ -46,9 +46,6 @@
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/api_bundle.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
-#include "src/buildtool/execution_api/local/config.hpp"
-#include "src/buildtool/execution_api/local/context.hpp"
-#include "src/buildtool/execution_api/local/local_api.hpp"
#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
#include "src/buildtool/file_system/file_root.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
@@ -67,7 +64,6 @@
#include "src/buildtool/progress_reporting/base_progress_reporter.hpp"
#include "src/buildtool/progress_reporting/progress.hpp"
#include "src/buildtool/storage/storage.hpp"
-#include "src/buildtool/tree_structure/tree_structure_cache.hpp"
#include "src/buildtool/tree_structure/tree_structure_utils.hpp"
#include "src/utils/cpp/expected.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
@@ -384,73 +380,35 @@ void ComputeTreeStructureAndFill(
substitution_storage_config.has_value()
? substitution_storage_config.value()
: *storage_config;
- auto const storage = Storage::Create(&native_storage_config);
- LocalExecutionConfig const dummy_exec_config{};
- LocalContext const local_context{.exec_config = &dummy_exec_config,
- .storage_config = &native_storage_config,
- .storage = &storage};
- LocalApi const native_local_api(&local_context, /*repo_config=*/nullptr);
- TreeStructureCache const tree_structure_cache(&native_storage_config);
std::optional<std::string> resolved_hash;
-
- // Check the result is in cache already:
- if (auto const cache_entry = tree_structure_cache.Get(*digest)) {
- // Ensure the entry is present in git. If it is in cache, it must be
- // present in the CAS, so just import from CAS to git:
- auto to_git = TreeStructureUtils::ImportToGit(
- *cache_entry, native_local_api, native_storage_config, git_lock);
- if (not to_git.has_value()) {
- std::invoke(*logger, std::move(to_git).error(), /*fatal=*/true);
+ std::vector known_repositories{native_storage_config.GitRoot()};
+ if (not ref_root.IsAbsent()) {
+ auto const path_to_git_cas = ref_root.GetGitCasRoot();
+ if (not path_to_git_cas) {
+ std::invoke(
+ *logger,
+ fmt::format("Failed to get the path to the git cas for {}",
+ key.ToString()),
+ true);
return;
}
- resolved_hash = cache_entry->hash();
+ known_repositories.push_back(*path_to_git_cas);
}
- if (not resolved_hash) {
- // If the tree is not in the storage, it must be added:
- if (not storage.CAS().TreePath(*digest).has_value()) {
- std::vector<std::filesystem::path> known_repos;
- auto const path_to_git_cas = ref_root.GetGitCasRoot();
- if (not path_to_git_cas) {
- std::invoke(*logger,
- fmt::format("Failed to obtain the path to the git "
- "cas for {}",
- key.ToString()),
- true);
- return;
- }
- known_repos.push_back(*path_to_git_cas);
- known_repos.push_back(native_storage_config.GitRoot());
-
- auto in_cas = TreeStructureUtils::ExportFromGit(
- *digest, known_repos, native_local_api);
- if (not in_cas.has_value()) {
- std::invoke(*logger, std::move(in_cas).error(), /*fatal=*/true);
- return;
- }
- }
-
- if (storage.CAS().TreePath(*digest).has_value()) {
- // Compute tree structure and add it to the cache:
- auto const tree_structure = TreeStructureUtils::Compute(
- *digest, storage, tree_structure_cache);
- if (not tree_structure) {
- std::invoke(*logger, tree_structure.error(), /*fatal=*/true);
- return;
- }
-
- auto to_git = TreeStructureUtils::ImportToGit(*tree_structure,
- native_local_api,
- native_storage_config,
- git_lock);
- if (not to_git.has_value()) {
- std::invoke(*logger, std::move(to_git).error(), /*fatal=*/true);
- return;
- }
+ // Try to compute the tree structure locally:
+ if (auto from_local = TreeStructureUtils::ComputeStructureLocally(
+ *digest, known_repositories, native_storage_config, git_lock)) {
+ std::optional<ArtifactDigest> const& tree_structure = *from_local;
+ if (tree_structure.has_value()) {
resolved_hash = tree_structure->hash();
}
}
+ else {
+ // A critical error occurred:
+ std::invoke(*logger, std::move(from_local).error(), /*fatal=*/true);
+ return;
+ }
if (not resolved_hash) {
std::invoke(*logger,
diff --git a/src/buildtool/tree_structure/TARGETS b/src/buildtool/tree_structure/TARGETS
index 0c9dc676..c88585c3 100644
--- a/src/buildtool/tree_structure/TARGETS
+++ b/src/buildtool/tree_structure/TARGETS
@@ -42,6 +42,9 @@
, ["src/buildtool/common", "protocol_traits"]
, ["src/buildtool/crypto", "hash_function"]
, ["src/buildtool/execution_api/git", "git"]
+ , ["src/buildtool/execution_api/local", "config"]
+ , ["src/buildtool/execution_api/local", "context"]
+ , ["src/buildtool/execution_api/local", "local"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/file_system", "git_repo"]
, ["src/buildtool/file_system", "object_type"]
diff --git a/src/buildtool/tree_structure/tree_structure_utils.cpp b/src/buildtool/tree_structure/tree_structure_utils.cpp
index 7cc04b6f..f09d627b 100644
--- a/src/buildtool/tree_structure/tree_structure_utils.cpp
+++ b/src/buildtool/tree_structure/tree_structure_utils.cpp
@@ -31,6 +31,9 @@
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/git/git_api.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/context.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/file_system/object_type.hpp"
@@ -236,3 +239,75 @@ auto TreeStructureUtils::ExportFromGit(
return git_api.RetrieveToCas({Artifact::ObjectInfo{tree, ObjectType::Tree}},
target_api);
}
+
+auto TreeStructureUtils::ComputeStructureLocally(
+ ArtifactDigest const& tree,
+ std::vector<std::filesystem::path> const& known_repositories,
+ StorageConfig const& storage_config,
+ gsl::not_null<std::mutex*> const& tagging_lock)
+ -> expected<std::optional<ArtifactDigest>, std::string> {
+ if (not ProtocolTraits::IsNative(tree.GetHashType()) or not tree.IsTree()) {
+ return unexpected{fmt::format("Not a git tree: {}", tree.hash())};
+ }
+
+ if (not ProtocolTraits::IsNative(storage_config.hash_function.GetType())) {
+ return unexpected{fmt::format("Not a native storage config")};
+ }
+
+ auto const storage = Storage::Create(&storage_config);
+ LocalExecutionConfig const dummy_exec_config{};
+ LocalContext const local_context{.exec_config = &dummy_exec_config,
+ .storage_config = &storage_config,
+ .storage = &storage};
+ LocalApi const local_api{&local_context};
+
+ // First check the result is in cache already:
+ TreeStructureCache const tree_structure_cache{&storage_config};
+ if (auto const from_cache = tree_structure_cache.Get(tree)) {
+ auto to_git =
+ ImportToGit(*from_cache, local_api, storage_config, tagging_lock);
+ if (not to_git.has_value()) {
+ return unexpected{fmt::format("While importing {} to git:\n{}",
+ from_cache->hash(),
+ std::move(to_git).error())};
+ }
+ return std::make_optional<ArtifactDigest>(std::move(to_git).value());
+ }
+
+ // If the tree is not in the storage, it must be present in git:
+ if (not storage.CAS().TreePath(tree).has_value()) {
+ auto in_cas = ExportFromGit(tree, known_repositories, local_api);
+ if (not in_cas.has_value()) {
+ return unexpected{
+ fmt::format("While exporting {} from git to CAS:\n{}",
+ tree.hash(),
+ std::move(in_cas).error())};
+ }
+
+ // If the tree hasn't been found neither in CAS, nor in git, there's
+ // nothing else to do:
+ if (not storage.CAS().TreePath(tree).has_value()) {
+ return std::optional<ArtifactDigest>{};
+ }
+ }
+
+ // Compute tree structure and add it to the storage and cache:
+ auto tree_structure = Compute(tree, storage, tree_structure_cache);
+ if (not tree_structure) {
+ return unexpected{
+ fmt::format("Failed to compute tree structure of {}:\n{}",
+ tree.hash(),
+ std::move(tree_structure).error())};
+ }
+
+ // Import the result to git:
+ auto to_git =
+ ImportToGit(*tree_structure, local_api, storage_config, tagging_lock);
+ if (not to_git.has_value()) {
+ return unexpected{fmt::format(
+ "While importing the resulting tree structure {} to git:\n{}",
+ tree_structure->hash(),
+ std::move(to_git).error())};
+ }
+ return std::make_optional<ArtifactDigest>(*std::move(tree_structure));
+}
diff --git a/src/buildtool/tree_structure/tree_structure_utils.hpp b/src/buildtool/tree_structure/tree_structure_utils.hpp
index 4f908640..7cecf7f8 100644
--- a/src/buildtool/tree_structure/tree_structure_utils.hpp
+++ b/src/buildtool/tree_structure/tree_structure_utils.hpp
@@ -17,6 +17,7 @@
#include <filesystem>
#include <mutex>
+#include <optional>
#include <string>
#include <vector>
@@ -74,6 +75,22 @@ class TreeStructureUtils final {
std::vector<std::filesystem::path> const& source_repos,
IExecutionApi const& target_api) noexcept
-> expected<bool, std::string>;
+
+ /// \brief Find git tree locally and compute its tree structure.
+ /// \param tree Git tree to process
+ /// \param known_repositories Known git repositories to check
+ /// \param storage_config Storage to use for lookup and import.
+ /// \param tagging_lock Mutex to protect critical git operations
+ /// \return Digest of the tree structure that is available in
+ /// storage_config's git repo and in storage_config's CAS; std::nullopt if
+ /// the search failed to locate the tree's sources locally; an error string
+ /// on critical failure.
+ [[nodiscard]] static auto ComputeStructureLocally(
+ ArtifactDigest const& tree,
+ std::vector<std::filesystem::path> const& known_repositories,
+ StorageConfig const& storage_config,
+ gsl::not_null<std::mutex*> const& tagging_lock)
+ -> expected<std::optional<ArtifactDigest>, std::string>;
};
#endif // INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 15:46:23 +0000