summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/computed_roots/evaluate.cpp77
-rw-r--r--src/buildtool/tree_structure/TARGETS4
-rw-r--r--src/buildtool/tree_structure/tree_structure_utils.cpp65
-rw-r--r--src/buildtool/tree_structure/tree_structure_utils.hpp20
4 files changed, 108 insertions, 58 deletions
diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp
index 332b079e..0905c240 100644
--- a/src/buildtool/computed_roots/evaluate.cpp
+++ b/src/buildtool/computed_roots/evaluate.cpp
@@ -46,12 +46,10 @@
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/api_bundle.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
-#include "src/buildtool/execution_api/common/tree_reader.hpp"
#include "src/buildtool/execution_api/git/git_api.hpp"
#include "src/buildtool/execution_api/local/config.hpp"
#include "src/buildtool/execution_api/local/context.hpp"
#include "src/buildtool/execution_api/local/local_api.hpp"
-#include "src/buildtool/execution_api/local/local_cas_reader.hpp"
#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
#include "src/buildtool/file_system/file_root.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
@@ -387,33 +385,31 @@ void ComputeTreeStructureAndFill(
substitution_storage_config.has_value()
? substitution_storage_config.value()
: *storage_config;
+ auto const storage = Storage::Create(&native_storage_config);
+ LocalExecutionConfig const dummy_exec_config{};
+ LocalContext const local_context{.exec_config = &dummy_exec_config,
+ .storage_config = &native_storage_config,
+ .storage = &storage};
+ LocalApi const native_local_api(&local_context, /*repo_config=*/nullptr);
TreeStructureCache const tree_structure_cache(&native_storage_config);
std::optional<std::string> resolved_hash;
// Check the result is in cache already:
if (auto const cache_entry = tree_structure_cache.Get(*digest)) {
- auto const cached_hash = cache_entry->hash();
- auto tree_present =
- GitRepo::IsTreeInRepo(native_storage_config.GitRoot(), cached_hash);
- if (not tree_present) {
- std::invoke(*logger,
- fmt::format("While checking presence of tree {} in "
- "local git repo:\n{}",
- cached_hash,
- std::move(tree_present).error()),
- /*fatal=*/true);
+ // Ensure the entry is present in git. If it is in cache, it must be
+ // present in the CAS, so just import from CAS to git:
+ auto to_git = TreeStructureUtils::ImportToGit(
+ *cache_entry, native_local_api, native_storage_config, git_lock);
+ if (not to_git.has_value()) {
+ std::invoke(*logger, std::move(to_git).error(), /*fatal=*/true);
return;
}
-
- if (*tree_present) {
- resolved_hash = cached_hash;
- }
+ resolved_hash = cache_entry->hash();
}
if (not resolved_hash) {
// If the tree is not in the storage, it must be added:
- auto const storage = Storage::Create(&native_storage_config);
if (not storage.CAS().TreePath(*digest).has_value()) {
auto const path_to_git_cas = ref_root.GetGitCasRoot();
if (not path_to_git_cas) {
@@ -435,16 +431,9 @@ void ComputeTreeStructureAndFill(
}
GitApi const git_api{&root_config};
- LocalExecutionConfig const dummy_exec_config{};
- LocalContext const local_context{
- .exec_config = &dummy_exec_config,
- .storage_config = &native_storage_config,
- .storage = &storage};
- LocalApi const local_api(&local_context, /*repo_config=*/nullptr);
-
if (not git_api.RetrieveToCas(
{Artifact::ObjectInfo{*digest, ObjectType::Tree}},
- local_api) or
+ native_local_api) or
not storage.CAS().TreePath(*digest).has_value()) {
std::invoke(*logger,
fmt::format("Failed to retrieve {} to CAS for {}",
@@ -463,41 +452,13 @@ void ComputeTreeStructureAndFill(
return;
}
- auto const tmp_dir =
- native_storage_config.CreateTypedTmpDir("stage_tree_structure");
- if (tmp_dir == nullptr) {
- std::invoke(
- *logger,
- fmt::format("Failed to create temporary directory for {}",
- key.ToString()),
- true);
- return;
- }
-
- // Stage the resulting tree structure to a temporary directory:
- auto const root_dir = tmp_dir->GetPath() / "root";
- if (auto reader = TreeReader<LocalCasReader>(&storage.CAS());
- not reader.StageTo(
- {Artifact::ObjectInfo{*tree_structure, ObjectType::Tree}},
- {root_dir})) {
- std::invoke(
- *logger,
- fmt::format("Failed to stage tree structure to a temporary "
- "location for {}.",
- key.ToString()),
- true);
- return;
- }
-
- // Import the result to git:
- auto result_tree = GitRepo::ImportToGit(
- native_storage_config, root_dir, "tree structure", git_lock);
- if (not result_tree) {
- std::invoke(
- *logger, std::move(result_tree).error(), /*fatal=*/true);
+ auto to_git = TreeStructureUtils::ImportToGit(
+ *tree_structure, native_local_api, native_storage_config, git_lock);
+ if (not to_git.has_value()) {
+ std::invoke(*logger, std::move(to_git).error(), /*fatal=*/true);
return;
}
- resolved_hash = *std::move(result_tree);
+ resolved_hash = tree_structure->hash();
}
if (not resolved_hash) {
diff --git a/src/buildtool/tree_structure/TARGETS b/src/buildtool/tree_structure/TARGETS
index 8f8ec9c2..45244119 100644
--- a/src/buildtool/tree_structure/TARGETS
+++ b/src/buildtool/tree_structure/TARGETS
@@ -27,7 +27,10 @@
, "srcs": ["tree_structure_utils.cpp"]
, "deps":
[ "tree_structure_cache"
+ , ["@", "gsl", "", "gsl"]
, ["src/buildtool/common", "common"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/storage", "config"]
, ["src/buildtool/storage", "storage"]
, ["src/utils/cpp", "expected"]
]
@@ -42,6 +45,7 @@
, ["src/buildtool/file_system", "object_type"]
, ["src/utils/cpp", "hex_string"]
, ["src/utils/cpp", "path"]
+ , ["src/utils/cpp", "tmp_dir"]
]
}
}
diff --git a/src/buildtool/tree_structure/tree_structure_utils.cpp b/src/buildtool/tree_structure/tree_structure_utils.cpp
index fd897672..b607cb9f 100644
--- a/src/buildtool/tree_structure/tree_structure_utils.cpp
+++ b/src/buildtool/tree_structure/tree_structure_utils.cpp
@@ -15,13 +15,16 @@
#include "src/buildtool/tree_structure/tree_structure_utils.hpp"
#include <algorithm>
+#include <filesystem>
#include <functional>
+#include <memory>
#include <optional>
#include <unordered_map>
#include <utility>
#include <vector>
#include "fmt/core.h"
+#include "src/buildtool/common/artifact.hpp"
#include "src/buildtool/common/artifact_digest_factory.hpp"
#include "src/buildtool/common/protocol_traits.hpp"
#include "src/buildtool/crypto/hash_function.hpp"
@@ -30,6 +33,7 @@
#include "src/buildtool/file_system/object_type.hpp"
#include "src/utils/cpp/hex_string.hpp"
#include "src/utils/cpp/path.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
auto TreeStructureUtils::Compute(ArtifactDigest const& tree,
Storage const& storage,
@@ -133,3 +137,64 @@ auto TreeStructureUtils::Compute(ArtifactDigest const& tree,
}
return *std::move(tree_structure);
}
+
+auto TreeStructureUtils::ImportToGit(
+ ArtifactDigest const& tree,
+ IExecutionApi const& source_api,
+ StorageConfig const& target_config,
+ gsl::not_null<std::mutex*> const& tagging_lock) noexcept
+ -> expected<ArtifactDigest, std::string> {
+ if (not tree.IsTree() or not ProtocolTraits::IsNative(tree.GetHashType())) {
+ return unexpected{fmt::format("Not a git tree: {}", tree.hash())};
+ }
+
+ // Check the source contains the tree:
+ if (not source_api.IsAvailable(tree)) {
+ return unexpected{
+ fmt::format("Source doesn't contain tree {}", tree.hash())};
+ }
+
+ // Check the tree is in the repository already:
+ if (auto in_repo =
+ GitRepo::IsTreeInRepo(target_config.GitRoot(), tree.hash())) {
+ if (*in_repo) {
+ return tree;
+ }
+ }
+ else {
+ return unexpected{std::move(in_repo).error()};
+ }
+
+ auto const tmp_dir =
+ target_config.CreateTypedTmpDir("import_from_cas_to_git");
+ if (tmp_dir == nullptr) {
+ return unexpected{fmt::format(
+ "Failed to create temporary directory for {}", tree.hash())};
+ }
+
+ // Stage the tree to a temporary directory:
+ if (not source_api.RetrieveToPaths(
+ {Artifact::ObjectInfo{tree, ObjectType::Tree}},
+ {tmp_dir->GetPath()})) {
+ return unexpected{fmt::format(
+ "Failed to stage {} to a temporary location.", tree.hash())};
+ }
+
+ // Import the result to git:
+ auto tree_hash = GitRepo::ImportToGit(
+ target_config,
+ tmp_dir->GetPath(),
+ /*commit_message=*/fmt::format("Import {}", tree.hash()),
+ tagging_lock);
+ if (not tree_hash) {
+ return unexpected{std::move(tree_hash).error()};
+ }
+ auto digest = ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1,
+ tree_hash.value(),
+ /*size_unknown=*/0,
+ /*is_tree=*/true);
+ if (not digest) {
+ return unexpected{std::move(digest).error()};
+ }
+ return *digest;
+}
diff --git a/src/buildtool/tree_structure/tree_structure_utils.hpp b/src/buildtool/tree_structure/tree_structure_utils.hpp
index 56dd1249..df2f9f5e 100644
--- a/src/buildtool/tree_structure/tree_structure_utils.hpp
+++ b/src/buildtool/tree_structure/tree_structure_utils.hpp
@@ -15,9 +15,13 @@
#ifndef INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP
#define INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP
+#include <mutex>
#include <string>
+#include "gsl/gsl"
#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "src/buildtool/storage/storage.hpp"
#include "src/buildtool/tree_structure/tree_structure_cache.hpp"
#include "src/utils/cpp/expected.hpp"
@@ -39,6 +43,22 @@ class TreeStructureUtils final {
Storage const& storage,
TreeStructureCache const& cache) noexcept
-> expected<ArtifactDigest, std::string>;
+
+ /// \brief Import a git tree from the given source to storage_config's git
+ /// repo.
+ /// \param tree GitSHA1 tree to import
+ /// \param source_api The source of the tree. Must be capable of
+ /// processing GitSHA1 trees.
+ /// \param target_config Config with target git repo.
+ /// \param tagging_lock Mutex to protect critical tagging operation
+ /// \return Digest of the tree that is available in the repo after the
+ /// call or an error message on failure.
+ [[nodiscard]] static auto ImportToGit(
+ ArtifactDigest const& tree,
+ IExecutionApi const& source_api,
+ StorageConfig const& target_config,
+ gsl::not_null<std::mutex*> const& tagging_lock) noexcept
+ -> expected<ArtifactDigest, std::string>;
};
#endif // INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_UTILS_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:58:28 +0000