diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/other_tools/ops_maps/TARGETS | 2 | ||||
-rw-r--r-- | src/other_tools/ops_maps/git_tree_fetch_map.cpp | 113 | ||||
-rw-r--r-- | src/other_tools/ops_maps/git_tree_fetch_map.hpp | 9 | ||||
-rw-r--r-- | src/other_tools/root_maps/TARGETS | 2 | ||||
-rw-r--r-- | src/other_tools/root_maps/tree_id_git_map.cpp | 75 | ||||
-rw-r--r-- | src/other_tools/utils/TARGETS | 3 | ||||
-rw-r--r-- | src/other_tools/utils/parse_git_tree.cpp | 14 |
7 files changed, 120 insertions, 98 deletions
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 0b7ac74f..9fa5dce7 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -117,11 +117,13 @@ , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/crypto", "hash_info"] ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": [ ["src/other_tools/ops_maps", "critical_git_op_map"] , ["src/buildtool/common", "config"] + , ["src/buildtool/common", "common"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/multithreading", "task_system"] diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp index 95582556..7e6da1dd 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp @@ -19,6 +19,7 @@ #include <utility> // std::move #include "fmt/core.h" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/execution_api/common/execution_common.hpp" @@ -30,7 +31,7 @@ namespace { -void BackupToRemote(std::string const& tree_id, +void BackupToRemote(ArtifactDigest const& digest, StorageConfig const& storage_config, IExecutionApi const& remote_api, GitTreeFetchMap::LoggerPtr const& logger) { @@ -39,14 +40,13 @@ void BackupToRemote(std::string const& tree_id, if (repo.SetGitCAS(storage_config.GitRoot())) { auto git_api = GitApi{&repo}; if (not git_api.RetrieveToCas( - {Artifact::ObjectInfo{ - .digest = ArtifactDigest{tree_id, 0, /*is_tree=*/true}, - .type = ObjectType::Tree}}, + {Artifact::ObjectInfo{.digest = digest, + .type = ObjectType::Tree}}, remote_api)) { // give a warning (*logger)(fmt::format( "Failed to back up tree {} from local CAS to remote", - tree_id), + digest.hash()), /*fatal=*/false); } } @@ -60,8 +60,7 @@ void BackupToRemote(std::string const& tree_id, /// \brief Moves the root tree from local CAS to the Git cache and sets the /// root. -void MoveCASTreeToGit(std::string const& tree_id, - ArtifactDigest const& digest, +void MoveCASTreeToGit(ArtifactDigest const& digest, gsl::not_null<ImportToGitMap*> const& import_to_git_map, gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<IExecutionApi const*> const& local_api, @@ -83,17 +82,17 @@ void MoveCASTreeToGit(std::string const& tree_id, {Artifact::ObjectInfo{.digest = digest, .type = ObjectType::Tree}}, {tmp_dir->GetPath()})) { (*logger)(fmt::format("Failed to copy git-tree {} to {}", - tree_id, + digest.hash(), tmp_dir->GetPath().string()), true); return; } - CommitInfo c_info{tmp_dir->GetPath(), "tree", tree_id}; + CommitInfo c_info{tmp_dir->GetPath(), "tree", digest.hash()}; import_to_git_map->ConsumeAfterKeysReady( ts, {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive - tree_id, + digest, storage_config, remote_api, backup_to_remote, @@ -106,21 +105,21 @@ void MoveCASTreeToGit(std::string const& tree_id, } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(tree_id, *storage_config, *remote_api, logger); + BackupToRemote(digest, *storage_config, *remote_api, logger); } (*setter)(false /*no cache hit*/); }, - [logger, tmp_dir, tree_id](auto const& msg, bool fatal) { + [logger, tmp_dir, digest](auto const& msg, bool fatal) { (*logger)(fmt::format( "While moving git-tree {} from {} to local git:\n{}", - tree_id, + digest.hash(), tmp_dir->GetPath().string(), msg), fatal); }); } -void TagAndSetRoot(std::string tree_id, +void TagAndSetRoot(ArtifactDigest const& digest, gsl::not_null<StorageConfig const*> const& storage_config, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, IExecutionApi const* remote_api, @@ -132,14 +131,14 @@ void TagAndSetRoot(std::string tree_id, GitOpKey op_key = {.params = { repo, // target_path - tree_id, // git_hash + digest.hash(), // git_hash "Keep referenced tree alive" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [tree_id, backup_to_remote, storage_config, remote_api, logger, setter]( + [digest, backup_to_remote, storage_config, remote_api, logger, setter]( auto const& values) { GitOpValue op_result = *values[0]; if (not op_result.result) { @@ -149,14 +148,14 @@ void TagAndSetRoot(std::string tree_id, } // backup to remote if needed and in compatibility mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(tree_id, *storage_config, *remote_api, logger); + BackupToRemote(digest, *storage_config, *remote_api, logger); } (*setter)(false /*no cache hit*/); }, - [logger, repo, tree_id](auto const& msg, bool fatal) { + [logger, repo, digest](auto const& msg, bool fatal) { (*logger)( fmt::format("While tagging tree {} in {} to keep it alive:\n{}", - tree_id, + digest.hash(), repo.string(), msg), fatal); @@ -165,7 +164,7 @@ void TagAndSetRoot(std::string tree_id, void TakeTreeFromOlderGeneration( std::size_t generation, - std::string tree_id, + ArtifactDigest const& digest, gsl::not_null<StorageConfig const*> const& storage_config, GitCASPtr const& git_cas, gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map, @@ -178,14 +177,14 @@ void TakeTreeFromOlderGeneration( GitOpKey op_key = {.params = { source, // target_path - tree_id, // git_hash + digest.hash(), // git_hash "Tag commit for fetching" // message }, .op_type = GitOpType::KEEP_TREE}; critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [tree_id, + [digest, git_cas, critical_git_op_map, remote_api, @@ -218,7 +217,7 @@ void TakeTreeFromOlderGeneration( *storage_config, source, tag, fetch_logger)) { return; } - TagAndSetRoot(tree_id, + TagAndSetRoot(digest, storage_config, critical_git_op_map, remote_api, @@ -227,11 +226,11 @@ void TakeTreeFromOlderGeneration( setter, logger); }, - [logger, source, tree_id](auto const& msg, bool fatal) { + [logger, source, digest](auto const& msg, bool fatal) { (*logger)( fmt::format("While tagging tree {} in {} for fetching:\n{}", source.string(), - tree_id, + digest.hash(), msg), fatal); }); @@ -318,8 +317,8 @@ auto CreateGitTreeFetchMap( fatal); }); // check if the desired tree ID is in Git cache - auto tree_found = - git_repo->CheckTreeExists(key.hash, wrapped_logger); + auto tree_found = git_repo->CheckTreeExists( + key.tree_hash.Hash(), wrapped_logger); if (not tree_found) { // errors encountered return; @@ -327,8 +326,10 @@ auto CreateGitTreeFetchMap( if (*tree_found) { // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote( - key.hash, *storage_config, *remote_api, logger); + BackupToRemote(ArtifactDigest{key.tree_hash, 0}, + *storage_config, + *remote_api, + logger); } // success (*setter)(true /*cache hit*/); @@ -346,19 +347,20 @@ auto CreateGitTreeFetchMap( std::make_shared<AsyncMapConsumerLogger>( [](auto /*unused*/, auto /*unused*/) {}); if (old_repo) { - auto check_result = - old_repo->CheckTreeExists(key.hash, no_logging); + auto check_result = old_repo->CheckTreeExists( + key.tree_hash.Hash(), no_logging); if (check_result and *check_result) { - TakeTreeFromOlderGeneration(generation, - key.hash, - storage_config, - op_result.git_cas, - critical_git_op_map, - remote_api, - backup_to_remote, - ts, - setter, - logger); + TakeTreeFromOlderGeneration( + generation, + ArtifactDigest{key.tree_hash, 0}, + storage_config, + op_result.git_cas, + critical_git_op_map, + remote_api, + backup_to_remote, + ts, + setter, + logger); return; } } @@ -366,11 +368,10 @@ auto CreateGitTreeFetchMap( } // check if tree is known to local CAS - auto digest = ArtifactDigest{key.hash, 0, /*is_tree=*/true}; + auto const digest = ArtifactDigest{key.tree_hash, 0}; if (local_api->IsAvailable(digest)) { // import tree to Git cache - MoveCASTreeToGit(key.hash, - digest, + MoveCASTreeToGit(digest, import_to_git_map, storage_config, local_api, @@ -389,7 +390,7 @@ auto CreateGitTreeFetchMap( // as we anyway interrogate the remote execution endpoint, // we're only interested here in the serve endpoint making // an attempt to upload the tree, if known, to remote CAS - std::ignore = serve->TreeInRemoteCAS(key.hash); + std::ignore = serve->TreeInRemoteCAS(key.tree_hash.Hash()); } // check if tree is in remote CAS, if a remote is given if (remote_api != nullptr and @@ -399,7 +400,6 @@ auto CreateGitTreeFetchMap( *local_api)) { progress->TaskTracker().Stop(key.origin); MoveCASTreeToGit( - key.hash, digest, import_to_git_map, storage_config, @@ -432,7 +432,7 @@ auto CreateGitTreeFetchMap( return; } // execute command in temporary location - SystemCommand system{key.hash}; + SystemCommand system{key.tree_hash.Hash()}; auto cmdline = launcher; std::copy(key.command.begin(), key.command.end(), @@ -468,8 +468,8 @@ auto CreateGitTreeFetchMap( repo_dir->GetPath(), // target_path "", // git_hash fmt::format("Content of tree {}", - key.hash), // message - content_dir->GetPath() // source_path + key.tree_hash.Hash()), // message + content_dir->GetPath() // source_path }, .op_type = GitOpType::INITIAL_COMMIT}; critical_git_op_map->ConsumeAfterKeysReady( @@ -518,8 +518,8 @@ auto CreateGitTreeFetchMap( fatal); }); // check that the desired tree ID is part of the repo - auto tree_check = - git_repo->CheckTreeExists(key.hash, wrapped_logger); + auto tree_check = git_repo->CheckTreeExists( + key.tree_hash.Hash(), wrapped_logger); if (not tree_check) { // errors encountered return; @@ -548,7 +548,7 @@ auto CreateGitTreeFetchMap( fmt::format("Executing {} did not create " "specified tree {}{}", nlohmann::json(cmdline).dump(), - key.hash, + key.tree_hash.Hash(), output), /*fatal=*/true); return; @@ -638,10 +638,11 @@ auto CreateGitTreeFetchMap( // backup to remote if needed and in native mode if (backup_to_remote and remote_api != nullptr) { - BackupToRemote(key.hash, - *storage_config, - *remote_api, - logger); + BackupToRemote( + ArtifactDigest{key.tree_hash, 0}, + *storage_config, + *remote_api, + logger); } // success (*setter)(false /*no cache hit*/); diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp index b7400ae2..04d28ea2 100644 --- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp +++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp @@ -23,6 +23,7 @@ #include <vector> #include "gsl/gsl" +#include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/config.hpp" @@ -32,7 +33,7 @@ // Stores all the information needed to make a Git tree available struct GitTreeInfo { - std::string hash{}; /* key */ + HashInfo tree_hash{}; /* key */ std::map<std::string, std::string> env_vars{}; std::vector<std::string> inherit_env{}; std::vector<std::string> command{}; @@ -40,7 +41,7 @@ struct GitTreeInfo { std::string origin{}; [[nodiscard]] auto operator==(const GitTreeInfo& other) const -> bool { - return hash == other.hash; + return tree_hash.Hash() == other.tree_hash.Hash(); } }; @@ -49,7 +50,7 @@ template <> struct hash<GitTreeInfo> { [[nodiscard]] auto operator()(const GitTreeInfo& gti) const noexcept -> std::size_t { - return std::hash<std::string>{}(gti.hash); + return std::hash<std::string>{}(gti.tree_hash.Hash()); } }; } // namespace std @@ -74,6 +75,6 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>; // use explicit cast to std::function to allow template deduction when used static const std::function<std::string(GitTreeInfo const&)> kGitTreeInfoPrinter = - [](GitTreeInfo const& x) -> std::string { return x.hash; }; + [](GitTreeInfo const& x) -> std::string { return x.tree_hash.Hash(); }; #endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 92c606e7..84eef4a5 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -175,8 +175,10 @@ [ ["@", "fmt", "", "fmt"] , "root_utils" , ["src/buildtool/common", "config"] + , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/git", "git"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/crypto", "hash_info"] ] } , "root_utils": diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp index 78c1cffd..76b29045 100644 --- a/src/other_tools/root_maps/tree_id_git_map.cpp +++ b/src/other_tools/root_maps/tree_id_git_map.cpp @@ -15,7 +15,9 @@ #include "src/other_tools/root_maps/tree_id_git_map.hpp" #include "fmt/core.h" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" +#include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/other_tools/root_maps/root_utils.hpp" @@ -173,8 +175,8 @@ auto CreateTreeIdGitMap( if (key.absent and not fetch_absent) { if (serve != nullptr) { // check serve endpoint - auto has_tree = - CheckServeHasAbsentRoot(*serve, key.tree_info.hash, logger); + auto has_tree = CheckServeHasAbsentRoot( + *serve, key.tree_info.tree_hash.Hash(), logger); if (not has_tree) { return; } @@ -184,7 +186,7 @@ auto CreateTreeIdGitMap( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, - key.tree_info.hash}); + key.tree_info.tree_hash.Hash()}); (*setter)( std::pair(std::move(root), /*is_cache_hit=*/false)); return; @@ -194,18 +196,17 @@ auto CreateTreeIdGitMap( (*logger)( fmt::format("Cannot create workspace root {} as absent " "for the provided serve endpoint.", - key.tree_info.hash), + key.tree_info.tree_hash.Hash()), /*fatal=*/true); return; } // check if tree in already in remote CAS - auto digest = - ArtifactDigest{key.tree_info.hash, 0, /*is_tree=*/true}; + auto const digest = ArtifactDigest{key.tree_info.tree_hash, 0}; if (remote_api->IsAvailable({digest})) { // tell serve to set up the root from the remote CAS tree; // upload can be skipped if (EnsureAbsentRootOnServe(*serve, - key.tree_info.hash, + key.tree_info.tree_hash.Hash(), /*repo_path=*/"", /*remote_api=*/nullptr, logger, @@ -215,7 +216,7 @@ auto CreateTreeIdGitMap( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, - key.tree_info.hash}); + key.tree_info.tree_hash.Hash()}); (*setter)( std::pair(std::move(root), /*is_cache_hit=*/false)); return; @@ -223,7 +224,7 @@ auto CreateTreeIdGitMap( (*logger)( fmt::format("Serve endpoint failed to create workspace " "root {} that locally was marked absent.", - key.tree_info.hash), + key.tree_info.tree_hash.Hash()), /*fatal=*/true); return; } @@ -282,7 +283,7 @@ auto CreateTreeIdGitMap( }); // check if the desired tree ID is in Git cache auto tree_found = git_repo->CheckTreeExists( - key.tree_info.hash, wrapped_logger); + key.tree_info.tree_hash.Hash(), wrapped_logger); if (not tree_found) { // errors encountered return; @@ -291,14 +292,15 @@ auto CreateTreeIdGitMap( // upload tree from Git cache to remote CAS and tell // serve to set up the root from the remote CAS // tree, then set root as absent - UploadToServeAndSetRoot(*serve, - *storage_config, - key.tree_info.hash, - digest, - *remote_api, - key.ignore_special, - setter, - logger); + UploadToServeAndSetRoot( + *serve, + *storage_config, + key.tree_info.tree_hash.Hash(), + digest, + *remote_api, + key.ignore_special, + setter, + logger); // done! return; } @@ -306,17 +308,18 @@ auto CreateTreeIdGitMap( if (local_api->IsAvailable(digest)) { // Move tree locally from CAS to Git cache, then // continue processing it by UploadToServeAndSetRoot - MoveCASTreeToGitAndProcess(*serve, - storage_config, - key.tree_info.hash, - digest, - import_to_git_map, - local_api, - remote_api, - key.ignore_special, - ts, - setter, - logger); + MoveCASTreeToGitAndProcess( + *serve, + storage_config, + key.tree_info.tree_hash.Hash(), + digest, + import_to_git_map, + local_api, + remote_api, + key.ignore_special, + ts, + setter, + logger); // done! return; } @@ -326,7 +329,7 @@ auto CreateTreeIdGitMap( (*logger)(fmt::format("Cannot create workspace root " "{} as absent for the provided " "serve endpoint.", - key.tree_info.hash), + key.tree_info.tree_hash.Hash()), /*fatal=*/true); }, [logger, target_path = storage_config->GitRoot()]( @@ -344,13 +347,13 @@ auto CreateTreeIdGitMap( // give warning that serve endpoint is missing (*logger)(fmt::format("Workspace root {} marked absent but no " "suitable serve endpoint provided.", - key.tree_info.hash), + key.tree_info.tree_hash.Hash()), /*fatal=*/false); // set workspace root as absent auto root = nlohmann::json::array( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, - key.tree_info.hash}); + key.tree_info.tree_hash.Hash()}); (*setter)(std::pair(std::move(root), false)); return; } @@ -370,15 +373,15 @@ auto CreateTreeIdGitMap( {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, - key.tree_info.hash, + key.tree_info.tree_hash.Hash(), storage_config->GitRoot().string()}), is_cache_hit)); }, - [logger, tree_id = key.tree_info.hash](auto const& msg, - bool fatal) { + [logger, hash = key.tree_info.tree_hash.Hash()](auto const& msg, + bool fatal) { (*logger)(fmt::format( "While ensuring git-tree {} is in Git cache:\n{}", - tree_id, + hash, msg), fatal); }); diff --git a/src/other_tools/utils/TARGETS b/src/other_tools/utils/TARGETS index accde39f..6b4e00bd 100644 --- a/src/other_tools/utils/TARGETS +++ b/src/other_tools/utils/TARGETS @@ -76,7 +76,8 @@ , ["src/other_tools/ops_maps", "git_tree_fetch_map"] , ["src/utils/cpp", "expected"] ] - , "private-deps": [["@", "fmt", "", "fmt"]] + , "private-deps": + [["@", "fmt", "", "fmt"], ["src/buildtool/crypto", "hash_info"]] , "stage": ["src", "other_tools", "utils"] } } diff --git a/src/other_tools/utils/parse_git_tree.cpp b/src/other_tools/utils/parse_git_tree.cpp index dde759b9..0e58676d 100644 --- a/src/other_tools/utils/parse_git_tree.cpp +++ b/src/other_tools/utils/parse_git_tree.cpp @@ -19,6 +19,7 @@ #include <vector> #include "fmt/core.h" +#include "src/buildtool/crypto/hash_info.hpp" [[nodiscard]] auto ParseGitTree(ExpressionPtr const& repo_desc, std::optional<std::string> origin) @@ -34,6 +35,17 @@ repo_desc_hash->get()->ToString())}; } + auto repo_desc_hash_info = HashInfo::Create(HashFunction::Type::GitSHA1, + repo_desc_hash->get()->String(), + /*is_tree=*/true); + if (not repo_desc_hash_info) { + return unexpected{ + fmt::format("Unsupported value {} for " + "mandatory field \"id\"\n{}", + repo_desc_hash->get()->ToString(), + std::move(repo_desc_hash_info).error())}; + } + auto repo_desc_cmd = repo_desc->At("cmd"); if (not repo_desc_cmd) { return unexpected<std::string>{"Mandatory field \"cmd\" is missing"}; @@ -90,7 +102,7 @@ } } // populate struct - auto info = GitTreeInfo{.hash = repo_desc_hash->get()->String(), + auto info = GitTreeInfo{.tree_hash = *std::move(repo_desc_hash_info), .env_vars = std::move(env), .inherit_env = std::move(inherit_env), .command = std::move(cmd)}; |