diff options
22 files changed, 213 insertions, 110 deletions
diff --git a/src/buildtool/build_engine/expression/TARGETS b/src/buildtool/build_engine/expression/TARGETS index 30e64466..85df889f 100644 --- a/src/buildtool/build_engine/expression/TARGETS +++ b/src/buildtool/build_engine/expression/TARGETS @@ -51,6 +51,7 @@ , ["src/utils/cpp", "concepts"] , ["src/utils/cpp", "gsl"] , ["@", "gsl", "", "gsl"] + , ["src/buildtool/crypto", "hash_function"] ] , "stage": ["src", "buildtool", "build_engine", "expression"] , "private-deps": @@ -59,7 +60,6 @@ , ["src/utils/cpp", "type_safe_arithmetic"] , ["src/utils/cpp", "path"] , ["src/buildtool/crypto", "hasher"] - , ["src/buildtool/crypto", "hash_function"] ] } } diff --git a/src/buildtool/build_engine/expression/target_result.cpp b/src/buildtool/build_engine/expression/target_result.cpp index c4f43f89..61f5a17f 100644 --- a/src/buildtool/build_engine/expression/target_result.cpp +++ b/src/buildtool/build_engine/expression/target_result.cpp @@ -180,6 +180,7 @@ auto SerializeTargetResultWithReplacement( // NOLINTNEXTLINE(misc-no-recursion) [[nodiscard]] auto DeserializeExpression( + HashFunction::Type hash_type, nlohmann::json const& entry, nlohmann::json const& nodes, std::unordered_set<std::string> const& provided_artifacts, @@ -198,7 +199,8 @@ auto SerializeTargetResultWithReplacement( auto const& json = nodes.at(id); if (json.is_object()) { if (provided_artifacts.contains(id)) { - if (auto artifact = ArtifactDescription::FromJson(json)) { + if (auto artifact = + ArtifactDescription::FromJson(hash_type, json)) { auto result = ExpressionPtr{*artifact}; sofar->emplace(id, result); return result; @@ -209,14 +211,16 @@ auto SerializeTargetResultWithReplacement( if (json["type"] == "ABSTRACT_NODE") { auto node_type = json["node_type"].get<std::string>(); auto target_fields = - DeserializeExpression(json["target_fields"], + DeserializeExpression(hash_type, + json["target_fields"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); auto string_fields = - DeserializeExpression(json["string_fields"], + DeserializeExpression(hash_type, + json["string_fields"], nodes, provided_artifacts, provided_nodes, @@ -230,7 +234,8 @@ auto SerializeTargetResultWithReplacement( return result; } if (json["type"] == "VALUE_NODE") { - auto value = DeserializeExpression(json["result"], + auto value = DeserializeExpression(hash_type, + json["result"], nodes, provided_artifacts, provided_nodes, @@ -243,19 +248,22 @@ auto SerializeTargetResultWithReplacement( return ExpressionPtr{nullptr}; } if (provided_results.contains(id)) { - auto artifact_stage = DeserializeExpression(json["artifact_stage"], + auto artifact_stage = DeserializeExpression(hash_type, + json["artifact_stage"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); - auto runfiles = DeserializeExpression(json["runfiles"], + auto runfiles = DeserializeExpression(hash_type, + json["runfiles"], nodes, provided_artifacts, provided_nodes, provided_results, sofar); - auto provides = DeserializeExpression(json["provides"], + auto provides = DeserializeExpression(hash_type, + json["provides"], nodes, provided_artifacts, provided_nodes, @@ -273,7 +281,8 @@ auto SerializeTargetResultWithReplacement( Expression::map_t::underlying_map_t map{}; for (auto const& [key, val] : json.items()) { - auto new_val = DeserializeExpression(val.get<std::string>(), + auto new_val = DeserializeExpression(hash_type, + val.get<std::string>(), nodes, provided_artifacts, provided_nodes, @@ -293,7 +302,8 @@ auto SerializeTargetResultWithReplacement( Expression::list_t list{}; list.reserve(json.size()); for (auto const& val : json) { - auto new_val = DeserializeExpression(val.get<std::string>(), + auto new_val = DeserializeExpression(hash_type, + val.get<std::string>(), nodes, provided_artifacts, provided_nodes, @@ -335,12 +345,13 @@ auto SerializeTargetResultWithReplacement( return artifacts; } -[[nodiscard]] auto DeserializeArtifactMap(nlohmann::json const& json) +[[nodiscard]] auto DeserializeArtifactMap(HashFunction::Type hash_type, + nlohmann::json const& json) -> ExpressionPtr { if (json.is_object()) { Expression::map_t::underlying_map_t map{}; for (auto const& [key, val] : json.items()) { - auto artifact = ArtifactDescription::FromJson(val); + auto artifact = ArtifactDescription::FromJson(hash_type, val); if (not artifact) { return ExpressionPtr{nullptr}; } @@ -388,10 +399,12 @@ auto JsonSet(nlohmann::json const& j) -> std::unordered_set<std::string> { } // NOLINTNEXTLINE(misc-no-recursion) -[[nodiscard]] auto DeserializeProvidesMap(nlohmann::json const& json) +[[nodiscard]] auto DeserializeProvidesMap(HashFunction::Type hash_type, + nlohmann::json const& json) -> ExpressionPtr { std::unordered_map<std::string, ExpressionPtr> sofar{}; - return DeserializeExpression(json["entry"], + return DeserializeExpression(hash_type, + json["entry"], json["nodes"], JsonSet(json["provided_artifacts"]), JsonSet(json["provided_nodes"]), @@ -434,12 +447,13 @@ auto TargetResult::ReplaceNonKnownAndToJson( } // NOLINTNEXTLINE(misc-no-recursion) -auto TargetResult::FromJson(nlohmann::json const& json) noexcept +auto TargetResult::FromJson(HashFunction::Type hash_type, + nlohmann::json const& json) noexcept -> std::optional<TargetResult> { try { - auto artifacts = DeserializeArtifactMap(json["artifacts"]); - auto runfiles = DeserializeArtifactMap(json["runfiles"]); - auto provides = DeserializeProvidesMap(json["provides"]); + auto artifacts = DeserializeArtifactMap(hash_type, json["artifacts"]); + auto runfiles = DeserializeArtifactMap(hash_type, json["runfiles"]); + auto provides = DeserializeProvidesMap(hash_type, json["provides"]); if (artifacts and runfiles and provides) { return TargetResult{artifacts, provides, runfiles}; } diff --git a/src/buildtool/build_engine/expression/target_result.hpp b/src/buildtool/build_engine/expression/target_result.hpp index dcf201ee..eba780c4 100644 --- a/src/buildtool/build_engine/expression/target_result.hpp +++ b/src/buildtool/build_engine/expression/target_result.hpp @@ -22,6 +22,7 @@ #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/artifact_description.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/hash_combine.hpp" struct TargetResult { @@ -30,7 +31,8 @@ struct TargetResult { ExpressionPtr runfiles{}; bool is_cacheable{provides.IsCacheable()}; - [[nodiscard]] static auto FromJson(nlohmann::json const& json) noexcept + [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, + nlohmann::json const& json) noexcept -> std::optional<TargetResult>; [[nodiscard]] auto ToJson() const -> nlohmann::json; diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index aff9861d..450b8037 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -109,9 +109,9 @@ , ["@", "json", "", "json"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "logging"] + , ["src/buildtool/crypto", "hash_function"] ] - , "private-deps": - [["src/utils/cpp", "json"], ["src/buildtool/crypto", "hash_function"]] + , "private-deps": [["src/utils/cpp", "json"], "artifact_digest_factory"] , "stage": ["src", "buildtool", "common"] } , "action_description": @@ -125,6 +125,7 @@ , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "json"] + , ["src/buildtool/crypto", "hash_function"] ] , "stage": ["src", "buildtool", "common"] } diff --git a/src/buildtool/common/action_description.hpp b/src/buildtool/common/action_description.hpp index 2ae7ad8a..f4756688 100644 --- a/src/buildtool/common/action_description.hpp +++ b/src/buildtool/common/action_description.hpp @@ -26,6 +26,7 @@ #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact_description.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/json.hpp" @@ -45,7 +46,8 @@ class ActionDescription { action_{std::move(action)}, inputs_{std::move(inputs)} {} - [[nodiscard]] static auto FromJson(std::string const& id, + [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, + std::string const& id, nlohmann::json const& desc) noexcept -> std::optional<ActionDescription::Ptr> { try { @@ -119,7 +121,8 @@ class ActionDescription { inputs_t inputs{}; for (auto const& [path, input_desc] : input.items()) { - auto artifact = ArtifactDescription::FromJson(input_desc); + auto artifact = + ArtifactDescription::FromJson(hash_type, input_desc); if (not artifact) { return std::nullopt; } diff --git a/src/buildtool/common/artifact_description.cpp b/src/buildtool/common/artifact_description.cpp index a36d91e5..e5a67155 100644 --- a/src/buildtool/common/artifact_description.cpp +++ b/src/buildtool/common/artifact_description.cpp @@ -17,7 +17,7 @@ #include <cstddef> #include "nlohmann/json.hpp" -#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/json.hpp" @@ -42,7 +42,8 @@ namespace { [[nodiscard]] auto CreateLocalArtifactDescription(nlohmann::json const& data) -> std::optional<ArtifactDescription>; -[[nodiscard]] auto CreateKnownArtifactDescription(nlohmann::json const& data) +[[nodiscard]] auto CreateKnownArtifactDescription(HashFunction::Type hash_type, + nlohmann::json const& data) -> std::optional<ArtifactDescription>; [[nodiscard]] auto CreateActionArtifactDescription(nlohmann::json const& data) @@ -79,7 +80,8 @@ auto ArtifactDescription::CreateTree(std::string tree_id) noexcept return ArtifactDescription{std::move(tree_id)}; } -auto ArtifactDescription::FromJson(nlohmann::json const& json) noexcept +auto ArtifactDescription::FromJson(HashFunction::Type hash_type, + nlohmann::json const& json) noexcept -> std::optional<ArtifactDescription> { try { auto const type = ExtractValueAs<std::string>( @@ -107,7 +109,7 @@ auto ArtifactDescription::FromJson(nlohmann::json const& json) noexcept return CreateLocalArtifactDescription(*data); } if (*type == "KNOWN") { - return CreateKnownArtifactDescription(*data); + return CreateKnownArtifactDescription(hash_type, *data); } if (*type == "ACTION") { return CreateActionArtifactDescription(*data); @@ -259,7 +261,8 @@ auto CreateLocalArtifactDescription(nlohmann::json const& data) return std::nullopt; } -auto CreateKnownArtifactDescription(nlohmann::json const& data) +auto CreateKnownArtifactDescription(HashFunction::Type hash_type, + nlohmann::json const& data) -> std::optional<ArtifactDescription> { auto const blob_id = ExtractValueAs<std::string>(data, "id", [](std::string const& error) { @@ -284,9 +287,15 @@ auto CreateKnownArtifactDescription(nlohmann::json const& data) }); if (blob_id.has_value() and size.has_value() and file_type.has_value() and file_type->size() == 1) { - auto const& object_type = FromChar((*file_type)[0]); - ArtifactDigest digest{*blob_id, *size, IsTreeObject(object_type)}; - return ArtifactDescription::CreateKnown(std::move(digest), object_type); + auto const object_type = FromChar((*file_type)[0]); + + auto digest = ArtifactDigestFactory::Create( + hash_type, *blob_id, *size, IsTreeObject(object_type)); + if (not digest) { + return std::nullopt; + } + return ArtifactDescription::CreateKnown(*std::move(digest), + object_type); } return std::nullopt; } diff --git a/src/buildtool/common/artifact_description.hpp b/src/buildtool/common/artifact_description.hpp index 53d3adcc..c63a2dec 100644 --- a/src/buildtool/common/artifact_description.hpp +++ b/src/buildtool/common/artifact_description.hpp @@ -23,6 +23,7 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" class ArtifactDescription final { @@ -65,7 +66,8 @@ class ArtifactDescription final { return std::holds_alternative<Tree>(data_); } - [[nodiscard]] static auto FromJson(nlohmann::json const& json) noexcept + [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, + nlohmann::json const& json) noexcept -> std::optional<ArtifactDescription>; [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json; diff --git a/src/buildtool/common/tree.hpp b/src/buildtool/common/tree.hpp index 53333901..55922346 100644 --- a/src/buildtool/common/tree.hpp +++ b/src/buildtool/common/tree.hpp @@ -54,13 +54,15 @@ class Tree { return ArtifactDescription::CreateTree(id_); } - [[nodiscard]] static auto FromJson(std::string const& id, + [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, + std::string const& id, nlohmann::json const& json) -> std::optional<Tree::Ptr> { auto inputs = inputs_t{}; inputs.reserve(json.size()); for (auto const& [path, artifact] : json.items()) { - auto artifact_desc = ArtifactDescription::FromJson(artifact); + auto artifact_desc = + ArtifactDescription::FromJson(hash_type, artifact); if (not artifact_desc) { return std::nullopt; } diff --git a/src/buildtool/graph_traverser/TARGETS b/src/buildtool/graph_traverser/TARGETS index f2f77dd2..c3ed60ec 100644 --- a/src/buildtool/graph_traverser/TARGETS +++ b/src/buildtool/graph_traverser/TARGETS @@ -7,6 +7,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "artifact_digest_factory"] , ["src/buildtool/common", "tree"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/execution_engine/dag", "dag"] , ["src/buildtool/execution_engine/executor", "context"] , ["src/buildtool/execution_engine/executor", "executor"] diff --git a/src/buildtool/graph_traverser/graph_traverser.hpp b/src/buildtool/graph_traverser/graph_traverser.hpp index 51a1976d..b62bd08a 100644 --- a/src/buildtool/graph_traverser/graph_traverser.hpp +++ b/src/buildtool/graph_traverser/graph_traverser.hpp @@ -35,6 +35,7 @@ #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/tree.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/artifact_blob_container.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" @@ -179,10 +180,13 @@ class GraphTraverser { } auto const [blobs, tree_descs, actions] = *desc; + HashFunction::Type const hash_type = + context_.apis->hash_function.GetType(); std::vector<ActionDescription::Ptr> action_descriptions{}; action_descriptions.reserve(actions.size()); for (auto const& [id, description] : actions.items()) { - auto action = ActionDescription::FromJson(id, description); + auto action = + ActionDescription::FromJson(hash_type, id, description); if (not action) { return std::nullopt; // Error already logged } @@ -191,7 +195,7 @@ class GraphTraverser { std::vector<Tree::Ptr> trees{}; for (auto const& [id, description] : tree_descs.items()) { - auto tree = Tree::FromJson(id, description); + auto tree = Tree::FromJson(hash_type, id, description); if (not tree) { return std::nullopt; } @@ -200,7 +204,8 @@ class GraphTraverser { std::map<std::string, ArtifactDescription> artifact_descriptions{}; for (auto const& [rel_path, description] : artifacts.items()) { - auto artifact = ArtifactDescription::FromJson(description); + auto artifact = + ArtifactDescription::FromJson(hash_type, description); if (not artifact) { return std::nullopt; // Error already logged } diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 9c9cebfb..894b5486 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -317,6 +317,7 @@ [ ["src/buildtool/multithreading", "async_map_utils"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/logging", "log_level"] + , ["src/buildtool/crypto", "hash_function"] ] } , "archive": diff --git a/src/buildtool/main/build_utils.cpp b/src/buildtool/main/build_utils.cpp index 97b9e1b7..cbd5873e 100644 --- a/src/buildtool/main/build_utils.cpp +++ b/src/buildtool/main/build_utils.cpp @@ -14,6 +14,7 @@ #include "src/buildtool/main/build_utils.hpp" #ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/multithreading/async_map_utils.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" @@ -94,7 +95,8 @@ auto CreateTargetCacheWriterMap( return; } auto const& target = cache_targets.at(tc_key); - auto entry = TargetCacheEntry::FromTarget(target, extra_infos); + auto entry = TargetCacheEntry::FromTarget( + apis->hash_function.GetType(), target, extra_infos); if (not entry) { (*logger)( fmt::format("Failed creating target cache entry for key {}", diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS index c211d26f..acbe40e1 100644 --- a/src/buildtool/serve_api/remote/TARGETS +++ b/src/buildtool/serve_api/remote/TARGETS @@ -76,6 +76,7 @@ , ["src/buildtool/common", "artifact_digest_factory"] , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] + , ["src/buildtool/crypto", "hash_function"] ] } , "configuration_client": diff --git a/src/buildtool/serve_api/remote/target_client.cpp b/src/buildtool/serve_api/remote/target_client.cpp index 32d7377f..f339172f 100644 --- a/src/buildtool/serve_api/remote/target_client.cpp +++ b/src/buildtool/serve_api/remote/target_client.cpp @@ -24,6 +24,7 @@ #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/common/remote/client_common.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" namespace { @@ -173,7 +174,8 @@ auto TargetClient::ServeTarget(const TargetCacheKey& key, obj_info.ToString())}; } try { - auto const& result = TargetCacheEntry::FromJson( + auto const result = TargetCacheEntry::FromJson( + storage_.GetHashFunction().GetType(), nlohmann::json::parse(*target_value_str)); // return the target cache value information return serve_target_result_t{std::in_place_index<3>, diff --git a/src/buildtool/storage/target_cache.tpp b/src/buildtool/storage/target_cache.tpp index f19a37d5..c31fa1cc 100644 --- a/src/buildtool/storage/target_cache.tpp +++ b/src/buildtool/storage/target_cache.tpp @@ -19,6 +19,7 @@ #include <tuple> //std::ignore #include "nlohmann/json.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/target_cache.hpp" @@ -92,7 +93,8 @@ auto TargetCache<kDoGlobalUplink>::Read( if (auto value = FileSystemManager::ReadFile(*path)) { try { return std::make_pair( - TargetCacheEntry{nlohmann::json::parse(*value)}, + TargetCacheEntry{hash_type, + nlohmann::json::parse(*value)}, std::move(*info)); } catch (std::exception const& ex) { logger_->Emit(LogLevel::Warning, @@ -161,7 +163,8 @@ auto TargetCache<kDoGlobalUplink>::LocalUplinkEntry( } catch (std::exception const& ex) { return false; } - auto entry = TargetCacheEntry::FromJson(json_desc); + auto entry = + TargetCacheEntry::FromJson(cas_.GetHashFunction().GetType(), json_desc); // Uplink the implied export targets first for (auto const& implied_digest : entry.ToImplied()) { diff --git a/src/buildtool/storage/target_cache_entry.cpp b/src/buildtool/storage/target_cache_entry.cpp index becc6a13..f6659256 100644 --- a/src/buildtool/storage/target_cache_entry.cpp +++ b/src/buildtool/storage/target_cache_entry.cpp @@ -25,6 +25,7 @@ #include "src/utils/cpp/gsl.hpp" auto TargetCacheEntry::FromTarget( + HashFunction::Type hash_type, AnalysedTargetPtr const& target, std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& replacements) noexcept -> std::optional<TargetCacheEntry> { @@ -42,17 +43,18 @@ auto TargetCacheEntry::FromTarget( if (not implied.empty()) { (*desc)["implied export targets"] = implied; } - return TargetCacheEntry{*desc}; + return TargetCacheEntry{hash_type, *desc}; } -auto TargetCacheEntry::FromJson(nlohmann::json desc) noexcept +auto TargetCacheEntry::FromJson(HashFunction::Type hash_type, + nlohmann::json desc) noexcept -> TargetCacheEntry { - return TargetCacheEntry(std::move(desc)); + return TargetCacheEntry(hash_type, std::move(desc)); } auto TargetCacheEntry::ToResult() const noexcept -> std::optional<TargetResult> { - return TargetResult::FromJson(desc_); + return TargetResult::FromJson(hash_type_, desc_); } auto TargetCacheEntry::ToImplied() const noexcept -> std::set<std::string> { @@ -93,9 +95,10 @@ auto TargetCacheEntry::ToImpliedIds(std::string const& entry_key_hash) return result; } -[[nodiscard]] auto ToObjectInfo(nlohmann::json const& json) +[[nodiscard]] static auto ToObjectInfo(HashFunction::Type hash_type, + nlohmann::json const& json) -> Artifact::ObjectInfo { - auto const& desc = ArtifactDescription::FromJson(json); + auto const desc = ArtifactDescription::FromJson(hash_type, json); // The assumption is that all artifacts mentioned in a target cache // entry are KNOWN to the remote side. ExpectsAudit(desc and desc->IsKnown()); @@ -104,7 +107,8 @@ auto TargetCacheEntry::ToImpliedIds(std::string const& entry_key_hash) return *info; } -[[nodiscard]] auto ScanArtifactMap( +[[nodiscard]] static auto ScanArtifactMap( + HashFunction::Type hash_type, gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, nlohmann::json const& json) -> bool { if (not json.is_object()) { @@ -114,11 +118,14 @@ auto TargetCacheEntry::ToImpliedIds(std::string const& entry_key_hash) std::transform(json.begin(), json.end(), std::back_inserter(*infos), - [](auto const& item) { return ToObjectInfo(item); }); + [hash_type](auto const& item) { + return ToObjectInfo(hash_type, item); + }); return true; } -[[nodiscard]] auto ScanProvidesMap( +[[nodiscard]] static auto ScanProvidesMap( + HashFunction::Type hash_type, gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, nlohmann::json const& json) -> bool { if (not json.is_object()) { @@ -127,13 +134,13 @@ auto TargetCacheEntry::ToImpliedIds(std::string const& entry_key_hash) auto const& nodes = json["nodes"]; auto const& provided_artifacts = json["provided_artifacts"]; infos->reserve(infos->size() + provided_artifacts.size()); - std::transform( - provided_artifacts.begin(), - provided_artifacts.end(), - std::back_inserter(*infos), - [&nodes](auto const& item) { - return ToObjectInfo(nodes[item.template get<std::string>()]); - }); + std::transform(provided_artifacts.begin(), + provided_artifacts.end(), + std::back_inserter(*infos), + [hash_type, &nodes](auto const& item) { + return ToObjectInfo( + hash_type, nodes[item.template get<std::string>()]); + }); return true; } @@ -141,9 +148,9 @@ auto TargetCacheEntry::ToArtifacts( gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) const noexcept -> bool { try { - if (ScanArtifactMap(infos, desc_["artifacts"]) and - ScanArtifactMap(infos, desc_["runfiles"]) and - ScanProvidesMap(infos, desc_["provides"])) { + if (ScanArtifactMap(hash_type_, infos, desc_["artifacts"]) and + ScanArtifactMap(hash_type_, infos, desc_["runfiles"]) and + ScanProvidesMap(hash_type_, infos, desc_["provides"])) { return true; } } catch (std::exception const& ex) { diff --git a/src/buildtool/storage/target_cache_entry.hpp b/src/buildtool/storage/target_cache_entry.hpp index b0bd1020..a1cddf62 100644 --- a/src/buildtool/storage/target_cache_entry.hpp +++ b/src/buildtool/storage/target_cache_entry.hpp @@ -27,21 +27,25 @@ #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_description.hpp" +#include "src/buildtool/crypto/hash_function.hpp" // Entry for target cache. Created from target, contains TargetResult. class TargetCacheEntry { public: - explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} + explicit TargetCacheEntry(HashFunction::Type hash_type, nlohmann::json desc) + : hash_type_{hash_type}, desc_(std::move(desc)) {} // Create the entry from target with replacement artifacts/infos. // Replacement artifacts must replace all non-known artifacts by known. [[nodiscard]] static auto FromTarget( + HashFunction::Type hash_type, AnalysedTargetPtr const& target, std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& replacements) noexcept -> std::optional<TargetCacheEntry>; // Create a target-cache entry from a json description. - [[nodiscard]] static auto FromJson(nlohmann::json desc) noexcept + [[nodiscard]] static auto FromJson(HashFunction::Type hash_type, + nlohmann::json desc) noexcept -> TargetCacheEntry; // Obtain TargetResult from cache entry. @@ -70,6 +74,7 @@ class TargetCacheEntry { } private: + HashFunction::Type hash_type_; nlohmann::json desc_; }; diff --git a/test/buildtool/common/TARGETS b/test/buildtool/common/TARGETS index dc6b3056..ef1fff25 100644 --- a/test/buildtool/common/TARGETS +++ b/test/buildtool/common/TARGETS @@ -11,6 +11,7 @@ , ["@", "src", "src/buildtool/common", "artifact_digest_factory"] , ["@", "src", "src/buildtool/crypto", "hash_function"] , ["@", "src", "src/buildtool/file_system", "object_type"] + , ["@", "src", "src/buildtool/compatibility", "compatibility"] ] , "stage": ["test", "buildtool", "common"] } @@ -24,6 +25,8 @@ , ["@", "json", "", "json"] , ["@", "src", "src/buildtool/common", "action_description"] , ["@", "src", "src/buildtool/common", "common"] + , ["@", "src", "src/buildtool/crypto", "hash_function"] + , ["@", "src", "src/buildtool/compatibility", "compatibility"] ] , "stage": ["test", "buildtool", "common"] } diff --git a/test/buildtool/common/action_description.test.cpp b/test/buildtool/common/action_description.test.cpp index eab2cc48..7d5a72d7 100644 --- a/test/buildtool/common/action_description.test.cpp +++ b/test/buildtool/common/action_description.test.cpp @@ -20,6 +20,14 @@ #include "nlohmann/json.hpp" #include "src/buildtool/common/action.hpp" #include "src/buildtool/common/artifact_description.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/crypto/hash_function.hpp" + +[[nodiscard]] static inline auto GetHashType(bool compatible) noexcept + -> HashFunction::Type { + return compatible ? HashFunction::Type::PlainSHA256 + : HashFunction::Type::GitSHA1; +} TEST_CASE("From JSON", "[action_description]") { using path = std::filesystem::path; @@ -37,56 +45,57 @@ TEST_CASE("From JSON", "[action_description]") { desc.Inputs()} .ToJson(); + auto const hash_type = GetHashType(Compatibility::IsCompatible()); SECTION("Parse full action") { - auto description = ActionDescription::FromJson("id", json); + auto description = ActionDescription::FromJson(hash_type, "id", json); REQUIRE(description); CHECK((*description)->ToJson() == json); } SECTION("Parse action without optional input") { json["input"] = nlohmann::json::object(); - CHECK(ActionDescription::FromJson("id", json)); + CHECK(ActionDescription::FromJson(hash_type, "id", json)); json["input"] = nlohmann::json::array(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("input"); - CHECK(ActionDescription::FromJson("id", json)); + CHECK(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without optional env") { json["env"] = nlohmann::json::object(); - CHECK(ActionDescription::FromJson("id", json)); + CHECK(ActionDescription::FromJson(hash_type, "id", json)); json["env"] = nlohmann::json::array(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("env"); - CHECK(ActionDescription::FromJson("id", json)); + CHECK(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without mandatory outputs") { json["output"] = nlohmann::json::array(); json["output_dirs"] = nlohmann::json::array(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json["output"] = nlohmann::json::object(); json["output_dirs"] = nlohmann::json::object(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("output"); json.erase("output_dirs"); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); } SECTION("Parse action without mandatory command") { json["command"] = nlohmann::json::array(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json["command"] = nlohmann::json::object(); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); json.erase("command"); - CHECK_FALSE(ActionDescription::FromJson("id", json)); + CHECK_FALSE(ActionDescription::FromJson(hash_type, "id", json)); } } diff --git a/test/buildtool/common/artifact_description.test.cpp b/test/buildtool/common/artifact_description.test.cpp index f7af5159..a05baa66 100644 --- a/test/buildtool/common/artifact_description.test.cpp +++ b/test/buildtool/common/artifact_description.test.cpp @@ -22,6 +22,7 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/file_system/object_type.hpp" @@ -39,10 +40,17 @@ static auto NamedDigest(std::string const& str) -> ArtifactDigest { lhs.Info() == rhs.Info(); } +[[nodiscard]] static inline auto GetHashType(bool compatible) noexcept + -> HashFunction::Type { + return compatible ? HashFunction::Type::PlainSHA256 + : HashFunction::Type::GitSHA1; +} + TEST_CASE("Local artifact", "[artifact_description]") { auto local_desc = ArtifactDescription::CreateLocal( std::filesystem::path{"local_path"}, "repo"); - auto from_json = ArtifactDescription::FromJson(local_desc.ToJson()); + auto from_json = ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), local_desc.ToJson()); CHECK(local_desc == *from_json); } @@ -50,19 +58,22 @@ TEST_CASE("Known artifact", "[artifact_description]") { SECTION("File object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("f_fake_hash"), ObjectType::File); - auto from_json = ArtifactDescription::FromJson(known_desc.ToJson()); + auto from_json = ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), known_desc.ToJson()); CHECK(known_desc == *from_json); } SECTION("Executable object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("x_fake_hash"), ObjectType::Executable); - auto from_json = ArtifactDescription::FromJson(known_desc.ToJson()); + auto from_json = ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), known_desc.ToJson()); CHECK(known_desc == *from_json); } SECTION("Symlink object") { auto known_desc = ArtifactDescription::CreateKnown( NamedDigest("l_fake_hash"), ObjectType::Symlink); - auto from_json = ArtifactDescription::FromJson(known_desc.ToJson()); + auto from_json = ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), known_desc.ToJson()); CHECK(known_desc == *from_json); } } @@ -70,7 +81,8 @@ TEST_CASE("Known artifact", "[artifact_description]") { TEST_CASE("Action artifact", "[artifact_description]") { auto action_desc = ArtifactDescription::CreateAction( "action_id", std::filesystem::path{"out_path"}); - auto from_json = ArtifactDescription::FromJson(action_desc.ToJson()); + auto from_json = ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), action_desc.ToJson()); CHECK(action_desc == *from_json); } @@ -81,76 +93,77 @@ TEST_CASE("From JSON", "[artifact_description]") { .ToJson(); auto action = ArtifactDescription::CreateAction("id", "output").ToJson(); + auto const hash_type = GetHashType(Compatibility::IsCompatible()); SECTION("Parse artifacts") { - CHECK(ArtifactDescription::FromJson(local)); - CHECK(ArtifactDescription::FromJson(known)); - CHECK(ArtifactDescription::FromJson(action)); + CHECK(ArtifactDescription::FromJson(hash_type, local)); + CHECK(ArtifactDescription::FromJson(hash_type, known)); + CHECK(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse artifact without mandatory type") { local.erase("type"); known.erase("type"); action.erase("type"); - CHECK_FALSE(ArtifactDescription::FromJson(local)); - CHECK_FALSE(ArtifactDescription::FromJson(known)); - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse artifact without mandatory data") { local.erase("data"); known.erase("data"); action.erase("data"); - CHECK_FALSE(ArtifactDescription::FromJson(local)); - CHECK_FALSE(ArtifactDescription::FromJson(known)); - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("Parse local artifact without mandatory path") { local["data"]["path"] = 0; - CHECK_FALSE(ArtifactDescription::FromJson(local)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); local["data"].erase("path"); - CHECK_FALSE(ArtifactDescription::FromJson(local)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, local)); } SECTION("Parse known artifact") { SECTION("without mandatory id") { known["data"]["id"] = 0; - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("id"); - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } SECTION("without mandatory size") { known["data"]["size"] = "0"; - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("size"); - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } SECTION("without mandatory file_type") { known["data"]["file_type"] = "more_than_one_char"; - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); known["data"].erase("file_type"); - CHECK_FALSE(ArtifactDescription::FromJson(known)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, known)); } } SECTION("Parse action artifact") { SECTION("without mandatory id") { action["data"]["id"] = 0; - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); action["data"].erase("id"); - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } SECTION("without mandatory path") { action["data"]["path"] = 0; - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); action["data"].erase("path"); - CHECK_FALSE(ArtifactDescription::FromJson(action)); + CHECK_FALSE(ArtifactDescription::FromJson(hash_type, action)); } } } @@ -158,16 +171,19 @@ TEST_CASE("From JSON", "[artifact_description]") { TEST_CASE("Description missing mandatory key/value pair", "[artifact_description]") { nlohmann::json const missing_type = {{"data", {{"path", "some/path"}}}}; - CHECK(not ArtifactDescription::FromJson(missing_type)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), missing_type)); nlohmann::json const missing_data = {{"type", "LOCAL"}}; - CHECK(not ArtifactDescription::FromJson(missing_data)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), missing_data)); } TEST_CASE("Local artifact description contains incorrect value for \"data\"", "[artifact_description]") { nlohmann::json const local_art_missing_path = { {"type", "LOCAL"}, {"data", nlohmann::json::object()}}; - CHECK(not ArtifactDescription::FromJson(local_art_missing_path)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), local_art_missing_path)); } TEST_CASE("Known artifact description contains incorrect value for \"data\"", @@ -178,19 +194,24 @@ TEST_CASE("Known artifact description contains incorrect value for \"data\"", nlohmann::json const known_art_missing_id = { {"type", "KNOWN"}, {"data", {{"size", 15}, {"file_type", file_type}}}}; - CHECK(not ArtifactDescription::FromJson(known_art_missing_id)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), known_art_missing_id)); } SECTION("missing \"size\"") { nlohmann::json const known_art_missing_size = { {"type", "KNOWN"}, {"data", {{"id", "known_input"}, {"file_type", file_type}}}}; - CHECK(not ArtifactDescription::FromJson(known_art_missing_size)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), + known_art_missing_size)); } SECTION("missing \"file_type\"") { nlohmann::json const known_art_missing_file_type = { {"type", "KNOWN"}, {"data", {{"id", "known_input"}, {"size", 15}}}}; - CHECK(not ArtifactDescription::FromJson(known_art_missing_file_type)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), + known_art_missing_file_type)); } } @@ -198,9 +219,11 @@ TEST_CASE("Action artifact description contains incorrect value for \"data\"", "[artifact_description]") { nlohmann::json const action_art_missing_id = { {"type", "ACTION"}, {"data", {{"path", "output/path"}}}}; - CHECK(not ArtifactDescription::FromJson(action_art_missing_id)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), action_art_missing_id)); nlohmann::json const action_art_missing_path = { {"type", "ACTION"}, {"data", {{"id", "action_id"}}}}; - CHECK(not ArtifactDescription::FromJson(action_art_missing_path)); + CHECK(not ArtifactDescription::FromJson( + GetHashType(Compatibility::IsCompatible()), action_art_missing_path)); } diff --git a/test/buildtool/execution_engine/traverser/TARGETS b/test/buildtool/execution_engine/traverser/TARGETS index 821c4bc8..bdd7c4cc 100644 --- a/test/buildtool/execution_engine/traverser/TARGETS +++ b/test/buildtool/execution_engine/traverser/TARGETS @@ -10,6 +10,8 @@ , ["@", "src", "src/buildtool/common", "artifact_description"] , ["@", "src", "src/buildtool/execution_engine/dag", "dag"] , ["@", "src", "src/buildtool/execution_engine/traverser", "traverser"] + , ["@", "src", "src/buildtool/crypto", "hash_function"] + , ["@", "src", "src/buildtool/compatibility", "compatibility"] ] , "stage": ["test", "buildtool", "execution_engine", "traverser"] } diff --git a/test/buildtool/execution_engine/traverser/traverser.test.cpp b/test/buildtool/execution_engine/traverser/traverser.test.cpp index 97e2095e..70ab2465 100644 --- a/test/buildtool/execution_engine/traverser/traverser.test.cpp +++ b/test/buildtool/execution_engine/traverser/traverser.test.cpp @@ -24,6 +24,8 @@ #include "catch2/catch_test_macros.hpp" #include "src/buildtool/common/artifact_description.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "test/utils/container_matchers.hpp" @@ -182,8 +184,12 @@ class TestProject { auto inputs_desc = ActionDescription::inputs_t{}; if (not inputs.empty()) { command.emplace_back("FROM"); + auto const hash_type = Compatibility::IsCompatible() + ? HashFunction::Type::PlainSHA256 + : HashFunction::Type::GitSHA1; for (auto const& input_desc : inputs) { - auto artifact = ArtifactDescription::FromJson(input_desc); + auto artifact = + ArtifactDescription::FromJson(hash_type, input_desc); REQUIRE(artifact); auto const input_id = artifact->Id(); command.push_back(input_id); |