diff options
Diffstat (limited to 'src')
23 files changed, 143 insertions, 77 deletions
diff --git a/src/buildtool/build_engine/expression/TARGETS b/src/buildtool/build_engine/expression/TARGETS index ba030949..76a6058f 100644 --- a/src/buildtool/build_engine/expression/TARGETS +++ b/src/buildtool/build_engine/expression/TARGETS @@ -43,7 +43,7 @@ , "linked_map" , ["src/buildtool/build_engine/base_maps", "entity_name_data"] , ["src/buildtool/common", "artifact_description"] - , ["src/buildtool/crypto", "hash_generator"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/multithreading", "atomic_value"] , ["src/utils/cpp", "type_safe_arithmetic"] diff --git a/src/buildtool/build_engine/expression/expression.cpp b/src/buildtool/build_engine/expression/expression.cpp index 763f51f8..91d293bc 100644 --- a/src/buildtool/build_engine/expression/expression.cpp +++ b/src/buildtool/build_engine/expression/expression.cpp @@ -220,10 +220,10 @@ auto Expression::ComputeHash() const noexcept -> std::string { : IsNode() ? "#" : IsName() ? "$" : ""}; - hash = hash_gen_.Run(prefix + ToString()).Bytes(); + hash = HashFunction::ComputeHash(prefix + ToString()).Bytes(); } else { - auto hasher = hash_gen_.IncrementalHasher(); + auto hasher = HashFunction::Hasher(); if (IsList()) { auto list = Value<Expression::list_t>(); hasher.Update("["); @@ -235,7 +235,7 @@ auto Expression::ComputeHash() const noexcept -> std::string { auto map = Value<Expression::map_t>(); hasher.Update("{"); for (auto const& el : map->get()) { - hasher.Update(hash_gen_.Run(el.first).Bytes()); + hasher.Update(HashFunction::ComputeHash(el.first).Bytes()); hasher.Update(el.second->ToHash()); } } diff --git a/src/buildtool/build_engine/expression/expression.hpp b/src/buildtool/build_engine/expression/expression.hpp index 6acb1e0e..047089d2 100644 --- a/src/buildtool/build_engine/expression/expression.hpp +++ b/src/buildtool/build_engine/expression/expression.hpp @@ -19,7 +19,7 @@ #include "src/buildtool/build_engine/expression/target_node.hpp" #include "src/buildtool/build_engine/expression/target_result.hpp" #include "src/buildtool/common/artifact_description.hpp" -#include "src/buildtool/crypto/hash_generator.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/multithreading/atomic_value.hpp" #include "src/utils/cpp/hex_string.hpp" #include "src/utils/cpp/json.hpp" @@ -229,9 +229,6 @@ class Expression { Expression::FromJson(R"({"type": "empty_map"})"_json); private: - inline static HashGenerator const hash_gen_{ - HashGenerator::HashType::SHA256}; - std::variant<none_t, bool, number_t, diff --git a/src/buildtool/build_engine/target_map/built_in_rules.cpp b/src/buildtool/build_engine/target_map/built_in_rules.cpp index bfc7fd11..847bff8d 100644 --- a/src/buildtool/build_engine/target_map/built_in_rules.cpp +++ b/src/buildtool/build_engine/target_map/built_in_rules.cpp @@ -144,7 +144,8 @@ void FileGenRuleWithDeps( auto stage = ExpressionPtr{Expression::map_t{ file_name_val->String(), ExpressionPtr{ArtifactDescription{ - {ComputeHash(data_val->String()), data_val->String().size()}, + {HashFunction::ComputeBlobHash(data_val->String()).HexString(), + data_val->String().size()}, ObjectType::File}}}}; auto vars_set = std::unordered_set<std::string>{}; diff --git a/src/buildtool/build_engine/target_map/target_map.cpp b/src/buildtool/build_engine/target_map/target_map.cpp index 43d8be10..4f32cdf1 100644 --- a/src/buildtool/build_engine/target_map/target_map.cpp +++ b/src/buildtool/build_engine/target_map/target_map.cpp @@ -530,7 +530,8 @@ void withDependencies( } blobs.emplace_back(data->String()); return ExpressionPtr{ArtifactDescription{ - {ComputeHash(data->String()), data->String().size()}, + {HashFunction::ComputeBlobHash(data->String()).HexString(), + data->String().size()}, ObjectType::File}}; }}, {"TREE", diff --git a/src/buildtool/build_engine/target_map/utils.cpp b/src/buildtool/build_engine/target_map/utils.cpp index ea3ce0a5..136727bc 100644 --- a/src/buildtool/build_engine/target_map/utils.cpp +++ b/src/buildtool/build_engine/target_map/utils.cpp @@ -165,12 +165,9 @@ auto BuildMaps::Target::Utils::getTainted( namespace { auto hash_vector(std::vector<std::string> const& vec) -> std::string { - auto hasher = HashGenerator{BuildMaps::Target::Utils::kActionHash} - .IncrementalHasher(); + auto hasher = HashFunction::Hasher(); for (auto const& s : vec) { - hasher.Update(HashGenerator{BuildMaps::Target::Utils::kActionHash} - .Run(s) - .Bytes()); + hasher.Update(HashFunction::ComputeHash(s).Bytes()); } auto digest = std::move(hasher).Finalize(); if (not digest) { @@ -189,8 +186,7 @@ auto BuildMaps::Target::Utils::createAction( std::optional<std::string> may_fail, bool no_cache, const ExpressionPtr& inputs_exp) -> ActionDescription::Ptr { - auto hasher = HashGenerator{BuildMaps::Target::Utils::kActionHash} - .IncrementalHasher(); + auto hasher = HashFunction::Hasher(); hasher.Update(hash_vector(output_files)); hasher.Update(hash_vector(output_dirs)); hasher.Update(hash_vector(command)); diff --git a/src/buildtool/build_engine/target_map/utils.hpp b/src/buildtool/build_engine/target_map/utils.hpp index 2d6690c1..629f7492 100644 --- a/src/buildtool/build_engine/target_map/utils.hpp +++ b/src/buildtool/build_engine/target_map/utils.hpp @@ -17,8 +17,6 @@ namespace BuildMaps::Target::Utils { -constexpr HashGenerator::HashType kActionHash = HashGenerator::HashType::SHA256; - auto obtainTargetByName(const SubExprEvaluator&, const ExpressionPtr&, const Configuration&, diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index 3967e54b..4f297a12 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -31,7 +31,7 @@ ] , "deps": [ "bazel_types" - , ["src/buildtool/crypto", "hash_generator"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "object_type"] , ["src/utils/cpp", "type_safe_arithmetic"] , ["src/utils/cpp", "hash_combine"] diff --git a/src/buildtool/common/artifact_description.hpp b/src/buildtool/common/artifact_description.hpp index 2d361f8c..81f7780e 100644 --- a/src/buildtool/common/artifact_description.hpp +++ b/src/buildtool/common/artifact_description.hpp @@ -177,15 +177,13 @@ class ArtifactDescription { } private: - inline static HashGenerator const hash_gen_{ - HashGenerator::HashType::SHA256}; std::variant<Local, Known, Action, Tree> data_; ArtifactIdentifier id_{ComputeId(ToJson())}; [[nodiscard]] static auto ComputeId(nlohmann::json const& desc) noexcept -> ArtifactIdentifier { try { - return hash_gen_.Run(desc.dump()).Bytes(); + return HashFunction::ComputeHash(desc.dump()).Bytes(); } catch (std::exception const& ex) { Logger::Log(LogLevel::Error, "Computing artifact id failed with error:\n{}", diff --git a/src/buildtool/common/artifact_digest.hpp b/src/buildtool/common/artifact_digest.hpp index 7499e975..92bbde92 100644 --- a/src/buildtool/common/artifact_digest.hpp +++ b/src/buildtool/common/artifact_digest.hpp @@ -6,7 +6,7 @@ #include "gsl-lite/gsl-lite.hpp" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/crypto/hash_generator.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/utils/cpp/hash_combine.hpp" // Wrapper for bazel_re::Digest. Can be implicitly cast to bazel_re::Digest. @@ -41,7 +41,8 @@ class ArtifactDigest { [[nodiscard]] static auto Create(std::string const& content) noexcept -> ArtifactDigest { - return ArtifactDigest{ComputeHash(content), content.size()}; + return ArtifactDigest{ + HashFunction::ComputeBlobHash(content).HexString(), content.size()}; } private: diff --git a/src/buildtool/common/repository_config.cpp b/src/buildtool/common/repository_config.cpp index 1366d8f9..eb72b1c6 100644 --- a/src/buildtool/common/repository_config.cpp +++ b/src/buildtool/common/repository_config.cpp @@ -60,7 +60,8 @@ auto RepositoryConfig::DeduplicateRepo(std::string const& repo) const // content-fixed. if (data.base_desc) { // Use hash of content-fixed base description as content id - auto hash = ComputeHashDigest(data.base_desc->dump()).Bytes(); + auto hash = + HashFunction::ComputeHash(data.base_desc->dump()).Bytes(); // Add state with name, transitions, and content id minimizer.AddState(repo, data.info.name_mapping, hash); } diff --git a/src/buildtool/common/tree.hpp b/src/buildtool/common/tree.hpp index 6b76f8f1..cbfe721c 100644 --- a/src/buildtool/common/tree.hpp +++ b/src/buildtool/common/tree.hpp @@ -67,7 +67,8 @@ class Tree { } static auto ComputeId(inputs_t const& inputs) -> std::string { - return ComputeHash(ComputeDescription(inputs).dump()); + return HashFunction::ComputeHash(ComputeDescription(inputs).dump()) + .HexString(); } }; diff --git a/src/buildtool/compatibility/TARGETS b/src/buildtool/compatibility/TARGETS index 277ca22b..fbae7b48 100644 --- a/src/buildtool/compatibility/TARGETS +++ b/src/buildtool/compatibility/TARGETS @@ -2,7 +2,7 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["compatibility"] , "hdrs": ["compatibility.hpp"] - , "deps": [["src/buildtool/crypto", "hash_generator"]] + , "deps": [["src/buildtool/crypto", "hash_function"]] , "stage": ["src", "buildtool", "compatibility"] } } diff --git a/src/buildtool/compatibility/compatibility.hpp b/src/buildtool/compatibility/compatibility.hpp index 271d590e..930013f9 100644 --- a/src/buildtool/compatibility/compatibility.hpp +++ b/src/buildtool/compatibility/compatibility.hpp @@ -4,15 +4,15 @@ #include <unordered_map> #include <utility> -#include "src/buildtool/crypto/hash_generator.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/logger.hpp" class Compatibility { using git_hash = std::string; - using sha256_hash = std::string; + using compat_hash = std::string; using git_repo = std::string; - using GitToComaptibleMap = std::unordered_map<git_hash, sha256_hash>; + using GitToCompatibleMap = std::unordered_map<git_hash, compat_hash>; using CompatibleToGitMap = - std::unordered_map<sha256_hash, std::pair<git_hash, git_repo>>; + std::unordered_map<compat_hash, std::pair<git_hash, git_repo>>; public: [[nodiscard]] static auto Instance() noexcept -> Compatibility& { @@ -24,9 +24,10 @@ class Compatibility { } static void SetCompatible() noexcept { Instance().compatible_ = true; } - static auto RegisterGitEntry(std::string const& git_hash, - std::string const& data, - std::string const& repo) -> sha256_hash { + [[nodiscard]] static auto RegisterGitEntry(std::string const& git_hash, + std::string const& data, + std::string const& repo) + -> compat_hash { { auto& git_to_compatible = Instance().git_to_compatible_; @@ -36,14 +37,16 @@ class Compatibility { return it->second; } } - auto compatible_hash = ComputeHash(data); + // This is only used in compatible mode. Therefore, the default hash + // function produces the compatible hash. + auto compatible_hash = HashFunction::ComputeHash(data).HexString(); std::unique_lock lock_{Instance().mutex_}; Instance().git_to_compatible_[git_hash] = compatible_hash; Instance().compatible_to_git_[compatible_hash] = {git_hash, repo}; return compatible_hash; } - static auto GetGitEntry(std::string const& compatible_hash) + [[nodiscard]] static auto GetGitEntry(std::string const& compatible_hash) -> std::optional<std::pair<git_hash const&, git_repo const&>> { auto const& compatible_to_git = Instance().compatible_to_git_; std::shared_lock lock_{Instance().mutex_}; @@ -59,8 +62,7 @@ class Compatibility { } private: - GitToComaptibleMap git_to_compatible_{}; - + GitToCompatibleMap git_to_compatible_{}; CompatibleToGitMap compatible_to_git_{}; bool compatible_{false}; std::shared_mutex mutex_; diff --git a/src/buildtool/crypto/TARGETS b/src/buildtool/crypto/TARGETS index 57bc3608..c48bf596 100644 --- a/src/buildtool/crypto/TARGETS +++ b/src/buildtool/crypto/TARGETS @@ -28,4 +28,11 @@ , "deps": ["hash_impl"] , "stage": ["src", "buildtool", "crypto"] } +, "hash_function": + { "type": ["@", "rules", "CC", "library"] + , "name": ["hash_function"] + , "hdrs": ["hash_function.hpp"] + , "deps": ["hash_generator"] + , "stage": ["src", "buildtool", "crypto"] + } } diff --git a/src/buildtool/crypto/hash_function.hpp b/src/buildtool/crypto/hash_function.hpp new file mode 100644 index 00000000..4030c7a7 --- /dev/null +++ b/src/buildtool/crypto/hash_function.hpp @@ -0,0 +1,87 @@ +#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP +#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP + +#include <cstdint> +#include <functional> +#include <optional> +#include <string> + +#include "src/buildtool/crypto/hash_generator.hpp" + +/// \brief Hash function used for the entire buildtool. +class HashFunction { + public: + enum class JustHash : std::uint8_t { + Native, ///< SHA1 for plain hashes, and Git for blobs and trees. + Compatible ///< SHA256 for all hashes. + }; + + /// \brief Set globally used hash type. + static void SetHashType(JustHash type) { + [[maybe_unused]] auto _ = HashType(type); + } + + /// \brief Compute a plain hash. + [[nodiscard]] static auto ComputeHash(std::string const& data) noexcept + -> HashGenerator::HashDigest { + return ComputeTaggedHash(data); + } + + /// \brief Compute a blob hash. + [[nodiscard]] static auto ComputeBlobHash(std::string const& data) noexcept + -> HashGenerator::HashDigest { + static auto const kBlobTagCreator = + [](std::string const& data) -> std::string { + return {"blob " + std::to_string(data.size()) + '\0'}; + }; + return ComputeTaggedHash(data, kBlobTagCreator); + } + + /// \brief Compute a tree hash. + [[nodiscard]] static auto ComputeTreeHash(std::string const& data) noexcept + -> HashGenerator::HashDigest { + static auto const kTreeTagCreator = + [](std::string const& data) -> std::string { + return {"tree " + std::to_string(data.size()) + '\0'}; + }; + return ComputeTaggedHash(data, kTreeTagCreator); + } + + /// \brief Obtain incremental hasher for computing plain hashes. + [[nodiscard]] static auto Hasher() noexcept -> HashGenerator::Hasher { + switch (HashType()) { + case JustHash::Native: + return HashGenerator{HashGenerator::HashType::SHA1} + .IncrementalHasher(); + case JustHash::Compatible: + return HashGenerator{HashGenerator::HashType::SHA256} + .IncrementalHasher(); + } + } + + private: + static constexpr auto kDefaultType = JustHash::Native; + + [[nodiscard]] static auto HashType( + std::optional<JustHash> type = std::nullopt) -> JustHash { + static JustHash type_{kDefaultType}; + if (type) { + type_ = *type; + } + return type_; + } + + [[nodiscard]] static auto ComputeTaggedHash( + std::string const& data, + std::function<std::string(std::string const&)> const& tag_creator = + {}) noexcept -> HashGenerator::HashDigest { + auto hasher = Hasher(); + if (tag_creator and HashType() == JustHash::Native) { + hasher.Update(tag_creator(data)); + } + hasher.Update(data); + return *std::move(hasher).Finalize(); + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP diff --git a/src/buildtool/crypto/hash_generator.hpp b/src/buildtool/crypto/hash_generator.hpp index dc720912..d8b59631 100644 --- a/src/buildtool/crypto/hash_generator.hpp +++ b/src/buildtool/crypto/hash_generator.hpp @@ -20,19 +20,6 @@ class HashGenerator { /// \brief Types of hash implementations supported by generator. enum class HashType { MD5, SHA1, SHA256, GIT }; - // this function is called for the first time from the main to initialize - // the HashGenerator according to the command line flag --compatible - // note that, once the HashGenerator is initialized, it cannot be changed - [[maybe_unused]] static auto SetHashGenerator( - std::optional<HashType> const x = std::nullopt) -> HashGenerator& { - static HashGenerator gen{x.value_or(HashType::GIT)}; - return gen; - } - [[nodiscard]] static auto Instance() -> HashGenerator& { - return SetHashGenerator(); // return the already initialized - // HashGenerator - } - /// \brief The universal hash digest. /// The type of hash and the digest length depends on the hash /// implementation used to generated this digest. @@ -137,15 +124,4 @@ class HashGenerator { } }; -[[nodiscard]] static inline auto ComputeHashDigest( - std::string const& data) noexcept -> HashGenerator::HashDigest { - return HashGenerator::Instance().Run(data); -} - -/// \brief Hash function used for the entire buildtool -[[nodiscard]] static inline auto ComputeHash(std::string const& data) noexcept - -> std::string { - return ComputeHashDigest(data).HexString(); -} - #endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_GENERATOR_HPP diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS index 5adca546..d2e7aff2 100644 --- a/src/buildtool/execution_api/bazel_msg/TARGETS +++ b/src/buildtool/execution_api/bazel_msg/TARGETS @@ -4,7 +4,7 @@ , "hdrs": ["bazel_blob.hpp", "bazel_blob_container.hpp", "bazel_common.hpp"] , "deps": [ ["src/buildtool/common", "common"] - , ["src/buildtool/crypto", "hash_generator"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/utils/cpp", "concepts"] , ["src/utils/cpp", "type_safe_arithmetic"] diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 6994e247..e6043889 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -11,7 +11,7 @@ , "deps": [ ["@", "gsl-lite", "", "gsl-lite"] , ["src/buildtool/common", "common"] - , ["src/buildtool/crypto", "hash_generator"] + , ["src/buildtool/crypto", "hash_function"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] diff --git a/src/buildtool/execution_api/common/execution_common.hpp b/src/buildtool/execution_api/common/execution_common.hpp index 1208fc99..a3b364d9 100644 --- a/src/buildtool/execution_api/common/execution_common.hpp +++ b/src/buildtool/execution_api/common/execution_common.hpp @@ -17,7 +17,7 @@ #include <thread> #include "gsl-lite/gsl-lite.hpp" -#include "src/buildtool/crypto/hash_generator.hpp" +#include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/utils/cpp/hex_string.hpp" @@ -89,7 +89,7 @@ static void EncodeUUIDVariant1(std::string* uuid) { constexpr auto kHexDashPos = std::array{8UL, 12UL, 16UL, 20UL}; auto value = fmt::format("{}-{}", std::to_string(kRandomConstant), seed); - auto uuid = HashGenerator::Instance().Run(value).Bytes(); + auto uuid = HashFunction::ComputeHash(value).Bytes(); EncodeUUIDVersion4(&uuid); EncodeUUIDVariant1(&uuid); gsl_Expects(uuid.size() >= kRawLength); diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp index ebe7dd54..9e83129a 100644 --- a/src/buildtool/execution_engine/executor/executor.hpp +++ b/src/buildtool/execution_engine/executor/executor.hpp @@ -221,7 +221,9 @@ class ExecutorImpl { if (not content.has_value()) { return std::nullopt; } - auto digest = ArtifactDigest{ComputeHash(*content), content->size()}; + auto digest = + ArtifactDigest{HashFunction::ComputeBlobHash(*content).HexString(), + content->size()}; if (not api->Upload( BlobContainer{{BazelBlob{digest, std::move(*content)}}})) { return std::nullopt; diff --git a/src/buildtool/graph_traverser/graph_traverser.hpp b/src/buildtool/graph_traverser/graph_traverser.hpp index 7de74ad3..fac93e27 100644 --- a/src/buildtool/graph_traverser/graph_traverser.hpp +++ b/src/buildtool/graph_traverser/graph_traverser.hpp @@ -261,7 +261,8 @@ class GraphTraverser { std::vector<std::string> const& blobs) const noexcept -> bool { BlobContainer container; for (auto const& blob : blobs) { - auto digest = ArtifactDigest{ComputeHash(blob), blob.size()}; + auto digest = ArtifactDigest{ + HashFunction::ComputeBlobHash(blob).HexString(), blob.size()}; Logger::Log(LogLevel::Trace, [&]() { return fmt::format( "Uploaded blob {}, its digest has id {} and size {}.", diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 4f074eb4..ba8bcb7d 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -249,13 +249,10 @@ void SetupExecutionConfig(EndpointArguments const& eargs, } } -void SetupHashGenerator() { - if (Compatibility::IsCompatible()) { - HashGenerator::SetHashGenerator(HashGenerator::HashType::SHA256); - } - else { - HashGenerator::SetHashGenerator(HashGenerator::HashType::GIT); - } +void SetupHashFunction() { + HashFunction::SetHashType(Compatibility::IsCompatible() + ? HashFunction::JustHash::Compatible + : HashFunction::JustHash::Native); } #endif @@ -1324,7 +1321,7 @@ auto main(int argc, char* argv[]) -> int { SetupLogging(arguments.common); #ifndef BOOTSTRAP_BUILD_TOOL - SetupHashGenerator(); + SetupHashFunction(); SetupExecutionConfig( arguments.endpoint, arguments.build, arguments.rebuild); #endif |