From a8112042b57b7a1be8b37cce46d279fd0ff9e861 Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Mon, 18 Nov 2024 16:36:30 +0100 Subject: Rename MRApiUtils to RehashUtils --- src/buildtool/execution_api/serve/TARGETS | 29 +---- src/buildtool/execution_api/serve/mr_git_api.cpp | 16 +-- src/buildtool/execution_api/serve/mr_local_api.cpp | 14 +-- src/buildtool/execution_api/serve/utils.cpp | 118 --------------------- src/buildtool/execution_api/serve/utils.hpp | 64 ----------- src/buildtool/execution_api/utils/TARGETS | 19 ++++ src/buildtool/execution_api/utils/rehash_utils.cpp | 118 +++++++++++++++++++++ src/buildtool/execution_api/utils/rehash_utils.hpp | 64 +++++++++++ 8 files changed, 221 insertions(+), 221 deletions(-) delete mode 100644 src/buildtool/execution_api/serve/utils.cpp delete mode 100644 src/buildtool/execution_api/serve/utils.hpp create mode 100644 src/buildtool/execution_api/utils/rehash_utils.cpp create mode 100644 src/buildtool/execution_api/utils/rehash_utils.hpp (limited to 'src/buildtool/execution_api') diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS index f4d999b2..cfc23515 100644 --- a/src/buildtool/execution_api/serve/TARGETS +++ b/src/buildtool/execution_api/serve/TARGETS @@ -1,23 +1,4 @@ -{ "utils": - { "type": ["@", "rules", "CC", "library"] - , "name": ["utils"] - , "hdrs": ["utils.hpp"] - , "srcs": ["utils.cpp"] - , "deps": - [ ["src/buildtool/common", "common"] - , ["src/buildtool/file_system", "object_type"] - , ["src/buildtool/storage", "config"] - , ["src/utils/cpp", "expected"] - ] - , "stage": ["src", "buildtool", "execution_api", "serve"] - , "private-deps": - [ ["@", "fmt", "", "fmt"] - , ["src/buildtool/crypto", "hash_function"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["src/buildtool/storage", "fs_utils"] - ] - } -, "mr_git_api": +{ "mr_git_api": { "type": ["@", "rules", "CC", "library"] , "name": ["mr_git_api"] , "hdrs": ["mr_git_api.hpp"] @@ -34,9 +15,9 @@ ] , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": - [ "utils" - , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + [ ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] @@ -58,9 +39,9 @@ ] , "stage": ["src", "buildtool", "execution_api", "serve"] , "private-deps": - [ "utils" - , ["src/buildtool/common", "protocol_traits"] + [ ["src/buildtool/common", "protocol_traits"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + , ["src/buildtool/execution_api/utils", "rehash_utils"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] diff --git a/src/buildtool/execution_api/serve/mr_git_api.cpp b/src/buildtool/execution_api/serve/mr_git_api.cpp index 46bc4c83..6630e891 100644 --- a/src/buildtool/execution_api/serve/mr_git_api.cpp +++ b/src/buildtool/execution_api/serve/mr_git_api.cpp @@ -19,7 +19,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" -#include "src/buildtool/execution_api/serve/utils.hpp" +#include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -56,7 +56,7 @@ auto MRGitApi::RetrieveToCas( [native_sc = native_storage_config_, compat_sc = compat_storage_config_](ArtifactDigest const& digest) -> expected, std::string> { - return MRApiUtils::ReadRehashedDigest( + return RehashUtils::ReadRehashedDigest( digest, *native_sc, *compat_sc, /*from_git=*/true); }; auto store_rehashed = @@ -65,12 +65,12 @@ auto MRGitApi::RetrieveToCas( ArtifactDigest const& source_digest, ArtifactDigest const& target_digest, ObjectType obj_type) -> std::optional { - return MRApiUtils::StoreRehashedDigest(source_digest, - target_digest, - obj_type, - *native_sc, - *compat_sc, - /*from_git=*/true); + return RehashUtils::StoreRehashedDigest(source_digest, + target_digest, + obj_type, + *native_sc, + *compat_sc, + /*from_git=*/true); }; // collect the native blobs and rehash them as compatible to be able to diff --git a/src/buildtool/execution_api/serve/mr_local_api.cpp b/src/buildtool/execution_api/serve/mr_local_api.cpp index 7a8eb61a..55dddafc 100644 --- a/src/buildtool/execution_api/serve/mr_local_api.cpp +++ b/src/buildtool/execution_api/serve/mr_local_api.cpp @@ -19,7 +19,7 @@ #include "src/buildtool/common/protocol_traits.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" -#include "src/buildtool/execution_api/serve/utils.hpp" +#include "src/buildtool/execution_api/utils/rehash_utils.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -89,7 +89,7 @@ auto MRLocalApi::RetrieveToCas( compat_storage = compat_context_->storage_config]( ArtifactDigest const& digest) -> expected, std::string> { - return MRApiUtils::ReadRehashedDigest( + return RehashUtils::ReadRehashedDigest( digest, *native_storage, *compat_storage); }; auto store_rehashed = @@ -98,11 +98,11 @@ auto MRLocalApi::RetrieveToCas( ArtifactDigest const& source_digest, ArtifactDigest const& target_digest, ObjectType obj_type) -> std::optional { - return MRApiUtils::StoreRehashedDigest(source_digest, - target_digest, - obj_type, - *native_storage, - *compat_storage); + return RehashUtils::StoreRehashedDigest(source_digest, + target_digest, + obj_type, + *native_storage, + *compat_storage); }; // collect the native blobs and rehash them as compatible to be able to diff --git a/src/buildtool/execution_api/serve/utils.cpp b/src/buildtool/execution_api/serve/utils.cpp deleted file mode 100644 index 39c4f7d3..00000000 --- a/src/buildtool/execution_api/serve/utils.cpp +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/execution_api/serve/utils.hpp" - -#include // std::size_t -#include -#include -#include // std::move - -#include "fmt/core.h" -#include "src/buildtool/crypto/hash_function.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/storage/fs_utils.hpp" - -namespace MRApiUtils { - -auto ReadRehashedDigest(ArtifactDigest const& digest, - StorageConfig const& source_config, - StorageConfig const& target_config, - bool from_git) noexcept - -> expected, std::string> { - // check for mapping file in all generations - std::size_t generation = 0; - std::optional rehash_id_file = std::nullopt; - auto const compat_hash_type = target_config.hash_function.GetType(); - for (; generation < source_config.num_generations; ++generation) { - auto path = StorageUtils::GetRehashIDFile(source_config, - compat_hash_type, - digest.hash(), - from_git, - generation); - if (FileSystemManager::Exists(path)) { - rehash_id_file = std::move(path); - break; // found the generation - } - } - if (rehash_id_file) { - // read id file - auto compat_obj_str = FileSystemManager::ReadFile(*rehash_id_file); - if (not compat_obj_str) { - return unexpected{fmt::format("failed to read rehash id file {}", - rehash_id_file->string())}; - } - // get artifact object from content - auto compat_obj = - Artifact::ObjectInfo::FromString(compat_hash_type, *compat_obj_str); - if (not compat_obj) { - // handle nullopt value explicitly - return unexpected{ - fmt::format("failed to read rehashed artifact from id file {}", - rehash_id_file->string())}; - } - // ensure the id file is in generation 0 for future calls - if (generation != 0) { - auto dest_id_file = StorageUtils::GetRehashIDFile( - source_config, compat_hash_type, digest.hash(), from_git); - auto ok = FileSystemManager::CreateFileHardlink(*rehash_id_file, - dest_id_file); - if (not ok) { - auto const& err = ok.error(); - if (err != std::errc::too_many_links) { - return unexpected{ - fmt::format("failed to link rehash id file {}:\n{} {}", - dest_id_file.string(), - err.value(), - err.message())}; - } - // if too many links reported, write id file ourselves - if (not StorageUtils::WriteTreeIDFile(dest_id_file, - *compat_obj_str)) { - return unexpected{ - fmt::format("failed to write rehash id file {}", - dest_id_file.string())}; - } - } - } - return std::move(compat_obj); // not dereferenced to assist type - // deduction in variant - } - // no mapping file found - return std::optional{std::nullopt}; -} - -auto StoreRehashedDigest(ArtifactDigest const& source_digest, - ArtifactDigest const& target_digest, - ObjectType obj_type, - StorageConfig const& source_config, - StorageConfig const& target_config, - bool from_git) noexcept -> std::optional { - // write mapping - auto const rehash_id_file = - StorageUtils::GetRehashIDFile(source_config, - target_config.hash_function.GetType(), - source_digest.hash(), - from_git); - if (not StorageUtils::WriteTreeIDFile( - rehash_id_file, - Artifact::ObjectInfo{.digest = target_digest, .type = obj_type} - .ToString())) { - return fmt::format("failed to write rehash id to file {}", - rehash_id_file.string()); - } - return std::nullopt; // a-ok -} - -} // namespace MRApiUtils diff --git a/src/buildtool/execution_api/serve/utils.hpp b/src/buildtool/execution_api/serve/utils.hpp deleted file mode 100644 index 3b588cf3..00000000 --- a/src/buildtool/execution_api/serve/utils.hpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_UTILS_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_UTILS_HPP - -#include -#include - -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/common/artifact_digest.hpp" -#include "src/buildtool/file_system/object_type.hpp" -#include "src/buildtool/storage/config.hpp" -#include "src/utils/cpp/expected.hpp" - -namespace MRApiUtils { - -/// \brief Get a corresponding known object from a different local CAS, as -/// stored in a mapping file, if exists. -/// \param digest Source digest. -/// \param source_config Storage config corresponding to source digest. -/// \param target_config Storage config corresponding to target digest. -/// \param from_git Specify if source digest comes from a Git location instead -/// of CAS. -/// \returns The target artifact info on successfully reading an existing -/// mapping file, nullopt if no mapping file exists, or the error message on -/// failure. -[[nodiscard]] auto ReadRehashedDigest(ArtifactDigest const& digest, - StorageConfig const& source_config, - StorageConfig const& target_config, - bool from_git = false) noexcept - -> expected, std::string>; - -/// \brief Write the mapping file linking two digests hashing the same content. -/// \param source_digest Source digest. -/// \param target_digest Target digest. -/// \param obj_type Object type of the content represented by the two digests. -/// \param source_config Storage config corresponding to source digest. -/// \param target_config Storage config corresponding to target digest. -/// \param from_git Specify if source digest comes from a Git location instead -/// of CAS. -/// \returns nullopt on success, error message on failure. -[[nodiscard]] auto StoreRehashedDigest(ArtifactDigest const& source_digest, - ArtifactDigest const& target_digest, - ObjectType obj_type, - StorageConfig const& source_config, - StorageConfig const& target_config, - bool from_git = false) noexcept - -> std::optional; - -} // namespace MRApiUtils - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_UTILS_HPP diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS index bff656ec..afaabeb0 100644 --- a/src/buildtool/execution_api/utils/TARGETS +++ b/src/buildtool/execution_api/utils/TARGETS @@ -31,4 +31,23 @@ ] , "stage": ["src", "buildtool", "execution_api", "utils"] } +, "rehash_utils": + { "type": ["@", "rules", "CC", "library"] + , "name": ["utils"] + , "hdrs": ["rehash_utils.hpp"] + , "srcs": ["rehash_utils.cpp"] + , "deps": + [ ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/storage", "config"] + , ["src/utils/cpp", "expected"] + ] + , "stage": ["src", "buildtool", "execution_api", "utils"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/storage", "fs_utils"] + ] + } } diff --git a/src/buildtool/execution_api/utils/rehash_utils.cpp b/src/buildtool/execution_api/utils/rehash_utils.cpp new file mode 100644 index 00000000..660d8cec --- /dev/null +++ b/src/buildtool/execution_api/utils/rehash_utils.cpp @@ -0,0 +1,118 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/utils/rehash_utils.hpp" + +#include // std::size_t +#include +#include +#include // std::move + +#include "fmt/core.h" +#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/storage/fs_utils.hpp" + +namespace RehashUtils { + +auto ReadRehashedDigest(ArtifactDigest const& digest, + StorageConfig const& source_config, + StorageConfig const& target_config, + bool from_git) noexcept + -> expected, std::string> { + // check for mapping file in all generations + std::size_t generation = 0; + std::optional rehash_id_file = std::nullopt; + auto const compat_hash_type = target_config.hash_function.GetType(); + for (; generation < source_config.num_generations; ++generation) { + auto path = StorageUtils::GetRehashIDFile(source_config, + compat_hash_type, + digest.hash(), + from_git, + generation); + if (FileSystemManager::Exists(path)) { + rehash_id_file = std::move(path); + break; // found the generation + } + } + if (rehash_id_file) { + // read id file + auto compat_obj_str = FileSystemManager::ReadFile(*rehash_id_file); + if (not compat_obj_str) { + return unexpected{fmt::format("failed to read rehash id file {}", + rehash_id_file->string())}; + } + // get artifact object from content + auto compat_obj = + Artifact::ObjectInfo::FromString(compat_hash_type, *compat_obj_str); + if (not compat_obj) { + // handle nullopt value explicitly + return unexpected{ + fmt::format("failed to read rehashed artifact from id file {}", + rehash_id_file->string())}; + } + // ensure the id file is in generation 0 for future calls + if (generation != 0) { + auto dest_id_file = StorageUtils::GetRehashIDFile( + source_config, compat_hash_type, digest.hash(), from_git); + auto ok = FileSystemManager::CreateFileHardlink(*rehash_id_file, + dest_id_file); + if (not ok) { + auto const& err = ok.error(); + if (err != std::errc::too_many_links) { + return unexpected{ + fmt::format("failed to link rehash id file {}:\n{} {}", + dest_id_file.string(), + err.value(), + err.message())}; + } + // if too many links reported, write id file ourselves + if (not StorageUtils::WriteTreeIDFile(dest_id_file, + *compat_obj_str)) { + return unexpected{ + fmt::format("failed to write rehash id file {}", + dest_id_file.string())}; + } + } + } + return std::move(compat_obj); // not dereferenced to assist type + // deduction in variant + } + // no mapping file found + return std::optional{std::nullopt}; +} + +auto StoreRehashedDigest(ArtifactDigest const& source_digest, + ArtifactDigest const& target_digest, + ObjectType obj_type, + StorageConfig const& source_config, + StorageConfig const& target_config, + bool from_git) noexcept -> std::optional { + // write mapping + auto const rehash_id_file = + StorageUtils::GetRehashIDFile(source_config, + target_config.hash_function.GetType(), + source_digest.hash(), + from_git); + if (not StorageUtils::WriteTreeIDFile( + rehash_id_file, + Artifact::ObjectInfo{.digest = target_digest, .type = obj_type} + .ToString())) { + return fmt::format("failed to write rehash id to file {}", + rehash_id_file.string()); + } + return std::nullopt; // a-ok +} + +} // namespace RehashUtils diff --git a/src/buildtool/execution_api/utils/rehash_utils.hpp b/src/buildtool/execution_api/utils/rehash_utils.hpp new file mode 100644 index 00000000..7ed7cb11 --- /dev/null +++ b/src/buildtool/execution_api/utils/rehash_utils.hpp @@ -0,0 +1,64 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP + +#include +#include + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/utils/cpp/expected.hpp" + +namespace RehashUtils { + +/// \brief Get a corresponding known object from a different local CAS, as +/// stored in a mapping file, if exists. +/// \param digest Source digest. +/// \param source_config Storage config corresponding to source digest. +/// \param target_config Storage config corresponding to target digest. +/// \param from_git Specify if source digest comes from a Git location instead +/// of CAS. +/// \returns The target artifact info on successfully reading an existing +/// mapping file, nullopt if no mapping file exists, or the error message on +/// failure. +[[nodiscard]] auto ReadRehashedDigest(ArtifactDigest const& digest, + StorageConfig const& source_config, + StorageConfig const& target_config, + bool from_git = false) noexcept + -> expected, std::string>; + +/// \brief Write the mapping file linking two digests hashing the same content. +/// \param source_digest Source digest. +/// \param target_digest Target digest. +/// \param obj_type Object type of the content represented by the two digests. +/// \param source_config Storage config corresponding to source digest. +/// \param target_config Storage config corresponding to target digest. +/// \param from_git Specify if source digest comes from a Git location instead +/// of CAS. +/// \returns nullopt on success, error message on failure. +[[nodiscard]] auto StoreRehashedDigest(ArtifactDigest const& source_digest, + ArtifactDigest const& target_digest, + ObjectType obj_type, + StorageConfig const& source_config, + StorageConfig const& target_config, + bool from_git = false) noexcept + -> std::optional; + +} // namespace RehashUtils + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_REHASH_UTILS_HPP -- cgit v1.2.3