diff options
| -rw-r--r-- | src/buildtool/execution_api/serve/TARGETS | 27 | ||||
| -rw-r--r-- | src/buildtool/execution_api/serve/mr_git_api.cpp | 178 | ||||
| -rw-r--r-- | src/buildtool/execution_api/serve/mr_git_api.hpp | 143 |
3 files changed, 348 insertions, 0 deletions
diff --git a/src/buildtool/execution_api/serve/TARGETS b/src/buildtool/execution_api/serve/TARGETS index 087c32a7..d54b01d9 100644 --- a/src/buildtool/execution_api/serve/TARGETS +++ b/src/buildtool/execution_api/serve/TARGETS @@ -16,4 +16,31 @@ , ["src/buildtool/storage", "fs_utils"] ] } +, "mr_git_api": + { "type": ["@", "rules", "CC", "library"] + , "name": ["mr_git_api"] + , "hdrs": ["mr_git_api.hpp"] + , "srcs": ["mr_git_api.cpp"] + , "deps": + [ ["@", "gsl", "", "gsl"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/common", "config"] + , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/execution_api/common", "artifact_blob_container"] + , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/execution_engine/dag", "dag"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] + ] + , "stage": ["src", "buildtool", "execution_api", "serve"] + , "private-deps": + [ "utils" + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + , ["src/buildtool/execution_api/git", "git"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/logging", "log_level"] + , ["src/buildtool/logging", "logging"] + , ["src/utils/cpp", "expected"] + ] + } } diff --git a/src/buildtool/execution_api/serve/mr_git_api.cpp b/src/buildtool/execution_api/serve/mr_git_api.cpp new file mode 100644 index 00000000..1e433fff --- /dev/null +++ b/src/buildtool/execution_api/serve/mr_git_api.cpp @@ -0,0 +1,178 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/serve/mr_git_api.hpp" + +#include <utility> + +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/git/git_api.hpp" +#include "src/buildtool/execution_api/serve/utils.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/expected.hpp" + +MRGitApi::MRGitApi( + gsl::not_null<RepositoryConfig const*> const& repo_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compatible_storage_config, + Storage const* compatible_storage, + IExecutionApi const* compatible_local_api) noexcept + : repo_config_{repo_config}, + native_storage_config_{native_storage_config}, + compat_storage_config_{compatible_storage_config}, + compat_storage_{compatible_storage}, + compat_local_api_{compatible_local_api} {} + +auto MRGitApi::RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + IExecutionApi const& api) const noexcept -> bool { + // Return immediately if target CAS is this CAS + if (this == &api) { + return true; + } + + // in native mode: dispatch to regular GitApi + if (compat_storage_config_ == nullptr) { + GitApi const git_api{repo_config_}; + return git_api.RetrieveToCas(artifacts_info, api); + } + + // in compatible mode: set up needed callbacks for caching digest mappings + auto read_rehashed = + [native_sc = native_storage_config_, + compat_sc = compat_storage_config_](ArtifactDigest const& digest) + -> expected<std::optional<Artifact::ObjectInfo>, std::string> { + return MRApiUtils::ReadRehashedDigest( + digest, *native_sc, *compat_sc, /*from_git=*/true); + }; + auto store_rehashed = + [native_sc = native_storage_config_, + compat_sc = compat_storage_config_]( + ArtifactDigest const& source_digest, + ArtifactDigest const& target_digest, + ObjectType obj_type) -> std::optional<std::string> { + return MRApiUtils::StoreRehashedDigest(source_digest, + target_digest, + obj_type, + *native_sc, + *compat_sc, + /*from_git=*/true); + }; + + // collect the native blobs and rehash them as compatible to be able to + // check what is missing in the other api + std::vector<Artifact::ObjectInfo> compat_artifacts; + compat_artifacts.reserve(artifacts_info.size()); + for (auto const& native_obj : artifacts_info) { + // check if we know already the compatible digest + auto cached_obj = read_rehashed(native_obj.digest); + if (not cached_obj) { + Logger::Log( + LogLevel::Error, "MRGitApi: {}", std::move(cached_obj).error()); + return false; + } + if (*cached_obj) { + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back(std::move(cached_obj)->value()); + } + else { + // process object; trees need to be handled appropriately + if (IsTreeObject(native_obj.type)) { + // set up all the callbacks needed + auto read_git = [repo_config = repo_config_]( + ArtifactDigest const& digest, + ObjectType /*type*/) + -> std::optional< + std::variant<std::filesystem::path, std::string>> { + return repo_config->ReadBlobFromGitCAS(digest.hash()); + }; + auto store_file = + [cas = &compat_storage_->CAS()]( + std::variant<std::filesystem::path, std::string> const& + data, + bool is_exec) -> std::optional<ArtifactDigest> { + if (not std::holds_alternative<std::string>(data)) { + return std::nullopt; + } + return cas->StoreBlob(std::get<std::string>(data), is_exec); + }; + BazelMsgFactory::TreeStoreFunc store_dir = + [cas = &compat_storage_->CAS()](std::string const& content) + -> std::optional<ArtifactDigest> { + return cas->StoreTree(content); + }; + BazelMsgFactory::SymlinkStoreFunc store_symlink = + [cas = &compat_storage_->CAS()](std::string const& content) + -> std::optional<ArtifactDigest> { + return cas->StoreBlob(content); + }; + // get the directory digest + auto tree_digest = + BazelMsgFactory::CreateDirectoryDigestFromGitTree( + native_obj.digest, + read_git, + store_file, + store_dir, + store_symlink, + read_rehashed, + store_rehashed); + if (not tree_digest) { + Logger::Log(LogLevel::Error, + "MRGitApi: {}", + std::move(tree_digest).error()); + return false; + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back( + Artifact::ObjectInfo{.digest = *std::move(tree_digest), + .type = ObjectType::Tree}); + } + else { + // blobs are read from repo and added to compatible CAS + auto const blob_content = + repo_config_->ReadBlobFromGitCAS(native_obj.digest.hash()); + if (not blob_content) { + Logger::Log(LogLevel::Error, + "MRGitApi: failed reading Git entry {}", + native_obj.digest.hash()); + return false; + } + auto blob_digest = compat_storage_->CAS().StoreBlob( + *blob_content, IsExecutableObject(native_obj.type)); + if (not blob_digest) { + Logger::Log(LogLevel::Error, + "MRGitApi: failed to rehash Git entry {}", + native_obj.digest.hash()); + return false; + } + // cache the digest association + if (auto error_msg = store_rehashed( + native_obj.digest, *blob_digest, native_obj.type)) { + Logger::Log( + LogLevel::Error, "MRGitApi: {}", *std::move(error_msg)); + return false; + } + // add object to the vector of compatible artifacts + compat_artifacts.emplace_back( + Artifact::ObjectInfo{.digest = *std::move(blob_digest), + .type = native_obj.type}); + } + } + } + // now that we have gathered all the compatible object infos, simply pass + // them to a local api that can interact with the remote + return compat_local_api_->RetrieveToCas(compat_artifacts, api); +} diff --git a/src/buildtool/execution_api/serve/mr_git_api.hpp b/src/buildtool/execution_api/serve/mr_git_api.hpp new file mode 100644 index 00000000..74864218 --- /dev/null +++ b/src/buildtool/execution_api/serve/mr_git_api.hpp @@ -0,0 +1,143 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP + +#include <filesystem> +#include <map> +#include <optional> +#include <string> +#include <vector> + +#include "gsl/gsl" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/common/repository_config.hpp" +#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/common/artifact_blob_container.hpp" +#include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/execution_engine/dag/dag.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" + +/// \brief Multi-repo-specific implementation of the abstract Execution API. +/// Handles interaction between the Git CAS and another api, irrespective of the +/// remote-execution protocol used. This instance cannot create actions or store +/// anything to the Git CAS, but has access to local storages. +class MRGitApi final : public IExecutionApi { + public: + MRGitApi(gsl::not_null<RepositoryConfig const*> const& repo_config, + gsl::not_null<StorageConfig const*> const& native_storage_config, + StorageConfig const* compatible_storage_config = nullptr, + Storage const* compatible_storage = nullptr, + IExecutionApi const* compatible_local_api = nullptr) noexcept; + + /// \brief Not supported. + [[nodiscard]] auto CreateAction( + ArtifactDigest const& /*root_digest*/, + std::vector<std::string> const& /*command*/, + std::string const& /*cwd*/, + std::vector<std::string> const& /*output_files*/, + std::vector<std::string> const& /*output_dirs*/, + std::map<std::string, std::string> const& /*env_vars*/, + std::map<std::string, std::string> const& /*properties*/) const noexcept + -> IExecutionAction::Ptr final { + // Execution not supported. + return nullptr; + } + + /// \brief Not supported. + // NOLINTNEXTLINE(google-default-arguments) + [[nodiscard]] auto RetrieveToPaths( + std::vector<Artifact::ObjectInfo> const& /*artifacts_info*/, + std::vector<std::filesystem::path> const& /*output_paths*/, + IExecutionApi const* /*alternative*/ = nullptr) const noexcept + -> bool final { + // Retrieval to paths not suported. + return false; + } + + /// \brief Not supported. + [[nodiscard]] auto RetrieveToFds( + std::vector<Artifact::ObjectInfo> const& /*artifacts_info*/, + std::vector<int> const& /*fds*/, + bool /*raw_tree*/) const noexcept -> bool final { + // Retrieval to file descriptors not supported. + return false; + } + + /// \brief Passes artifacts from Git CAS to specified (remote) api. In + /// compatible mode, it must rehash the native digests to be able to upload + /// to a compatible remote. Expects native digests. + /// \note Caller is responsible for passing vectors with artifacts of the + /// same digest type. + [[nodiscard]] auto RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + IExecutionApi const& api) const noexcept -> bool final; + + /// \brief Not supported. + [[nodiscard]] auto RetrieveToMemory( + Artifact::ObjectInfo const& /*artifact_info*/) const noexcept + -> std::optional<std::string> final { + // Retrieval to memory not supported. + return std::nullopt; + } + + /// \brief Not supported. + // NOLINTNEXTLINE(google-default-arguments) + [[nodiscard]] auto Upload(ArtifactBlobContainer&& /*blobs*/, + bool /*skip_find_missing*/ = false) const noexcept + -> bool final { + // Upload not suppoorted. + return false; + } + + /// \brief Not supported. + [[nodiscard]] auto UploadTree( + std::vector<DependencyGraph::NamedArtifactNodePtr> const& /*artifacts*/) + const noexcept -> std::optional<ArtifactDigest> final { + // Upload tree not supported. + return std::nullopt; + } + + /// \brief Not supported. + [[nodiscard]] auto IsAvailable( + ArtifactDigest const& /*digest*/) const noexcept -> bool final { + // Not supported. + return false; + } + + /// \brief Not implemented. + [[nodiscard]] auto IsAvailable( + std::vector<ArtifactDigest> const& /*digests*/) const noexcept + -> std::vector<ArtifactDigest> final { + // Not supported. + return {}; + } + + private: + gsl::not_null<const RepositoryConfig*> repo_config_; + + // retain references to needed storages and configs + gsl::not_null<StorageConfig const*> native_storage_config_; + StorageConfig const* compat_storage_config_; + Storage const* compat_storage_; + + // an api accessing compatible storage, used purely to communicate with a + // compatible remote; only instantiated if in compatible mode + IExecutionApi const* compat_local_api_; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_SERVE_MR_GIT_API_HPP |