From 93b92ef2669bed7345eadabbf71ee30c4b7929af Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 11 Dec 2023 12:25:55 +0100 Subject: serve target: Move server-side helper methods in own library --- .../serve_api/serve_service/target_utils.cpp | 283 +++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 src/buildtool/serve_api/serve_service/target_utils.cpp (limited to 'src/buildtool/serve_api/serve_service/target_utils.cpp') diff --git a/src/buildtool/serve_api/serve_service/target_utils.cpp b/src/buildtool/serve_api/serve_service/target_utils.cpp new file mode 100644 index 00000000..c9268f63 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/target_utils.cpp @@ -0,0 +1,283 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/serve_service/target_utils.hpp" + +#include +#include +#include + +#include "fmt/core.h" +#include "nlohmann/json.hpp" +#include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/file_system/git_cas.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" + +auto IsTreeInRepo(std::string const& tree_id, + std::filesystem::path const& repo_path, + std::shared_ptr const& logger) -> bool { + if (auto git_cas = GitCAS::Open(repo_path)) { + if (auto repo = GitRepo::Open(git_cas)) { + // wrap logger for GitRepo call + auto wrapped_logger = std::make_shared( + [logger, repo_path, tree_id](auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeTarget: While checking existence of tree {} " + "in repository {}:\n{}", + tree_id, + repo_path.string(), + msg); + logger->Emit(LogLevel::Info, err); + } + }); + if (auto res = repo->CheckTreeExists(tree_id, wrapped_logger)) { + return *res; + } + } + } + return false; // tree not found +} + +auto GetServingRepository(std::string const& tree_id, + std::shared_ptr const& logger) + -> std::optional { + // try the Git cache repository + if (IsTreeInRepo(tree_id, StorageConfig::GitRoot(), logger)) { + return StorageConfig::GitRoot(); + } + // check the known repositories + for (auto const& path : RemoteServeConfig::KnownRepositories()) { + if (IsTreeInRepo(tree_id, path, logger)) { + return path; + } + } + return std::nullopt; // tree cannot be served +} + +auto DetermineRoots(std::string const& main_repo, + std::filesystem::path const& repo_config_path, + gsl::not_null const& repository_config, + std::shared_ptr const& logger) + -> std::optional { + // parse repository configuration file + auto repos = nlohmann::json::object(); + try { + std::ifstream fs(repo_config_path); + repos = nlohmann::json::parse(fs); + if (not repos.is_object()) { + return fmt::format( + "Repository configuration file {} does not contain a map.", + repo_config_path.string()); + } + } catch (std::exception const& ex) { + return fmt::format("Parsing repository config file {} failed with:\n{}", + repo_config_path.string(), + ex.what()); + } + if (not repos.contains(main_repo)) { + return fmt::format( + "Repository configuration does not contain expected main " + "repository {}", + main_repo); + } + // populate RepositoryConfig instance + std::string error_msg; + for (auto const& [repo, desc] : repos.items()) { + // root parser + auto parse_keyword_root = + [&desc = desc, &repo = repo, &error_msg = error_msg, logger]( + std::string const& keyword) -> std::optional { + auto it = desc.find(keyword); + if (it != desc.end()) { + if (auto parsed_root = + FileRoot::ParseRoot(repo, keyword, *it, &error_msg)) { + // check that root has absent-like format + if (not parsed_root->first.IsAbsent()) { + error_msg = fmt::format( + "Expected {} to have absent Git tree format, but " + "found {}", + keyword, + it->dump()); + return std::nullopt; + } + // find the serving repository for the root tree + auto tree_id = *parsed_root->first.GetAbsentTreeId(); + auto repo_path = GetServingRepository(tree_id, logger); + if (not repo_path) { + error_msg = fmt::format( + "{} tree {} is not known", keyword, tree_id); + return std::nullopt; + } + // set the root as present + if (auto root = FileRoot::FromGit( + *repo_path, + tree_id, + parsed_root->first.IgnoreSpecial())) { + return root; + } + } + error_msg = + fmt::format("Failed to parse {} {}", keyword, it->dump()); + return std::nullopt; + } + error_msg = + fmt::format("Missing {} for repository {}", keyword, repo); + return std::nullopt; + }; + + std::optional ws_root = parse_keyword_root("workspace_root"); + if (not ws_root) { + return error_msg; + } + auto info = RepositoryConfig::RepositoryInfo{std::move(*ws_root)}; + + if (auto target_root = parse_keyword_root("target_root")) { + info.target_root = std::move(*target_root); + } + else { + return error_msg; + } + + if (auto rule_root = parse_keyword_root("rule_root")) { + info.rule_root = std::move(*rule_root); + } + else { + return error_msg; + } + + if (auto expression_root = parse_keyword_root("expression_root")) { + info.expression_root = std::move(*expression_root); + } + else { + return error_msg; + } + + auto it_bindings = desc.find("bindings"); + if (it_bindings != desc.end()) { + if (not it_bindings->is_object()) { + return fmt::format( + "bindings has to be a string-string map, but found {}", + it_bindings->dump()); + } + for (auto const& [local_name, global_name] : it_bindings->items()) { + if (not repos.contains(global_name)) { + return fmt::format( + "Binding {} for {} in {} does not refer to a " + "defined repository.", + global_name, + local_name, + repo); + } + info.name_mapping[local_name] = global_name; + } + } + else { + return fmt::format("Missing bindings for repository {}", repo); + } + + auto parse_keyword_file_name = + [&desc = desc, &repo = repo, &error_msg = error_msg]( + std::string* keyword_file_name, + std::string const& keyword) -> bool { + auto it = desc.find(keyword); + if (it != desc.end()) { + *keyword_file_name = *it; + return true; + } + error_msg = + fmt::format("Missing {} for repository {}", keyword, repo); + return false; + }; + + if (not parse_keyword_file_name(&info.target_file_name, + "target_file_name")) { + return error_msg; + } + + if (not parse_keyword_file_name(&info.rule_file_name, + "rule_file_name")) { + return error_msg; + } + + if (not parse_keyword_file_name(&info.expression_file_name, + "expression_file_name")) { + return error_msg; + } + + repository_config->SetInfo(repo, std::move(info)); + } + // success + return std::nullopt; +} + +auto GetBlobContent(std::filesystem::path const& repo_path, + std::string const& tree_id, + std::string const& rel_path, + std::shared_ptr const& logger) + -> std::optional>> { + if (auto git_cas = GitCAS::Open(repo_path)) { + if (auto repo = GitRepo::Open(git_cas)) { + // check if tree exists + auto wrapped_logger = std::make_shared( + [logger, repo_path, tree_id](auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeTargetVariables: While checking if tree {} " + "exists in repository {}:\n{}", + tree_id, + repo_path.string(), + msg); + logger->Emit(LogLevel::Debug, err); + } + }); + if (repo->CheckTreeExists(tree_id, wrapped_logger) == true) { + // get tree entry by path + if (auto entry_info = + repo->GetObjectByPathFromTree(tree_id, rel_path)) { + wrapped_logger = std::make_shared( + [logger, repo_path, blob_id = entry_info->id]( + auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeTargetVariables: While retrieving " + "blob {} from repository {}:\n{}", + blob_id, + repo_path.string(), + msg); + logger->Emit(LogLevel::Debug, err); + } + }); + // get blob content + return repo->TryReadBlob(entry_info->id, wrapped_logger); + } + // trace failure to get entry + auto err = fmt::format( + "ServeTargetVariables: Failed to retrieve entry {} in " + "tree {} from repository {}", + rel_path, + tree_id, + repo_path.string()); + logger->Emit(LogLevel::Debug, err); + return std::pair(false, std::nullopt); // could not read blob + } + } + } + return std::nullopt; // tree not found or errors while retrieving tree +} -- cgit v1.2.3