// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/setup_utils.hpp" #include #include #include #include #include #include #include #include #include #include #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/precomputed_root.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/utils/parse_precomputed_root.hpp" #include "src/utils/cpp/expected.hpp" namespace { void WarnUnknownKeys(std::string const& name, ExpressionPtr const& repo_def) { if (not repo_def->IsMap()) { return; } for (auto const& [key, value] : repo_def->Map()) { if (not kRepositoryExpectedFields.contains(key)) { Logger::Log(std::any_of(kRepositoryPossibleFieldTrunks.begin(), kRepositoryPossibleFieldTrunks.end(), [k = key](auto const& trunk) { return k.find(trunk) != std::string::npos; }) ? LogLevel::Debug : LogLevel::Warning, "Ignoring unknown field {} in repository {}", key, name); } } } [[nodiscard]] auto GetTargetRepoIfPrecomputed(ExpressionPtr const& repos, std::string const& name) -> std::optional { // Resolve indirections while the root's workspace root is declared // implicitly: ExpressionPtr root{name}; while (root.IsNotNull() and root->IsString()) { auto const repo = repos->Get(root->String(), Expression::none_t{}); if (not repo.IsNotNull() or not repo->IsMap()) { return std::nullopt; } root = repo->Get("repository", Expression::none_t{}); } // Check the root is a precomputed root: if (auto const precomputed = ParsePrecomputedRoot(root)) { return precomputed->GetReferencedRepository(); } return std::nullopt; } [[nodiscard]] auto IsAbsent(ExpressionPtr const& repo_def) -> bool { if (repo_def.IsNotNull() and repo_def->IsMap()) { if (auto repo = repo_def->Get("repository", Expression::none_t{}); repo.IsNotNull() and repo->IsMap()) { if (auto pragma = repo->Get("pragma", Expression::none_t{}); pragma.IsNotNull() and pragma->IsMap()) { auto absent = pragma->Get("absent", Expression::none_t{}); return absent.IsNotNull() and absent->IsBool() and absent->Bool(); } } } return false; } [[nodiscard]] auto IsNotContentFixed(ExpressionPtr const& repo_def) -> bool { if (not repo_def.IsNotNull() or not repo_def->IsMap()) { return false; } if (auto repo = repo_def->Get("repository", Expression::none_t{}); repo.IsNotNull() and repo->IsMap()) { // Check if type == "file" auto type = repo->Get("type", Expression::none_t{}); if (not type.IsNotNull() or not type->IsString()) { return false; } if (type->String() == "file") { auto pragma = repo->Get("pragma", Expression::none_t{}); if (not pragma.IsNotNull() or not pragma->IsMap()) { return true; // not content-fixed if not to_git } // Check for explicit to_git == true if (auto to_git = pragma->Get("to_git", Expression::none_t{}); to_git.IsNotNull() and to_git->IsBool() and to_git->Bool()) { return false; } // Check for implicit to_git == true if (auto special = pragma->Get("special", Expression::none_t{}); special.IsNotNull() and special->IsString()) { auto const& special_str = special->String(); if (special_str == "resolve-partially" or special_str == "resolve-completely") { return false; } } return true; // not content-fixed if not to_git } } return false; } } // namespace namespace JustMR::Utils { void ReachableRepositories( ExpressionPtr const& repos, std::string const& main, std::shared_ptr const& setup_repos) { // use temporary sets to avoid duplicates std::unordered_set include_repos_set; std::unordered_set setup_repos_set; bool absent_main = IsAbsent(repos->Get(main, Expression::none_t{})); // traverse all bindings of main repository for (std::queue to_process({main}); not to_process.empty(); to_process.pop()) { auto const& repo_name = to_process.front(); // Check the repo hasn't been processed yet if (not include_repos_set.insert(repo_name).second) { continue; } auto const repos_repo_name = repos->Get(repo_name, Expression::none_t{}); if (not repos_repo_name.IsNotNull()) { continue; } WarnUnknownKeys(repo_name, repos_repo_name); // Warn if main repo is marked absent and current repo (including main) // is not content-fixed if (absent_main and IsNotContentFixed(repos_repo_name)) { Logger::Log(LogLevel::Warning, "Found non-content-fixed repository {} as dependency " "of absent main repository {}", nlohmann::json(repo_name).dump(), nlohmann::json(main).dump()); } // If the current repo is a computed one, process its target repo if (auto precomputed = GetTargetRepoIfPrecomputed(repos, repo_name)) { to_process.push(*std::move(precomputed)); } // check bindings auto const bindings = repos_repo_name->Get("bindings", Expression::none_t{}); if (bindings.IsNotNull() and bindings->IsMap()) { for (auto const& bound : bindings->Map().Values()) { if (bound.IsNotNull() and bound->IsString()) { to_process.push(bound->String()); } } } for (auto const& layer : kAltDirs) { auto const layer_val = repos_repo_name->Get(layer, Expression::none_t{}); if (layer_val.IsNotNull() and layer_val->IsString()) { auto const layer_repo_name = layer_val->String(); setup_repos_set.insert(layer_repo_name); // If the overlay repo is a computed one, process its target // repo if (auto precomputed = GetTargetRepoIfPrecomputed(repos, layer_repo_name)) { to_process.push(*std::move(precomputed)); } } } } setup_repos_set.insert(include_repos_set.begin(), include_repos_set.end()); // copy to vectors setup_repos->to_setup.clear(); setup_repos->to_setup.reserve(setup_repos_set.size()); std::copy( setup_repos_set.begin(), setup_repos_set.end(), std::inserter(setup_repos->to_setup, setup_repos->to_setup.end())); setup_repos->to_include.clear(); setup_repos->to_include.reserve(include_repos_set.size()); std::copy( include_repos_set.begin(), include_repos_set.end(), std::inserter(setup_repos->to_include, setup_repos->to_include.end())); } void DefaultReachableRepositories( ExpressionPtr const& repos, std::shared_ptr const& setup_repos) { setup_repos->to_setup = repos->Map().Keys(); setup_repos->to_include = setup_repos->to_setup; } auto ReadConfiguration( std::optional const& config_file_opt, std::optional const& absent_file_opt) -> std::shared_ptr { if (not config_file_opt) { Logger::Log(LogLevel::Error, "Cannot find repository configuration."); std::exit(kExitConfigError); } auto const& config_file = *config_file_opt; auto config = nlohmann::json::object(); if (not FileSystemManager::IsFile(config_file)) { Logger::Log(LogLevel::Error, "Cannot read config file {}.", config_file.string()); std::exit(kExitConfigError); } try { std::ifstream fs(config_file); config = nlohmann::json::parse(fs); if (not config.is_object()) { Logger::Log(LogLevel::Error, "Config file {} does not contain a JSON object.", config_file.string()); std::exit(kExitConfigError); } } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing config file {} failed with error:\n{}", config_file.string(), e.what()); std::exit(kExitConfigError); } if (absent_file_opt) { if (not FileSystemManager::IsFile(*absent_file_opt)) { Logger::Log(LogLevel::Error, "Not file specifying the absent repositories: {}", absent_file_opt->string()); std::exit(kExitConfigError); } try { std::ifstream fs(*absent_file_opt); auto absent = nlohmann::json::parse(fs); if (not absent.is_array()) { Logger::Log(LogLevel::Error, "Expected {} to contain a list of repository " "names, but found {}", absent_file_opt->string(), absent.dump()); std::exit(kExitConfigError); } std::unordered_set absent_set{}; for (auto const& repo : absent) { if (not repo.is_string()) { Logger::Log(LogLevel::Error, "Repositories names have to be strings, but " "found entry {} in {}", repo.dump(), absent_file_opt->string()); std::exit(kExitConfigError); } absent_set.insert(repo.get()); } auto new_repos = nlohmann::json::object(); auto repos = config.value("repositories", nlohmann::json::object()); for (auto const& [key, val] : repos.items()) { new_repos[key] = val; auto ws = val.value("repository", nlohmann::json::object()); if (ws.is_object()) { auto pragma = ws.value("pragma", nlohmann::json::object()); pragma["absent"] = absent_set.contains(key); ws["pragma"] = pragma; new_repos[key]["repository"] = ws; } } config["repositories"] = new_repos; } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing absent-repos file {} failed with error:\n{}", absent_file_opt->string(), e.what()); std::exit(kExitConfigError); } } try { return std::make_shared(Expression::FromJson(config)); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing configuration file failed with error:\n{}", e.what()); std::exit(kExitConfigError); } } auto CreateAuthConfig(MultiRepoRemoteAuthArguments const& authargs) noexcept -> std::optional { Auth::TLS::Builder tls_builder; tls_builder.SetCACertificate(authargs.tls_ca_cert) .SetClientCertificate(authargs.tls_client_cert) .SetClientKey(authargs.tls_client_key); // create auth config (including validation) auto result = tls_builder.Build(); if (result) { if (*result) { // correctly configured TLS/SSL certification return *std::move(*result); } Logger::Log(LogLevel::Error, result->error()); return std::nullopt; } // no TLS/SSL configuration was given, and we currently support no other // certification method, so return an empty config (no certification) return Auth{}; } auto CreateLocalExecutionConfig(MultiRepoCommonArguments const& cargs) noexcept -> std::optional { LocalExecutionConfig::Builder builder; if (cargs.local_launcher.has_value()) { builder.SetLauncher(*cargs.local_launcher); } auto config = builder.Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } auto CreateRemoteExecutionConfig( std::optional const& remote_exec_addr, std::optional const& remote_serve_addr) noexcept -> std::optional { // if only a serve endpoint address is given, we assume it is one that acts // also as remote-execution auto remote_addr = remote_exec_addr ? remote_exec_addr : remote_serve_addr; RemoteExecutionConfig::Builder builder; auto config = builder.SetRemoteAddress(remote_addr).Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } auto CreateServeConfig( std::optional const& remote_serve_addr) noexcept -> std::optional { RemoteServeConfig::Builder builder; auto config = builder.SetRemoteAddress(remote_serve_addr).Build(); if (config) { return *std::move(config); } Logger::Log(LogLevel::Error, config.error()); return std::nullopt; } } // namespace JustMR::Utils