// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/other_tools/just_mr/launch.hpp" #ifdef __unix__ #include #else #error "Non-unix is not supported yet" #endif #include #include // for errno #include #include // for strerror() #include #include #include #include #include #include #include #include #include #include #include #include "fmt/chrono.h" #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/expression/expression_ptr.hpp" #include "src/buildtool/common/clidefaults.hpp" #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/execution_api/common/ids.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #include "src/buildtool/storage/repository_garbage_collector.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/setup.hpp" #include "src/other_tools/just_mr/setup_utils.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/file_locking.hpp" #include "src/utils/cpp/path.hpp" auto CallJust(std::optional const& config_file, InvocationLogArguments const& invocation_log, MultiRepoCommonArguments const& common_args, MultiRepoSetupArguments const& setup_args, MultiRepoJustSubCmdsArguments const& just_cmd_args, MultiRepoLogArguments const& log_args, MultiRepoRemoteAuthArguments const& auth_args, RetryArguments const& retry_args, ForwardOnlyArguments const& launch_fwd, StorageConfig const& storage_config, Storage const& storage, bool forward_build_root, std::string const& multi_repo_tool_name) -> int { // check if subcmd_name can be taken from additional args auto additional_args_offset = 0U; auto subcommand = just_cmd_args.subcmd_name; if (not subcommand and not just_cmd_args.additional_just_args.empty()) { subcommand = just_cmd_args.additional_just_args[0]; additional_args_offset++; } bool use_config{false}; bool use_build_root{false}; bool use_launcher{false}; bool supports_defines{false}; bool supports_remote{false}; bool supports_remote_properties{false}; bool supports_serve{false}; bool supports_dispatch{false}; bool does_build{false}; std::optional> mr_config_pair{ std::nullopt}; // If gc locks are needed, ensure to keep them alive also for the exec call std::optional lock{}; std::optional repo_lock{}; if (subcommand and kKnownJustSubcommands.contains(*subcommand)) { auto const& flags = kKnownJustSubcommands.at(*subcommand); // Read the config file if needed if (flags.config) { repo_lock = RepositoryGarbageCollector::SharedLock(storage_config); if (not repo_lock) { return kExitGenericFailure; } lock = GarbageCollector::SharedLock(storage_config); if (not lock) { return kExitGenericFailure; } auto config = JustMR::Utils::ReadConfiguration( config_file, common_args.absent_repository_file); use_config = true; mr_config_pair = MultiRepoSetup(config, common_args, setup_args, just_cmd_args, auth_args, retry_args, storage_config, storage, /*interactive=*/false, multi_repo_tool_name); if (not mr_config_pair) { Logger::Log(LogLevel::Error, "Failed to setup config for calling \"{} {}\"", common_args.just_path ? common_args.just_path->string() : kDefaultJustPath, *subcommand); return kExitSetupError; } } use_build_root = flags.build_root; use_launcher = flags.launch; supports_defines = flags.defines; supports_remote = flags.remote; supports_remote_properties = flags.remote_props; supports_serve = flags.serve; supports_dispatch = flags.dispatch; does_build = flags.does_build; } // build just command std::vector cmd = {common_args.just_path->string()}; if (subcommand) { cmd.emplace_back(*subcommand); } if (use_config) { cmd.emplace_back("-C"); cmd.emplace_back(mr_config_pair->first.string()); } if (use_build_root and forward_build_root) { cmd.emplace_back("--local-build-root"); cmd.emplace_back(*common_args.just_mr_paths->root); } if (use_launcher and common_args.local_launcher and (common_args.local_launcher != kDefaultLauncher)) { cmd.emplace_back("--local-launcher"); cmd.emplace_back(nlohmann::json(*common_args.local_launcher).dump()); } // forward logging arguments if (not log_args.log_files.empty()) { cmd.emplace_back("--log-append"); for (auto const& log_file : log_args.log_files) { cmd.emplace_back("-f"); cmd.emplace_back(log_file.string()); } } if (log_args.log_limit and *log_args.log_limit != kDefaultLogLevel) { cmd.emplace_back("--log-limit"); cmd.emplace_back( std::to_string(static_cast>( *log_args.log_limit))); } if (log_args.restrict_stderr_log_limit) { cmd.emplace_back("--restrict-stderr-log-limit"); cmd.emplace_back( std::to_string(static_cast>( *log_args.restrict_stderr_log_limit))); } if (log_args.plain_log) { cmd.emplace_back("--plain-log"); } if (supports_defines) { if (just_cmd_args.config) { cmd.emplace_back("-c"); cmd.emplace_back(just_cmd_args.config->string()); } auto overlay_config = Configuration(); for (auto const& s : common_args.defines) { try { auto map = Expression::FromJson(nlohmann::json::parse(s)); if (not map->IsMap()) { Logger::Log(LogLevel::Error, "Defines entry {} does not contain a map.", nlohmann::json(s).dump()); std::exit(kExitClargsError); } overlay_config = overlay_config.Update(map); } catch (std::exception const& e) { Logger::Log(LogLevel::Error, "Parsing defines entry {} failed with error:\n{}", nlohmann::json(s).dump(), e.what()); std::exit(kExitClargsError); } } if (not overlay_config.Expr()->Map().empty()) { cmd.emplace_back("-D"); cmd.emplace_back(overlay_config.ToString()); } } // forward remote execution and mutual TLS arguments if (supports_remote) { if (common_args.compatible) { cmd.emplace_back("--compatible"); } if (common_args.remote_execution_address) { cmd.emplace_back("-r"); cmd.emplace_back(*common_args.remote_execution_address); } if (auth_args.tls_ca_cert) { cmd.emplace_back("--tls-ca-cert"); cmd.emplace_back(auth_args.tls_ca_cert->string()); } if (auth_args.tls_client_cert) { cmd.emplace_back("--tls-client-cert"); cmd.emplace_back(auth_args.tls_client_cert->string()); } if (auth_args.tls_client_key) { cmd.emplace_back("--tls-client-key"); cmd.emplace_back(auth_args.tls_client_key->string()); } if (retry_args.max_attempts) { cmd.emplace_back("--max-attempts"); cmd.emplace_back(std::to_string(*retry_args.max_attempts)); } if (retry_args.initial_backoff_seconds) { cmd.emplace_back("--initial-backoff-seconds"); cmd.emplace_back( std::to_string(*retry_args.initial_backoff_seconds)); } if (retry_args.max_backoff_seconds) { cmd.emplace_back("--max-backoff-seconds"); cmd.emplace_back(std::to_string(*retry_args.max_backoff_seconds)); } } if (supports_dispatch and just_cmd_args.endpoint_configuration) { cmd.emplace_back("--endpoint-configuration"); cmd.emplace_back(*just_cmd_args.endpoint_configuration); } if (supports_serve and common_args.remote_serve_address) { cmd.emplace_back("-R"); cmd.emplace_back(*common_args.remote_serve_address); } // forward-only arguments, still to come before the just-arguments if (supports_remote_properties) { for (auto const& prop : launch_fwd.remote_execution_properties) { cmd.emplace_back("--remote-execution-property"); cmd.emplace_back(prop); } } // add args read from just-mrrc if (subcommand and just_cmd_args.just_args.contains(*subcommand)) { for (auto const& subcmd_arg : just_cmd_args.just_args.at(*subcommand)) { cmd.emplace_back(subcmd_arg); } } std::optional log_dir = std::nullopt; auto invocation_time = std::time(nullptr); // Check invocation logging if (invocation_log.directory) { auto dir = *invocation_log.directory; if (invocation_log.project_id) { if (not IsValidFileName(*invocation_log.project_id)) { Logger::Log(LogLevel::Error, "Invalid file name for poject id: {}", nlohmann::json(*invocation_log.project_id).dump()); std::exit(kExitClargsError); } dir = dir / *invocation_log.project_id; } else { dir = dir / "unknown"; } std::string uuid = CreateUUID(); auto invocation_id = fmt::format( "{:%Y-%m-%d-%H:%M:%S}-{}", fmt::gmtime(invocation_time), uuid); dir = dir / invocation_id; if (FileSystemManager::CreateDirectoryExclusive(dir)) { if (invocation_log.invocation_msg) { Logger::Log(LogLevel::Info, "{}{}", *invocation_log.invocation_msg, invocation_id); } Logger::Log( LogLevel::Info, "Invocation logged at {}", dir.string()); log_dir = dir; } else { Logger::Log(LogLevel::Warning, "Failed to create directory {} for invocation logging", nlohmann::json(dir.string()).dump()); } } // invocation-specific if (log_dir and supports_remote_properties) { if (invocation_log.graph_file) { if (not IsValidFileName(*invocation_log.graph_file)) { Logger::Log(LogLevel::Error, "Invalid file name for option --dump-graph: {}", nlohmann::json(*invocation_log.graph_file).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-graph"); cmd.emplace_back(*log_dir / *invocation_log.graph_file); } if (invocation_log.graph_file_plain) { if (not IsValidFileName(*invocation_log.graph_file_plain)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-plain-graph: {}", nlohmann::json(*invocation_log.graph_file_plain).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-plain-graph"); cmd.emplace_back(*log_dir / *invocation_log.graph_file_plain); } if (invocation_log.dump_artifacts_to_build) { if (not IsValidFileName(*invocation_log.dump_artifacts_to_build)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-artifacts_to_build: " "{}", nlohmann::json(*invocation_log.dump_artifacts_to_build) .dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-artifacts-to-build"); cmd.emplace_back(*log_dir / *invocation_log.dump_artifacts_to_build); } if (does_build and invocation_log.dump_artifacts) { if (not IsValidFileName(*invocation_log.dump_artifacts)) { Logger::Log( LogLevel::Error, "Invalid file name for option --dump-artifacts: {}", nlohmann::json(*invocation_log.dump_artifacts).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--dump-artifacts"); cmd.emplace_back(*log_dir / *invocation_log.dump_artifacts); } if (invocation_log.profile) { if (not IsValidFileName(*invocation_log.profile)) { Logger::Log(LogLevel::Error, "Invalid file name for option --profile: {}", nlohmann::json(*invocation_log.profile).dump()); std::exit(kExitClargsError); } cmd.emplace_back("--profile"); cmd.emplace_back(*log_dir / *invocation_log.profile); } } // add (remaining) args given by user as clargs for (auto it = just_cmd_args.additional_just_args.begin() + additional_args_offset; it != just_cmd_args.additional_just_args.end(); ++it) { cmd.emplace_back(*it); } // Write invocation metadata, if requested if (log_dir and invocation_log.metadata) { if (not IsValidFileName(*invocation_log.metadata)) { Logger::Log(LogLevel::Error, "Invlaid file name for metadata file: {}", nlohmann::json(*invocation_log.metadata).dump()); std::exit(kExitClargsError); } auto meta = nlohmann::json::object(); meta["time"] = invocation_time; if (mr_config_pair) { meta["configuration"] = mr_config_pair->second; } meta["cmdline"] = cmd; if (not invocation_log.context_vars.empty()) { auto context = nlohmann::json::object(); for (auto const& env_var : invocation_log.context_vars) { auto* env_value = std::getenv(env_var.c_str()); context[env_var] = env_value != nullptr ? env_value : nlohmann::json{}; } meta["context"] = context; } // "configuration" -- the blob-identifier of the multi-repo // configuration auto file_name = *log_dir / *invocation_log.metadata; if (not FileSystemManager::WriteFile(meta.dump(2), file_name)) { Logger::Log(LogLevel::Warning, "Failed to write metadata file {}.", nlohmann::json(file_name).dump()); } } Logger::Log( LogLevel::Info, "Setup finished, exec {}", nlohmann::json(cmd).dump()); // create argv std::vector argv{}; std::transform(std::begin(cmd), std::end(cmd), std::back_inserter(argv), [](auto& str) { return str.data(); }); argv.push_back(nullptr); // run execvp; will only return if failure [[maybe_unused]] auto res = execvp(argv[0], static_cast(argv.data())); // execvp returns only if command errored out Logger::Log(LogLevel::Error, "execvp failed with:\n{}", strerror(errno)); return kExitExecError; }