From 619def44c1cca9f3cdf63544d5f24f2c7a7d9b77 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 22 Feb 2022 17:03:21 +0100 Subject: Initial self-hosting commit This is the initial version of our tool that is able to build itself. In can be bootstrapped by ./bin/bootstrap.py Co-authored-by: Oliver Reiche Co-authored-by: Victor Moreno --- src/buildtool/common/cli.hpp | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 src/buildtool/common/cli.hpp (limited to 'src/buildtool/common/cli.hpp') diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp new file mode 100644 index 00000000..b4710147 --- /dev/null +++ b/src/buildtool/common/cli.hpp @@ -0,0 +1,365 @@ +#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP +#define INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP + +#include +#include +#include +#include +#include + +#include "CLI/CLI.hpp" +#include "fmt/core.h" +#include "nlohmann/json.hpp" +#include "gsl-lite/gsl-lite.hpp" +#include "src/buildtool/logging/log_level.hpp" + +constexpr auto kDefaultLogLevel = LogLevel::Info; + +/// \brief Arguments common to all commands. +struct CommonArguments { + std::optional workspace_root{}; + std::optional repository_config{}; + std::optional main{}; + std::optional log_file{}; + std::size_t jobs{std::max(1U, std::thread::hardware_concurrency())}; + LogLevel log_limit{kDefaultLogLevel}; +}; + +/// \brief Arguments required for analysing targets. +struct AnalysisArguments { + std::filesystem::path config_file{}; + std::optional target{}; + std::optional target_file_name{}; + std::optional rule_file_name{}; + std::optional expression_file_name{}; + std::optional target_root{}; + std::optional rule_root{}; + std::optional expression_root{}; + std::optional graph_file{}; + std::optional artifacts_to_build_file{}; +}; + +/// \brief Arguments required for running diagnostics. +struct DiagnosticArguments { + std::optional dump_actions{std::nullopt}; + std::optional dump_blobs{std::nullopt}; + std::optional dump_trees{std::nullopt}; + std::optional dump_targets{std::nullopt}; + std::optional dump_anonymous{std::nullopt}; + std::optional dump_nodes{std::nullopt}; +}; + +/// \brief Arguments required for specifying cache/build endpoint. +struct EndpointArguments { + std::optional local_root{}; + std::optional remote_execution_address; +}; + +/// \brief Arguments required for building. +struct BuildArguments { + std::optional> local_launcher{std::nullopt}; + std::map platform_properties; + std::size_t build_jobs{}; + std::optional dump_artifacts{std::nullopt}; + std::optional print_to_stdout{std::nullopt}; + bool persistent_build_dir{false}; + bool show_runfiles{false}; +}; + +/// \brief Arguments required for staging. +struct StageArguments { + std::filesystem::path output_dir{}; +}; + +/// \brief Arguments required for rebuilding. +struct RebuildArguments { + std::optional cache_endpoint{}; + std::optional dump_flaky{}; +}; + +/// \brief Arguments for fetching artifacts from CAS. +struct FetchArguments { + std::string object_id{}; + std::optional output_path{}; +}; + +/// \brief Arguments required for running from graph file. +struct GraphArguments { + nlohmann::json artifacts{}; + std::filesystem::path graph_file{}; + std::optional git_cas{}; +}; + +static inline auto SetupCommonArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option("-C,--repository_config", + clargs->repository_config, + "Path to configuration file for multi-repository builds.") + ->type_name("PATH"); + app->add_option( + "--main", clargs->main, "The repository to take the target from.") + ->type_name("NAME"); + app->add_option_function( + "-w,--workspace_root", + [clargs](auto const& workspace_root_raw) { + clargs->workspace_root = std::filesystem::canonical( + std::filesystem::absolute(workspace_root_raw)); + }, + "Path of the workspace's root directory.") + ->type_name("PATH"); + app->add_option("-j,--jobs", + clargs->jobs, + "Number of jobs to run (Default: Number of cores).") + ->type_name("NUM"); + app->add_option( + "-f,--log-file", clargs->log_file, "Path to local log file.") + ->type_name("PATH"); + app->add_option_function>( + "--log-limit", + [clargs](auto const& limit) { + clargs->log_limit = ToLogLevel(limit); + }, + fmt::format("Log limit in interval [{},{}] (Default: {}).", + kFirstLogLevel, + kLastLogLevel, + kDefaultLogLevel)) + ->type_name("NUM"); +} + +static inline auto SetupAnalysisArguments( + gsl::not_null const& app, + gsl::not_null const& clargs, + bool with_graph = true) { + app->add_option( + "-c,--config", clargs->config_file, "Path to configuration file.") + ->type_name("PATH"); + app->add_option_function>( + "target", + [clargs](auto const& target_raw) { + if (target_raw.size() > 1) { + clargs->target = + nlohmann::json{target_raw[0], target_raw[1]}; + } + else { + clargs->target = nlohmann::json{target_raw[0]}[0]; + } + }, + "Module and target name to build.\n" + "Assumes current module if module name is omitted.") + ->expected(2); + app->add_option("--target_root", + clargs->target_root, + "Path of the target files' root directory.\n" + "Default: Same as --workspace_root") + ->type_name("PATH"); + app->add_option("--rule_root", + clargs->rule_root, + "Path of the rule files' root directory.\n" + "Default: Same as --target_root") + ->type_name("PATH"); + app->add_option("--expression_root", + clargs->expression_root, + "Path of the expression files' root directory.\n" + "Default: Same as --rule_root") + ->type_name("PATH"); + app->add_option("--target_file_name", + clargs->target_file_name, + "Name of the targets file."); + app->add_option( + "--rule_file_name", clargs->rule_file_name, "Name of the rules file."); + app->add_option("--expression_file_name", + clargs->expression_file_name, + "Name of the expressions file."); + if (with_graph) { + app->add_option( + "--dump_graph", + clargs->graph_file, + "File path for writing the action graph description to.") + ->type_name("PATH"); + app->add_option("--dump_artifacts_to_build", + clargs->artifacts_to_build_file, + "File path for writing the artifacts to build to.") + ->type_name("PATH"); + } +} + +static inline auto SetupDiagnosticArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option("--dump_actions", + clargs->dump_actions, + "Dump actions to file (use - for stdout).") + ->type_name("PATH"); + app->add_option("--dump_trees", + clargs->dump_trees, + "Dump trees to file (use - for stdout).") + ->type_name("PATH"); + app->add_option("--dump_blobs", + clargs->dump_blobs, + "Dump blobs to file (use - for stdout).") + ->type_name("PATH"); + app->add_option("--dump_targets", + clargs->dump_targets, + "Dump targets to file (use - for stdout).") + ->type_name("PATH"); + app->add_option("--dump_anonymous", + clargs->dump_anonymous, + "Dump anonymous targets to file (use - for stdout).") + ->type_name("PATH"); + app->add_option("--dump_nodes", + clargs->dump_nodes, + "Dump nodes of target to file (use - for stdout).") + ->type_name("PATH"); +} + +static inline auto SetupEndpointArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option_function( + "--local_build_root", + [clargs](auto const& build_root_raw) { + clargs->local_root = std::filesystem::weakly_canonical( + std::filesystem::absolute(build_root_raw)); + }, + "Root for local CAS, cache, and build directories.") + ->type_name("PATH"); + + app->add_option("-r,--remote_execution_address", + clargs->remote_execution_address, + "Address of the remote execution service.") + ->type_name("NAME:PORT"); +} + +static inline auto SetupBuildArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_flag("-p,--persistent", + clargs->persistent_build_dir, + "Do not clean build directory after execution."); + + app->add_option_function( + "-L,--local_launcher", + [clargs](auto const& launcher_raw) { + clargs->local_launcher = + nlohmann::json::parse(launcher_raw) + .template get>(); + }, + "JSON array with the list of strings representing the launcher to " + "prepend actions' commands before being executed locally.") + ->type_name("JSON") + ->default_val(nlohmann::json{"env", "--"}.dump()); + + app->add_option_function( + "--remote_execution_property", + [clargs](auto const& property) { + std::istringstream pss(property); + std::string key; + std::string val; + if (not std::getline(pss, key, ':') or + not std::getline(pss, val, ':')) { + throw CLI::ConversionError{property, + "--remote_execution_property"}; + } + clargs->platform_properties.emplace(std::move(key), + std::move(val)); + }, + "Property for remote execution as key-value pair.") + ->type_name("KEY:VAL") + ->allow_extra_args(false) + ->expected(1, 1); + + app->add_option( + "-J,--build_jobs", + clargs->build_jobs, + "Number of jobs to run during build phase (Default: same as jobs).") + ->type_name("NUM"); + app->add_option("--dump_artifacts", + clargs->dump_artifacts, + "Dump artifacts to file (use - for stdout).") + ->type_name("PATH"); + + app->add_flag("-s,--show_runfiles", + clargs->show_runfiles, + "Do not omit runfiles in build report."); + + app->add_option("-P,--print_to_stdout", + clargs->print_to_stdout, + "After building, print the specified artifact to stdout.") + ->type_name("LOGICAL_PATH"); +} + +static inline auto SetupStageArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option_function( + "-o,--output_dir", + [clargs](auto const& output_dir_raw) { + clargs->output_dir = std::filesystem::weakly_canonical( + std::filesystem::absolute(output_dir_raw)); + }, + "Path of the directory where outputs will be copied.") + ->type_name("PATH") + ->required(); +} + +static inline auto SetupRebuildArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option_function( + "--vs", + [clargs](auto const& cache_endpoint) { + clargs->cache_endpoint = cache_endpoint; + }, + "Cache endpoint to compare against (use \"local\" for local cache).") + ->type_name("NAME:PORT|\"local\""); + + app->add_option( + "--dump_flaky", clargs->dump_flaky, "Dump flaky actions to file.") + ->type_name("PATH"); +} + +static inline auto SetupFetchArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option( + "object_id", + clargs->object_id, + "Object identifier with the format '[::]'.") + ->required(); + + app->add_option_function( + "-o,--output_path", + [clargs](auto const& output_path_raw) { + clargs->output_path = std::filesystem::weakly_canonical( + std::filesystem::absolute(output_path_raw)); + }, + "Install path for the artifact. (omit to dump to stdout)") + ->type_name("PATH"); +} + +static inline auto SetupGraphArguments( + gsl::not_null const& app, + gsl::not_null const& clargs) { + app->add_option_function( + "-a,--artifacts", + [clargs](auto const& artifact_map_raw) { + clargs->artifacts = nlohmann::json::parse(artifact_map_raw); + }, + "Json object with key/value pairs formed by the relative path in which " + "artifact is to be copied and the description of the artifact as json " + "object as well."); + + app->add_option("-g,--graph_file", + clargs->graph_file, + "Path of the file containing the description of the " + "actions.") + ->required(); + + app->add_option("--git_cas", + clargs->git_cas, + "Path to a Git repository, containing blobs of potentially " + "missing KNOWN artifacts."); +} + +#endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP -- cgit v1.2.3