diff options
| -rw-r--r-- | src/buildtool/common/cli.hpp | 34 | ||||
| -rw-r--r-- | src/buildtool/main/TARGETS | 17 | ||||
| -rw-r--r-- | src/buildtool/main/cli.cpp | 17 | ||||
| -rw-r--r-- | src/buildtool/main/cli.hpp | 6 | ||||
| -rw-r--r-- | src/buildtool/main/main.cpp | 28 | ||||
| -rw-r--r-- | src/buildtool/main/serve.cpp | 276 | ||||
| -rw-r--r-- | src/buildtool/main/serve.hpp | 29 |
7 files changed, 386 insertions, 21 deletions
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp index a49b2f6b..c90e54f9 100644 --- a/src/buildtool/common/cli.hpp +++ b/src/buildtool/common/cli.hpp @@ -150,7 +150,7 @@ struct ServerAuthArguments { std::optional<std::filesystem::path> tls_server_key{std::nullopt}; }; -struct ExecutionServiceArguments { +struct ServiceArguments { std::optional<int> port{std::nullopt}; std::optional<std::filesystem::path> info_file{std::nullopt}; std::optional<std::string> interface{std::nullopt}; @@ -158,6 +158,12 @@ struct ExecutionServiceArguments { std::optional<uint8_t> op_exponent; }; +struct ServeArguments { + std::filesystem::path config{}; + // repositories populated from just-serve config file + std::vector<std::filesystem::path> repositories{}; +}; + static inline auto SetupCommonArguments( gsl::not_null<CLI::App*> const& app, gsl::not_null<CommonArguments*> const& clargs) { @@ -601,33 +607,43 @@ static inline auto SetupServerAuthArguments( "Path to the TLS server key."); } -static inline auto SetupExecutionServiceArguments( +static inline auto SetupServiceArguments( gsl::not_null<CLI::App*> const& app, - gsl::not_null<ExecutionServiceArguments*> const& es_args) { + gsl::not_null<ServiceArguments*> const& service_args) { app->add_option("-p,--port", - es_args->port, - "Execution service will listen to this port. If unset, the " + service_args->port, + "The service will listen to this port. If unset, the " "service will listen to the first available one."); app->add_option("--info-file", - es_args->info_file, + service_args->info_file, "Write the used port, interface, and pid to this file in " "JSON format. If the file exists, it " "will be overwritten."); app->add_option("-i,--interface", - es_args->interface, + service_args->interface, "Interface to use. If unset, the loopback device is used."); app->add_option( "--pid-file", - es_args->pid_file, + service_args->pid_file, "Write pid to this file in plain txt. If the file exists, it " "will be overwritten."); app->add_option( "--log-operations-threshold", - es_args->op_exponent, + service_args->op_exponent, "Once the number of operations stored exceeds twice 2^n, where n is " "given by the option --log-operations-threshold, at most 2^n " "operations will be removed, in a FIFO scheme. If unset, defaults to " "14. Must be in the range [0,255]"); } + +static inline auto SetupServeArguments( + gsl::not_null<CLI::App*> const& app, + gsl::not_null<ServeArguments*> const& serve_args) { + app->add_option("config", + serve_args->config, + "Configuration file for the subcommand.") + ->required(); +} + #endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 974c0e61..1264bfb5 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -30,6 +30,7 @@ , "describe" , "diagnose" , "constants" + , "serve" ] , "stage": ["src", "buildtool", "main"] , "private-ldflags": @@ -179,4 +180,20 @@ , "hdrs": ["constants.hpp"] , "stage": ["src", "buildtool", "main"] } +, "serve": + { "type": ["@", "rules", "CC", "library"] + , "name": ["serve"] + , "hdrs": ["serve.hpp"] + , "srcs": ["serve.cpp"] + , "deps": [["@", "gsl", "", "gsl"], "cli"] + , "stage": ["src", "buildtool", "main"] + , "private-deps": + [ ["@", "json", "", "json"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/logging", "log_level"] + , ["src/buildtool/logging", "logging"] + , "common" + ] + } } diff --git a/src/buildtool/main/cli.cpp b/src/buildtool/main/cli.cpp index cb53acde..f7e1ea71 100644 --- a/src/buildtool/main/cli.cpp +++ b/src/buildtool/main/cli.cpp @@ -120,11 +120,20 @@ auto SetupExecutionServiceCommandArguments( SetupCompatibilityArguments(app); SetupCommonBuildArguments(app, &clargs->build); SetupCacheArguments(app, &clargs->endpoint); - SetupExecutionServiceArguments(app, &clargs->es); + SetupServiceArguments(app, &clargs->service); SetupLogArguments(app, &clargs->log); SetupCommonAuthArguments(app, &clargs->auth); SetupServerAuthArguments(app, &clargs->sauth); } + +/// \brief Setup arguments for sub command "just serve". +auto SetupServeServiceCommandArguments( + gsl::not_null<CLI::App*> const& app, + gsl::not_null<CommandLineArguments*> const& clargs) { + // all other arguments will be read from config file + SetupServeArguments(app, &clargs->serve); +} + } // namespace auto ParseCommandLineArguments(int argc, char const* const* argv) @@ -149,6 +158,8 @@ auto ParseCommandLineArguments(int argc, char const* const* argv) app.add_subcommand("gc", "Trigger garbage collection of local cache."); auto* cmd_execution = app.add_subcommand( "execute", "Start single node execution service on this machine."); + auto* cmd_serve = + app.add_subcommand("serve", "Provide target dependencies for a build."); auto* cmd_traverse = app.group("") // group for creating hidden options ->add_subcommand("traverse", @@ -165,6 +176,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv) SetupTraverseCommandArguments(cmd_traverse, &clargs); SetupGcCommandArguments(cmd_gc, &clargs); SetupExecutionServiceCommandArguments(cmd_execution, &clargs); + SetupServeServiceCommandArguments(cmd_serve, &clargs); try { app.parse(argc, argv); } catch (CLI::Error& e) { @@ -204,6 +216,9 @@ auto ParseCommandLineArguments(int argc, char const* const* argv) else if (*cmd_execution) { clargs.cmd = SubCommand::kExecute; } + else if (*cmd_serve) { + clargs.cmd = SubCommand::kServe; + } return clargs; } diff --git a/src/buildtool/main/cli.hpp b/src/buildtool/main/cli.hpp index 7a580ff6..b2945757 100644 --- a/src/buildtool/main/cli.hpp +++ b/src/buildtool/main/cli.hpp @@ -28,7 +28,8 @@ enum class SubCommand { kInstallCas, kTraverse, kGc, - kExecute + kExecute, + kServe }; struct CommandLineArguments { @@ -47,7 +48,8 @@ struct CommandLineArguments { CommonAuthArguments auth; ClientAuthArguments cauth; ServerAuthArguments sauth; - ExecutionServiceArguments es; + ServiceArguments service; + ServeArguments serve; }; auto ParseCommandLineArguments(int argc, char const* const* argv) diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index e362b7ef..b2c04be7 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -47,9 +47,10 @@ #include "src/buildtool/execution_api/execution_service/server_implementation.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/graph_traverser/graph_traverser.hpp" +#include "src/buildtool/main/serve.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" #include "src/buildtool/storage/garbage_collector.hpp" -#endif +#endif // BOOTSTRAP_BUILD_TOOL #include "src/buildtool/logging/log_config.hpp" #include "src/buildtool/logging/log_sink_cmdline.hpp" #include "src/buildtool/logging/log_sink_file.hpp" @@ -191,7 +192,7 @@ void SetupAuthConfig(CommonAuthArguments const& authargs, } } -void SetupExecutionServiceConfig(ExecutionServiceArguments const& args) { +void SetupExecutionServiceConfig(ServiceArguments const& args) { if (args.port) { if (!ServerImpl::SetPort(*args.port)) { Logger::Log(LogLevel::Error, "Invalid port '{}'", *args.port); @@ -233,7 +234,7 @@ void SetupHashFunction() { : HashFunction::JustHash::Native); } -#endif +#endif // BOOTSTRAP_BUILD_TOOL // returns path relative to `root`. [[nodiscard]] auto FindRoot(std::filesystem::path const& subdir, @@ -809,7 +810,8 @@ void WriteTargetCacheEntries( }); } } -#endif + +#endif // BOOTSTRAP_BUILD_TOOL } // namespace @@ -823,6 +825,14 @@ auto main(int argc, char* argv[]) -> int { return kExitSuccess; } + // just serve configures all from its config file, so parse that before + // doing further setup steps +#ifndef BOOTSTRAP_BUILD_TOOL + if (arguments.cmd == SubCommand::kServe) { + ReadJustServeConfig(&arguments); + } +#endif // BOOTSTRAP_BUILD_TOOL + SetupLogging(arguments.log); if (arguments.analysis.expression_log_limit) { Evaluator::SetExpressionLogLimit( @@ -851,13 +861,13 @@ auto main(int argc, char* argv[]) -> int { } if (arguments.cmd == SubCommand::kExecute) { - SetupExecutionServiceConfig(arguments.es); + SetupExecutionServiceConfig(arguments.service); if (!ServerImpl::Instance().Run()) { return kExitFailure; } return kExitSuccess; } -#endif +#endif // BOOTSTRAP_BUILD_TOOL auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs : arguments.common.jobs; @@ -893,7 +903,7 @@ auto main(int argc, char* argv[]) -> int { ? kExitSuccess : kExitFailure; } -#endif +#endif // BOOTSTRAP_BUILD_TOOL auto [main_repo, main_ws_root] = DetermineRoots(arguments.common, arguments.analysis); @@ -942,7 +952,7 @@ auto main(int argc, char* argv[]) -> int { } else { -#endif +#endif // BOOTSTRAP_BUILD_TOOL BuildMaps::Target::ResultTargetMap result_map{ arguments.common.jobs}; auto id = ReadConfiguredTarget( @@ -1039,7 +1049,7 @@ auto main(int argc, char* argv[]) -> int { : kExitSuccess; } } -#endif +#endif // BOOTSTRAP_BUILD_TOOL } } catch (std::exception const& ex) { Logger::Log( diff --git a/src/buildtool/main/serve.cpp b/src/buildtool/main/serve.cpp new file mode 100644 index 00000000..da7ed62a --- /dev/null +++ b/src/buildtool/main/serve.cpp @@ -0,0 +1,276 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/main/serve.hpp" + +#ifndef BOOTSTRAP_BUILD_TOOL + +#include <fstream> + +#include "nlohmann/json.hpp" +#include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/build_engine/expression/expression.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/main/exit_codes.hpp" + +void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) { + Configuration serve_config{}; + if (FileSystemManager::IsFile(clargs->serve.config)) { + // json::parse may throw + try { + std::ifstream fs(clargs->serve.config); + auto map = Expression::FromJson(nlohmann::json::parse(fs)); + if (not map->IsMap()) { + Logger::Log(LogLevel::Error, + "In just-serve config file {}: expected an object " + "but found:\n{}", + clargs->serve.config.string(), + map->ToString()); + std::exit(kExitFailure); + } + serve_config = Configuration{map}; + } catch (std::exception const& e) { + Logger::Log(LogLevel::Error, + "Parsing just-serve config file {} as JSON failed with " + "error:\n{}", + clargs->serve.config.string(), + e.what()); + std::exit(kExitFailure); + } + } + else { + Logger::Log(LogLevel::Error, + "Cannot read just-serve config file {}", + clargs->serve.config.string()); + std::exit(kExitFailure); + } + // read paths of additional lookup repositories + auto repositories = serve_config["repositories"]; + if (repositories.IsNotNull()) { + if (not repositories->IsList()) { + Logger::Log( + LogLevel::Error, + "just-serve: configuration-file provided repositories has " + "to be a list of strings, but found {}", + repositories->ToString()); + std::exit(kExitFailure); + } + auto const& repos_list = repositories->List(); + clargs->serve.repositories.reserve(clargs->serve.repositories.size() + + repos_list.size()); + for (auto const& repo : repos_list) { + if (not repo->IsString()) { + Logger::Log(LogLevel::Error, + "just-serve: expected each repository path to be a " + "string, but found {}", + repo->ToString()); + std::exit(kExitFailure); + } + clargs->serve.repositories.emplace_back(repo->String()); + } + } + // read logging arguments + auto logging = serve_config["logging"]; + if (logging.IsNotNull()) { + if (not logging->IsMap()) { + Logger::Log( + LogLevel::Error, + "just-serve: configuration-file provided logging arguments has " + "to be a map, but found {}", + logging->ToString()); + std::exit(kExitFailure); + } + // read in first the append flag + auto append = logging->Get("append", Expression::none_t{}); + if (append.IsNotNull()) { + if (not append->IsBool()) { + Logger::Log(LogLevel::Error, + "Configuration-provided log append has to be a " + "flag, but found {}", + append->ToString()); + std::exit(kExitFailure); + } + clargs->log.log_append = append->Bool(); + } + // read in the plain flag + auto plain = logging->Get("plain", Expression::none_t{}); + if (plain.IsNotNull()) { + if (not plain->IsBool()) { + Logger::Log(LogLevel::Error, + "Configuration-provided plain log has to be a " + "flag, but found {}", + plain->ToString()); + std::exit(kExitFailure); + } + clargs->log.plain_log = plain->Bool(); + } + // read in files field + auto files = logging->Get("files", Expression::none_t{}); + if (files.IsNotNull()) { + if (not files->IsList()) { + Logger::Log(LogLevel::Error, + "Configuration-provided log files has to be a " + "list, but found {}", + files->ToString()); + std::exit(kExitFailure); + } + auto const& files_list = files->List(); + clargs->log.log_files.reserve(clargs->log.log_files.size() + + files_list.size()); + for (auto const& file : files_list) { + if (not file->IsString()) { + Logger::Log(LogLevel::Error, + "just-serve: expected each log file path to be " + "a string, but found {}", + file->ToString()); + std::exit(kExitFailure); + } + clargs->log.log_files.emplace_back(file->String()); + } + } + // read in limit field + auto limit = logging->Get("limit", Expression::none_t{}); + if (limit.IsNotNull()) { + if (not limit->IsNumber()) { + Logger::Log(LogLevel::Error, + "Configuration-provided log limit has to be " + "numeric, but found {}", + limit->ToString()); + std::exit(kExitFailure); + } + clargs->log.log_limit = ToLogLevel(limit->Number()); + } + } + // read client TLS authentication arguments + auto auth_args = serve_config["authentication"]; + if (auth_args.IsNotNull()) { + if (not auth_args->IsMap()) { + Logger::Log(LogLevel::Error, + "just-serve: configuration-file provided " + "authentication has to be a map, but found {}", + auth_args->ToString()); + std::exit(kExitFailure); + } + // read the TLS CA certificate + auto cacert = auth_args->Get("ca cert", Expression::none_t{}); + if (cacert.IsNotNull()) { + if (not cacert->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided TLS CA certificate has to " + "be a string, but found {}", + cacert->ToString()); + std::exit(kExitFailure); + } + clargs->auth.tls_ca_cert = cacert->String(); + } + } + // read remote service arguments + auto remote_service = serve_config["remote service"]; + if (remote_service.IsNotNull()) { + if (not remote_service->IsMap()) { + Logger::Log(LogLevel::Error, + "just-serve: configuration-file provided remote " + "service has to be a map, but found {}", + remote_service->ToString()); + std::exit(kExitFailure); + } + // read the interface + auto interface = remote_service->Get("interface", Expression::none_t{}); + if (interface.IsNotNull()) { + if (not interface->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided remote service interface " + "has to be a string, but found {}", + interface->ToString()); + std::exit(kExitFailure); + } + clargs->service.interface = interface->String(); + } + // read the port + auto port = remote_service->Get("port", Expression::none_t{}); + if (port.IsNotNull()) { + if (not port->IsNumber()) { + Logger::Log(LogLevel::Error, + "Configuration-provided remote service port has to " + "be numeric, but found {}", + port->ToString()); + std::exit(kExitFailure); + } + double val{}; + if (std::modf(port->Number(), &val) != 0.0) { + Logger::Log(LogLevel::Error, + "Configuration-provided remote service port has to " + "be an integer, but found {}", + interface->ToString()); + std::exit(kExitFailure); + } + // we are sure now that the port is an integer + clargs->service.port = std::nearbyint(val); + } + // read the pid file + auto pid_file = remote_service->Get("pid file", Expression::none_t{}); + if (pid_file.IsNotNull()) { + if (not pid_file->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided remote service pid file " + "has to be a string, but found {}", + pid_file->ToString()); + std::exit(kExitFailure); + } + clargs->service.pid_file = pid_file->String(); + } + // read the info file + auto info_file = remote_service->Get("info file", Expression::none_t{}); + if (info_file.IsNotNull()) { + if (not info_file->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided remote service info file " + "has to be a string, but found {}", + info_file->ToString()); + std::exit(kExitFailure); + } + clargs->service.info_file = info_file->String(); + } + // read the TLS server certificate + auto server_cert = + remote_service->Get("server cert", Expression::none_t{}); + if (server_cert.IsNotNull()) { + if (not server_cert->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided TLS server certificate has " + "to be a string, but found {}", + server_cert->ToString()); + std::exit(kExitFailure); + } + clargs->sauth.tls_server_cert = server_cert->String(); + } + // read the TLS server key + auto server_key = + remote_service->Get("server key", Expression::none_t{}); + if (server_key.IsNotNull()) { + if (not server_key->IsString()) { + Logger::Log(LogLevel::Error, + "Configuration-provided TLS server key has to be a " + "string, but found {}", + server_key->ToString()); + std::exit(kExitFailure); + } + clargs->sauth.tls_server_key = server_key->String(); + } + } +} + +#endif // BOOTSTRAP_BUILD_TOOL diff --git a/src/buildtool/main/serve.hpp b/src/buildtool/main/serve.hpp new file mode 100644 index 00000000..bc7c4664 --- /dev/null +++ b/src/buildtool/main/serve.hpp @@ -0,0 +1,29 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP +#define INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP + +#ifndef BOOTSTRAP_BUILD_TOOL + +#include "gsl/gsl" +#include "src/buildtool/main/cli.hpp" + +/// \brief Parse the "just serve" config file. +/// While having a separate config file, almost all fields are already used by +/// "just" itself, so we can populate the respective known command-line fields. +void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs); + +#endif // BOOTSTRAP_BUILD_TOOL +#endif // INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP |