just: Add handling of 'just serve' configuration

7 files changed, 386 insertions, 21 deletions

diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
index a49b2f6b..c90e54f9 100644
--- a/src/buildtool/common/cli.hpp
+++ b/src/buildtool/common/cli.hpp
@@ -150,7 +150,7 @@ struct ServerAuthArguments {
     std::optional<std::filesystem::path> tls_server_key{std::nullopt};
 };
 
-struct ExecutionServiceArguments {
+struct ServiceArguments {
     std::optional<int> port{std::nullopt};
     std::optional<std::filesystem::path> info_file{std::nullopt};
     std::optional<std::string> interface{std::nullopt};
@@ -158,6 +158,12 @@ struct ExecutionServiceArguments {
     std::optional<uint8_t> op_exponent;
 };
 
+struct ServeArguments {
+    std::filesystem::path config{};
+    // repositories populated from just-serve config file
+    std::vector<std::filesystem::path> repositories{};
+};
+
 static inline auto SetupCommonArguments(
     gsl::not_null<CLI::App*> const& app,
     gsl::not_null<CommonArguments*> const& clargs) {
@@ -601,33 +607,43 @@ static inline auto SetupServerAuthArguments(
                     "Path to the TLS server key.");
 }
 
-static inline auto SetupExecutionServiceArguments(
+static inline auto SetupServiceArguments(
     gsl::not_null<CLI::App*> const& app,
-    gsl::not_null<ExecutionServiceArguments*> const& es_args) {
+    gsl::not_null<ServiceArguments*> const& service_args) {
     app->add_option("-p,--port",
-                    es_args->port,
-                    "Execution service will listen to this port. If unset, the "
+                    service_args->port,
+                    "The service will listen to this port. If unset, the "
                     "service will listen to the first available one.");
     app->add_option("--info-file",
-                    es_args->info_file,
+                    service_args->info_file,
                     "Write the used port, interface, and pid to this file in "
                     "JSON format. If the file exists, it "
                     "will be overwritten.");
     app->add_option("-i,--interface",
-                    es_args->interface,
+                    service_args->interface,
                     "Interface to use. If unset, the loopback device is used.");
     app->add_option(
         "--pid-file",
-        es_args->pid_file,
+        service_args->pid_file,
         "Write pid to this file in plain txt. If the file exists, it "
         "will be overwritten.");
 
     app->add_option(
         "--log-operations-threshold",
-        es_args->op_exponent,
+        service_args->op_exponent,
         "Once the number of operations stored exceeds twice 2^n, where n is "
         "given by the option --log-operations-threshold, at most 2^n "
         "operations will be removed, in a FIFO scheme. If unset, defaults to "
         "14. Must be in the range [0,255]");
 }
+
+static inline auto SetupServeArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<ServeArguments*> const& serve_args) {
+    app->add_option("config",
+                    serve_args->config,
+                    "Configuration file for the subcommand.")
+        ->required();
+}
+
 #endif  // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index 974c0e61..1264bfb5 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -30,6 +30,7 @@
     , "describe"
     , "diagnose"
     , "constants"
+    , "serve"
     ]
   , "stage": ["src", "buildtool", "main"]
   , "private-ldflags":
@@ -179,4 +180,20 @@
   , "hdrs": ["constants.hpp"]
   , "stage": ["src", "buildtool", "main"]
   }
+, "serve":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["serve"]
+  , "hdrs": ["serve.hpp"]
+  , "srcs": ["serve.cpp"]
+  , "deps": [["@", "gsl", "", "gsl"], "cli"]
+  , "stage": ["src", "buildtool", "main"]
+  , "private-deps":
+    [ ["@", "json", "", "json"]
+    , ["src/buildtool/build_engine/expression", "expression"]
+    , ["src/buildtool/file_system", "file_system_manager"]
+    , ["src/buildtool/logging", "log_level"]
+    , ["src/buildtool/logging", "logging"]
+    , "common"
+    ]
+  }
 }
diff --git a/src/buildtool/main/cli.cpp b/src/buildtool/main/cli.cpp
index cb53acde..f7e1ea71 100644
--- a/src/buildtool/main/cli.cpp
+++ b/src/buildtool/main/cli.cpp
@@ -120,11 +120,20 @@ auto SetupExecutionServiceCommandArguments(
     SetupCompatibilityArguments(app);
     SetupCommonBuildArguments(app, &clargs->build);
     SetupCacheArguments(app, &clargs->endpoint);
-    SetupExecutionServiceArguments(app, &clargs->es);
+    SetupServiceArguments(app, &clargs->service);
     SetupLogArguments(app, &clargs->log);
     SetupCommonAuthArguments(app, &clargs->auth);
     SetupServerAuthArguments(app, &clargs->sauth);
 }
+
+/// \brief Setup arguments for sub command "just serve".
+auto SetupServeServiceCommandArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<CommandLineArguments*> const& clargs) {
+    // all other arguments will be read from config file
+    SetupServeArguments(app, &clargs->serve);
+}
+
 }  // namespace
 
 auto ParseCommandLineArguments(int argc, char const* const* argv)
@@ -149,6 +158,8 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
         app.add_subcommand("gc", "Trigger garbage collection of local cache.");
     auto* cmd_execution = app.add_subcommand(
         "execute", "Start single node execution service on this machine.");
+    auto* cmd_serve =
+        app.add_subcommand("serve", "Provide target dependencies for a build.");
     auto* cmd_traverse =
         app.group("")  // group for creating hidden options
             ->add_subcommand("traverse",
@@ -165,6 +176,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     SetupTraverseCommandArguments(cmd_traverse, &clargs);
     SetupGcCommandArguments(cmd_gc, &clargs);
     SetupExecutionServiceCommandArguments(cmd_execution, &clargs);
+    SetupServeServiceCommandArguments(cmd_serve, &clargs);
     try {
         app.parse(argc, argv);
     } catch (CLI::Error& e) {
@@ -204,6 +216,9 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     else if (*cmd_execution) {
         clargs.cmd = SubCommand::kExecute;
     }
+    else if (*cmd_serve) {
+        clargs.cmd = SubCommand::kServe;
+    }
 
     return clargs;
 }
diff --git a/src/buildtool/main/cli.hpp b/src/buildtool/main/cli.hpp
index 7a580ff6..b2945757 100644
--- a/src/buildtool/main/cli.hpp
+++ b/src/buildtool/main/cli.hpp
@@ -28,7 +28,8 @@ enum class SubCommand {
     kInstallCas,
     kTraverse,
     kGc,
-    kExecute
+    kExecute,
+    kServe
 };
 
 struct CommandLineArguments {
@@ -47,7 +48,8 @@ struct CommandLineArguments {
     CommonAuthArguments auth;
     ClientAuthArguments cauth;
     ServerAuthArguments sauth;
-    ExecutionServiceArguments es;
+    ServiceArguments service;
+    ServeArguments serve;
 };
 
 auto ParseCommandLineArguments(int argc, char const* const* argv)
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index e362b7ef..b2c04be7 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -47,9 +47,10 @@
 #include "src/buildtool/execution_api/execution_service/server_implementation.hpp"
 #include "src/buildtool/execution_api/remote/config.hpp"
 #include "src/buildtool/graph_traverser/graph_traverser.hpp"
+#include "src/buildtool/main/serve.hpp"
 #include "src/buildtool/progress_reporting/progress_reporter.hpp"
 #include "src/buildtool/storage/garbage_collector.hpp"
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
 #include "src/buildtool/logging/log_config.hpp"
 #include "src/buildtool/logging/log_sink_cmdline.hpp"
 #include "src/buildtool/logging/log_sink_file.hpp"
@@ -191,7 +192,7 @@ void SetupAuthConfig(CommonAuthArguments const& authargs,
     }
 }
 
-void SetupExecutionServiceConfig(ExecutionServiceArguments const& args) {
+void SetupExecutionServiceConfig(ServiceArguments const& args) {
     if (args.port) {
         if (!ServerImpl::SetPort(*args.port)) {
             Logger::Log(LogLevel::Error, "Invalid port '{}'", *args.port);
@@ -233,7 +234,7 @@ void SetupHashFunction() {
                                   : HashFunction::JustHash::Native);
 }
 
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
 
 // returns path relative to `root`.
 [[nodiscard]] auto FindRoot(std::filesystem::path const& subdir,
@@ -809,7 +810,8 @@ void WriteTargetCacheEntries(
             });
     }
 }
-#endif
+
+#endif  // BOOTSTRAP_BUILD_TOOL
 
 }  // namespace
 
@@ -823,6 +825,14 @@ auto main(int argc, char* argv[]) -> int {
             return kExitSuccess;
         }
 
+        // just serve configures all from its config file, so parse that before
+        // doing further setup steps
+#ifndef BOOTSTRAP_BUILD_TOOL
+        if (arguments.cmd == SubCommand::kServe) {
+            ReadJustServeConfig(&arguments);
+        }
+#endif  // BOOTSTRAP_BUILD_TOOL
+
         SetupLogging(arguments.log);
         if (arguments.analysis.expression_log_limit) {
             Evaluator::SetExpressionLogLimit(
@@ -851,13 +861,13 @@ auto main(int argc, char* argv[]) -> int {
         }
 
         if (arguments.cmd == SubCommand::kExecute) {
-            SetupExecutionServiceConfig(arguments.es);
+            SetupExecutionServiceConfig(arguments.service);
             if (!ServerImpl::Instance().Run()) {
                 return kExitFailure;
             }
             return kExitSuccess;
         }
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
 
         auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs
                                                    : arguments.common.jobs;
@@ -893,7 +903,7 @@ auto main(int argc, char* argv[]) -> int {
                        ? kExitSuccess
                        : kExitFailure;
         }
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
 
         auto [main_repo, main_ws_root] =
             DetermineRoots(arguments.common, arguments.analysis);
@@ -942,7 +952,7 @@ auto main(int argc, char* argv[]) -> int {
         }
 
         else {
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
             BuildMaps::Target::ResultTargetMap result_map{
                 arguments.common.jobs};
             auto id = ReadConfiguredTarget(
@@ -1039,7 +1049,7 @@ auto main(int argc, char* argv[]) -> int {
                                : kExitSuccess;
                 }
             }
-#endif
+#endif  // BOOTSTRAP_BUILD_TOOL
         }
     } catch (std::exception const& ex) {
         Logger::Log(
diff --git a/src/buildtool/main/serve.cpp b/src/buildtool/main/serve.cpp
new file mode 100644
index 00000000..da7ed62a
--- /dev/null
+++ b/src/buildtool/main/serve.cpp
@@ -0,0 +1,276 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/main/serve.hpp"
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+
+#include <fstream>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/main/exit_codes.hpp"
+
+void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
+    Configuration serve_config{};
+    if (FileSystemManager::IsFile(clargs->serve.config)) {
+        // json::parse may throw
+        try {
+            std::ifstream fs(clargs->serve.config);
+            auto map = Expression::FromJson(nlohmann::json::parse(fs));
+            if (not map->IsMap()) {
+                Logger::Log(LogLevel::Error,
+                            "In just-serve config file {}: expected an object "
+                            "but found:\n{}",
+                            clargs->serve.config.string(),
+                            map->ToString());
+                std::exit(kExitFailure);
+            }
+            serve_config = Configuration{map};
+        } catch (std::exception const& e) {
+            Logger::Log(LogLevel::Error,
+                        "Parsing just-serve config file {} as JSON failed with "
+                        "error:\n{}",
+                        clargs->serve.config.string(),
+                        e.what());
+            std::exit(kExitFailure);
+        }
+    }
+    else {
+        Logger::Log(LogLevel::Error,
+                    "Cannot read just-serve config file {}",
+                    clargs->serve.config.string());
+        std::exit(kExitFailure);
+    }
+    // read paths of additional lookup repositories
+    auto repositories = serve_config["repositories"];
+    if (repositories.IsNotNull()) {
+        if (not repositories->IsList()) {
+            Logger::Log(
+                LogLevel::Error,
+                "just-serve: configuration-file provided repositories has "
+                "to be a list of strings, but found {}",
+                repositories->ToString());
+            std::exit(kExitFailure);
+        }
+        auto const& repos_list = repositories->List();
+        clargs->serve.repositories.reserve(clargs->serve.repositories.size() +
+                                           repos_list.size());
+        for (auto const& repo : repos_list) {
+            if (not repo->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "just-serve: expected each repository path to be a "
+                            "string, but found {}",
+                            repo->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->serve.repositories.emplace_back(repo->String());
+        }
+    }
+    // read logging arguments
+    auto logging = serve_config["logging"];
+    if (logging.IsNotNull()) {
+        if (not logging->IsMap()) {
+            Logger::Log(
+                LogLevel::Error,
+                "just-serve: configuration-file provided logging arguments has "
+                "to be a map, but found {}",
+                logging->ToString());
+            std::exit(kExitFailure);
+        }
+        // read in first the append flag
+        auto append = logging->Get("append", Expression::none_t{});
+        if (append.IsNotNull()) {
+            if (not append->IsBool()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided log append has to be a "
+                            "flag, but found {}",
+                            append->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->log.log_append = append->Bool();
+        }
+        // read in the plain flag
+        auto plain = logging->Get("plain", Expression::none_t{});
+        if (plain.IsNotNull()) {
+            if (not plain->IsBool()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided plain log has to be a "
+                            "flag, but found {}",
+                            plain->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->log.plain_log = plain->Bool();
+        }
+        // read in files field
+        auto files = logging->Get("files", Expression::none_t{});
+        if (files.IsNotNull()) {
+            if (not files->IsList()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided log files has to be a "
+                            "list, but found {}",
+                            files->ToString());
+                std::exit(kExitFailure);
+            }
+            auto const& files_list = files->List();
+            clargs->log.log_files.reserve(clargs->log.log_files.size() +
+                                          files_list.size());
+            for (auto const& file : files_list) {
+                if (not file->IsString()) {
+                    Logger::Log(LogLevel::Error,
+                                "just-serve: expected each log file path to be "
+                                "a string, but found {}",
+                                file->ToString());
+                    std::exit(kExitFailure);
+                }
+                clargs->log.log_files.emplace_back(file->String());
+            }
+        }
+        // read in limit field
+        auto limit = logging->Get("limit", Expression::none_t{});
+        if (limit.IsNotNull()) {
+            if (not limit->IsNumber()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided log limit has to be "
+                            "numeric, but found {}",
+                            limit->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->log.log_limit = ToLogLevel(limit->Number());
+        }
+    }
+    // read client TLS authentication arguments
+    auto auth_args = serve_config["authentication"];
+    if (auth_args.IsNotNull()) {
+        if (not auth_args->IsMap()) {
+            Logger::Log(LogLevel::Error,
+                        "just-serve: configuration-file provided "
+                        "authentication has to be a map, but found {}",
+                        auth_args->ToString());
+            std::exit(kExitFailure);
+        }
+        // read the TLS CA certificate
+        auto cacert = auth_args->Get("ca cert", Expression::none_t{});
+        if (cacert.IsNotNull()) {
+            if (not cacert->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided TLS CA certificate has to "
+                            "be a string, but found {}",
+                            cacert->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->auth.tls_ca_cert = cacert->String();
+        }
+    }
+    // read remote service arguments
+    auto remote_service = serve_config["remote service"];
+    if (remote_service.IsNotNull()) {
+        if (not remote_service->IsMap()) {
+            Logger::Log(LogLevel::Error,
+                        "just-serve: configuration-file provided remote "
+                        "service has to be a map, but found {}",
+                        remote_service->ToString());
+            std::exit(kExitFailure);
+        }
+        // read the interface
+        auto interface = remote_service->Get("interface", Expression::none_t{});
+        if (interface.IsNotNull()) {
+            if (not interface->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided remote service interface "
+                            "has to be a string, but found {}",
+                            interface->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->service.interface = interface->String();
+        }
+        // read the port
+        auto port = remote_service->Get("port", Expression::none_t{});
+        if (port.IsNotNull()) {
+            if (not port->IsNumber()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided remote service port has to "
+                            "be numeric, but found {}",
+                            port->ToString());
+                std::exit(kExitFailure);
+            }
+            double val{};
+            if (std::modf(port->Number(), &val) != 0.0) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided remote service port has to "
+                            "be an integer, but found {}",
+                            interface->ToString());
+                std::exit(kExitFailure);
+            }
+            // we are sure now that the port is an integer
+            clargs->service.port = std::nearbyint(val);
+        }
+        // read the pid file
+        auto pid_file = remote_service->Get("pid file", Expression::none_t{});
+        if (pid_file.IsNotNull()) {
+            if (not pid_file->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided remote service pid file "
+                            "has to be a string, but found {}",
+                            pid_file->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->service.pid_file = pid_file->String();
+        }
+        // read the info file
+        auto info_file = remote_service->Get("info file", Expression::none_t{});
+        if (info_file.IsNotNull()) {
+            if (not info_file->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided remote service info file "
+                            "has to be a string, but found {}",
+                            info_file->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->service.info_file = info_file->String();
+        }
+        // read the TLS server certificate
+        auto server_cert =
+            remote_service->Get("server cert", Expression::none_t{});
+        if (server_cert.IsNotNull()) {
+            if (not server_cert->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided TLS server certificate has "
+                            "to be a string, but found {}",
+                            server_cert->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->sauth.tls_server_cert = server_cert->String();
+        }
+        // read the TLS server key
+        auto server_key =
+            remote_service->Get("server key", Expression::none_t{});
+        if (server_key.IsNotNull()) {
+            if (not server_key->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided TLS server key has to be a "
+                            "string, but found {}",
+                            server_key->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->sauth.tls_server_key = server_key->String();
+        }
+    }
+}
+
+#endif  // BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/main/serve.hpp b/src/buildtool/main/serve.hpp
new file mode 100644
index 00000000..bc7c4664
--- /dev/null
+++ b/src/buildtool/main/serve.hpp
@@ -0,0 +1,29 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP
+#define INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+
+#include "gsl/gsl"
+#include "src/buildtool/main/cli.hpp"
+
+/// \brief Parse the "just serve" config file.
+/// While having a separate config file, almost all fields are already used by
+/// "just" itself, so we can populate the respective known command-line fields.
+void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs);
+
+#endif  // BOOTSTRAP_BUILD_TOOL
+#endif  // INCLUDED_SRC_BUILDTOOL_MAIN_SERVE_HPP