diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/buildtool/main/TARGETS | 2 | ||||
| -rw-r--r-- | src/buildtool/main/main.cpp | 54 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/TARGETS | 8 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/config.hpp | 48 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/TARGETS | 41 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/just_serve.proto | 52 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/serve_server_implementation.cpp | 126 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/serve_server_implementation.hpp | 60 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/target_level_cache_server.cpp | 78 | ||||
| -rw-r--r-- | src/buildtool/serve_api/serve_service/target_level_cache_server.hpp | 56 |
10 files changed, 525 insertions, 0 deletions
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 1264bfb5..40d19095 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -22,6 +22,8 @@ , ["src/buildtool/execution_api/execution_service", "operation_cache"] , ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/execution_api/remote", "config"] + , ["src/buildtool/serve_api/remote", "config"] + , ["src/buildtool/serve_api/serve_service", "serve_server_implementation"] , "common" , "cli" , "version" diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index b2c04be7..8cc81449 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -49,6 +49,8 @@ #include "src/buildtool/graph_traverser/graph_traverser.hpp" #include "src/buildtool/main/serve.hpp" #include "src/buildtool/progress_reporting/progress_reporter.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/serve_api/serve_service/serve_server_implementation.hpp" #include "src/buildtool/storage/garbage_collector.hpp" #endif // BOOTSTRAP_BUILD_TOOL #include "src/buildtool/logging/log_config.hpp" @@ -132,6 +134,16 @@ void SetupExecutionConfig(EndpointArguments const& eargs, } } +void SetupServeConfig(ServeArguments const& srvargs) { + using RemoteConfig = RemoteServeConfig; + if (not srvargs.repositories.empty() and + not RemoteConfig::SetKnownRepositories(srvargs.repositories)) { + Logger::Log(LogLevel::Error, + "setting serve service repositories failed."); + std::exit(kExitFailure); + } +} + void SetupAuthConfig(CommonAuthArguments const& authargs, ClientAuthArguments const& client_authargs, ServerAuthArguments const& server_authargs) { @@ -228,6 +240,39 @@ void SetupExecutionServiceConfig(ServiceArguments const& args) { } } +void SetupServeServiceConfig(ServiceArguments const& args) { + if (args.port) { + if (!ServeServerImpl::SetPort(*args.port)) { + Logger::Log(LogLevel::Error, "Invalid port '{}'", *args.port); + std::exit(kExitFailure); + } + } + if (args.info_file) { + if (!ServeServerImpl::SetInfoFile(*args.info_file)) { + Logger::Log(LogLevel::Error, + "Invalid info-file '{}'", + args.info_file->string()); + std::exit(kExitFailure); + } + } + if (args.interface) { + if (!ServeServerImpl::SetInterface(*args.interface)) { + Logger::Log(LogLevel::Error, + "Invalid interface '{}'", + args.info_file->string()); + std::exit(kExitFailure); + } + } + if (args.pid_file) { + if (!ServeServerImpl::SetPidFile(*args.pid_file)) { + Logger::Log(LogLevel::Error, + "Invalid pid-file '{}'", + args.info_file->string()); + std::exit(kExitFailure); + } + } +} + void SetupHashFunction() { HashFunction::SetHashType(Compatibility::IsCompatible() ? HashFunction::JustHash::Compatible @@ -851,6 +896,7 @@ auto main(int argc, char* argv[]) -> int { SetupHashFunction(); SetupExecutionConfig( arguments.endpoint, arguments.build, arguments.rebuild); + SetupServeConfig(arguments.serve); SetupAuthConfig(arguments.auth, arguments.cauth, arguments.sauth); if (arguments.cmd == SubCommand::kGc) { @@ -867,6 +913,14 @@ auto main(int argc, char* argv[]) -> int { } return kExitSuccess; } + + if (arguments.cmd == SubCommand::kServe) { + SetupServeServiceConfig(arguments.service); + if (!ServeServerImpl::Instance().Run()) { + return kExitFailure; + } + return kExitSuccess; + } #endif // BOOTSTRAP_BUILD_TOOL auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS new file mode 100644 index 00000000..a56adf60 --- /dev/null +++ b/src/buildtool/serve_api/remote/TARGETS @@ -0,0 +1,8 @@ +{ "config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config"] + , "hdrs": ["config.hpp"] + , "deps": [["src/buildtool/execution_api/remote", "config"]] + , "stage": ["src", "buildtool", "serve_api", "remote"] + } +} diff --git a/src/buildtool/serve_api/remote/config.hpp b/src/buildtool/serve_api/remote/config.hpp new file mode 100644 index 00000000..dac7fabf --- /dev/null +++ b/src/buildtool/serve_api/remote/config.hpp @@ -0,0 +1,48 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <filesystem> +#include <iterator> +#include <vector> + +#include "src/buildtool/execution_api/remote/config.hpp" + +class RemoteServeConfig : public RemoteExecutionConfig { + public: + // Obtain global instance + [[nodiscard]] static auto Instance() noexcept -> RemoteServeConfig& { + static RemoteServeConfig config; + return config; + } + + // Set the list of known repositories + [[nodiscard]] static auto SetKnownRepositories( + std::vector<std::filesystem::path> const& repos) noexcept -> bool { + auto& inst = Instance(); + inst.repositories_ = std::vector<std::filesystem::path>( + std::make_move_iterator(repos.begin()), + std::make_move_iterator(repos.end())); + return repos.size() == inst.repositories_.size(); + } + + // Repositories known to 'just serve' + [[nodiscard]] static auto KnownRepositories() noexcept + -> const std::vector<std::filesystem::path>& { + return Instance().repositories_; + } + + private: + // Known Git repositories to serve server. + std::vector<std::filesystem::path> repositories_{}; +}; diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS new file mode 100644 index 00000000..93155ae7 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/TARGETS @@ -0,0 +1,41 @@ +{ "just_serve_proto": + { "type": ["@", "rules", "proto", "library"] + , "name": ["just_serve_proto"] + , "service": ["yes"] + , "srcs": ["just_serve.proto"] + , "deps": [["@", "bazel_remote_apis", "", "remote_execution_proto"]] + , "stage": ["justbuild", "just_serve"] + } +, "target_level_cache_server": + { "type": ["@", "rules", "CC", "library"] + , "name": ["target_level_cache_server"] + , "hdrs": ["target_level_cache_server.hpp"] + , "srcs": ["target_level_cache_server.cpp"] + , "proto": ["just_serve_proto"] + , "deps": [["src/buildtool/logging", "logging"]] + , "stage": ["src", "buildtool", "serve_api", "serve_service"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/serve_api/remote", "config"] + , ["src/buildtool/storage", "config"] + ] + } +, "serve_server_implementation": + { "type": ["@", "rules", "CC", "library"] + , "name": ["serve_server_implementation"] + , "hdrs": ["serve_server_implementation.hpp"] + , "srcs": ["serve_server_implementation.cpp"] + , "deps": [["src/buildtool/logging", "logging"]] + , "stage": ["src", "buildtool", "serve_api", "serve_service"] + , "private-deps": + [ "target_level_cache_server" + , ["@", "fmt", "", "fmt"] + , ["@", "grpc", "", "grpc++"] + , ["@", "json", "", "json"] + , ["src/buildtool/auth", "auth"] + , ["src/buildtool/common/remote", "port"] + , ["src/buildtool/compatibility", "compatibility"] + ] + } +} diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto new file mode 100644 index 00000000..3eca2954 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/just_serve.proto @@ -0,0 +1,52 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package justbuild.just_serve; + +import "google/rpc/status.proto"; + +// A request message for +// [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree]. +message ServeCommitTreeRequest { + // The Git commit identifier to be searched on the server. + string commit = 1; + + // Relative path of requested tree with respect to the commit root. + string subdir = 2; +} + +// A response message for +// [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree]. +message ServeCommitTreeResponse { + // The requested Git tree hash. + string tree = 1; + + // If the status has a code other than `OK`, it indicates that the tree hash + // could not be computed. In this case, the `tree` field is optional. + // + // If the status code is `NOT_FOUND`, it indicates that either the commit was + // not found, or the commit root tree does not contain the given relative + // path. + google.rpc.Status status = 2; +} + +// Services for improved interaction with the target-level cache. +service TargetLevelCache { + // Retrieve the Git-subtree identifier from a given Git commit. + // + // There are no method-specific errors. + rpc ServeCommitTree(ServeCommitTreeRequest) returns (ServeCommitTreeResponse) {} +} diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp new file mode 100644 index 00000000..3c9f5f21 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp @@ -0,0 +1,126 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/serve_service/serve_server_implementation.hpp" + +#include <iostream> +#include <memory> + +#include <sys/types.h> + +#include "fmt/core.h" +#include "grpcpp/grpcpp.h" +#include "nlohmann/json.hpp" +#include "src/buildtool/auth/authentication.hpp" +#include "src/buildtool/common/remote/port.hpp" +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/serve_api/serve_service/target_level_cache_server.hpp" + +namespace { +template <typename T> +auto TryWrite(std::string const& file, T const& content) noexcept -> bool { + std::ofstream of{file}; + if (!of.good()) { + Logger::Log(LogLevel::Error, + "Could not open {}. Make sure to have write permissions", + file); + return false; + } + of << content; + return true; +} +} // namespace + +auto ServeServerImpl::Run() -> bool { + TargetLevelCacheService tlc{}; + + grpc::ServerBuilder builder; + + builder.RegisterService(&tlc); + + std::shared_ptr<grpc::ServerCredentials> creds; + if (Auth::GetAuthMethod() == AuthMethod::kTLS) { + auto tls_opts = grpc::SslServerCredentialsOptions{}; + + tls_opts.pem_root_certs = Auth::TLS::CACert(); + grpc::SslServerCredentialsOptions::PemKeyCertPair keycert = { + Auth::TLS::ServerKey(), Auth::TLS::ServerCert()}; + + tls_opts.pem_key_cert_pairs.emplace_back(keycert); + + creds = grpc::SslServerCredentials(tls_opts); + } + else { + creds = grpc::InsecureServerCredentials(); + } + + builder.AddListeningPort( + fmt::format("{}:{}", interface_, port_), creds, &port_); + + auto server = builder.BuildAndStart(); + if (!server) { + Logger::Log(LogLevel::Error, "Could not start serve service"); + return false; + } + + auto pid = getpid(); + + nlohmann::json const& info = { + {"interface", interface_}, {"port", port_}, {"pid", pid}}; + + if (!pid_file_.empty()) { + if (!TryWrite(pid_file_, pid)) { + server->Shutdown(); + return false; + } + } + + auto const& info_str = nlohmann::to_string(info); + Logger::Log(LogLevel::Info, + fmt::format("{}serve service started: {}", + Compatibility::IsCompatible() ? "compatible " : "", + info_str)); + + if (!info_file_.empty()) { + if (!TryWrite(info_file_, info_str)) { + server->Shutdown(); + return false; + } + } + + server->Wait(); + return true; +} + +[[nodiscard]] auto ServeServerImpl::SetInfoFile(std::string const& x) noexcept + -> bool { + Instance().info_file_ = x; + return true; +} + +[[nodiscard]] auto ServeServerImpl::SetPidFile(std::string const& x) noexcept + -> bool { + Instance().pid_file_ = x; + return true; +} + +[[nodiscard]] auto ServeServerImpl::SetPort(int const x) noexcept -> bool { + auto port_num = ParsePort(x); + if (!port_num) { + return false; + } + Instance().port_ = static_cast<int>(*port_num); + return true; +} diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.hpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.hpp new file mode 100644 index 00000000..f756a104 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.hpp @@ -0,0 +1,60 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVE_SERVER_IMPLEMENTATION_HPP +#define SERVE_SERVER_IMPLEMENTATION_HPP + +#include <string> + +#include "src/buildtool/logging/logger.hpp" + +class ServeServerImpl { + public: + ServeServerImpl() noexcept = default; + [[nodiscard]] static auto Instance() noexcept -> ServeServerImpl& { + static ServeServerImpl x; + return x; + } + + [[nodiscard]] static auto SetInterface(std::string const& x) noexcept + -> bool { + Instance().interface_ = x; + return true; + } + + [[nodiscard]] static auto SetPidFile(std::string const& x) noexcept -> bool; + + [[nodiscard]] static auto SetPort(int x) noexcept -> bool; + + [[nodiscard]] static auto SetInfoFile(std::string const& x) noexcept + -> bool; + + ServeServerImpl(ServeServerImpl const&) = delete; + auto operator=(ServeServerImpl const&) noexcept + -> ServeServerImpl& = delete; + + ServeServerImpl(ServeServerImpl&&) noexcept = delete; + auto operator=(ServeServerImpl&&) noexcept -> ServeServerImpl& = delete; + + auto Run() -> bool; + ~ServeServerImpl() = default; + + private: + std::string interface_{"127.0.0.1"}; + int port_{0}; + std::string info_file_{}; + std::string pid_file_{}; +}; + +#endif // SERVE_SERVER_IMPLEMENTATION_HPP diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp new file mode 100644 index 00000000..688a5c32 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp @@ -0,0 +1,78 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/serve_service/target_level_cache_server.hpp" + +#include "fmt/core.h" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/storage/config.hpp" + +auto TargetLevelCacheService::GetTreeFromCommit( + std::filesystem::path const& repo_path, + std::string const& commit, + std::string const& subdir, + std::shared_ptr<Logger> const& logger) -> std::optional<std::string> { + if (auto git_cas = GitCAS::Open(repo_path)) { + if (auto repo = GitRepo::Open(git_cas)) { + // wrap logger for GitRepo call + auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>( + [logger, repo_path, commit](auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeCommitTree: While retrieving tree of commit " + "{} from repository {}:\n{}", + commit, + repo_path.string(), + msg); + logger->Emit(LogLevel::Trace, err); + } + }); + if (auto tree_id = repo->GetSubtreeFromCommit( + commit, subdir, wrapped_logger)) { + return tree_id; + } + } + } + return std::nullopt; +} + +auto TargetLevelCacheService::ServeCommitTree( + ::grpc::ServerContext* /* context */, + const ::justbuild::just_serve::ServeCommitTreeRequest* request, + ::justbuild::just_serve::ServeCommitTreeResponse* response) + -> ::grpc::Status { + auto const& commit{request->commit()}; + auto const& subdir{request->subdir()}; + // try in local build root Git cache + if (auto tree_id = GetTreeFromCommit( + StorageConfig::GitRoot(), commit, subdir, logger_)) { + *(response->mutable_tree()) = std::move(*tree_id); + response->mutable_status()->CopyFrom(google::rpc::Status{}); + return ::grpc::Status::OK; + } + // try given extra repositories, in order + for (auto const& path : RemoteServeConfig::KnownRepositories()) { + if (auto tree_id = GetTreeFromCommit(path, commit, subdir, logger_)) { + *(response->mutable_tree()) = std::move(*tree_id); + response->mutable_status()->CopyFrom(google::rpc::Status{}); + return ::grpc::Status::OK; + } + } + // commit not found + google::rpc::Status status; + status.set_code(grpc::StatusCode::NOT_FOUND); + response->mutable_status()->CopyFrom(status); + return ::grpc::Status::OK; +} diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp new file mode 100644 index 00000000..80d09647 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp @@ -0,0 +1,56 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TARGET_LEVEL_CACHE_SERVER_HPP +#define TARGET_LEVEL_CACHE_SERVER_HPP + +#include <filesystem> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include "justbuild/just_serve/just_serve.grpc.pb.h" +#include "src/buildtool/logging/logger.hpp" + +class TargetLevelCacheService final + : public justbuild::just_serve::TargetLevelCache::Service { + + public: + // Retrieve the tree of a commit. + // + // This request interrogates the service whether it knows a given Git + // commit. If requested commit is found, it provides the commit's + // Git-tree identifier. + // + // Errors: + // + // * `NOT_FOUND`: The requested commit could not be found. + auto ServeCommitTree( + ::grpc::ServerContext* context, + const ::justbuild::just_serve::ServeCommitTreeRequest* request, + ::justbuild::just_serve::ServeCommitTreeResponse* response) + -> ::grpc::Status override; + + private: + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("serve-service")}; + + [[nodiscard]] static auto GetTreeFromCommit( + std::filesystem::path const& repo_path, + std::string const& commit, + std::string const& subdir, + std::shared_ptr<Logger> const& logger) -> std::optional<std::string>; +}; + +#endif // TARGET_LEVEL_CACHE_SERVER_HPP |