diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/buildtool/serve_api/remote/TARGETS | 41 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/configuration_client.cpp | 52 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/configuration_client.hpp | 46 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/serve_api.cpp | 41 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/serve_api.hpp | 64 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/target_client.cpp | 118 | ||||
| -rw-r--r-- | src/buildtool/serve_api/remote/target_client.hpp | 65 |
7 files changed, 373 insertions, 54 deletions
diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS index 2cd24545..f77fac04 100644 --- a/src/buildtool/serve_api/remote/TARGETS +++ b/src/buildtool/serve_api/remote/TARGETS @@ -23,12 +23,51 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["serve_api"] , "hdrs": ["serve_api.hpp"] - , "srcs": ["serve_api.cpp"] , "deps": [ ["src/buildtool/common/remote", "port"] , ["src/buildtool/file_system/symlinks_map", "pragma_special"] , "source_tree_client" + , "target_client" + , "configuration_client" + , "config" ] , "stage": ["src", "buildtool", "serve_api", "remote"] } +, "target_client": + { "type": ["@", "rules", "CC", "library"] + , "name": ["target_client"] + , "hdrs": ["target_client.hpp"] + , "srcs": ["target_client.cpp"] + , "deps": + [ ["src/buildtool/common/remote", "port"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/logging", "logging"] + , ["src/buildtool/execution_api/common", "create_execution_api"] + , ["src/buildtool/storage", "storage"] + ] + , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] + , "stage": ["src", "buildtool", "serve_api", "remote"] + , "private-deps": + [ ["src/buildtool/common/remote", "client_common"] + , ["src/buildtool/common", "bazel_types"] + , ["@", "json", "", "json"] + ] + } +, "configuration_client": + { "type": ["@", "rules", "CC", "library"] + , "name": ["configuration_client"] + , "hdrs": ["configuration_client.hpp"] + , "srcs": ["configuration_client.cpp"] + , "deps": + [ ["src/buildtool/common/remote", "port"] + , ["src/buildtool/logging", "logging"] + , ["src/buildtool/common/remote", "client_common"] + ] + , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]] + , "stage": ["src", "buildtool", "serve_api", "remote"] + , "private-deps": + [ ["@", "json", "", "json"] + , ["src/buildtool/execution_api/remote", "config"] + ] + } } diff --git a/src/buildtool/serve_api/remote/configuration_client.cpp b/src/buildtool/serve_api/remote/configuration_client.cpp new file mode 100644 index 00000000..25c38d79 --- /dev/null +++ b/src/buildtool/serve_api/remote/configuration_client.cpp @@ -0,0 +1,52 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/remote/configuration_client.hpp" + +#include <nlohmann/json.hpp> + +#include "src/buildtool/execution_api/remote/config.hpp" + +auto ConfigurationClient::CheckServeRemoteExecution() -> bool { + auto client_remote_address = RemoteExecutionConfig::RemoteAddress(); + if (!client_remote_address) { + logger_.Emit(LogLevel::Error, + "In order to use just-serve, also the " + "--remote-execution-address option must be given."); + return false; + } + + grpc::ClientContext context; + justbuild::just_serve::RemoteExecutionEndpointRequest request{}; + justbuild::just_serve::RemoteExecutionEndpointResponse response{}; + grpc::Status status = + stub_->RemoteExecutionEndpoint(&context, request, &response); + + if (not status.ok()) { + LogStatus(&logger_, LogLevel::Error, status); + return false; + } + auto serve_remote_endpoint = nlohmann::json::parse(response.address()); + if (serve_remote_endpoint != client_remote_address->ToJson()) { + Logger::Log(LogLevel::Error, + "Different execution endpoint detected.\nIn order to " + "correctly use just serve, its remote execution endpoint " + "must be the same used by the client.\nserve remote " + "endpoint: {}\nclient remote endpoint: {}", + serve_remote_endpoint.dump(), + client_remote_address->ToJson().dump()); + return false; + } + return true; +} diff --git a/src/buildtool/serve_api/remote/configuration_client.hpp b/src/buildtool/serve_api/remote/configuration_client.hpp new file mode 100644 index 00000000..13584832 --- /dev/null +++ b/src/buildtool/serve_api/remote/configuration_client.hpp @@ -0,0 +1,46 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP +#define INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP + +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "justbuild/just_serve/just_serve.grpc.pb.h" +#include "src/buildtool/common/remote/client_common.hpp" +#include "src/buildtool/common/remote/port.hpp" +#include "src/buildtool/logging/logger.hpp" + +/// Implements client side for Configuration service defined in: +/// src/buildtool/serve_api/serve_service/just_serve.proto +class ConfigurationClient { + public: + ConfigurationClient(std::string const& server, Port port) noexcept + : + + stub_{justbuild::just_serve::Configuration::NewStub( + CreateChannelWithCredentials(server, port))} {} + + auto CheckServeRemoteExecution() -> bool; + + private: + std::unique_ptr<justbuild::just_serve::Configuration::Stub> stub_; + Logger logger_{"RemoteTargetClient"}; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP diff --git a/src/buildtool/serve_api/remote/serve_api.cpp b/src/buildtool/serve_api/remote/serve_api.cpp deleted file mode 100644 index 0c5c531c..00000000 --- a/src/buildtool/serve_api/remote/serve_api.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/buildtool/serve_api/remote/serve_api.hpp" - -ServeApi::ServeApi(std::string const& host, Port port) noexcept - : stc_{std::make_unique<SourceTreeClient>(host, port)} {} - -// implement move constructor in cpp, where all members are complete types -ServeApi::ServeApi(ServeApi&& other) noexcept = default; - -// implement destructor in cpp, where all members are complete types -ServeApi::~ServeApi() = default; - -auto ServeApi::RetrieveTreeFromCommit(std::string const& commit, - std::string const& subdir, - bool sync_tree) - -> std::optional<std::string> { - return stc_->ServeCommitTree(commit, subdir, sync_tree); -} - -auto ServeApi::RetrieveTreeFromArchive( - std::string const& content, - std::string const& archive_type, - std::string const& subdir, - std::optional<PragmaSpecial> const& resolve_symlinks, - bool sync_tree) -> std::optional<std::string> { - return stc_->ServeArchiveTree( - content, archive_type, subdir, resolve_symlinks, sync_tree); -} diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp index a0d50ea9..48ba4098 100644 --- a/src/buildtool/serve_api/remote/serve_api.hpp +++ b/src/buildtool/serve_api/remote/serve_api.hpp @@ -21,37 +21,77 @@ #include "src/buildtool/common/remote/port.hpp" #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/serve_api/remote/configuration_client.hpp" #include "src/buildtool/serve_api/remote/source_tree_client.hpp" +#include "src/buildtool/serve_api/remote/target_client.hpp" class ServeApi final { public: - using Ptr = std::unique_ptr<ServeApi>; - - ServeApi(std::string const& host, Port port) noexcept; - ServeApi(ServeApi const&) = delete; - ServeApi(ServeApi&& other) noexcept; + ~ServeApi() = default; auto operator=(ServeApi const&) -> ServeApi& = delete; auto operator=(ServeApi&&) -> ServeApi& = delete; - ~ServeApi(); + [[nodiscard]] static auto Instance() noexcept -> ServeApi& { + static ServeApi instance = ServeApi::init(); + return instance; + } - [[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit, - std::string const& subdir = ".", - bool sync_tree = false) - -> std::optional<std::string>; + [[nodiscard]] static auto RetrieveTreeFromCommit( + std::string const& commit, + std::string const& subdir = ".", + bool sync_tree = false) -> std::optional<std::string> { + return Instance().stc_->ServeCommitTree(commit, subdir, sync_tree); + } - [[nodiscard]] auto RetrieveTreeFromArchive( + [[nodiscard]] static auto RetrieveTreeFromArchive( std::string const& content, std::string const& archive_type = "archive", std::string const& subdir = ".", std::optional<PragmaSpecial> const& resolve_symlinks = std::nullopt, - bool sync_tree = false) -> std::optional<std::string>; + bool sync_tree = false) -> std::optional<std::string> { + return Instance().stc_->ServeArchiveTree( + content, archive_type, subdir, resolve_symlinks, sync_tree); + } + + [[nodiscard]] static auto ServeTargetVariables( + std::string const& target_root_id, + std::string const& target_file, + std::string const& target) -> std::optional<std::vector<std::string>> { + return Instance().tc_->ServeTargetVariables( + target_root_id, target_file, target); + } + + [[nodiscard]] static auto ServeTarget(const TargetCacheKey& key) + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> { + return Instance().tc_->ServeTarget(key); + } + + [[nodiscard]] static auto CheckServeRemoteExecution() -> bool { + return Instance().cc_->CheckServeRemoteExecution(); + } private: + ServeApi(std::string const& host, Port port) noexcept + : stc_{std::make_unique<SourceTreeClient>(host, port)}, + tc_{std::make_unique<TargetClient>(host, port)}, + cc_{std::make_unique<ConfigurationClient>(host, port)} {} + + ServeApi(ServeApi&& other) noexcept = default; + + [[nodiscard]] static auto init() noexcept -> ServeApi { + auto sadd = RemoteServeConfig::RemoteAddress(); + return ServeApi{sadd->host, sadd->port}; + } + // source tree service client std::unique_ptr<SourceTreeClient> stc_; + // target service client + std::unique_ptr<TargetClient> tc_; + // configuration service client + std::unique_ptr<ConfigurationClient> cc_; }; #endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_SERVE_API_HPP diff --git a/src/buildtool/serve_api/remote/target_client.cpp b/src/buildtool/serve_api/remote/target_client.cpp new file mode 100644 index 00000000..bb30907d --- /dev/null +++ b/src/buildtool/serve_api/remote/target_client.cpp @@ -0,0 +1,118 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/remote/target_client.hpp" + +#include <utility> + +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/common/remote/client_common.hpp" + +TargetClient::TargetClient(std::string const& server, Port port) noexcept { + stub_ = justbuild::just_serve::Target::NewStub( + CreateChannelWithCredentials(server, port)); +} + +auto TargetClient::ServeTarget(const TargetCacheKey& key) + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> { + // make sure the blob containing the key is in the remote cas + if (!local_api_->RetrieveToCas({key.Id()}, &*remote_api_)) { + logger_.Emit(LogLevel::Error, + "failed to retrieve to remote cas ObjectInfo {}", + key.Id().ToString()); + return std::nullopt; + } + + bazel_re::Digest key_dgst{key.Id().digest}; + justbuild::just_serve::ServeTargetRequest request{}; + *(request.mutable_target_cache_key_id()) = std::move(key_dgst); + + auto execution_backend_dgst = ArtifactDigest::Create<ObjectType::File>( + StorageConfig::ExecutionBackendDescription()); + auto const& execution_info = + Artifact::ObjectInfo{.digest = ArtifactDigest{execution_backend_dgst}, + .type = ObjectType::File}; + if (!local_api_->RetrieveToCas({execution_info}, &*remote_api_)) { + logger_.Emit(LogLevel::Error, + "failed to upload blob {} to remote cas", + execution_info.ToString()); + return std::nullopt; + } + + *(request.mutable_execution_backend_description_id()) = + std::move(execution_backend_dgst); + + grpc::ClientContext context; + justbuild::just_serve::ServeTargetResponse response; + auto const& status = stub_->ServeTarget(&context, request, &response); + if (!status.ok()) { + LogStatus(&logger_, LogLevel::Error, status); + return std::nullopt; + } + auto const& target_value_dgst = ArtifactDigest{response.target_value()}; + auto const& obj_info = Artifact::ObjectInfo{.digest = target_value_dgst, + .type = ObjectType::File}; + if (!local_api_->IsAvailable(target_value_dgst)) { + if (!remote_api_->RetrieveToCas({obj_info}, &*local_api_)) { + logger_.Emit(LogLevel::Error, + "failed to retrieve blob {} from remote cas", + obj_info.ToString()); + return std::nullopt; + } + } + + auto const& target_value_str = local_api_->RetrieveToMemory(obj_info); + if (!target_value_str) { + logger_.Emit(LogLevel::Error, + "failed to retrieve blob {} from local cas", + obj_info.ToString()); + return std::nullopt; + } + auto const& result = + TargetCacheEntry::FromJson(nlohmann::json::parse(*target_value_str)); + + return std::make_pair(result, obj_info); +} + +auto TargetClient::ServeTargetVariables(std::string const& target_root_id, + std::string const& target_file, + std::string const& target) + -> std::optional<std::vector<std::string>> { + justbuild::just_serve::ServeTargetVariablesRequest request{}; + request.set_root_tree(target_root_id); + request.set_target_file(target_file); + request.set_target(target); + + grpc::ClientContext context; + justbuild::just_serve::ServeTargetVariablesResponse response; + grpc::Status status = + stub_->ServeTargetVariables(&context, request, &response); + + if (not status.ok()) { + LogStatus(&logger_, LogLevel::Error, status); + return std::nullopt; + } + auto size = response.flexible_config_size(); + if (size == 0) { + return std::vector<std::string>(); + } + std::vector<std::string> res{}; + res.reserve(size); + for (auto const& var : response.flexible_config()) { + res.emplace_back(var); + } + return res; +} diff --git a/src/buildtool/serve_api/remote/target_client.hpp b/src/buildtool/serve_api/remote/target_client.hpp new file mode 100644 index 00000000..ac52a468 --- /dev/null +++ b/src/buildtool/serve_api/remote/target_client.hpp @@ -0,0 +1,65 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP +#define INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP + +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "justbuild/just_serve/just_serve.grpc.pb.h" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/remote/port.hpp" +#include "src/buildtool/execution_api/common/create_execution_api.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" + +/// Implements client side for Target service defined in: +/// src/buildtool/serve_api/serve_service/just_serve.proto +class TargetClient { + public: + TargetClient(std::string const& server, Port port) noexcept; + + /// \brief Retrieve the pair of TargetCacheEntry and ObjectInfo associated + /// to the given key. + /// \param[in] key The TargetCacheKey of an export target + /// \returns Pair of cache entry and its object info on success or nullopt. + [[nodiscard]] auto ServeTarget(const TargetCacheKey& key) + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; + + /// \brief Retrieve the flexible config variables of an export target. + /// \param[in] target_root_id Hash of target-level root tree. + /// \param[in] target_file Relative path of the target file. + /// \param[in] target Name of the target to interrogate. + /// \returns The list of flexible config variables, or nullopt on errors. + [[nodiscard]] auto ServeTargetVariables(std::string const& target_root_id, + std::string const& target_file, + std::string const& target) + -> std::optional<std::vector<std::string>>; + + private: + std::unique_ptr<justbuild::just_serve::Target::Stub> stub_; + Logger logger_{"RemoteTargetClient"}; + gsl::not_null<IExecutionApi::Ptr> const remote_api_{ + CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(), + "remote-execution")}; + gsl::not_null<IExecutionApi::Ptr> const local_api_{ + CreateExecutionApi(std::nullopt)}; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP |