diff options
Diffstat (limited to 'src')
6 files changed, 549 insertions, 0 deletions
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS index 5e00b3d1..c92a8758 100644 --- a/src/buildtool/serve_api/serve_service/TARGETS +++ b/src/buildtool/serve_api/serve_service/TARGETS @@ -46,6 +46,8 @@ , "stage": ["src", "buildtool", "serve_api", "serve_service"] , "private-deps": [ "source_tree" + , "target_service" + , "configuration_service" , ["@", "fmt", "", "fmt"] , ["@", "grpc", "", "grpc++"] , ["@", "json", "", "json"] @@ -61,4 +63,41 @@ , ["src/buildtool/execution_api/execution_service", "operations_server"] ] } +, "target_service": + { "type": ["@", "rules", "CC", "library"] + , "name": ["target_service"] + , "hdrs": ["target.hpp"] + , "srcs": ["target.cpp"] + , "proto": ["just_serve_proto"] + , "deps": + [ ["src/buildtool/logging", "logging"] + , ["@", "gsl", "", "gsl"] + , ["src/buildtool/execution_api/remote", "config"] + , ["src/buildtool/execution_api/common", "create_execution_api"] + ] + , "stage": ["src", "buildtool", "serve_api", "serve_service"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["@", "json", "", "json"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] + , ["src/buildtool/file_system", "git_cas"] + , ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/serve_api/remote", "config"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/storage", "storage"] + , ["src/utils/cpp", "verify_hash"] + ] + } +, "configuration_service": + { "type": ["@", "rules", "CC", "library"] + , "name": ["configuration_service"] + , "hdrs": ["configuration.hpp"] + , "srcs": ["configuration.cpp"] + , "proto": ["just_serve_proto"] + , "stage": ["src", "buildtool", "serve_api", "serve_service"] + , "private-deps": [["src/buildtool/execution_api/remote", "config"]] + } } diff --git a/src/buildtool/serve_api/serve_service/configuration.cpp b/src/buildtool/serve_api/serve_service/configuration.cpp new file mode 100644 index 00000000..4a103ea4 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/configuration.cpp @@ -0,0 +1,34 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/serve_service/configuration.hpp" + +#include <optional> + +#include "src/buildtool/execution_api/remote/config.hpp" + +auto ConfigurationService::RemoteExecutionEndpoint( + ::grpc::ServerContext* /*context*/, + const ::justbuild::just_serve::RemoteExecutionEndpointRequest* /*request*/, + ::justbuild::just_serve::RemoteExecutionEndpointResponse* response) + -> ::grpc::Status { + auto address = RemoteExecutionConfig::RemoteAddress(); + if (!address) { + return grpc::Status{grpc::StatusCode::INTERNAL, + "could not obtain the address of the associated " + "remote execution endpoint"}; + } + response->set_address(address->ToJson().dump()); + return ::grpc::Status::OK; +} diff --git a/src/buildtool/serve_api/serve_service/configuration.hpp b/src/buildtool/serve_api/serve_service/configuration.hpp new file mode 100644 index 00000000..9bd3c892 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/configuration.hpp @@ -0,0 +1,30 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP +#define INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP + +#include "justbuild/just_serve/just_serve.grpc.pb.h" + +class ConfigurationService final + : public justbuild::just_serve::Configuration::Service { + public: + auto RemoteExecutionEndpoint( + ::grpc::ServerContext* context, + const ::justbuild::just_serve::RemoteExecutionEndpointRequest* request, + ::justbuild::just_serve::RemoteExecutionEndpointResponse* response) + -> ::grpc::Status override; +}; + +#endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_CONFIGURATION_HPP diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp index 7eea7d91..16aa6466 100644 --- a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp +++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp @@ -34,7 +34,9 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/serve_api/serve_service/configuration.hpp" #include "src/buildtool/serve_api/serve_service/source_tree.hpp" +#include "src/buildtool/serve_api/serve_service/target.hpp" #include "src/buildtool/storage/config.hpp" namespace { @@ -68,10 +70,14 @@ auto ServeServerImpl::Run(bool with_execute) -> bool { } SourceTreeService sts{}; + TargetService ts{}; + ConfigurationService cs{}; grpc::ServerBuilder builder; builder.RegisterService(&sts); + builder.RegisterService(&ts); + builder.RegisterService(&cs); // the user has not given any remote-execution endpoint // so we start a "just-execute instance" on the same process diff --git a/src/buildtool/serve_api/serve_service/target.cpp b/src/buildtool/serve_api/serve_service/target.cpp new file mode 100644 index 00000000..364f0420 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/target.cpp @@ -0,0 +1,349 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/serve_api/serve_service/target.hpp" + +#include <string> +#include <utility> + +#include <fmt/core.h> +#include <fmt/format.h> +#include <nlohmann/json.hpp> + +#include "src/buildtool/build_engine/expression/expression.hpp" +#include "src/buildtool/build_engine/expression/expression_ptr.hpp" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/file_system/git_cas.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/serve_api/remote/config.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" +#include "src/utils/cpp/verify_hash.hpp" + +auto TargetService::ServeTarget( + ::grpc::ServerContext* /*context*/, + const ::justbuild::just_serve::ServeTargetRequest* request, + ::justbuild::just_serve::ServeTargetResponse* response) -> ::grpc::Status { + if (auto error_msg = IsAHash(request->target_cache_key_id().hash()); + error_msg) { + logger_->Emit(LogLevel::Debug, *error_msg); + return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, *error_msg}; + } + if (auto error_msg = + IsAHash(request->execution_backend_description_id().hash()); + error_msg) { + logger_->Emit(LogLevel::Debug, *error_msg); + return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, *error_msg}; + } + auto const& target_cache_key_digest = + ArtifactDigest{request->target_cache_key_id()}; + + auto const& tc = Storage::Instance().TargetCache().WithShard( + ArtifactDigest{request->execution_backend_description_id()}.hash()); + auto const& tc_key = + TargetCacheKey{{target_cache_key_digest, ObjectType::File}}; + + // check if target-level cache entry has already been computed + if (auto target_entry = tc->Read(tc_key); target_entry) { + + // make sure all artifacts referenced in the target cache value are in + // the remote cas + std::vector<Artifact::ObjectInfo> artifacts; + if (!target_entry->first.ToArtifacts(&artifacts)) { + auto msg = fmt::format( + "Failed to extract artifacts from target cache entry {}", + target_entry->second.ToString()); + logger_->Emit(LogLevel::Error, msg); + return grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, msg}; + } + if (!local_api_->RetrieveToCas(artifacts, &*remote_api_)) { + auto msg = fmt::format( + "Failed to upload to remote cas the artifacts referenced in " + "the target cache entry {}", + target_entry->second.ToString()); + logger_->Emit(LogLevel::Error, msg); + return grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, msg}; + } + + // make sure the target cache value is in the remote cas + if (!local_api_->RetrieveToCas({target_entry->second}, &*remote_api_)) { + auto msg = fmt::format( + "Failed to upload to remote cas the target cache entry {}", + target_entry->second.ToString()); + logger_->Emit(LogLevel::Error, msg); + return grpc::Status{grpc::StatusCode::UNAVAILABLE, msg}; + } + + *(response->mutable_target_value()) = + std::move(target_entry->second.digest); + return ::grpc::Status::OK; + } + + // get target description from remote cas + auto const& target_cache_key_info = Artifact::ObjectInfo{ + .digest = target_cache_key_digest, .type = ObjectType::File}; + + if (!local_api_->IsAvailable(target_cache_key_digest) and + !remote_api_->RetrieveToCas({target_cache_key_info}, &*local_api_)) { + auto msg = fmt::format( + "could not retrieve from remote-execution end point blob {}", + target_cache_key_info.ToString()); + logger_->Emit(LogLevel::Error, msg); + return grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, msg}; + } + + auto const& target_description_str = + local_api_->RetrieveToMemory(target_cache_key_info); + + auto const& target_description_dict = + nlohmann::json::parse(*target_description_str); + + // utility function to check the correctness of the TargetCacheKey + [[maybe_unused]] std::string error_msg; + auto check_key = [&target_description_dict, + this, + &target_cache_key_digest, + &error_msg](std::string const& key) -> bool { + if (!target_description_dict.contains(key)) { + error_msg = + fmt::format("TargetCacheKey {} does not contain key \"{}\"", + target_cache_key_digest.hash(), + key); + logger_->Emit(LogLevel::Error, error_msg); + return false; + } + return true; + }; + + if (!check_key("repo_key") or !check_key("target_name") or + !check_key("effective_config")) { + return grpc::Status{grpc::StatusCode::FAILED_PRECONDITION, error_msg}; + } + + // TODO(asartori): coordinate the build on the remote + // for now we return an error + const auto* msg = "orchestration of remote build not yet implemented"; + logger_->Emit(LogLevel::Error, msg); + return grpc::Status{grpc::StatusCode::UNIMPLEMENTED, msg}; +} + +auto TargetService::GetBlobContent(std::filesystem::path const& repo_path, + std::string const& tree_id, + std::string const& rel_path, + std::shared_ptr<Logger> const& logger) + -> std::optional<std::pair<bool, std::optional<std::string>>> { + if (auto git_cas = GitCAS::Open(repo_path)) { + if (auto repo = GitRepo::Open(git_cas)) { + // check if tree exists + auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>( + [logger, repo_path, tree_id](auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeTargetVariables: While checking if tree {} " + "exists in repository {}:\n{}", + tree_id, + repo_path.string(), + msg); + logger->Emit(LogLevel::Debug, err); + } + }); + if (repo->CheckTreeExists(tree_id, wrapped_logger) == true) { + // get tree entry by path + if (auto entry_info = + repo->GetObjectByPathFromTree(tree_id, rel_path)) { + wrapped_logger = std::make_shared<GitRepo::anon_logger_t>( + [logger, repo_path, blob_id = entry_info->id]( + auto const& msg, bool fatal) { + if (fatal) { + auto err = fmt::format( + "ServeTargetVariables: While retrieving " + "blob {} from repository {}:\n{}", + blob_id, + repo_path.string(), + msg); + logger->Emit(LogLevel::Debug, err); + } + }); + // get blob content + return repo->TryReadBlob(entry_info->id, wrapped_logger); + } + // trace failure to get entry + auto err = fmt::format( + "ServeTargetVariables: Failed to retrieve entry {} in " + "tree {} from repository {}", + rel_path, + tree_id, + repo_path.string()); + logger->Emit(LogLevel::Debug, err); + return std::pair(false, std::nullopt); // could not read blob + } + } + } + return std::nullopt; // tree not found or errors while retrieving tree +} + +auto TargetService::ServeTargetVariables( + ::grpc::ServerContext* /*context*/, + const ::justbuild::just_serve::ServeTargetVariablesRequest* request, + ::justbuild::just_serve::ServeTargetVariablesResponse* response) + -> ::grpc::Status { + auto const& root_tree{request->root_tree()}; + auto const& target_file{request->target_file()}; + auto const& target{request->target()}; + // retrieve content of target file + std::optional<std::string> target_file_content{std::nullopt}; + bool tree_found{false}; + // try in local build root Git cache + if (auto res = GetBlobContent( + StorageConfig::GitRoot(), root_tree, target_file, logger_)) { + tree_found = true; + if (res->first) { + if (not res->second) { + // tree exists, but does not contain target file + auto err = fmt::format( + "Target-root {} found, but does not contain target file {}", + root_tree, + target_file); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, + err}; + } + target_file_content = *res->second; + } + } + if (not target_file_content) { + // try given extra repositories, in order + for (auto const& path : RemoteServeConfig::KnownRepositories()) { + if (auto res = + GetBlobContent(path, root_tree, target_file, logger_)) { + tree_found = true; + if (res->first) { + if (not res->second) { + // tree exists, but does not contain target file + auto err = fmt::format( + "Target-root {} found, but does not contain target " + "file {}", + root_tree, + target_file); + return ::grpc::Status{ + ::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + target_file_content = *res->second; + break; + } + } + } + } + // report if failed to find root tree + if (not target_file_content) { + if (tree_found) { + // something went wrong trying to read the target file blob + auto err = + fmt::format("Could not read target file {}", target_file); + return ::grpc::Status{::grpc::StatusCode::INTERNAL, err}; + } + // tree not found + auto err = fmt::format("Missing target-root tree {}", root_tree); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + // parse target file as json + ExpressionPtr map{nullptr}; + try { + map = Expression::FromJson(nlohmann::json::parse(*target_file_content)); + } catch (std::exception const& e) { + auto err = fmt::format( + "Failed to parse target file {} as json with error:\n{}", + target_file, + e.what()); + logger_->Emit(LogLevel::Error, err); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + if (not map->IsMap()) { + auto err = + fmt::format("Target file {} should contain a map, but found:\n{}", + target_file, + map->ToString()); + logger_->Emit(LogLevel::Error, err); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + // do validity checks (target present, is export, flexible_config valid) + auto target_desc = map->At(target); + if (not target_desc) { + // target is not present + auto err = fmt::format( + "Missing target {} in target file {}", target, target_file); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + auto export_desc = target_desc->get()->At("type"); + if (not export_desc) { + auto err = fmt::format( + "Missing \"type\" field for target {} in target file {}.", + target, + target_file); + logger_->Emit(LogLevel::Error, err); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + if (not export_desc->get()->IsString()) { + auto err = fmt::format( + "Field \"type\" for target {} in target file {} should be a " + "string, but found:\n{}", + target, + target_file, + export_desc->get()->ToString()); + logger_->Emit(LogLevel::Error, err); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + if (export_desc->get()->String() != "export") { + // target is not of "type" : "export" + auto err = + fmt::format(R"(target {} is not of "type" : "export")", target); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + auto flexible_config = target_desc->get()->At("flexible_config"); + if (not flexible_config) { + // respond with success and an empty flexible_config list + return ::grpc::Status::OK; + } + if (not flexible_config->get()->IsList()) { + auto err = fmt::format( + "Field \"flexible_config\" for target {} in target file {} should " + "be a list, but found {}", + target, + target_file, + flexible_config->get()->ToString()); + logger_->Emit(LogLevel::Error, err); + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + // populate response with flexible_config list + auto flexible_config_list = flexible_config->get()->List(); + std::vector<std::string> fclist{}; + for (auto const& elem : flexible_config_list) { + if (not elem->IsString()) { + auto err = fmt::format( + "Field \"flexible_config\" for target {} in target file {} has " + "non-string entry {}", + target, + target_file, + elem->ToString()); + logger_->Emit(LogLevel::Error, err); + response->clear_flexible_config(); // must be unset + return ::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION, err}; + } + response->add_flexible_config(elem->String()); + } + // respond with success + return ::grpc::Status::OK; +} diff --git a/src/buildtool/serve_api/serve_service/target.hpp b/src/buildtool/serve_api/serve_service/target.hpp new file mode 100644 index 00000000..0f3f4ae2 --- /dev/null +++ b/src/buildtool/serve_api/serve_service/target.hpp @@ -0,0 +1,91 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP +#define INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP + +#include <filesystem> +#include <memory> +#include <optional> + +#include "justbuild/just_serve/just_serve.grpc.pb.h" +#include "src/buildtool/common/remote/remote_common.hpp" +#include "src/buildtool/execution_api/common/create_execution_api.hpp" +#include "src/buildtool/execution_api/remote/config.hpp" +#include "src/buildtool/logging/logger.hpp" + +#include <gsl/gsl> + +class TargetService final : public justbuild::just_serve::Target::Service { + public: + // Given a target-level caching key, returns the computed value. In doing + // so, it can build on the associated end-point passing the + // RemoteExecutionProperties contained in the ServeTargetRequest. + // + // If the status has a code different from `OK`, the response MUST not be + // used. + // + // Errors: + // * `FAILED_PRECONDITION`: Failed to find required information in the CAS + // or the target cache key is malformed. + // * `UNAVAILABLE`: Could not communicate with the remote execution + // endpoint. + // * `INTERNAL`: Internally, something is very broken. + auto ServeTarget( + ::grpc::ServerContext* /*context*/, + const ::justbuild::just_serve::ServeTargetRequest* /*request*/, + ::justbuild::just_serve::ServeTargetResponse* /*response*/) + -> ::grpc::Status override; + + // Given the target-level root tree and the name of an export target, + // returns the list of flexible variables from that target's description. + // + // If the status has a code different from `OK`, the response MUST not be + // used. + // + // Errors: + // * `FAILED_PRECONDITION`: An error occurred in retrieving the + // configuration of the requested target, such as missing entries + // (target-root, target file, target name), unparsable target file, or + // requested target not being of "type" : "export". + // * `INTERNAL`: Internally, something is very broken. + auto ServeTargetVariables( + ::grpc::ServerContext* /*context*/, + const ::justbuild::just_serve::ServeTargetVariablesRequest* request, + ::justbuild::just_serve::ServeTargetVariablesResponse* response) + -> ::grpc::Status override; + + private: + std::shared_ptr<Logger> logger_{std::make_shared<Logger>("target-service")}; + + // remote execution endpoint used for remote building + gsl::not_null<IExecutionApi::Ptr> const remote_api_{ + CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(), + "serve-remote-execution")}; + // used for storing and retrieving target-level cache entries + gsl::not_null<IExecutionApi::Ptr> const local_api_{ + CreateExecutionApi(std::nullopt)}; + + /// \brief Get the blob content at given path inside a Git tree. + /// \returns If tree found, pair of "no-internal-errors" flag and content of + /// blob at the path specified if blob exists, nullopt otherwise. + [[nodiscard]] static auto GetBlobContent( + std::filesystem::path const& repo_path, + std::string const& tree_id, + std::string const& rel_path, + std::shared_ptr<Logger> const& logger) + -> std::optional<std::pair<bool, std::optional<std::string>>>; +}; + +#endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP |