just-serve: Extend client and API for services Target and Configuration

Co-authored-by: Alberto Sartori <alberto.sartori@huawei.com>

7 files changed, 373 insertions, 54 deletions

diff --git a/src/buildtool/serve_api/remote/TARGETS b/src/buildtool/serve_api/remote/TARGETS
index 2cd24545..f77fac04 100644
--- a/src/buildtool/serve_api/remote/TARGETS
+++ b/src/buildtool/serve_api/remote/TARGETS
@@ -23,12 +23,51 @@
   { "type": ["@", "rules", "CC", "library"]
   , "name": ["serve_api"]
   , "hdrs": ["serve_api.hpp"]
-  , "srcs": ["serve_api.cpp"]
   , "deps":
     [ ["src/buildtool/common/remote", "port"]
     , ["src/buildtool/file_system/symlinks_map", "pragma_special"]
     , "source_tree_client"
+    , "target_client"
+    , "configuration_client"
+    , "config"
     ]
   , "stage": ["src", "buildtool", "serve_api", "remote"]
   }
+, "target_client":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["target_client"]
+  , "hdrs": ["target_client.hpp"]
+  , "srcs": ["target_client.cpp"]
+  , "deps":
+    [ ["src/buildtool/common/remote", "port"]
+    , ["src/buildtool/common", "common"]
+    , ["src/buildtool/logging", "logging"]
+    , ["src/buildtool/execution_api/common", "create_execution_api"]
+    , ["src/buildtool/storage", "storage"]
+    ]
+  , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]]
+  , "stage": ["src", "buildtool", "serve_api", "remote"]
+  , "private-deps":
+    [ ["src/buildtool/common/remote", "client_common"]
+    , ["src/buildtool/common", "bazel_types"]
+    , ["@", "json", "", "json"]
+    ]
+  }
+, "configuration_client":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["configuration_client"]
+  , "hdrs": ["configuration_client.hpp"]
+  , "srcs": ["configuration_client.cpp"]
+  , "deps":
+    [ ["src/buildtool/common/remote", "port"]
+    , ["src/buildtool/logging", "logging"]
+    , ["src/buildtool/common/remote", "client_common"]
+    ]
+  , "proto": [["src/buildtool/serve_api/serve_service", "just_serve_proto"]]
+  , "stage": ["src", "buildtool", "serve_api", "remote"]
+  , "private-deps":
+    [ ["@", "json", "", "json"]
+    , ["src/buildtool/execution_api/remote", "config"]
+    ]
+  }
 }
diff --git a/src/buildtool/serve_api/remote/configuration_client.cpp b/src/buildtool/serve_api/remote/configuration_client.cpp
new file mode 100644
index 00000000..25c38d79
--- /dev/null
+++ b/src/buildtool/serve_api/remote/configuration_client.cpp
@@ -0,0 +1,52 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/serve_api/remote/configuration_client.hpp"
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/execution_api/remote/config.hpp"
+
+auto ConfigurationClient::CheckServeRemoteExecution() -> bool {
+    auto client_remote_address = RemoteExecutionConfig::RemoteAddress();
+    if (!client_remote_address) {
+        logger_.Emit(LogLevel::Error,
+                     "In order to use just-serve, also the "
+                     "--remote-execution-address option must be given.");
+        return false;
+    }
+
+    grpc::ClientContext context;
+    justbuild::just_serve::RemoteExecutionEndpointRequest request{};
+    justbuild::just_serve::RemoteExecutionEndpointResponse response{};
+    grpc::Status status =
+        stub_->RemoteExecutionEndpoint(&context, request, &response);
+
+    if (not status.ok()) {
+        LogStatus(&logger_, LogLevel::Error, status);
+        return false;
+    }
+    auto serve_remote_endpoint = nlohmann::json::parse(response.address());
+    if (serve_remote_endpoint != client_remote_address->ToJson()) {
+        Logger::Log(LogLevel::Error,
+                    "Different execution endpoint detected.\nIn order to "
+                    "correctly use just serve, its remote execution endpoint "
+                    "must be the same used by the client.\nserve remote "
+                    "endpoint:  {}\nclient remote endpoint: {}",
+                    serve_remote_endpoint.dump(),
+                    client_remote_address->ToJson().dump());
+        return false;
+    }
+    return true;
+}
diff --git a/src/buildtool/serve_api/remote/configuration_client.hpp b/src/buildtool/serve_api/remote/configuration_client.hpp
new file mode 100644
index 00000000..13584832
--- /dev/null
+++ b/src/buildtool/serve_api/remote/configuration_client.hpp
@@ -0,0 +1,46 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "justbuild/just_serve/just_serve.grpc.pb.h"
+#include "src/buildtool/common/remote/client_common.hpp"
+#include "src/buildtool/common/remote/port.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// Implements client side for Configuration service defined in:
+/// src/buildtool/serve_api/serve_service/just_serve.proto
+class ConfigurationClient {
+  public:
+    ConfigurationClient(std::string const& server, Port port) noexcept
+        :
+
+          stub_{justbuild::just_serve::Configuration::NewStub(
+              CreateChannelWithCredentials(server, port))} {}
+
+    auto CheckServeRemoteExecution() -> bool;
+
+  private:
+    std::unique_ptr<justbuild::just_serve::Configuration::Stub> stub_;
+    Logger logger_{"RemoteTargetClient"};
+};
+
+#endif  // INCLUDED_SRC_BUILDTOOL_SERVE_API_CONFIGURATION_CLIENT_HPP
diff --git a/src/buildtool/serve_api/remote/serve_api.cpp b/src/buildtool/serve_api/remote/serve_api.cpp
deleted file mode 100644
index 0c5c531c..00000000
--- a/src/buildtool/serve_api/remote/serve_api.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "src/buildtool/serve_api/remote/serve_api.hpp"
-
-ServeApi::ServeApi(std::string const& host, Port port) noexcept
-    : stc_{std::make_unique<SourceTreeClient>(host, port)} {}
-
-// implement move constructor in cpp, where all members are complete types
-ServeApi::ServeApi(ServeApi&& other) noexcept = default;
-
-// implement destructor in cpp, where all members are complete types
-ServeApi::~ServeApi() = default;
-
-auto ServeApi::RetrieveTreeFromCommit(std::string const& commit,
-                                      std::string const& subdir,
-                                      bool sync_tree)
-    -> std::optional<std::string> {
-    return stc_->ServeCommitTree(commit, subdir, sync_tree);
-}
-
-auto ServeApi::RetrieveTreeFromArchive(
-    std::string const& content,
-    std::string const& archive_type,
-    std::string const& subdir,
-    std::optional<PragmaSpecial> const& resolve_symlinks,
-    bool sync_tree) -> std::optional<std::string> {
-    return stc_->ServeArchiveTree(
-        content, archive_type, subdir, resolve_symlinks, sync_tree);
-}
diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp
index a0d50ea9..48ba4098 100644
--- a/src/buildtool/serve_api/remote/serve_api.hpp
+++ b/src/buildtool/serve_api/remote/serve_api.hpp
@@ -21,37 +21,77 @@
 
 #include "src/buildtool/common/remote/port.hpp"
 #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp"
+#include "src/buildtool/serve_api/remote/config.hpp"
+#include "src/buildtool/serve_api/remote/configuration_client.hpp"
 #include "src/buildtool/serve_api/remote/source_tree_client.hpp"
+#include "src/buildtool/serve_api/remote/target_client.hpp"
 
 class ServeApi final {
   public:
-    using Ptr = std::unique_ptr<ServeApi>;
-
-    ServeApi(std::string const& host, Port port) noexcept;
-
     ServeApi(ServeApi const&) = delete;
-    ServeApi(ServeApi&& other) noexcept;
+    ~ServeApi() = default;
 
     auto operator=(ServeApi const&) -> ServeApi& = delete;
     auto operator=(ServeApi&&) -> ServeApi& = delete;
 
-    ~ServeApi();
+    [[nodiscard]] static auto Instance() noexcept -> ServeApi& {
+        static ServeApi instance = ServeApi::init();
+        return instance;
+    }
 
-    [[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit,
-                                              std::string const& subdir = ".",
-                                              bool sync_tree = false)
-        -> std::optional<std::string>;
+    [[nodiscard]] static auto RetrieveTreeFromCommit(
+        std::string const& commit,
+        std::string const& subdir = ".",
+        bool sync_tree = false) -> std::optional<std::string> {
+        return Instance().stc_->ServeCommitTree(commit, subdir, sync_tree);
+    }
 
-    [[nodiscard]] auto RetrieveTreeFromArchive(
+    [[nodiscard]] static auto RetrieveTreeFromArchive(
         std::string const& content,
         std::string const& archive_type = "archive",
         std::string const& subdir = ".",
         std::optional<PragmaSpecial> const& resolve_symlinks = std::nullopt,
-        bool sync_tree = false) -> std::optional<std::string>;
+        bool sync_tree = false) -> std::optional<std::string> {
+        return Instance().stc_->ServeArchiveTree(
+            content, archive_type, subdir, resolve_symlinks, sync_tree);
+    }
+
+    [[nodiscard]] static auto ServeTargetVariables(
+        std::string const& target_root_id,
+        std::string const& target_file,
+        std::string const& target) -> std::optional<std::vector<std::string>> {
+        return Instance().tc_->ServeTargetVariables(
+            target_root_id, target_file, target);
+    }
+
+    [[nodiscard]] static auto ServeTarget(const TargetCacheKey& key)
+        -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> {
+        return Instance().tc_->ServeTarget(key);
+    }
+
+    [[nodiscard]] static auto CheckServeRemoteExecution() -> bool {
+        return Instance().cc_->CheckServeRemoteExecution();
+    }
 
   private:
+    ServeApi(std::string const& host, Port port) noexcept
+        : stc_{std::make_unique<SourceTreeClient>(host, port)},
+          tc_{std::make_unique<TargetClient>(host, port)},
+          cc_{std::make_unique<ConfigurationClient>(host, port)} {}
+
+    ServeApi(ServeApi&& other) noexcept = default;
+
+    [[nodiscard]] static auto init() noexcept -> ServeApi {
+        auto sadd = RemoteServeConfig::RemoteAddress();
+        return ServeApi{sadd->host, sadd->port};
+    }
+
     // source tree service client
     std::unique_ptr<SourceTreeClient> stc_;
+    // target service client
+    std::unique_ptr<TargetClient> tc_;
+    // configuration service client
+    std::unique_ptr<ConfigurationClient> cc_;
 };
 
 #endif  // INCLUDED_SRC_BUILDTOOL_SERVE_API_REMOTE_SERVE_API_HPP
diff --git a/src/buildtool/serve_api/remote/target_client.cpp b/src/buildtool/serve_api/remote/target_client.cpp
new file mode 100644
index 00000000..bb30907d
--- /dev/null
+++ b/src/buildtool/serve_api/remote/target_client.cpp
@@ -0,0 +1,118 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/serve_api/remote/target_client.hpp"
+
+#include <utility>
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/common/remote/client_common.hpp"
+
+TargetClient::TargetClient(std::string const& server, Port port) noexcept {
+    stub_ = justbuild::just_serve::Target::NewStub(
+        CreateChannelWithCredentials(server, port));
+}
+
+auto TargetClient::ServeTarget(const TargetCacheKey& key)
+    -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> {
+    // make sure the blob containing the key is in the remote cas
+    if (!local_api_->RetrieveToCas({key.Id()}, &*remote_api_)) {
+        logger_.Emit(LogLevel::Error,
+                     "failed to retrieve to remote cas ObjectInfo {}",
+                     key.Id().ToString());
+        return std::nullopt;
+    }
+
+    bazel_re::Digest key_dgst{key.Id().digest};
+    justbuild::just_serve::ServeTargetRequest request{};
+    *(request.mutable_target_cache_key_id()) = std::move(key_dgst);
+
+    auto execution_backend_dgst = ArtifactDigest::Create<ObjectType::File>(
+        StorageConfig::ExecutionBackendDescription());
+    auto const& execution_info =
+        Artifact::ObjectInfo{.digest = ArtifactDigest{execution_backend_dgst},
+                             .type = ObjectType::File};
+    if (!local_api_->RetrieveToCas({execution_info}, &*remote_api_)) {
+        logger_.Emit(LogLevel::Error,
+                     "failed to upload blob {} to remote cas",
+                     execution_info.ToString());
+        return std::nullopt;
+    }
+
+    *(request.mutable_execution_backend_description_id()) =
+        std::move(execution_backend_dgst);
+
+    grpc::ClientContext context;
+    justbuild::just_serve::ServeTargetResponse response;
+    auto const& status = stub_->ServeTarget(&context, request, &response);
+    if (!status.ok()) {
+        LogStatus(&logger_, LogLevel::Error, status);
+        return std::nullopt;
+    }
+    auto const& target_value_dgst = ArtifactDigest{response.target_value()};
+    auto const& obj_info = Artifact::ObjectInfo{.digest = target_value_dgst,
+                                                .type = ObjectType::File};
+    if (!local_api_->IsAvailable(target_value_dgst)) {
+        if (!remote_api_->RetrieveToCas({obj_info}, &*local_api_)) {
+            logger_.Emit(LogLevel::Error,
+                         "failed to retrieve blob {} from remote cas",
+                         obj_info.ToString());
+            return std::nullopt;
+        }
+    }
+
+    auto const& target_value_str = local_api_->RetrieveToMemory(obj_info);
+    if (!target_value_str) {
+        logger_.Emit(LogLevel::Error,
+                     "failed to retrieve blob {} from local cas",
+                     obj_info.ToString());
+        return std::nullopt;
+    }
+    auto const& result =
+        TargetCacheEntry::FromJson(nlohmann::json::parse(*target_value_str));
+
+    return std::make_pair(result, obj_info);
+}
+
+auto TargetClient::ServeTargetVariables(std::string const& target_root_id,
+                                        std::string const& target_file,
+                                        std::string const& target)
+    -> std::optional<std::vector<std::string>> {
+    justbuild::just_serve::ServeTargetVariablesRequest request{};
+    request.set_root_tree(target_root_id);
+    request.set_target_file(target_file);
+    request.set_target(target);
+
+    grpc::ClientContext context;
+    justbuild::just_serve::ServeTargetVariablesResponse response;
+    grpc::Status status =
+        stub_->ServeTargetVariables(&context, request, &response);
+
+    if (not status.ok()) {
+        LogStatus(&logger_, LogLevel::Error, status);
+        return std::nullopt;
+    }
+    auto size = response.flexible_config_size();
+    if (size == 0) {
+        return std::vector<std::string>();
+    }
+    std::vector<std::string> res{};
+    res.reserve(size);
+    for (auto const& var : response.flexible_config()) {
+        res.emplace_back(var);
+    }
+    return res;
+}
diff --git a/src/buildtool/serve_api/remote/target_client.hpp b/src/buildtool/serve_api/remote/target_client.hpp
new file mode 100644
index 00000000..ac52a468
--- /dev/null
+++ b/src/buildtool/serve_api/remote/target_client.hpp
@@ -0,0 +1,65 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "justbuild/just_serve/just_serve.grpc.pb.h"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/remote/port.hpp"
+#include "src/buildtool/execution_api/common/create_execution_api.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/storage/target_cache_entry.hpp"
+#include "src/buildtool/storage/target_cache_key.hpp"
+
+/// Implements client side for Target service defined in:
+/// src/buildtool/serve_api/serve_service/just_serve.proto
+class TargetClient {
+  public:
+    TargetClient(std::string const& server, Port port) noexcept;
+
+    /// \brief Retrieve the pair of TargetCacheEntry and ObjectInfo associated
+    /// to the given key.
+    /// \param[in] key The TargetCacheKey of an export target
+    /// \returns Pair of cache entry and its object info on success or nullopt.
+    [[nodiscard]] auto ServeTarget(const TargetCacheKey& key)
+        -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>;
+
+    /// \brief Retrieve the flexible config variables of an export target.
+    /// \param[in] target_root_id Hash of target-level root tree.
+    /// \param[in] target_file Relative path of the target file.
+    /// \param[in] target Name of the target to interrogate.
+    /// \returns The list of flexible config variables, or nullopt on errors.
+    [[nodiscard]] auto ServeTargetVariables(std::string const& target_root_id,
+                                            std::string const& target_file,
+                                            std::string const& target)
+        -> std::optional<std::vector<std::string>>;
+
+  private:
+    std::unique_ptr<justbuild::just_serve::Target::Stub> stub_;
+    Logger logger_{"RemoteTargetClient"};
+    gsl::not_null<IExecutionApi::Ptr> const remote_api_{
+        CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(),
+                           "remote-execution")};
+    gsl::not_null<IExecutionApi::Ptr> const local_api_{
+        CreateExecutionApi(std::nullopt)};
+};
+
+#endif  // INCLUDED_SRC_BUILDTOOL_SERVE_API_TARGET_CLIENT_HPP