From 7d2f632b1dd1fe2ca01ef89716efe355e4d32687 Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Mon, 16 Sep 2024 15:32:59 +0200 Subject: Introduce ByteStreamUtils, wrap kChunkSize ...and use the qualified name ByteStreamUtils::kChunkSize --- src/buildtool/execution_api/common/TARGETS | 6 ++--- .../execution_api/common/bytestream_common.hpp | 23 ------------------- .../execution_api/common/bytestream_utils.hpp | 26 ++++++++++++++++++++++ .../execution_api/execution_service/TARGETS | 2 +- .../execution_service/bytestream_server.cpp | 6 ++--- src/buildtool/execution_api/remote/TARGETS | 2 +- .../remote/bazel/bytestream_client.hpp | 9 ++++---- 7 files changed, 39 insertions(+), 35 deletions(-) delete mode 100644 src/buildtool/execution_api/common/bytestream_common.hpp create mode 100644 src/buildtool/execution_api/common/bytestream_utils.hpp (limited to 'src') diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 0f3288e5..a060d0fa 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -30,10 +30,10 @@ , "private-deps": [["src/buildtool/common", "artifact_digest_factory"]] , "stage": ["src", "buildtool", "execution_api", "common"] } -, "bytestream-common": +, "bytestream_utils": { "type": ["@", "rules", "CC", "library"] - , "name": ["bytestream-common"] - , "hdrs": ["bytestream_common.hpp"] + , "name": ["bytestream_utils"] + , "hdrs": ["bytestream_utils.hpp"] , "stage": ["src", "buildtool", "execution_api", "common"] } , "api_bundle": diff --git a/src/buildtool/execution_api/common/bytestream_common.hpp b/src/buildtool/execution_api/common/bytestream_common.hpp deleted file mode 100644 index 5c7cf3cb..00000000 --- a/src/buildtool/execution_api/common/bytestream_common.hpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// settings common for server and client -#ifndef BYTESTREAM_COMMON_HPP -#define BYTESTREAM_COMMON_HPP -#include - -// Chunk size for uploads (default size used by BuildBarn) -constexpr static std::size_t kChunkSize = 64 * 1024; - -#endif diff --git a/src/buildtool/execution_api/common/bytestream_utils.hpp b/src/buildtool/execution_api/common/bytestream_utils.hpp new file mode 100644 index 00000000..c28c9627 --- /dev/null +++ b/src/buildtool/execution_api/common/bytestream_utils.hpp @@ -0,0 +1,26 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP + +#include + +class ByteStreamUtils final { + public: + // Chunk size for uploads (default size used by BuildBarn) + static constexpr std::size_t kChunkSize = 64 * 1024; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_BYTESTREAM_UTILS_HPP diff --git a/src/buildtool/execution_api/execution_service/TARGETS b/src/buildtool/execution_api/execution_service/TARGETS index 7f8e0e0d..26be0be8 100644 --- a/src/buildtool/execution_api/execution_service/TARGETS +++ b/src/buildtool/execution_api/execution_service/TARGETS @@ -118,7 +118,7 @@ , ["src/buildtool/storage", "config"] ] , "private-deps": - [ ["src/buildtool/execution_api/common", "bytestream-common"] + [ ["src/buildtool/execution_api/common", "bytestream_utils"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/common", "artifact_digest_factory"] , ["src/buildtool/common", "common"] diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index fb54ea2b..985f41a9 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -23,7 +23,7 @@ #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/common/bazel_types.hpp" -#include "src/buildtool/execution_api/common/bytestream_common.hpp" +#include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -104,10 +104,10 @@ auto BytestreamServiceImpl::Read( ::google::bytestream::ReadResponse response; std::string& buffer = *response.mutable_data(); - buffer.resize(kChunkSize); + buffer.resize(ByteStreamUtils::kChunkSize); while (not stream.eof()) { - stream.read(buffer.data(), kChunkSize); + stream.read(buffer.data(), ByteStreamUtils::kChunkSize); if (stream.bad()) { auto const str = fmt::format("Failed to read data for {}", read_digest->hash()); diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index ee1de5cb..2ba50578 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -33,7 +33,7 @@ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/auth", "auth"] - , ["src/buildtool/execution_api/common", "bytestream-common"] + , ["src/buildtool/execution_api/common", "bytestream_utils"] , ["src/utils/cpp", "expected"] , ["src/utils/cpp", "gsl"] , ["src/buildtool/common/remote", "client_common"] diff --git a/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp index d5485c9f..a444e61b 100644 --- a/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp +++ b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp @@ -27,7 +27,7 @@ #include "src/buildtool/auth/authentication.hpp" #include "src/buildtool/common/remote/client_common.hpp" #include "src/buildtool/common/remote/port.hpp" -#include "src/buildtool/execution_api/common/bytestream_common.hpp" +#include "src/buildtool/execution_api/common/bytestream_utils.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -115,11 +115,12 @@ class ByteStreamClient { google::bytestream::WriteRequest request{}; request.set_resource_name(resource_name); - request.mutable_data()->resize(kChunkSize, '\0'); + request.mutable_data()->resize(ByteStreamUtils::kChunkSize, '\0'); std::size_t pos{}; do { - auto const size = std::min(data.size() - pos, kChunkSize); + auto const size = + std::min(data.size() - pos, ByteStreamUtils::kChunkSize); request.mutable_data()->resize(size); data.copy(request.mutable_data()->data(), size, pos); request.set_write_offset(static_cast(pos)); @@ -141,7 +142,7 @@ class ByteStreamClient { pos = gsl::narrow(committed_size); } else { - pos += kChunkSize; + pos += ByteStreamUtils::kChunkSize; } } while (pos < data.size()); if (not writer->WritesDone()) { -- cgit v1.2.3