From ebe7695ee5803dd3c2bb4f22f5e12d776c985d56 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 15 Feb 2023 14:47:15 +0100 Subject: structure cleanup: move libcurl utilities to other_tools... ...in order to not include unwanted dependencies in just proper. As the whole other_tools folder is meant to be excluded from bootstrapping, also remove the bootstrap guards. --- src/other_tools/ops_maps/TARGETS | 2 +- src/other_tools/ops_maps/content_cas_map.cpp | 2 +- src/other_tools/utils/TARGETS | 21 ++++ src/other_tools/utils/curl_context.cpp | 34 ++++++ src/other_tools/utils/curl_context.hpp | 36 ++++++ src/other_tools/utils/curl_easy_handle.cpp | 151 +++++++++++++++++++++++ src/other_tools/utils/curl_easy_handle.hpp | 86 +++++++++++++ src/utils/cpp/TARGETS | 21 ---- src/utils/cpp/curl_context.cpp | 40 ------ src/utils/cpp/curl_context.hpp | 36 ------ src/utils/cpp/curl_easy_handle.cpp | 175 --------------------------- src/utils/cpp/curl_easy_handle.hpp | 86 ------------- test/other_tools/utils/TARGETS | 27 ++++- test/other_tools/utils/curl_usage.test.cpp | 60 +++++++++ test/other_tools/utils/curl_usage_test.sh | 60 +++++++++ test/utils/cpp/TARGETS | 25 +--- test/utils/cpp/curl_usage.test.cpp | 60 --------- test/utils/cpp/curl_usage_test.sh | 60 --------- 18 files changed, 476 insertions(+), 506 deletions(-) create mode 100644 src/other_tools/utils/curl_context.cpp create mode 100644 src/other_tools/utils/curl_context.hpp create mode 100644 src/other_tools/utils/curl_easy_handle.cpp create mode 100644 src/other_tools/utils/curl_easy_handle.hpp delete mode 100644 src/utils/cpp/curl_context.cpp delete mode 100644 src/utils/cpp/curl_context.hpp delete mode 100644 src/utils/cpp/curl_easy_handle.cpp delete mode 100644 src/utils/cpp/curl_easy_handle.hpp create mode 100644 test/other_tools/utils/curl_usage.test.cpp create mode 100644 test/other_tools/utils/curl_usage_test.sh delete mode 100644 test/utils/cpp/curl_usage.test.cpp delete mode 100644 test/utils/cpp/curl_usage_test.sh diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 72002975..bedf8cba 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -59,7 +59,7 @@ ] , "stage": ["src", "other_tools", "ops_maps"] , "private-deps": - [ ["src/utils/cpp", "curl_easy_handle"] + [ ["src/other_tools/utils", "curl_easy_handle"] , ["src/buildtool/crypto", "hasher"] , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/file_system", "file_storage"] diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 55aa78a7..7ed1f3df 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -17,7 +17,7 @@ #include "src/buildtool/crypto/hasher.hpp" #include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/file_system/file_storage.hpp" -#include "src/utils/cpp/curl_easy_handle.hpp" +#include "src/other_tools/utils/curl_easy_handle.hpp" namespace { diff --git a/src/other_tools/utils/TARGETS b/src/other_tools/utils/TARGETS index 0893821d..883ab841 100644 --- a/src/other_tools/utils/TARGETS +++ b/src/other_tools/utils/TARGETS @@ -8,4 +8,25 @@ , "private-deps": [["src/buildtool/file_system", "file_system_manager"], ["", "libarchive"]] } +, "curl_context": + { "type": ["@", "rules", "CC", "library"] + , "name": ["curl_context"] + , "hdrs": ["curl_context.hpp"] + , "srcs": ["curl_context.cpp"] + , "stage": ["src", "other_tools", "utils"] + , "private-deps": [["src/buildtool/logging", "logging"], ["", "libcurl"]] + } +, "curl_easy_handle": + { "type": ["@", "rules", "CC", "library"] + , "name": ["curl_easy_handle"] + , "hdrs": ["curl_easy_handle.hpp"] + , "srcs": ["curl_easy_handle.cpp"] + , "deps": ["curl_context", ["@", "gsl-lite", "", "gsl-lite"]] + , "stage": ["src", "other_tools", "utils"] + , "private-deps": + [ ["src/buildtool/logging", "logging"] + , ["src/buildtool/file_system", "file_system_manager"] + , ["", "libcurl"] + ] + } } diff --git a/src/other_tools/utils/curl_context.cpp b/src/other_tools/utils/curl_context.cpp new file mode 100644 index 00000000..46f7cfac --- /dev/null +++ b/src/other_tools/utils/curl_context.cpp @@ -0,0 +1,34 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/other_tools/utils/curl_context.hpp" + +#include "src/buildtool/logging/logger.hpp" + +extern "C" { +#include +} + +CurlContext::CurlContext() noexcept { + // NOLINTNEXTLINE(hicpp-signed-bitwise) + if (not(initialized_ = (curl_global_init(CURL_GLOBAL_DEFAULT) >= 0))) { + Logger::Log(LogLevel::Error, "initializing libcurl failed"); + } +} + +CurlContext::~CurlContext() noexcept { + if (initialized_) { + curl_global_cleanup(); + } +} diff --git a/src/other_tools/utils/curl_context.hpp b/src/other_tools/utils/curl_context.hpp new file mode 100644 index 00000000..22e561bb --- /dev/null +++ b/src/other_tools/utils/curl_context.hpp @@ -0,0 +1,36 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP +#define INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP + +/// \brief Maintainer of a libcurl state. +/// Classes, static methods, and global functions dealing with curl operations +/// should create a CurlContext before using libcurl. +class CurlContext { + public: + // prohibit moves and copies + CurlContext(CurlContext const&) = delete; + CurlContext(CurlContext&& other) = delete; + auto operator=(CurlContext const&) = delete; + auto operator=(CurlContext&& other) = delete; + + CurlContext() noexcept; + ~CurlContext() noexcept; + + private: + bool initialized_{false}; +}; + +#endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_CONTEXT_HPP diff --git a/src/other_tools/utils/curl_easy_handle.cpp b/src/other_tools/utils/curl_easy_handle.cpp new file mode 100644 index 00000000..29e7670e --- /dev/null +++ b/src/other_tools/utils/curl_easy_handle.cpp @@ -0,0 +1,151 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/other_tools/utils/curl_easy_handle.hpp" + +#include + +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/logger.hpp" + +extern "C" { +#include "curl/curl.h" +} + +void curl_easy_closer(gsl::owner curl) { + curl_easy_cleanup(curl); +} + +auto CurlEasyHandle::Create() noexcept -> std::shared_ptr { + try { + auto curl = std::make_shared(); + auto* handle = curl_easy_init(); + if (handle == nullptr) { + return nullptr; + } + curl->handle_.reset(handle); + return curl; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "create curl easy handle failed with:\n{}", + ex.what()); + return nullptr; + } +} + +auto CurlEasyHandle::EasyWriteToFile(gsl::owner data, + size_t size, + size_t nmemb, + gsl::owner userptr) + -> std::streamsize { + auto actual_size = static_cast(size * nmemb); + auto* file = static_cast(userptr); + file->write(data, actual_size); // append chunk + return actual_size; +} + +auto CurlEasyHandle::EasyWriteToString(gsl::owner data, + size_t size, + size_t nmemb, + gsl::owner userptr) + -> std::streamsize { + size_t actual_size = size * nmemb; + (static_cast(userptr))->append(data, actual_size); + return static_cast(actual_size); +} + +auto CurlEasyHandle::DownloadToFile( + std::string const& url, + std::filesystem::path const& file_path) noexcept -> int { + try { + // set URL + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); + + // ensure redirects are allowed, otherwise it might simply read empty + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); + + // set callback for writing to file + std::ofstream file(file_path.c_str(), std::ios::binary); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToFile); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt( + handle_.get(), CURLOPT_WRITEDATA, static_cast(&file)); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); + + // perform download + auto res = curl_easy_perform(handle_.get()); + + // close file + file.close(); + + // check result + if (res != CURLE_OK) { + // cleanup failed downloaded file, if created + [[maybe_unused]] auto tmp_res = + FileSystemManager::RemoveFile(file_path); + } + return res; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "curl download to file failed with:\n{}", + ex.what()); + return 1; + } +} + +auto CurlEasyHandle::DownloadToString(std::string const& url) noexcept + -> std::optional { + try { + // set URL + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); + + // ensure redirects are allowed, otherwise it might simply read empty + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); + + // set callback for writing to string + std::string content{}; + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt( + handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToString); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt( + handle_.get(), CURLOPT_WRITEDATA, static_cast(&content)); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) + curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); + + // perform download + auto res = curl_easy_perform(handle_.get()); + + // check result + if (res != CURLE_OK) { + return std::nullopt; + } + return content; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "curl download to string failed with:\n{}", + ex.what()); + return std::nullopt; + } +} diff --git a/src/other_tools/utils/curl_easy_handle.hpp b/src/other_tools/utils/curl_easy_handle.hpp new file mode 100644 index 00000000..4f6f33c9 --- /dev/null +++ b/src/other_tools/utils/curl_easy_handle.hpp @@ -0,0 +1,86 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP +#define INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP + +#include +#include +#include +#include + +#include "gsl-lite/gsl-lite.hpp" +#include "src/other_tools/utils/curl_context.hpp" + +extern "C" { +#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) +using CURL = struct Curl_easy; +#else +using CURL = void; +#endif +} + +void curl_easy_closer(gsl::owner curl); + +class CurlEasyHandle { + public: + CurlEasyHandle() noexcept = default; + ~CurlEasyHandle() noexcept = default; + + // prohibit moves and copies + CurlEasyHandle(CurlEasyHandle const&) = delete; + CurlEasyHandle(CurlEasyHandle&& other) = delete; + auto operator=(CurlEasyHandle const&) = delete; + auto operator=(CurlEasyHandle&& other) = delete; + + /// \brief Create a CurlEasyHandle object + [[nodiscard]] auto static Create() noexcept + -> std::shared_ptr; + + /// \brief Download file from URL into given file_path. + /// Will perform cleanup (i.e., remove empty file) in case download fails. + /// Returns 0 if successful. + [[nodiscard]] auto DownloadToFile( + std::string const& url, + std::filesystem::path const& file_path) noexcept -> int; + + /// \brief Download file from URL into string as binary. + /// Returns the content or nullopt if download failure. + [[nodiscard]] auto DownloadToString(std::string const& url) noexcept + -> std::optional; + + private: + // IMPORTANT: the CurlContext must to be initialized before any curl object! + CurlContext curl_context_{}; + std::unique_ptr handle_{ + nullptr, + curl_easy_closer}; + + /// \brief Overwrites write_callback to redirect to file instead of stdout. + [[nodiscard]] auto static EasyWriteToFile(gsl::owner data, + size_t size, + size_t nmemb, + gsl::owner userptr) + -> std::streamsize; + + /// \brief Overwrites write_callback to redirect to string instead of + /// stdout. + [[nodiscard]] auto static EasyWriteToString(gsl::owner data, + size_t size, + size_t nmemb, + gsl::owner userptr) + -> std::streamsize; +}; + +#endif // INCLUDED_SRC_OTHER_TOOLS_UTILS_CURL_EASY_HANDLE_HPP \ No newline at end of file diff --git a/src/utils/cpp/TARGETS b/src/utils/cpp/TARGETS index 4bc79e8e..42d8ae9e 100644 --- a/src/utils/cpp/TARGETS +++ b/src/utils/cpp/TARGETS @@ -58,27 +58,6 @@ , "stage": ["src", "utils", "cpp"] , "private-deps": [["src/buildtool/file_system", "file_system_manager"]] } -, "curl_context": - { "type": ["@", "rules", "CC", "library"] - , "name": ["curl_context"] - , "hdrs": ["curl_context.hpp"] - , "srcs": ["curl_context.cpp"] - , "stage": ["src", "utils", "cpp"] - , "private-deps": [["src/buildtool/logging", "logging"], ["", "libcurl"]] - } -, "curl_easy_handle": - { "type": ["@", "rules", "CC", "library"] - , "name": ["curl_easy_handle"] - , "hdrs": ["curl_easy_handle.hpp"] - , "srcs": ["curl_easy_handle.cpp"] - , "deps": ["curl_context", ["@", "gsl-lite", "", "gsl-lite"]] - , "stage": ["src", "utils", "cpp"] - , "private-deps": - [ ["src/buildtool/logging", "logging"] - , ["src/buildtool/file_system", "file_system_manager"] - , ["", "libcurl"] - ] - } , "file_locking": { "type": ["@", "rules", "CC", "library"] , "name": ["file_locking"] diff --git a/src/utils/cpp/curl_context.cpp b/src/utils/cpp/curl_context.cpp deleted file mode 100644 index 3369b2d3..00000000 --- a/src/utils/cpp/curl_context.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/utils/cpp/curl_context.hpp" - -#include "src/buildtool/logging/logger.hpp" - -#ifndef BOOTSTRAP_BUILD_TOOL -extern "C" { -#include -} -#endif // BOOTSTRAP_BUILD_TOOL - -CurlContext::CurlContext() noexcept { -#ifndef BOOTSTRAP_BUILD_TOOL - // NOLINTNEXTLINE(hicpp-signed-bitwise) - if (not(initialized_ = (curl_global_init(CURL_GLOBAL_DEFAULT) >= 0))) { - Logger::Log(LogLevel::Error, "initializing libcurl failed"); - } -#endif // BOOTSTRAP_BUILD_TOOL -} - -CurlContext::~CurlContext() noexcept { -#ifndef BOOTSTRAP_BUILD_TOOL - if (initialized_) { - curl_global_cleanup(); - } -#endif // BOOTSTRAP_BUILD_TOOL -} \ No newline at end of file diff --git a/src/utils/cpp/curl_context.hpp b/src/utils/cpp/curl_context.hpp deleted file mode 100644 index e408b8bf..00000000 --- a/src/utils/cpp/curl_context.hpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_OTHER_TOOLS_CURL_CONTEXT_HPP -#define INCLUDED_SRC_OTHER_TOOLS_CURL_CONTEXT_HPP - -/// \brief Maintainer of a libcurl state. -/// Classes, static methods, and global functions dealing with curl operations -/// should create a CurlContext before using libcurl. -class CurlContext { - public: - // prohibit moves and copies - CurlContext(CurlContext const&) = delete; - CurlContext(CurlContext&& other) = delete; - auto operator=(CurlContext const&) = delete; - auto operator=(CurlContext&& other) = delete; - - CurlContext() noexcept; - ~CurlContext() noexcept; - - private: - bool initialized_{false}; -}; - -#endif // INCLUDED_SRC_OTHER_TOOLS_CURL_CONTEXT_HPP \ No newline at end of file diff --git a/src/utils/cpp/curl_easy_handle.cpp b/src/utils/cpp/curl_easy_handle.cpp deleted file mode 100644 index 7fac31fc..00000000 --- a/src/utils/cpp/curl_easy_handle.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "src/utils/cpp/curl_easy_handle.hpp" - -#include - -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/logger.hpp" - -#ifndef BOOTSTRAP_BUILD_TOOL -extern "C" { -#include "curl/curl.h" -} -#endif // BOOTSTRAP_BUILD_TOOL - -void curl_easy_closer(gsl::owner curl) { -#ifndef BOOTSTRAP_BUILD_TOOL - curl_easy_cleanup(curl); -#endif // BOOTSTRAP_BUILD_TOOL -} - -auto CurlEasyHandle::Create() noexcept -> std::shared_ptr { -#ifdef BOOTSTRAP_BUILD_TOOL - return nullptr; -#else - try { - auto curl = std::make_shared(); - auto* handle = curl_easy_init(); - if (handle == nullptr) { - return nullptr; - } - curl->handle_.reset(handle); - return curl; - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "create curl easy handle failed with:\n{}", - ex.what()); - return nullptr; - } -#endif // BOOTSTRAP_BUILD_TOOL -} - -auto CurlEasyHandle::EasyWriteToFile(gsl::owner data, - size_t size, - size_t nmemb, - gsl::owner userptr) - -> std::streamsize { -#ifdef BOOTSTRAP_BUILD_TOOL - return 0; -#else - auto actual_size = static_cast(size * nmemb); - auto* file = static_cast(userptr); - file->write(data, actual_size); // append chunk - return actual_size; -#endif // BOOTSTRAP_BUILD_TOOL -} - -auto CurlEasyHandle::EasyWriteToString(gsl::owner data, - size_t size, - size_t nmemb, - gsl::owner userptr) - -> std::streamsize { -#ifdef BOOTSTRAP_BUILD_TOOL - return 0; -#else - size_t actual_size = size * nmemb; - (static_cast(userptr))->append(data, actual_size); - return static_cast(actual_size); -#endif // BOOTSTRAP_BUILD_TOOL -} - -auto CurlEasyHandle::DownloadToFile( - std::string const& url, - std::filesystem::path const& file_path) noexcept -> int { -#ifdef BOOTSTRAP_BUILD_TOOL - return 1; -#else - try { - // set URL - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); - - // ensure redirects are allowed, otherwise it might simply read empty - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); - - // set callback for writing to file - std::ofstream file(file_path.c_str(), std::ios::binary); - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToFile); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt( - handle_.get(), CURLOPT_WRITEDATA, static_cast(&file)); - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); - - // perform download - auto res = curl_easy_perform(handle_.get()); - - // close file - file.close(); - - // check result - if (res != CURLE_OK) { - // cleanup failed downloaded file, if created - [[maybe_unused]] auto tmp_res = - FileSystemManager::RemoveFile(file_path); - } - return res; - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "curl download to file failed with:\n{}", - ex.what()); - return 1; - } -#endif // BOOTSTRAP_BUILD_TOOL -} - -auto CurlEasyHandle::DownloadToString(std::string const& url) noexcept - -> std::optional { -#ifdef BOOTSTRAP_BUILD_TOOL - return std::nullopt; -#else - try { - // set URL - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_URL, url.c_str()); - - // ensure redirects are allowed, otherwise it might simply read empty - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_FOLLOWLOCATION, 1); - - // set callback for writing to string - std::string content{}; - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt( - handle_.get(), CURLOPT_WRITEFUNCTION, EasyWriteToString); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt( - handle_.get(), CURLOPT_WRITEDATA, static_cast(&content)); - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg, hicpp-vararg) - curl_easy_setopt(handle_.get(), CURLOPT_VERBOSE, 1); - - // perform download - auto res = curl_easy_perform(handle_.get()); - - // check result - if (res != CURLE_OK) { - return std::nullopt; - } - return content; - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "curl download to string failed with:\n{}", - ex.what()); - return std::nullopt; - } -#endif // BOOTSTRAP_BUILD_TOOL -} diff --git a/src/utils/cpp/curl_easy_handle.hpp b/src/utils/cpp/curl_easy_handle.hpp deleted file mode 100644 index 5f318887..00000000 --- a/src/utils/cpp/curl_easy_handle.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INCLUDED_SRC_OTHER_TOOLS_CURL_EASY_HANDLE_HPP -#define INCLUDED_SRC_OTHER_TOOLS_CURL_EASY_HANDLE_HPP - -#include -#include -#include -#include - -#include "gsl-lite/gsl-lite.hpp" -#include "src/utils/cpp/curl_context.hpp" - -extern "C" { -#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) -using CURL = struct Curl_easy; -#else -using CURL = void; -#endif -} - -void curl_easy_closer(gsl::owner curl); - -class CurlEasyHandle { - public: - CurlEasyHandle() noexcept = default; - ~CurlEasyHandle() noexcept = default; - - // prohibit moves and copies - CurlEasyHandle(CurlEasyHandle const&) = delete; - CurlEasyHandle(CurlEasyHandle&& other) = delete; - auto operator=(CurlEasyHandle const&) = delete; - auto operator=(CurlEasyHandle&& other) = delete; - - /// \brief Create a CurlEasyHandle object - [[nodiscard]] auto static Create() noexcept - -> std::shared_ptr; - - /// \brief Download file from URL into given file_path. - /// Will perform cleanup (i.e., remove empty file) in case download fails. - /// Returns 0 if successful. - [[nodiscard]] auto DownloadToFile( - std::string const& url, - std::filesystem::path const& file_path) noexcept -> int; - - /// \brief Download file from URL into string as binary. - /// Returns the content or nullopt if download failure. - [[nodiscard]] auto DownloadToString(std::string const& url) noexcept - -> std::optional; - - private: - // IMPORTANT: the CurlContext must to be initialized before any curl object! - CurlContext curl_context_{}; - std::unique_ptr handle_{ - nullptr, - curl_easy_closer}; - - /// \brief Overwrites write_callback to redirrect to file instead of stdout. - [[nodiscard]] auto static EasyWriteToFile(gsl::owner data, - size_t size, - size_t nmemb, - gsl::owner userptr) - -> std::streamsize; - - /// \brief Overwrites write_callback to redirect to string instead of - /// stdout. - [[nodiscard]] auto static EasyWriteToString(gsl::owner data, - size_t size, - size_t nmemb, - gsl::owner userptr) - -> std::streamsize; -}; - -#endif // INCLUDED_SRC_OTHER_TOOLS_CURL_EASY_HANDLE_HPP \ No newline at end of file diff --git a/test/other_tools/utils/TARGETS b/test/other_tools/utils/TARGETS index d709b532..425015ce 100644 --- a/test/other_tools/utils/TARGETS +++ b/test/other_tools/utils/TARGETS @@ -12,6 +12,29 @@ ] , "stage": ["test", "other_tools", "utils"] } +, "curl_usage_install": + { "type": ["@", "rules", "CC", "binary"] + , "tainted": ["test"] + , "name": ["curl_usage"] + , "srcs": ["curl_usage.test.cpp"] + , "private-deps": + [ ["@", "catch2", "", "catch2"] + , ["test", "catch-main"] + , ["src/other_tools/utils", "curl_context"] + , ["src/other_tools/utils", "curl_easy_handle"] + , ["src/buildtool/file_system", "file_system_manager"] + ] + , "stage": ["test", "other_tools", "utils"] + } +, "curl_usage_test": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["curl_usage_test"] + , "test": ["curl_usage_test.sh"] + , "deps": [["test/utils", "test_utils_install"], "curl_usage_install"] + } , "TESTS": - {"type": "install", "tainted": ["test"], "deps": ["archive_usage"]} -} \ No newline at end of file + { "type": "install" + , "tainted": ["test"] + , "deps": ["archive_usage", "curl_usage_test"] + } +} diff --git a/test/other_tools/utils/curl_usage.test.cpp b/test/other_tools/utils/curl_usage.test.cpp new file mode 100644 index 00000000..eaa84c7d --- /dev/null +++ b/test/other_tools/utils/curl_usage.test.cpp @@ -0,0 +1,60 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "catch2/catch.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/other_tools/utils/curl_context.hpp" +#include "src/other_tools/utils/curl_easy_handle.hpp" + +// The caller of this test needs to make sure the port is given as content of +// the file "port.txt" in the directory where this test is run +[[nodiscard]] auto getPort() noexcept -> std::string { + // read file where port has to be given + auto port = FileSystemManager::ReadFile(std::filesystem::path("port.txt")); + REQUIRE(port); + // strip any end terminator + std::erase_if(*port, [](auto ch) { return (ch == '\n' or ch == '\r'); }); + return *port; +} + +TEST_CASE("Curl context", "[curl_context]") { + CurlContext curl_context{}; +} + +TEST_CASE("Curl easy handle", "[curl_easy_handle]") { + auto kServerUrl = std::string("http://127.0.0.1:") + getPort() + + std::string("/test_file.txt"); + auto kTargetDir = + std::filesystem::path(std::getenv("TEST_TMPDIR")) / "target_dir"; + + // make target dir + CHECK(FileSystemManager::CreateDirectory(kTargetDir)); + // create handle + auto curl_handle = CurlEasyHandle::Create(); + REQUIRE(curl_handle); + + SECTION("Curl download to file") { + // download test file from local HTTP server into new location + auto file_path = kTargetDir / "test_file.txt"; + REQUIRE(curl_handle->DownloadToFile(kServerUrl, file_path) == 0); + REQUIRE(FileSystemManager::IsFile(file_path)); + } + + SECTION("Curl download to string") { + // download test file from local HTTP server into string + auto content = curl_handle->DownloadToString(kServerUrl); + REQUIRE(content); + REQUIRE(*content == "test\n"); + } +} diff --git a/test/other_tools/utils/curl_usage_test.sh b/test/other_tools/utils/curl_usage_test.sh new file mode 100644 index 00000000..838b5c17 --- /dev/null +++ b/test/other_tools/utils/curl_usage_test.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +# cleanup of http.server; pass server_pid as arg +server_cleanup() { + echo "Shut down HTTP server" + # send SIGTERM + kill ${1} & res=$! + wait ${res} + echo "done" +} + +readonly ROOT=`pwd` + +readonly SERVER_ROOT="${TEST_TMPDIR}/server-root" + +echo "Create test file" +mkdir -p "${SERVER_ROOT}" +cd "${SERVER_ROOT}" +cat > test_file.txt <"${port_file}" & server_pid=$! +sleep 1s # give some time to set up properly +# set up cleanup of http server +trap "server_cleanup ${server_pid}" INT TERM EXIT + +cd "${ROOT}" + +echo "Run curl usage test" +error=false +test/other_tools/utils/curl_usage & res=$! +wait $res +if [ $? -ne 0 ]; then + error=true +fi + +# check test status +if [ $error = true ]; then + exit 1 +fi diff --git a/test/utils/cpp/TARGETS b/test/utils/cpp/TARGETS index 05e344cd..9da792dc 100644 --- a/test/utils/cpp/TARGETS +++ b/test/utils/cpp/TARGETS @@ -9,26 +9,6 @@ ] , "stage": ["test", "utils", "cpp"] } -, "curl_usage_install": - { "type": ["@", "rules", "CC", "binary"] - , "tainted": ["test"] - , "name": ["curl_usage"] - , "srcs": ["curl_usage.test.cpp"] - , "private-deps": - [ ["@", "catch2", "", "catch2"] - , ["test", "catch-main"] - , ["src/utils/cpp", "curl_context"] - , ["src/utils/cpp", "curl_easy_handle"] - , ["src/buildtool/file_system", "file_system_manager"] - ] - , "stage": ["test", "utils", "cpp"] - } -, "curl_usage_test": - { "type": ["@", "rules", "shell/test", "script"] - , "name": ["curl_usage_test"] - , "test": ["curl_usage_test.sh"] - , "deps": [["test/utils", "test_utils_install"], "curl_usage_install"] - } , "file_locking": { "type": ["@", "rules", "CC/test", "test"] , "name": ["file_locking"] @@ -43,8 +23,5 @@ , "stage": ["test", "utils", "cpp"] } , "TESTS": - { "type": "install" - , "tainted": ["test"] - , "deps": ["path", "curl_usage_test", "file_locking"] - } + {"type": "install", "tainted": ["test"], "deps": ["path", "file_locking"]} } diff --git a/test/utils/cpp/curl_usage.test.cpp b/test/utils/cpp/curl_usage.test.cpp deleted file mode 100644 index dc6666d4..00000000 --- a/test/utils/cpp/curl_usage.test.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "catch2/catch.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/utils/cpp/curl_context.hpp" -#include "src/utils/cpp/curl_easy_handle.hpp" - -// The caller of this test needs to make sure the port is given as content of -// the file "port.txt" in the directory where this test is run -[[nodiscard]] auto getPort() noexcept -> std::string { - // read file where port has to be given - auto port = FileSystemManager::ReadFile(std::filesystem::path("port.txt")); - REQUIRE(port); - // strip any end terminator - std::erase_if(*port, [](auto ch) { return (ch == '\n' or ch == '\r'); }); - return *port; -} - -TEST_CASE("Curl context", "[curl_context]") { - CurlContext curl_context{}; -} - -TEST_CASE("Curl easy handle", "[curl_easy_handle]") { - auto kServerUrl = std::string("http://127.0.0.1:") + getPort() + - std::string("/test_file.txt"); - auto kTargetDir = - std::filesystem::path(std::getenv("TEST_TMPDIR")) / "target_dir"; - - // make target dir - CHECK(FileSystemManager::CreateDirectory(kTargetDir)); - // create handle - auto curl_handle = CurlEasyHandle::Create(); - REQUIRE(curl_handle); - - SECTION("Curl download to file") { - // download test file from local HTTP server into new location - auto file_path = kTargetDir / "test_file.txt"; - REQUIRE(curl_handle->DownloadToFile(kServerUrl, file_path) == 0); - REQUIRE(FileSystemManager::IsFile(file_path)); - } - - SECTION("Curl download to string") { - // download test file from local HTTP server into string - auto content = curl_handle->DownloadToString(kServerUrl); - REQUIRE(content); - REQUIRE(*content == "test\n"); - } -} diff --git a/test/utils/cpp/curl_usage_test.sh b/test/utils/cpp/curl_usage_test.sh deleted file mode 100644 index 3ff874cb..00000000 --- a/test/utils/cpp/curl_usage_test.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -# cleanup of http.server; pass server_pid as arg -server_cleanup() { - echo "Shut down HTTP server" - # send SIGTERM - kill ${1} & res=$! - wait ${res} - echo "done" -} - -readonly ROOT=`pwd` - -readonly SERVER_ROOT="${TEST_TMPDIR}/server-root" - -echo "Create test file" -mkdir -p "${SERVER_ROOT}" -cd "${SERVER_ROOT}" -cat > test_file.txt <"${port_file}" & server_pid=$! -sleep 1s # give some time to set up properly -# set up cleanup of http server -trap "server_cleanup ${server_pid}" INT TERM EXIT - -cd "${ROOT}" - -echo "Run curl usage test" -error=false -test/utils/cpp/curl_usage & res=$! -wait $res -if [ $? -ne 0 ]; then - error=true -fi - -# check test status -if [ $error = true ]; then - exit 1 -fi -- cgit v1.2.3