diff options
| -rw-r--r-- | src/utils/cpp/TARGETS | 10 | ||||
| -rw-r--r-- | src/utils/cpp/archive_ops.cpp | 366 | ||||
| -rw-r--r-- | src/utils/cpp/archive_ops.hpp | 100 |
3 files changed, 476 insertions, 0 deletions
diff --git a/src/utils/cpp/TARGETS b/src/utils/cpp/TARGETS index e278210e..c3c513e2 100644 --- a/src/utils/cpp/TARGETS +++ b/src/utils/cpp/TARGETS @@ -79,4 +79,14 @@ , ["", "libcurl"] ] } +, "archive_ops": + { "type": ["@", "rules", "CC", "library"] + , "name": ["archive_ops"] + , "hdrs": ["archive_ops.hpp"] + , "srcs": ["archive_ops.cpp"] + , "deps": [["@", "gsl-lite", "", "gsl-lite"]] + , "stage": ["src", "utils", "cpp"] + , "private-deps": + [["src/buildtool/file_system", "file_system_manager"], ["", "libarchive"]] + } } diff --git a/src/utils/cpp/archive_ops.cpp b/src/utils/cpp/archive_ops.cpp new file mode 100644 index 00000000..3944e4d5 --- /dev/null +++ b/src/utils/cpp/archive_ops.cpp @@ -0,0 +1,366 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/utils/cpp/archive_ops.hpp" + +#include "src/buildtool/file_system/file_system_manager.hpp" + +#ifndef BOOTSTRAP_BUILD_TOOL + +extern "C" { +#include <archive.h> +#include <archive_entry.h> +} + +namespace { + +/// \brief Default block size for archive extraction. +constexpr size_t kArchiveBlockSize = 10240; + +/// \brief Clean-up function for archive entry objects. +void archive_entry_cleanup(archive_entry* entry) { + if (entry != nullptr) { + archive_entry_free(entry); + } +} + +/// \brief Clean-up function for archive objects open for writing. +void archive_write_closer(archive* a_out) { + if (a_out != nullptr) { + archive_write_close(a_out); // libarchive handles non-openness + archive_write_free(a_out); // also do cleanup! + } +} + +/// \brief Clean-up function for archive objects open for reading. +void archive_read_closer(archive* a_in) { + if (a_in != nullptr) { + archive_read_close(a_in); // libarchive handles non-openness + archive_read_free(a_in); // also do cleanup! + } +} + +} // namespace + +#endif // BOOTSTRAP_BUILD_TOOL + +auto ArchiveOps::WriteEntry(archive_entry* entry, archive* aw) + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + std::filesystem::path entry_path{archive_entry_sourcepath(entry)}; + // only write to archive if entry is file + if (FileSystemManager::IsFile(entry_path)) { + auto content = FileSystemManager::ReadFile(entry_path); + if (not content) { + return "ArchiveOps: failed to open file entry while creating " + "archive"; + } + if (not content->empty()) { + auto content_size = content->size(); + archive_write_data(aw, content->c_str(), content_size); + } + } + return std::nullopt; +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::CopyData(archive* ar, archive* aw) + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + int r{}; + const void* buff{nullptr}; + size_t size{}; + la_int64_t offset{}; + + while (true) { + r = archive_read_data_block(ar, &buff, &size, &offset); + if (r == ARCHIVE_EOF) { + return std::nullopt; // success! + } + if (r != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + if (archive_write_data_block(aw, buff, size, offset) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } + return std::nullopt; // success! +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + switch (type) { + case ArchiveType::kArchiveTypeZip: { + if (archive_write_set_format_zip(aw) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } break; + case ArchiveType::kArchiveTypeTar: { + if (archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } break; + case ArchiveType::kArchiveTypeTarGz: { + if ((archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) or + (archive_write_add_filter_gzip(aw) != ARCHIVE_OK)) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } break; + case ArchiveType::kArchiveTypeTarBz2: { + if ((archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) or + (archive_write_add_filter_bzip2(aw) != ARCHIVE_OK)) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } break; + } + return std::nullopt; // success! +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + switch (type) { + case ArchiveType::kArchiveTypeZip: { + if (archive_read_support_format_zip(ar) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; + case ArchiveType::kArchiveTypeTar: { + if (archive_read_support_format_tar(ar) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; + case ArchiveType::kArchiveTypeTarGz: { + if ((archive_read_support_format_tar(ar) != ARCHIVE_OK) or + (archive_read_support_filter_gzip(ar) != ARCHIVE_OK)) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; + case ArchiveType::kArchiveTypeTarBz2: { + if ((archive_read_support_format_tar(ar) != ARCHIVE_OK) or + (archive_read_support_filter_bzip2(ar) != ARCHIVE_OK)) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; + } + return std::nullopt; // success! +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::CreateArchive(ArchiveType type, + std::string const& name, + std::filesystem::path const& source) noexcept + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + return CreateArchive(type, name, source, std::filesystem::path(".")); +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::CreateArchive(ArchiveType type, + std::string const& name, + std::filesystem::path const& source, + std::filesystem::path const& destDir) noexcept + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + try { + // make sure paths will be relative wrt current dir + auto rel_source = std::filesystem::relative(source); + + std::unique_ptr<archive, decltype(&archive_write_closer)> a_out{ + archive_write_new(), archive_write_closer}; + if (a_out == nullptr) { + return std::string("ArchiveOps: archive_write_new failed"); + } + // enable the correct format for archive type + auto res = EnableWriteFormats(a_out.get(), type); + if (res != std::nullopt) { + return *res; + } + // open archive to write + if (not FileSystemManager::CreateDirectory(destDir)) { + return std::string( + "ArchiveOps: could not create destination directory ") + + destDir.string(); + } + if (archive_write_open_filename( + a_out.get(), (destDir / std::filesystem::path(name)).c_str()) != + ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(a_out.get())); + } + // open source + std::unique_ptr<archive, decltype(&archive_read_closer)> disk{ + archive_read_disk_new(), archive_read_closer}; + if (disk == nullptr) { + return std::string("ArchiveOps: archive_read_disk_new failed"); + } + archive_read_disk_set_standard_lookup(disk.get()); + if (archive_read_disk_open(disk.get(), rel_source.c_str()) != + ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(disk.get())); + } + // create archive + while (true) { + std::unique_ptr<archive_entry, decltype(&archive_entry_cleanup)> + entry{archive_entry_new(), archive_entry_cleanup}; + if (entry == nullptr) { + return std::string("ArchiveOps: archive_entry_new failed"); + } + int r = archive_read_next_header2(disk.get(), entry.get()); + if (r == ARCHIVE_EOF) { + return std::nullopt; // nothing left to archive; success! + } + if (r != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(disk.get())); + } + // if entry is a directory, make sure we descend into all its + // children + archive_read_disk_descend(disk.get()); + // get info on current entry + if (archive_write_header(a_out.get(), entry.get()) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(a_out.get())); + } + // write entry into archive + auto res = WriteEntry(entry.get(), a_out.get()); + if (res != std::nullopt) { + return *res; + } + } + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, "archive create failed with:\n{}", ex.what()); + return std::nullopt; + } +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::ExtractArchive(ArchiveType type, + std::filesystem::path const& source) noexcept + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + return ExtractArchive(type, source, std::filesystem::path(".")); +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto ArchiveOps::ExtractArchive(ArchiveType type, + std::filesystem::path const& source, + std::filesystem::path const& destDir) noexcept + -> std::optional<std::string> { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + try { + std::unique_ptr<archive, decltype(&archive_read_closer)> a_in{ + archive_read_new(), archive_read_closer}; + if (a_in == nullptr) { + return std::string("ArchiveOps: archive_read_new failed"); + } + // enable support for known formats + auto res = EnableReadFormats(a_in.get(), type); + if (res != std::nullopt) { + return *res; + } + // open archive for reading + if (archive_read_open_filename( + a_in.get(), source.c_str(), kArchiveBlockSize) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(a_in.get())); + } + // set up writer to disk + std::unique_ptr<archive, decltype(&archive_write_closer)> disk{ + archive_write_disk_new(), archive_write_closer}; + if (disk == nullptr) { + return std::string("ArchiveOps: archive_write_disk_new failed"); + } + // Select which attributes we want to restore. + uint flags = ARCHIVE_EXTRACT_TIME; + flags |= static_cast<uint>(ARCHIVE_EXTRACT_PERM); + flags |= static_cast<uint>(ARCHIVE_EXTRACT_FFLAGS); + archive_write_disk_set_options(disk.get(), static_cast<int>(flags)); + archive_write_disk_set_standard_lookup(disk.get()); + // make sure destination directory exists + if (not FileSystemManager::CreateDirectory(destDir)) { + return std::string( + "ArchiveOps: could not create destination directory ") + + destDir.string(); + } + // extract the archive + archive_entry* entry{nullptr}; + while (true) { + int r = archive_read_next_header(a_in.get(), &entry); + if (r == ARCHIVE_EOF) { + return std::nullopt; // nothing left to extract; success! + } + if (r != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(a_in.get())); + } + // set correct destination path + auto new_entry_path = + destDir / std::filesystem::path(archive_entry_pathname(entry)); + archive_entry_set_pathname(entry, new_entry_path.c_str()); + if (archive_write_header(disk.get(), entry) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(disk.get())); + } + // write to disk if file + if (archive_entry_size(entry) > 0) { + auto res = CopyData(a_in.get(), disk.get()); + if (res != std::nullopt) { + return *res; + } + } + // finish entry writing + if (archive_write_finish_entry(disk.get()) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(disk.get())); + } + } + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, "archive extract failed with:\n{}", ex.what()); + return std::nullopt; + } +#endif // BOOTSTRAP_BUILD_TOOL +} diff --git a/src/utils/cpp/archive_ops.hpp b/src/utils/cpp/archive_ops.hpp new file mode 100644 index 00000000..b5726f9d --- /dev/null +++ b/src/utils/cpp/archive_ops.hpp @@ -0,0 +1,100 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_OTHER_TOOLS_ARCHIVE_OPS_HPP +#define INCLUDED_SRC_OTHER_TOOLS_ARCHIVE_OPS_HPP + +#include <filesystem> +#include <optional> + +#include "gsl-lite/gsl-lite.hpp" + +extern "C" { +using archive = struct archive; +using archive_entry = struct archive_entry; +} + +enum class ArchiveType : size_t { + kArchiveTypeZip, + kArchiveTypeTar, + kArchiveTypeTarGz, + kArchiveTypeTarBz2 +}; + +/// \brief Class handling archiving and unarchiving operations via libarchive +class ArchiveOps { + public: + /// \brief Create archive of given type from file or directory at + /// source. All paths will be takes relative to current directory. + /// Destination folder is the current directory. Archive is stored under + /// given name. Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static CreateArchive( + ArchiveType type, + std::string const& name, + std::filesystem::path const& source) noexcept + -> std::optional<std::string>; + + /// \brief Create archive of given type from file or directory at source and + /// store it in destDir folder under given name. All paths will be taken as + /// relative to the current directory. Destination directory is created if + /// not present. Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static CreateArchive( + ArchiveType type, + std::string const& name, + std::filesystem::path const& source, + std::filesystem::path const& destDir) noexcept + -> std::optional<std::string>; + + /// \brief Extract archive pointed to by source into destDir folder. The + /// destination folder is the current directory and the type of archive is + /// specified from currently supported formats: tar, zip, tar.gz, tar.bz2. + /// Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static ExtractArchive( + ArchiveType type, + std::filesystem::path const& source) noexcept + -> std::optional<std::string>; + + /// \brief Extract archive pointed to by source into destDir folder. The + /// type of archive is specified from currently supported formats: tar, zip, + /// tar.gz, tar.bz2. Returns nullopt on success, or an error string if + /// failure. + [[nodiscard]] auto static ExtractArchive( + ArchiveType type, + std::filesystem::path const& source, + std::filesystem::path const& destDir) noexcept + -> std::optional<std::string>; + + private: + /// \brief Copy entry into archive object. + /// Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static WriteEntry(archive_entry* entry, archive* aw) + -> std::optional<std::string>; + + /// \brief Copy data blocks from one archive object to another. + /// Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static CopyData(archive* ar, archive* aw) + -> std::optional<std::string>; + + /// \brief Set up the appropriate supported format for writing an archive. + /// Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static EnableWriteFormats(archive* aw, ArchiveType type) + -> std::optional<std::string>; + + /// \brief Set up the supported formats for reading in an archive. + /// Returns nullopt on success, or an error string if failure. + [[nodiscard]] auto static EnableReadFormats(archive* ar, ArchiveType type) + -> std::optional<std::string>; +}; + +#endif // INCLUDED_SRC_OTHER_TOOLS_ARCHIVE_OPS_HPP
\ No newline at end of file |