LargeObjectUtils: Randomize large files and directories for testing purposes

3 files changed, 214 insertions, 0 deletions

diff --git a/test/utils/TARGETS b/test/utils/TARGETS
index 8cf95774..df238ca5 100644
--- a/test/utils/TARGETS
+++ b/test/utils/TARGETS
@@ -50,6 +50,17 @@
     ]
   , "stage": ["test", "utils"]
   }
+, "large_object_utils":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["large_object_utils"]
+  , "hdrs": ["large_objects/large_object_utils.hpp"]
+  , "srcs": ["large_objects/large_object_utils.cpp"]
+  , "private-deps":
+    [ ["@", "src", "src/utils/cpp", "gsl"]
+    , ["@", "src", "src/buildtool/file_system", "file_system_manager"]
+    ]
+  , "stage": ["test", "utils"]
+  }
 , "catch-main-remote-execution":
   { "type": ["@", "rules", "CC", "library"]
   , "name": ["catch-main-remote-execution"]
diff --git a/test/utils/large_objects/large_object_utils.cpp b/test/utils/large_objects/large_object_utils.cpp
new file mode 100644
index 00000000..b7bcebdc
--- /dev/null
+++ b/test/utils/large_objects/large_object_utils.cpp
@@ -0,0 +1,152 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/utils/large_objects/large_object_utils.hpp"
+
+#include <cstddef>
+#include <fstream>
+#include <limits>
+#include <random>
+#include <string>
+
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/utils/cpp/gsl.hpp"
+
+namespace {
+class Randomizer final {
+  public:
+    Randomizer(std::uint64_t min, std::uint64_t max) noexcept
+        : range_(std::random_device{}()), distribution_(min, max) {}
+
+    [[nodiscard]] inline auto Get() noexcept -> uint64_t {
+        return distribution_(range_);
+    }
+
+  private:
+    std::mt19937_64 range_;
+    std::uniform_int_distribution<std::mt19937_64::result_type> distribution_;
+};
+
+/// \brief Create a number of chunks of the predefined size.
+/// \tparam UChunkLength    Length of each chunk.
+/// \tparam USize           Number of chunks.
+template <size_t kChunkLength, size_t kPoolSize>
+class ChunkPool final {
+  public:
+    [[nodiscard]] static auto Instance() noexcept
+        -> ChunkPool<kChunkLength, kPoolSize> const& {
+        static ChunkPool<kChunkLength, kPoolSize> pool;
+        return pool;
+    }
+
+    [[nodiscard]] auto operator[](size_t index) const noexcept
+        -> std::string const& {
+        return gsl::at(pool_, static_cast<std::ptrdiff_t>(index));
+    }
+
+  private:
+    std::array<std::string, kPoolSize> pool_;
+
+    explicit ChunkPool() noexcept {
+        // Starts from 1 to exclude '\0' from randomization
+        Randomizer randomizer{1, std::numeric_limits<char>::max()};
+
+        for (size_t i = 0; i < pool_.size(); ++i) {
+            auto& chunk = gsl::at(pool_, static_cast<std::ptrdiff_t>(i));
+            chunk.resize(kChunkLength);
+            for (size_t j = 0; j < kChunkLength; ++j) {
+                chunk[j] = randomizer.Get();
+            }
+        }
+    }
+};
+}  // namespace
+
+auto LargeObjectUtils::GenerateFile(std::filesystem::path const& path,
+                                    std::uintmax_t size) noexcept -> bool {
+    // Remove the file, if exists:
+    if (not FileSystemManager::RemoveFile(path)) {
+        return false;
+    }
+
+    static constexpr size_t kChunkLength = 128;
+    static constexpr size_t kPoolSize = 64;
+    using Pool = ChunkPool<kChunkLength, kPoolSize>;
+
+    // To create a random file, the initial chunk position and the shift are
+    // randomized:
+    Randomizer randomizer{std::numeric_limits<size_t>::min(),
+                          std::numeric_limits<size_t>::max()};
+    const size_t pool_index = randomizer.Get() % kPoolSize;
+    const size_t pool_shift = randomizer.Get() % 10;
+    const size_t step_count = size / kChunkLength + 1;
+
+    try {
+        std::ofstream stream(path);
+        for (size_t i = 0; i < step_count && stream.good(); ++i) {
+            const size_t index = (pool_index + i * pool_shift) % kPoolSize;
+            if (i != step_count - 1) {
+                stream << Pool::Instance()[index];
+            }
+            else {
+                auto count = std::min(size - kChunkLength * i, kChunkLength);
+                stream << Pool::Instance()[index].substr(0, count);
+            }
+        }
+        if (not stream.good()) {
+            return false;
+        }
+        stream.close();
+    } catch (...) {
+        return false;
+    }
+    return true;
+}
+
+auto LargeObjectUtils::GenerateDirectory(std::filesystem::path const& path,
+                                         std::uintmax_t entries_count) noexcept
+    -> bool {
+    // Recreate the directory:
+    if (not FileSystemManager::RemoveDirectory(path) or
+        not FileSystemManager::CreateDirectory(path)) {
+        return false;
+    }
+
+    Randomizer randomizer{std::numeric_limits<size_t>::min(),
+                          std::numeric_limits<size_t>::max()};
+
+    std::uintmax_t entries = 0;
+    while (entries < entries_count) {
+        // Randomize the number for a file:
+        auto const random_number = randomizer.Get();
+        auto const file_name =
+            std::string(kTreeEntryPrefix) + std::to_string(random_number);
+        std::filesystem::path const file_path = path / file_name;
+
+        // Check file uniqueness:
+        if (FileSystemManager::IsFile(file_path)) {
+            continue;
+        }
+
+        try {
+            std::ofstream stream(file_path);
+            stream << random_number;
+            stream.close();
+        } catch (...) {
+            return false;
+        }
+        ++entries;
+    }
+    return true;
+}
diff --git a/test/utils/large_objects/large_object_utils.hpp b/test/utils/large_objects/large_object_utils.hpp
new file mode 100644
index 00000000..96619518
--- /dev/null
+++ b/test/utils/large_objects/large_object_utils.hpp
@@ -0,0 +1,51 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP
+#define INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP
+
+#include <cstdint>
+#include <filesystem>
+#include <string_view>
+
+/// \brief Provides an interface for randomizing large files and directories.
+class LargeObjectUtils {
+  public:
+    static constexpr std::string_view kTreeEntryPrefix =
+        "additional-large-prefix-to-make-tree-entry-larger";
+
+    /// \brief Generate a file of the specified size in the specified location.
+    /// If the file exists, it is overwritten. To reduce the number of
+    /// randomizations, a pool of pre-generated chunks is used.
+    /// \param path             Output path.
+    /// \param size             Size of the resulting file in bytes.
+    /// \return                 True if the file is generated properly.
+    [[nodiscard]] static auto GenerateFile(std::filesystem::path const& path,
+                                           std::uintmax_t size) noexcept
+        -> bool;
+
+    /// \brief Generate a directory in the specified location and fill it with a
+    /// number of randomized files. If the directory exists, it is overwritten.
+    /// The name of each file contains a random number and is prefixed with
+    /// kTreeEntryPrefix (to make tree entry larger for git). Each file contains
+    /// the same random number as in it's name.
+    /// \param path             Output path.
+    /// \param entries_count    Number of file entries in the directory.
+    /// \return                 True if the directory is generated properly.
+    [[nodiscard]] static auto GenerateDirectory(
+        std::filesystem::path const& path,
+        std::uintmax_t entries_count) noexcept -> bool;
+};
+
+#endif  // INCLUDED_SRC_TEST_UTILS_LARGE_OBJECTS_LARGE_OBJECT_UTILS_HPP