summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/tree_structure/TARGETS24
-rw-r--r--src/buildtool/tree_structure/tree_structure_cache.cpp151
-rw-r--r--src/buildtool/tree_structure/tree_structure_cache.hpp62
3 files changed, 237 insertions, 0 deletions
diff --git a/src/buildtool/tree_structure/TARGETS b/src/buildtool/tree_structure/TARGETS
new file mode 100644
index 00000000..e02f6e1c
--- /dev/null
+++ b/src/buildtool/tree_structure/TARGETS
@@ -0,0 +1,24 @@
+{ "tree_structure_cache":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["tree_structure_cache"]
+ , "hdrs": ["tree_structure_cache.hpp"]
+ , "srcs": ["tree_structure_cache.cpp"]
+ , "deps":
+ [ ["@", "gsl", "", "gsl"]
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/file_system", "file_storage"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/storage", "config"]
+ ]
+ , "stage": ["src", "buildtool", "tree_structure"]
+ , "private-deps":
+ [ ["src/buildtool/common", "artifact_digest_factory"]
+ , ["src/buildtool/crypto", "hash_function"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/storage", "storage"]
+ , ["src/utils/cpp", "expected"]
+ ]
+ }
+}
diff --git a/src/buildtool/tree_structure/tree_structure_cache.cpp b/src/buildtool/tree_structure/tree_structure_cache.cpp
new file mode 100644
index 00000000..21515e59
--- /dev/null
+++ b/src/buildtool/tree_structure/tree_structure_cache.cpp
@@ -0,0 +1,151 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/tree_structure/tree_structure_cache.hpp"
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/common/artifact_digest_factory.hpp"
+#include "src/buildtool/crypto/hash_function.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/storage/storage.hpp"
+#include "src/utils/cpp/expected.hpp"
+
+namespace {
+inline constexpr std::size_t kHash = 0;
+inline constexpr std::size_t kSize = 1;
+inline constexpr std::size_t kTree = 2;
+
+[[nodiscard]] auto IsInCas(Storage const& storage,
+ ArtifactDigest const& digest) -> bool {
+ if (digest.IsTree()) {
+ return storage.CAS().TreePath(digest).has_value();
+ }
+
+ static constexpr bool kIsExecutable = true;
+ return storage.CAS().BlobPathNoSync(digest, kIsExecutable).has_value() or
+ storage.CAS().BlobPathNoSync(digest, not kIsExecutable).has_value();
+}
+
+[[nodiscard]] auto ToJson(ArtifactDigest const& digest)
+ -> std::optional<nlohmann::json> {
+ try {
+ return nlohmann::json{digest.hash(), digest.size(), digest.IsTree()};
+ } catch (...) {
+ return std::nullopt;
+ }
+}
+
+[[nodiscard]] auto Parse(HashFunction::Type hash_type,
+ std::filesystem::path const& path) noexcept
+ -> std::optional<ArtifactDigest> {
+ try {
+ std::ifstream stream(path);
+ nlohmann::json j = nlohmann::json::parse(stream);
+ auto result = ArtifactDigestFactory::Create(
+ hash_type,
+ j.at(kHash).template get<std::string>(),
+ j.at(kSize).template get<std::size_t>(),
+ j.at(kTree).template get<bool>());
+ if (result) {
+ return *std::move(result);
+ }
+ } catch (...) {
+ return std::nullopt;
+ }
+ return std::nullopt;
+}
+} // namespace
+
+TreeStructureCache::TreeStructureCache(
+ gsl::not_null<StorageConfig const*> const& storage_config) noexcept
+ : TreeStructureCache(storage_config, 0, /*uplink=*/true) {}
+
+TreeStructureCache::TreeStructureCache(
+ gsl::not_null<StorageConfig const*> const& storage_config,
+ std::size_t generation,
+ bool uplink) noexcept
+ : storage_config_{*storage_config},
+ file_storage_{storage_config_.RepositoryGenerationRoot(generation) /
+ "tree_structure"},
+ uplink_{uplink} {}
+
+auto TreeStructureCache::Get(ArtifactDigest const& key) const noexcept
+ -> std::optional<ArtifactDigest> {
+ // Check key object is in the storage AND trigger uplinking if needed
+ auto const path = file_storage_.GetPath(key.hash());
+ if (uplink_ and not LocalUplinkObject(key, *this)) {
+ return std::nullopt;
+ }
+
+ if (not FileSystemManager::IsFile(path)) {
+ return std::nullopt;
+ }
+ return Parse(storage_config_.hash_function.GetType(), path);
+}
+
+auto TreeStructureCache::Set(ArtifactDigest const& key,
+ ArtifactDigest const& value) const noexcept
+ -> bool {
+ auto result = Get(key);
+ if (result) {
+ return *result == value;
+ }
+
+ auto const storage = Storage::Create(&storage_config_);
+ // Check both key and value are in the storage and in the latest generation.
+ if (not IsInCas(storage, key) or not IsInCas(storage, value)) {
+ return false;
+ }
+
+ try {
+ auto j = ToJson(value);
+ if (not j) {
+ return false;
+ }
+ return file_storage_.AddFromBytes(key.hash(), j->dump());
+ } catch (...) {
+ return false;
+ }
+}
+
+auto TreeStructureCache::LocalUplinkObject(
+ ArtifactDigest const& key,
+ TreeStructureCache const& latest) const noexcept -> bool {
+ auto const storage = Storage::Create(&storage_config_);
+ // ensure key is present in the storage AND is in the latest generation
+ if (not IsInCas(storage, key)) {
+ return false;
+ }
+ for (std::size_t i = 0; i < storage_config_.num_generations; ++i) {
+ TreeStructureCache const generation_cache(&storage_config_, i, false);
+ auto const path = generation_cache.file_storage_.GetPath(key.hash());
+ if (not FileSystemManager::IsFile(path)) {
+ continue;
+ }
+ auto digest = Parse(storage_config_.hash_function.GetType(), path);
+ // ensure value is present in the storage AND is in the latest
+ // generation
+ if (not digest or not IsInCas(storage, *digest)) {
+ return false;
+ }
+ static constexpr bool kIsOwner = true;
+ return latest.file_storage_.AddFromFile(key.hash(), path, kIsOwner);
+ }
+ return false;
+}
diff --git a/src/buildtool/tree_structure/tree_structure_cache.hpp b/src/buildtool/tree_structure/tree_structure_cache.hpp
new file mode 100644
index 00000000..4d57da7f
--- /dev/null
+++ b/src/buildtool/tree_structure/tree_structure_cache.hpp
@@ -0,0 +1,62 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP
+#define INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP
+
+#include <cstddef>
+#include <optional>
+
+#include "gsl/gsl"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/file_system/file_storage.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/storage/config.hpp"
+
+class TreeStructureCache final {
+ public:
+ explicit TreeStructureCache(
+ gsl::not_null<StorageConfig const*> const& storage_config) noexcept;
+
+ /// \brief Obtain the digest describing the tree structure of a key tree.
+ /// Can trigger deep uplinking of referenced objects (both key and value).
+ [[nodiscard]] auto Get(ArtifactDigest const& key) const noexcept
+ -> std::optional<ArtifactDigest>;
+
+ /// \brief Set coupling between key and value digest signalizing that the
+ /// value digest contains the tree structure of a key digest. Both key and
+ /// value are expected to be in the storage. Can trigger deep uplinking of
+ /// objects (both key and value).
+ /// \return True if the cache contains the key-value coupling. Fails if
+ /// there is the key in the storage, but it refers to another value. Fails
+ /// if key or value aren't present in the storage.
+ [[nodiscard]] auto Set(ArtifactDigest const& key,
+ ArtifactDigest const& value) const noexcept -> bool;
+
+ private:
+ StorageConfig const& storage_config_;
+ FileStorage<ObjectType::File, StoreMode::FirstWins, false> file_storage_;
+ bool uplink_;
+
+ explicit TreeStructureCache(
+ gsl::not_null<StorageConfig const*> const& storage_config,
+ std::size_t generation,
+ bool uplink) noexcept;
+
+ [[nodiscard]] auto LocalUplinkObject(
+ ArtifactDigest const& key,
+ TreeStructureCache const& latest) const noexcept -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_TREE_STRUCTURE_TREE_STRUCTURE_CACHE_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-06 12:14:19 +0000