1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP
#define INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP
#include <cstdint>
#include <filesystem>
#include <optional>
#include <string>
#include <utility>
#include <vector>
#include "gsl/gsl"
#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/storage/config.hpp"
#include "src/buildtool/storage/uplinker.hpp"
#include "src/utils/cpp/expected.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
template <bool>
class LocalCAS;
enum class LargeObjectErrorCode : std::uint8_t {
/// \brief An internal error occured.
Internal = 0,
/// \brief The digest is not in the CAS.
FileNotFound,
/// \brief The result is different from what was expected.
InvalidResult,
/// \brief Some parts of the tree are not in the storage.
InvalidTree
};
/// \brief Describes an error that occurred during split-splice.
class LargeObjectError final {
public:
LargeObjectError(LargeObjectErrorCode code, std::string message)
: code_(code), message_(std::move(message)) {}
/// \brief Obtain the error code.
[[nodiscard]] auto Code() const noexcept -> LargeObjectErrorCode {
return code_;
}
/// \brief Obtain the error message.
[[nodiscard]] auto Message() && noexcept -> std::string {
return std::move(message_);
}
private:
LargeObjectErrorCode const code_;
std::string message_;
};
/// \brief Stores auxiliary information for reconstructing large objects.
/// The entries are keyed by the hash of the spliced result and the value of an
/// entry is the concatenation of the hashes of chunks the large object is
/// composed of.
template <bool kDoGlobalUplink, ObjectType kType>
class LargeObjectCAS final {
public:
explicit LargeObjectCAS(
gsl::not_null<LocalCAS<kDoGlobalUplink> const*> const& local_cas,
GenerationConfig const& config,
gsl::not_null<Uplinker<kDoGlobalUplink> const*> const&
uplinker) noexcept
: local_cas_(*local_cas),
storage_config_{*config.storage_config},
uplinker_{*uplinker},
file_store_(IsTreeObject(kType) ? config.cas_large_t
: config.cas_large_f) {}
LargeObjectCAS(LargeObjectCAS const&) = delete;
LargeObjectCAS(LargeObjectCAS&&) = delete;
auto operator=(LargeObjectCAS const&) -> LargeObjectCAS& = delete;
auto operator=(LargeObjectCAS&&) -> LargeObjectCAS& = delete;
~LargeObjectCAS() noexcept = default;
/// \brief Obtain path to the storage root.
[[nodiscard]] auto StorageRoot() const noexcept
-> std::filesystem::path const& {
return file_store_.StorageRoot();
}
/// \brief Get the path to a large entry in the storage.
/// \param digest The digest of a large object.
/// \returns Path to the large entry if in the storage.
[[nodiscard]] auto GetEntryPath(ArtifactDigest const& digest) const noexcept
-> std::optional<std::filesystem::path>;
/// \brief Split an object from the main CAS into chunks. If the object had
/// been split before, it would not get split again.
/// \param digest The digest of the object to be split.
/// \return A set of chunks the resulting object is composed of
/// or an error on failure.
[[nodiscard]] auto Split(ArtifactDigest const& digest) const noexcept
-> expected<std::vector<ArtifactDigest>, LargeObjectError>;
/// \brief Splice an object based on the reconstruction rules from the
/// storage. This method doesn't check whether the result of splicing is
/// already in the CAS.
/// \param digest The digest of the object to be spliced.
/// \return A temporary directory that contains a single file
/// "result" on success or an error on failure.
[[nodiscard]] auto TrySplice(ArtifactDigest const& digest) const noexcept
-> expected<TmpFile::Ptr, LargeObjectError>;
/// \brief Splice an object from parts. This method doesn't check whether
/// the result of splicing is already in the CAS.
/// \param digest The digest of the resulting object.
/// \param parts Parts to be concatenated.
/// \return A temporary directory that contains a single file
/// "result" on success or an error on failure.
[[nodiscard]] auto Splice(ArtifactDigest const& digest,
std::vector<ArtifactDigest> const& parts)
const noexcept -> expected<TmpFile::Ptr, LargeObjectError>;
/// \brief Uplink large entry from this generation to latest LocalCAS
/// generation. For the large entry it's parts get promoted first and then
/// the entry itself. This function is only available for instances that are
/// used as local GC generations (i.e., disabled global uplink).
/// \tparam kIsLocalGeneration True if this instance is a local generation.
/// \param latest The latest LocalCAS generation.
/// \param latest_large The latest LargeObjectCAS
/// \param digest The digest of the large entry to uplink.
/// \returns True if the large entry was successfully uplinked.
template <bool kIsLocalGeneration = not kDoGlobalUplink>
requires(kIsLocalGeneration)
[[nodiscard]] auto LocalUplink(
LocalCAS<false> const& latest,
LargeObjectCAS<false, kType> const& latest_large,
ArtifactDigest const& digest) const noexcept -> bool;
private:
// By default, overwrite existing entries. Unless this is a generation
// (disabled global uplink), then we never want to overwrite any entries.
static constexpr auto kStoreMode =
kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins;
LocalCAS<kDoGlobalUplink> const& local_cas_;
StorageConfig const& storage_config_;
Uplinker<kDoGlobalUplink> const& uplinker_;
FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false>
file_store_;
/// \brief Obtain the information for reconstructing a large object.
/// \param digest The digest of a large object.
/// \returns Parts the large object is composed of, if present in
/// the storage.
[[nodiscard]] auto ReadEntry(ArtifactDigest const& digest) const noexcept
-> std::optional<std::vector<ArtifactDigest>>;
/// \brief Create a new entry description and add it to the storage.
/// \param digest The digest of the result.
/// \param parts Parts the resulting object is composed of.
/// \returns True if the entry exists afterwards.
[[nodiscard]] auto WriteEntry(
ArtifactDigest const& digest,
std::vector<ArtifactDigest> const& parts) const noexcept -> bool;
};
// NOLINTNEXTLINE(misc-header-include-cycle)
#include "src/buildtool/storage/large_object_cas.tpp" // IWYU pragma: export
#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_HPP
|
