1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
#include <filesystem>
#include <string>
#include "src/buildtool/execution_api/common/execution_common.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
enum class StoreMode {
// First thread to write conflicting file wins.
FirstWins,
// Last thread to write conflicting file wins, effectively overwriting
// existing entries. NOTE: This might cause races if hard linking from
// stored files due to an issue with the interaction of rename(2) and
// link(2) (see: https://stackoverflow.com/q/69076026/1107763).
LastWins
};
template <ObjectType kType = ObjectType::File,
StoreMode kMode = StoreMode::FirstWins>
class FileStorage {
public:
explicit FileStorage(std::filesystem::path storage_root) noexcept
: storage_root_{std::move(storage_root)} {}
/// \brief Add file to storage.
/// \returns true if file exists afterward.
[[nodiscard]] auto AddFromFile(std::string const& id,
std::filesystem::path const& source_path,
bool is_owner = false) const noexcept
-> bool {
return AtomicAdd(id, source_path, is_owner);
}
/// \brief Add bytes to storage.
/// \returns true if file exists afterward.
[[nodiscard]] auto AddFromBytes(std::string const& id,
std::string const& bytes) const noexcept
-> bool {
return AtomicAdd(id, bytes, /*is_owner=*/true);
}
[[nodiscard]] auto GetPath(std::string const& name) const noexcept
-> std::filesystem::path {
return storage_root_ / name;
}
private:
std::filesystem::path const storage_root_{};
static constexpr bool fd_less_{kType == ObjectType::Executable};
/// \brief Add file to storage via copy and atomic rename.
/// If a race-condition occurs, the winning thread will be the one
/// performing the rename operation first or last, depending on kMode being
/// set to FirstWins or LastWins, respectively. All threads will signal
/// success.
/// \returns true if file exists afterward.
template <class T>
[[nodiscard]] auto AtomicAdd(std::string const& id,
T const& data,
bool is_owner) const noexcept -> bool {
auto file_path = storage_root_ / id;
if (kMode == StoreMode::LastWins or
not FileSystemManager::Exists(file_path)) {
auto unique_path = CreateUniquePath(file_path);
if (unique_path and
FileSystemManager::CreateDirectory(file_path.parent_path()) and
CreateFileFromData(*unique_path, data, is_owner) and
StageFile(*unique_path, file_path)) {
Logger::Log(
LogLevel::Trace, "created entry {}.", file_path.string());
return true;
}
}
return FileSystemManager::IsFile(file_path);
}
/// \brief Create file from file path.
[[nodiscard]] static auto CreateFileFromData(
std::filesystem::path const& file_path,
std::filesystem::path const& other_path,
bool is_owner) noexcept -> bool {
// For files owned by us (e.g., generated files from the execution
// directory), prefer faster creation of hard links instead of a copy.
// Copy executables without opening any writeable file descriptors in
// this process to avoid those from being inherited by child processes.
return (is_owner and FileSystemManager::CreateFileHardlinkAs<kType>(
other_path, file_path)) or
FileSystemManager::CopyFileAs<kType>(
other_path, file_path, fd_less_);
}
/// \brief Create file from bytes.
[[nodiscard]] static auto CreateFileFromData(
std::filesystem::path const& file_path,
std::string const& bytes,
bool /*unused*/) noexcept -> bool {
// Write executables without opening any writeable file descriptors in
// this process to avoid those from being inherited by child processes.
return FileSystemManager::WriteFileAs<kType>(
bytes, file_path, fd_less_);
}
/// \brief Stage file from source path to target path.
[[nodiscard]] static auto StageFile(
std::filesystem::path const& src_path,
std::filesystem::path const& dst_path) noexcept -> bool {
switch (kMode) {
case StoreMode::FirstWins:
// try rename source or delete it if the target already exists
return FileSystemManager::Rename(
src_path, dst_path, /*no_clobber=*/true) or
(FileSystemManager::IsFile(dst_path) and
FileSystemManager::RemoveFile(src_path));
case StoreMode::LastWins:
return FileSystemManager::Rename(src_path, dst_path);
}
}
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
|
