summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_api/local/local_action.cpp174
-rw-r--r--src/buildtool/execution_api/local/local_action.hpp19
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp5
-rw-r--r--src/buildtool/execution_api/local/local_response.hpp29
-rw-r--r--src/buildtool/file_system/file_system_manager.hpp78
-rw-r--r--src/buildtool/storage/local_cas.tpp4
6 files changed, 248 insertions, 61 deletions
diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp
index e5457bcd..d42dcdd6 100644
--- a/src/buildtool/execution_api/local/local_action.cpp
+++ b/src/buildtool/execution_api/local/local_action.cpp
@@ -250,45 +250,79 @@ auto LocalAction::CreateDirectoryStructure(
});
}
-auto LocalAction::CollectOutputFile(std::filesystem::path const& exec_path,
- std::string const& local_path)
- const noexcept -> std::optional<bazel_re::OutputFile> {
+/// \brief We expect either a regular file, or a symlink.
+auto LocalAction::CollectOutputFileOrSymlink(
+ std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept
+ -> std::optional<OutputFileOrSymlink> {
auto file_path = exec_path / local_path;
auto type = FileSystemManager::Type(file_path);
- if ((not type) or (not IsFileObject(*type))) {
- Logger::Log(LogLevel::Error, "expected file at {}", local_path);
+ if (not type) {
+ Logger::Log(LogLevel::Error, "expected known type at {}", local_path);
return std::nullopt;
}
- bool is_executable = IsExecutableObject(*type);
- auto digest =
- storage_->CAS().StoreBlob</*kOwner=*/true>(file_path, is_executable);
- if (digest) {
- auto out_file = bazel_re::OutputFile{};
- out_file.set_path(local_path);
- out_file.set_allocated_digest(
- gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
- out_file.set_is_executable(is_executable);
- return out_file;
+ if (IsSymlinkObject(*type)) {
+ auto content = FileSystemManager::ReadSymlink(file_path);
+ if (content and PathIsNonUpwards(*content) and
+ storage_->CAS().StoreBlob(*content)) {
+ auto out_symlink = bazel_re::OutputSymlink{};
+ out_symlink.set_path(local_path);
+ out_symlink.set_target(*content);
+ return out_symlink;
+ }
+ }
+ else if (IsFileObject(*type)) {
+ bool is_executable = IsExecutableObject(*type);
+ auto digest = storage_->CAS().StoreBlob</*kOwner=*/true>(file_path,
+ is_executable);
+ if (digest) {
+ auto out_file = bazel_re::OutputFile{};
+ out_file.set_path(local_path);
+ out_file.set_allocated_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
+ out_file.set_is_executable(is_executable);
+ return out_file;
+ }
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error, "expected file or symlink at {}", local_path);
}
return std::nullopt;
}
-auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path,
- std::string const& local_path) const noexcept
- -> std::optional<bazel_re::OutputDirectory> {
+auto LocalAction::CollectOutputDirOrSymlink(
+ std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept
+ -> std::optional<OutputDirOrSymlink> {
auto dir_path = exec_path / local_path;
auto type = FileSystemManager::Type(dir_path);
- if ((not type) or (not IsTreeObject(*type))) {
- Logger::Log(LogLevel::Error, "expected directory at {}", local_path);
+ if (not type) {
+ Logger::Log(LogLevel::Error, "expected known type at {}", local_path);
return std::nullopt;
}
-
- if (auto digest = CreateDigestFromLocalOwnedTree(storage_, dir_path)) {
- auto out_dir = bazel_re::OutputDirectory{};
- out_dir.set_path(local_path);
- out_dir.set_allocated_tree_digest(
- gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
- return out_dir;
+ if (IsSymlinkObject(*type)) {
+ auto content = FileSystemManager::ReadSymlink(dir_path);
+ if (content and PathIsNonUpwards(*content) and
+ storage_->CAS().StoreBlob(*content)) {
+ auto out_symlink = bazel_re::OutputSymlink{};
+ out_symlink.set_path(local_path);
+ out_symlink.set_target(*content);
+ return out_symlink;
+ }
+ }
+ else if (IsTreeObject(*type)) {
+ if (auto digest = CreateDigestFromLocalOwnedTree(storage_, dir_path)) {
+ auto out_dir = bazel_re::OutputDirectory{};
+ out_dir.set_path(local_path);
+ out_dir.set_allocated_tree_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
+ return out_dir;
+ }
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error, "expected directory or symlink at {}", local_path);
}
return std::nullopt;
}
@@ -296,31 +330,73 @@ auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path,
auto LocalAction::CollectAndStoreOutputs(
bazel_re::ActionResult* result,
std::filesystem::path const& exec_path) const noexcept -> bool {
- logger_.Emit(LogLevel::Trace, "collecting outputs:");
- for (auto const& path : output_files_) {
- auto out_file = CollectOutputFile(exec_path, path);
- if (not out_file) {
- logger_.Emit(
- LogLevel::Error, "could not collect output file {}", path);
- return false;
+ try {
+ logger_.Emit(LogLevel::Trace, "collecting outputs:");
+ for (auto const& path : output_files_) {
+ auto out = CollectOutputFileOrSymlink(exec_path, path);
+ if (not out) {
+ logger_.Emit(LogLevel::Error,
+ "could not collect output file or symlink {}",
+ path);
+ return false;
+ }
+ if (std::holds_alternative<bazel_re::OutputSymlink>(*out)) {
+ auto out_symlink = std::get<bazel_re::OutputSymlink>(*out);
+ auto const& target = out_symlink.target();
+ if (not PathIsNonUpwards(target)) {
+ logger_.Emit(LogLevel::Error,
+ "collected upwards output symlink {}",
+ path);
+ return false;
+ }
+ logger_.Emit(
+ LogLevel::Trace, " - symlink {}: {}", path, target);
+ result->mutable_output_file_symlinks()->Add(
+ std::move(out_symlink));
+ }
+ else {
+ auto out_file = std::get<bazel_re::OutputFile>(*out);
+ auto const& digest = out_file.digest().hash();
+ logger_.Emit(LogLevel::Trace, " - file {}: {}", path, digest);
+ result->mutable_output_files()->Add(std::move(out_file));
+ }
}
- auto const& digest = out_file->digest().hash();
- logger_.Emit(LogLevel::Trace, " - file {}: {}", path, digest);
- result->mutable_output_files()->Add(std::move(*out_file));
- }
- for (auto const& path : output_dirs_) {
- auto out_dir = CollectOutputDir(exec_path, path);
- if (not out_dir) {
- logger_.Emit(
- LogLevel::Error, "could not collect output dir {}", path);
- return false;
+ for (auto const& path : output_dirs_) {
+ auto out = CollectOutputDirOrSymlink(exec_path, path);
+ if (not out) {
+ logger_.Emit(LogLevel::Error,
+ "could not collect output dir or symlink {}",
+ path);
+ return false;
+ }
+ if (std::holds_alternative<bazel_re::OutputSymlink>(*out)) {
+ auto out_symlink = std::get<bazel_re::OutputSymlink>(*out);
+ auto const& target = out_symlink.target();
+ if (not PathIsNonUpwards(target)) {
+ logger_.Emit(LogLevel::Error,
+ "collected upwards output symlink {}",
+ path);
+ return false;
+ }
+ logger_.Emit(
+ LogLevel::Trace, " - symlink {}: {}", path, target);
+ result->mutable_output_file_symlinks()->Add(
+ std::move(out_symlink));
+ }
+ else {
+ auto out_dir = std::get<bazel_re::OutputDirectory>(*out);
+ auto const& digest = out_dir.tree_digest().hash();
+ logger_.Emit(LogLevel::Trace, " - dir {}: {}", path, digest);
+ result->mutable_output_directories()->Add(std::move(out_dir));
+ }
}
- auto const& digest = out_dir->tree_digest().hash();
- logger_.Emit(LogLevel::Trace, " - dir {}: {}", path, digest);
- result->mutable_output_directories()->Add(std::move(*out_dir));
+ return true;
+ } catch (std::exception const& ex) {
+ // should never happen, but report nonetheless
+ logger_.Emit(
+ LogLevel::Error, "collecting outputs failed:\n{}", ex.what());
+ return false;
}
-
- return true;
}
auto LocalAction::DigestFromOwnedFile(std::filesystem::path const& file_path)
diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp
index a49060e4..95f95ff4 100644
--- a/src/buildtool/execution_api/local/local_action.hpp
+++ b/src/buildtool/execution_api/local/local_action.hpp
@@ -38,6 +38,11 @@ class LocalAction final : public IExecutionAction {
bool is_cached{};
};
+ using OutputFileOrSymlink =
+ std::variant<bazel_re::OutputFile, bazel_re::OutputSymlink>;
+ using OutputDirOrSymlink =
+ std::variant<bazel_re::OutputDirectory, bazel_re::OutputSymlink>;
+
auto Execute(Logger const* logger) noexcept
-> IExecutionResponse::Ptr final;
@@ -112,13 +117,15 @@ class LocalAction final : public IExecutionAction {
[[nodiscard]] auto CreateDirectoryStructure(
std::filesystem::path const& exec_path) const noexcept -> bool;
- [[nodiscard]] auto CollectOutputFile(std::filesystem::path const& exec_path,
- std::string const& local_path)
- const noexcept -> std::optional<bazel_re::OutputFile>;
+ [[nodiscard]] auto CollectOutputFileOrSymlink(
+ std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept
+ -> std::optional<OutputFileOrSymlink>;
- [[nodiscard]] auto CollectOutputDir(std::filesystem::path const& exec_path,
- std::string const& local_path)
- const noexcept -> std::optional<bazel_re::OutputDirectory>;
+ [[nodiscard]] auto CollectOutputDirOrSymlink(
+ std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept
+ -> std::optional<OutputDirOrSymlink>;
[[nodiscard]] auto CollectAndStoreOutputs(
bazel_re::ActionResult* result,
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
index 87296414..64e89203 100644
--- a/src/buildtool/execution_api/local/local_api.hpp
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -206,8 +206,9 @@ class LocalApi final : public IExecutionApi {
return false;
}
- // Read artifact content.
- auto const& content = FileSystemManager::ReadFile(*path);
+ // Read artifact content (file or symlink).
+ auto const& content =
+ FileSystemManager::ReadContentAtPath(*path, info.type);
if (not content) {
return false;
}
diff --git a/src/buildtool/execution_api/local/local_response.hpp b/src/buildtool/execution_api/local/local_response.hpp
index 135897d8..8e6cb04f 100644
--- a/src/buildtool/execution_api/local/local_response.hpp
+++ b/src/buildtool/execution_api/local/local_response.hpp
@@ -105,8 +105,7 @@ class LocalResponse final : public IExecutionResponse {
action_result.output_file_symlinks_size()) +
static_cast<std::size_t>(
action_result.output_directory_symlinks_size()) +
- static_cast<std::size_t>(
- action_result.output_directories().size()));
+ static_cast<std::size_t>(action_result.output_directories_size()));
DirSymlinks dir_symlinks{};
dir_symlinks_.reserve(static_cast<std::size_t>(
@@ -153,6 +152,32 @@ class LocalResponse final : public IExecutionResponse {
}
}
+ // collect all symlinks and store them
+ for (auto const& link : action_result.output_file_symlinks()) {
+ try {
+ artifacts.emplace(
+ link.path(),
+ Artifact::ObjectInfo{
+ .digest = ArtifactDigest::Create<ObjectType::File>(
+ link.target()),
+ .type = ObjectType::Symlink});
+ } catch (...) {
+ return false;
+ }
+ }
+ for (auto const& link : action_result.output_directory_symlinks()) {
+ try {
+ artifacts.emplace(
+ link.path(),
+ Artifact::ObjectInfo{
+ .digest = ArtifactDigest::Create<ObjectType::File>(
+ link.target()),
+ .type = ObjectType::Symlink});
+ } catch (...) {
+ return false;
+ }
+ }
+
// collect directories and store them
for (auto const& dir : action_result.output_directories()) {
try {
diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp
index a319cc0e..d5b4bf41 100644
--- a/src/buildtool/file_system/file_system_manager.hpp
+++ b/src/buildtool/file_system/file_system_manager.hpp
@@ -27,6 +27,8 @@
#include <fcntl.h>
#ifdef __unix__
+#include <fcntl.h>
+#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#endif
@@ -338,6 +340,7 @@ class FileSystemManager {
kSetEpochTime,
kSetWritable>(src, dst, fd_less, opt);
case ObjectType::Symlink:
+ return CopySymlinkAs<kSetEpochTime>(src, dst);
case ObjectType::Tree:
break;
}
@@ -381,6 +384,52 @@ class FileSystemManager {
}
}
+ /// \brief Create a symlink with option to set epoch time.
+ template <bool kSetEpochTime = false>
+ [[nodiscard]] static auto CreateSymlinkAs(
+ std::filesystem::path const& to,
+ std::filesystem::path const& link) noexcept -> bool {
+ return CreateSymlink(to, link) and
+ (not kSetEpochTime or SetEpochTime(link));
+ }
+
+ /// \brief Create symlink copy at location, with option to overwriting any
+ /// existing. Uses the content of src directly as the new target, whether
+ /// src is a regular file (CAS entry) or another symlink.
+ template <bool kSetEpochTime = false>
+ [[nodiscard]] static auto CopySymlinkAs(
+ std::filesystem::path const& src,
+ std::filesystem::path const& dst,
+ bool overwrite_existing = true) noexcept -> bool {
+ try {
+ if (overwrite_existing and Exists(dst) and
+ not std::filesystem::remove(dst)) {
+ Logger::Log(LogLevel::Debug,
+ "could not overwrite existing path {}",
+ dst.string());
+ return false;
+ }
+ if (std::filesystem::is_symlink(src)) {
+ if (auto content = ReadSymlink(src)) {
+ return CreateSymlinkAs<kSetEpochTime>(*content, dst);
+ }
+ }
+ else {
+ if (auto content = ReadFile(src)) {
+ return CreateSymlinkAs<kSetEpochTime>(*content, dst);
+ }
+ }
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "copying symlink from {} to {}:\n{}",
+ src.string(),
+ dst.string(),
+ e.what());
+ }
+
+ return false;
+ }
+
[[nodiscard]] static auto RemoveFile(
std::filesystem::path const& file) noexcept -> bool {
try {
@@ -946,11 +995,40 @@ class FileSystemManager {
}
}
+ /// \brief Set the last time of modification for a file (or symlink --
+ /// POSIX-only).
static auto SetEpochTime(std::filesystem::path const& file_path) noexcept
-> bool {
static auto const kPosixEpochTime =
System::GetPosixEpoch<std::chrono::file_clock>();
try {
+ if (std::filesystem::is_symlink(file_path)) {
+ // Because std::filesystem::last_write_time follows
+ // symlinks, one has instead to manually call utimensat with
+ // the AT_SYMLINK_NOFOLLOW flag. On non-POSIX systems, we
+ // return false by default for symlinks.
+#ifdef __unix__
+ std::array<timespec, 2> times{}; // default is POSIX epoch
+ if (utimensat(AT_FDCWD,
+ file_path.c_str(),
+ times.data(),
+ AT_SYMLINK_NOFOLLOW) != 0) {
+ Logger::Log(LogLevel::Error,
+ "Call to utimensat for symlink {} failed with "
+ "error: {}",
+ file_path.string(),
+ strerror(errno));
+ return false;
+ }
+ return true;
+#else
+ Logger::Log(
+ LogLevel::Warning,
+ "Setting the last modification time attribute for a "
+ "symlink is unsupported!");
+ return false;
+#endif
+ }
std::filesystem::last_write_time(file_path, kPosixEpochTime);
return true;
} catch (std::exception const& e) {
diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp
index ba5471e7..b3d95934 100644
--- a/src/buildtool/storage/local_cas.tpp
+++ b/src/buildtool/storage/local_cas.tpp
@@ -50,8 +50,8 @@ template <class T_CAS>
if (not link_path) {
return false;
}
- auto content =
- FileSystemManager::ReadSymlink(*link_path);
+ // in the local CAS we store as files
+ auto content = FileSystemManager::ReadFile(*link_path);
if (not content or not PathIsNonUpwards(*content)) {
return false;
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-08 17:07:01 +0000