summaryrefslogtreecommitdiff
path: root/src/buildtool/execution_api/common
diff options
context:
space:
mode:
Diffstat (limited to 'src/buildtool/execution_api/common')
-rw-r--r--src/buildtool/execution_api/common/TARGETS22
-rw-r--r--src/buildtool/execution_api/common/execution_action.hpp58
-rw-r--r--src/buildtool/execution_api/common/execution_api.hpp78
-rw-r--r--src/buildtool/execution_api/common/execution_common.hpp109
-rw-r--r--src/buildtool/execution_api/common/execution_response.hpp48
-rw-r--r--src/buildtool/execution_api/common/local_tree_map.hpp140
6 files changed, 455 insertions, 0 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS
new file mode 100644
index 00000000..aa3ad0bd
--- /dev/null
+++ b/src/buildtool/execution_api/common/TARGETS
@@ -0,0 +1,22 @@
+{ "common":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["common"]
+ , "hdrs":
+ [ "execution_common.hpp"
+ , "execution_api.hpp"
+ , "execution_action.hpp"
+ , "execution_response.hpp"
+ , "local_tree_map.hpp"
+ ]
+ , "deps":
+ [ ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/crypto", "hash_generator"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
+ , ["src/utils/cpp", "hex_string"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "common"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_api/common/execution_action.hpp b/src/buildtool/execution_api/common/execution_action.hpp
new file mode 100644
index 00000000..58176bda
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_action.hpp
@@ -0,0 +1,58 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
+
+#include <chrono>
+#include <memory>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/common/execution_response.hpp"
+
+class Logger;
+class ExecutionArtifactContainer;
+
+/// \brief Abstract action.
+/// Can execute multiple commands. Commands are executed in arbitrary order and
+/// cannot depend on each other.
+class IExecutionAction {
+ public:
+ using Ptr = std::unique_ptr<IExecutionAction>;
+
+ enum class CacheFlag {
+ CacheOutput, ///< run and cache, or serve from cache
+ DoNotCacheOutput, ///< run and do not cache, never served from cached
+ FromCacheOnly, ///< do not run, only serve from cache
+ PretendCached ///< always run, respond same action id as if cached
+ };
+
+ static constexpr std::chrono::milliseconds kDefaultTimeout{1000};
+
+ [[nodiscard]] static constexpr auto CacheEnabled(CacheFlag f) -> bool {
+ return f == CacheFlag::CacheOutput or f == CacheFlag::FromCacheOnly;
+ }
+
+ [[nodiscard]] static constexpr auto ExecutionEnabled(CacheFlag f) -> bool {
+ return f == CacheFlag::CacheOutput or
+ f == CacheFlag::DoNotCacheOutput or
+ f == CacheFlag::PretendCached;
+ }
+
+ IExecutionAction() = default;
+ IExecutionAction(IExecutionAction const&) = delete;
+ IExecutionAction(IExecutionAction&&) = delete;
+ auto operator=(IExecutionAction const&) -> IExecutionAction& = delete;
+ auto operator=(IExecutionAction &&) -> IExecutionAction& = delete;
+ virtual ~IExecutionAction() = default;
+
+ /// \brief Execute the action.
+ /// \returns Execution response, with commands' outputs and artifacts.
+ /// \returns nullptr if execution failed.
+ // NOLINTNEXTLINE(google-default-arguments)
+ [[nodiscard]] virtual auto Execute(Logger const* logger = nullptr) noexcept
+ -> IExecutionResponse::Ptr = 0;
+
+ virtual void SetCacheFlag(CacheFlag flag) noexcept = 0;
+
+ virtual void SetTimeout(std::chrono::milliseconds timeout) noexcept = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp
new file mode 100644
index 00000000..92002d48
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_api.hpp
@@ -0,0 +1,78 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp" // Artifact::ObjectInfo
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/common/execution_action.hpp"
+
+/// \brief Abstract remote execution API
+/// Can be used to create actions.
+class IExecutionApi {
+ public:
+ using Ptr = std::unique_ptr<IExecutionApi>;
+
+ IExecutionApi() = default;
+ IExecutionApi(IExecutionApi const&) = delete;
+ IExecutionApi(IExecutionApi&&) = default;
+ auto operator=(IExecutionApi const&) -> IExecutionApi& = delete;
+ auto operator=(IExecutionApi &&) -> IExecutionApi& = default;
+ virtual ~IExecutionApi() = default;
+
+ /// \brief Create a new action.
+ /// \param[in] root_digest Digest of the build root.
+ /// \param[in] command Command as argv vector
+ /// \param[in] output_files List of paths to output files.
+ /// \param[in] output_dirs List of paths to output directories.
+ /// \param[in] env_vars The environment variables to set.
+ /// \param[in] properties Platform properties to set.
+ /// \returns The new action.
+ [[nodiscard]] virtual auto CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ -> IExecutionAction::Ptr = 0;
+
+ /// \brief Retrieve artifacts from CAS and store to specified paths.
+ /// Tree artifacts are resolved its containing file artifacts are
+ /// recursively retrieved.
+ [[nodiscard]] virtual auto RetrieveToPaths(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<std::filesystem::path> const& output_paths) noexcept
+ -> bool = 0;
+
+ /// \brief Retrieve artifacts from CAS and write to file descriptors.
+ /// Tree artifacts are not resolved and instead the raw protobuf message
+ /// will be written to fd.
+ [[nodiscard]] virtual auto RetrieveToFds(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<int> const& fds) noexcept -> bool = 0;
+
+ /// \brief Upload blobs to CAS. Uploads only the blobs that are not yet
+ /// available in CAS, unless `skip_find_missing` is specified.
+ /// \param blobs Container of blobs to upload.
+ /// \param skip_find_missing Skip finding missing blobs, just upload all.
+ /// NOLINTNEXTLINE(google-default-arguments)
+ [[nodiscard]] virtual auto Upload(BlobContainer const& blobs,
+ bool skip_find_missing = false) noexcept
+ -> bool = 0;
+
+ [[nodiscard]] virtual auto UploadTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> = 0;
+
+ [[nodiscard]] virtual auto IsAvailable(
+ ArtifactDigest const& digest) const noexcept -> bool = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
diff --git a/src/buildtool/execution_api/common/execution_common.hpp b/src/buildtool/execution_api/common/execution_common.hpp
new file mode 100644
index 00000000..8b6aea40
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_common.hpp
@@ -0,0 +1,109 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
+
+#ifdef __unix__
+#include <sys/types.h>
+#include <unistd.h>
+#else
+#error "Non-unix is not supported yet"
+#endif
+
+#include <array>
+#include <filesystem>
+#include <optional>
+#include <random>
+#include <sstream>
+#include <string>
+#include <thread>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/crypto/hash_generator.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+/// \brief Create unique ID for current process and thread.
+[[nodiscard]] static inline auto CreateProcessUniqueId() noexcept
+ -> std::optional<std::string> {
+#ifdef __unix__
+ pid_t pid{};
+ try {
+ pid = getpid();
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return std::nullopt;
+ }
+#endif
+ auto tid = std::this_thread::get_id();
+ std::ostringstream id{};
+ id << pid << "-" << tid;
+ return id.str();
+}
+
+/// \brief Create unique path based on file_path.
+[[nodiscard]] static inline auto CreateUniquePath(
+ std::filesystem::path file_path) noexcept
+ -> std::optional<std::filesystem::path> {
+ auto id = CreateProcessUniqueId();
+ if (id) {
+ return file_path.concat("." + *id);
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] static auto GetNonDeterministicRandomNumber() -> unsigned int {
+ std::uniform_int_distribution<unsigned int> dist{};
+ std::random_device urandom{
+#ifdef __unix__
+ "/dev/urandom"
+#endif
+ };
+ return dist(urandom);
+}
+
+static auto kRandomConstant = GetNonDeterministicRandomNumber();
+
+static void EncodeUUIDVersion4(std::string* uuid) {
+ constexpr auto kVersionByte = 6UL;
+ constexpr auto kVersionBits = 0x40U; // version 4: 0100 xxxx
+ constexpr auto kClearMask = 0x0fU;
+ gsl_Expects(uuid->size() >= kVersionByte);
+ auto& byte = uuid->at(kVersionByte);
+ byte = static_cast<char>(kVersionBits |
+ (kClearMask & static_cast<std::uint8_t>(byte)));
+}
+
+static void EncodeUUIDVariant1(std::string* uuid) {
+ constexpr auto kVariantByte = 8UL;
+ constexpr auto kVariantBits = 0x80U; // variant 1: 10xx xxxx
+ constexpr auto kClearMask = 0x3fU;
+ gsl_Expects(uuid->size() >= kVariantByte);
+ auto& byte = uuid->at(kVariantByte);
+ byte = static_cast<char>(kVariantBits |
+ (kClearMask & static_cast<std::uint8_t>(byte)));
+}
+
+/// \brief Create UUID version 4 from seed.
+[[nodiscard]] static inline auto CreateUUIDVersion4(std::string const& seed)
+ -> std::string {
+ constexpr auto kRawLength = 16UL;
+ constexpr auto kHexDashPos = std::array{8UL, 12UL, 16UL, 20UL};
+
+ auto value = fmt::format("{}-{}", std::to_string(kRandomConstant), seed);
+ auto uuid = HashGenerator{HashGenerator::HashType::SHA1}.Run(value).Bytes();
+ EncodeUUIDVersion4(&uuid);
+ EncodeUUIDVariant1(&uuid);
+ gsl_Expects(uuid.size() >= kRawLength);
+
+ std::size_t cur{};
+ std::ostringstream ss{};
+ auto uuid_hex = ToHexString(uuid.substr(0, kRawLength));
+ for (auto pos : kHexDashPos) {
+ ss << uuid_hex.substr(cur, pos - cur) << '-';
+ cur = pos;
+ }
+ ss << uuid_hex.substr(cur);
+ gsl_EnsuresAudit(ss.str().size() == (2 * kRawLength) + kHexDashPos.size());
+ return ss.str();
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
diff --git a/src/buildtool/execution_api/common/execution_response.hpp b/src/buildtool/execution_api/common/execution_response.hpp
new file mode 100644
index 00000000..76349018
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_response.hpp
@@ -0,0 +1,48 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp"
+
+/// \brief Abstract response.
+/// Response of an action execution. Contains outputs from multiple commands and
+/// a single container with artifacts.
+class IExecutionResponse {
+ public:
+ using Ptr = std::unique_ptr<IExecutionResponse>;
+ using ArtifactInfos = std::unordered_map<std::string, Artifact::ObjectInfo>;
+
+ enum class StatusCode { Failed, Success };
+
+ IExecutionResponse() = default;
+ IExecutionResponse(IExecutionResponse const&) = delete;
+ IExecutionResponse(IExecutionResponse&&) = delete;
+ auto operator=(IExecutionResponse const&) -> IExecutionResponse& = delete;
+ auto operator=(IExecutionResponse &&) -> IExecutionResponse& = delete;
+ virtual ~IExecutionResponse() = default;
+
+ [[nodiscard]] virtual auto Status() const noexcept -> StatusCode = 0;
+
+ [[nodiscard]] virtual auto ExitCode() const noexcept -> int = 0;
+
+ [[nodiscard]] virtual auto IsCached() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto HasStdErr() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto HasStdOut() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto StdErr() noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto StdOut() noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto ActionDigest() const noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto Artifacts() const noexcept -> ArtifactInfos = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
diff --git a/src/buildtool/execution_api/common/local_tree_map.hpp b/src/buildtool/execution_api/common/local_tree_map.hpp
new file mode 100644
index 00000000..77de2d53
--- /dev/null
+++ b/src/buildtool/execution_api/common/local_tree_map.hpp
@@ -0,0 +1,140 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
+
+#include <filesystem>
+#include <shared_mutex>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Maps digest of `bazel_re::Directory` to `LocalTree`.
+class LocalTreeMap {
+ /// \brief Thread-safe pool of unique object infos.
+ class ObjectInfoPool {
+ public:
+ /// Get pointer to stored info, or a add new one and return its pointer.
+ [[nodiscard]] auto GetOrAdd(Artifact::ObjectInfo const& info)
+ -> Artifact::ObjectInfo const* {
+ { // get
+ std::shared_lock lock{mutex_};
+ auto it = infos_.find(info);
+ if (it != infos_.end()) {
+ return &(*it);
+ }
+ }
+ { // or add
+ std::unique_lock lock{mutex_};
+ return &(*infos_.emplace(info).first);
+ }
+ }
+
+ private:
+ std::unordered_set<Artifact::ObjectInfo> infos_;
+ mutable std::shared_mutex mutex_;
+ };
+
+ public:
+ /// \brief Maps blob locations to object infos.
+ class LocalTree {
+ friend class LocalTreeMap;
+
+ public:
+ /// \brief Add a new path and info pair to the tree.
+ /// Path must not be absolute, empty, or contain dot-segments.
+ /// \param path The location to add the object info.
+ /// \param info The object info to add.
+ /// \returns true if successfully inserted or info existed before.
+ [[nodiscard]] auto AddInfo(std::filesystem::path const& path,
+ Artifact::ObjectInfo const& info) noexcept
+ -> bool {
+ auto norm_path = path.lexically_normal();
+ if (norm_path.is_absolute() or norm_path.empty() or
+ *norm_path.begin() == "..") {
+ Logger::Log(LogLevel::Error,
+ "cannot add malformed path to local tree: {}",
+ path.string());
+ return false;
+ }
+ try {
+ if (entries_.contains(norm_path.string())) {
+ return true;
+ }
+ if (auto const* info_ptr = infos_->GetOrAdd(info)) {
+ entries_.emplace(norm_path.string(), info_ptr);
+ return true;
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "adding object info to tree failed with:\n{}",
+ ex.what());
+ }
+ return false;
+ }
+
+ [[nodiscard]] auto size() const noexcept { return entries_.size(); }
+ [[nodiscard]] auto begin() const noexcept { return entries_.begin(); }
+ [[nodiscard]] auto end() const noexcept { return entries_.end(); }
+
+ private:
+ gsl::not_null<ObjectInfoPool*> infos_;
+ std::unordered_map<std::string,
+ gsl::not_null<Artifact::ObjectInfo const*>>
+ entries_{};
+
+ explicit LocalTree(gsl::not_null<ObjectInfoPool*> infos) noexcept
+ : infos_{std::move(infos)} {}
+ };
+
+ /// \brief Create a new `LocalTree` object.
+ [[nodiscard]] auto CreateTree() noexcept -> LocalTree {
+ return LocalTree{&infos_};
+ }
+
+ /// \brief Get pointer to existing `LocalTree` object.
+ /// \param root_digest The root digest of the tree to lookup.
+ /// \returns nullptr if no tree was found for given root digest.
+ [[nodiscard]] auto GetTree(bazel_re::Digest const& root_digest)
+ const noexcept -> LocalTree const* {
+ std::shared_lock lock{mutex_};
+ auto it = trees_.find(root_digest);
+ return (it != trees_.end()) ? &(it->second) : nullptr;
+ }
+
+ /// \brief Checks if entry for root digest exists.
+ [[nodiscard]] auto HasTree(
+ bazel_re::Digest const& root_digest) const noexcept -> bool {
+ return GetTree(root_digest) != nullptr;
+ }
+
+ /// \brief Add new `LocalTree` for given root digest. Does not overwrite if
+ /// a tree for the given root digest already exists.
+ /// \param root_digest The root digest to add the new tree for.
+ /// \param tree The new tree to add.
+ /// \returns true if the tree was successfully added or existed before.
+ [[nodiscard]] auto AddTree(bazel_re::Digest const& root_digest,
+ LocalTree&& tree) noexcept -> bool {
+ if (not HasTree(root_digest)) {
+ try {
+ std::unique_lock lock{mutex_};
+ trees_.emplace(root_digest, std::move(tree));
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "adding local tree to tree map failed with:\n{}",
+ ex.what());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private:
+ ObjectInfoPool infos_; // pool to store each solid object info exactly once
+ std::unordered_map<bazel_re::Digest, LocalTree> trees_;
+ mutable std::shared_mutex mutex_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 18:27:57 +0000