diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-06-17 15:11:36 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-07-05 12:53:57 +0200 |
commit | 8c73c618d777a07c017ec0deefe88db83652827e (patch) | |
tree | 4165656a0a956a1aa0c7ffbaf555d25a5c5894b3 /src | |
parent | e8ab0db340f9d3f408c3f4d14b5b39c3a7cc7037 (diff) | |
download | justbuild-8c73c618d777a07c017ec0deefe88db83652827e.tar.gz |
Move functionality from StorageConfig to related classes.
Diffstat (limited to 'src')
-rw-r--r-- | src/buildtool/common/TARGETS | 1 | ||||
-rw-r--r-- | src/buildtool/common/location.cpp | 3 | ||||
-rw-r--r-- | src/buildtool/execution_api/remote/TARGETS | 5 | ||||
-rw-r--r-- | src/buildtool/execution_api/remote/config.cpp | 2 | ||||
-rw-r--r-- | src/buildtool/execution_api/remote/config.hpp | 42 | ||||
-rw-r--r-- | src/buildtool/file_system/file_system_manager.hpp | 26 | ||||
-rw-r--r-- | src/buildtool/storage/TARGETS | 5 | ||||
-rw-r--r-- | src/buildtool/storage/config.hpp | 74 | ||||
-rw-r--r-- | src/buildtool/storage/target_cache.hpp | 6 | ||||
-rw-r--r-- | src/other_tools/just_mr/TARGETS | 1 | ||||
-rw-r--r-- | src/other_tools/just_mr/utils.hpp | 7 |
11 files changed, 83 insertions, 89 deletions
diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS index dd8fdbd8..be4246f0 100644 --- a/src/buildtool/common/TARGETS +++ b/src/buildtool/common/TARGETS @@ -157,7 +157,6 @@ , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] - , ["src/buildtool/storage", "config"] ] } } diff --git a/src/buildtool/common/location.cpp b/src/buildtool/common/location.cpp index a266069d..597fc286 100644 --- a/src/buildtool/common/location.cpp +++ b/src/buildtool/common/location.cpp @@ -18,7 +18,6 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" -#include "src/buildtool/storage/config.hpp" auto ReadLocationObject(nlohmann::json const& location, std::optional<std::filesystem::path> const& ws_root) @@ -43,7 +42,7 @@ auto ReadLocationObject(nlohmann::json const& location, root_path = *ws_root; } if (root == "home") { - root_path = StorageConfig::GetUserHome(); + root_path = FileSystemManager::GetUserHome(); } if (root == "system") { root_path = FileSystemManager::GetCurrentDirectory().root_path(); diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index ef019f4d..f4aec5db 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -93,7 +93,10 @@ , "name": ["config"] , "hdrs": ["config.hpp"] , "srcs": ["config.cpp"] - , "deps": [["src/buildtool/common/remote", "remote_common"]] + , "deps": + [ ["src/buildtool/common/remote", "remote_common"] + , ["@", "json", "", "json"] + ] , "stage": ["src", "buildtool", "execution_api", "remote"] , "private-deps": [ ["src/buildtool/file_system", "file_system_manager"] diff --git a/src/buildtool/execution_api/remote/config.cpp b/src/buildtool/execution_api/remote/config.cpp index 2f166f8c..5ee0f32e 100644 --- a/src/buildtool/execution_api/remote/config.cpp +++ b/src/buildtool/execution_api/remote/config.cpp @@ -14,11 +14,9 @@ #include "src/buildtool/execution_api/remote/config.hpp" -#include <exception> #include <fstream> #include <utility> -#include "nlohmann/json.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" diff --git a/src/buildtool/execution_api/remote/config.hpp b/src/buildtool/execution_api/remote/config.hpp index bad729ab..7167a3ad 100644 --- a/src/buildtool/execution_api/remote/config.hpp +++ b/src/buildtool/execution_api/remote/config.hpp @@ -16,6 +16,7 @@ #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP #include <cstdint> +#include <exception> #include <filesystem> #include <map> #include <optional> @@ -24,6 +25,7 @@ #include <utility> #include <vector> +#include "nlohmann/json.hpp" #include "src/buildtool/common/remote/remote_common.hpp" class RemoteExecutionConfig { @@ -87,6 +89,46 @@ class RemoteExecutionConfig { return Instance().platform_properties_; } + /// \brief String representation of the used execution backend. + [[nodiscard]] static auto DescribeBackend() noexcept -> std::string { + auto address = RemoteAddress(); + auto properties = PlatformProperties(); + auto dispatch = DispatchList(); + auto description = nlohmann::json{ + {"remote_address", address ? address->ToJson() : nlohmann::json{}}, + {"platform_properties", properties}}; + if (!dispatch.empty()) { + try { + // only add the dispatch list, if not empty, so that keys remain + // not only more readable, but also backwards compatible with + // earlier versions. + auto dispatch_list = nlohmann::json::array(); + for (auto const& [props, endpoint] : dispatch) { + auto entry = nlohmann::json::array(); + entry.push_back(nlohmann::json(props)); + entry.push_back(endpoint.ToJson()); + dispatch_list.push_back(entry); + } + description["endpoint dispatch list"] = dispatch_list; + } catch (std::exception const& e) { + Logger::Log(LogLevel::Error, + "Failed to serialize endpoint dispatch list: {}", + e.what()); + } + } + try { + // json::dump with json::error_handler_t::replace will not throw an + // exception if invalid UTF-8 sequences are detected in the input. + // Instead, it will replace them with the UTF-8 replacement + // character, but still it needs to be inside a try-catch clause to + // ensure the noexcept modifier of the enclosing function. + return description.dump( + 2, ' ', false, nlohmann::json::error_handler_t::replace); + } catch (...) { + return ""; + } + } + private: // Server address of remote execution. std::optional<ServerAddress> remote_address_{}; diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp index 9e8826aa..93a911d8 100644 --- a/src/buildtool/file_system/file_system_manager.hpp +++ b/src/buildtool/file_system/file_system_manager.hpp @@ -18,7 +18,8 @@ #include <array> #include <chrono> #include <cstddef> -#include <cstdio> // for std::fopen +#include <cstdio> // for std::fopen +#include <cstdlib> // std::exit, std::getenv #include <cstring> #include <exception> #include <filesystem> @@ -28,10 +29,13 @@ #ifdef __unix__ #include <fcntl.h> +#include <pwd.h> #include <sys/stat.h> #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#else +#error "Non-unix is not supported yet" #endif #include "gsl/gsl" @@ -139,6 +143,26 @@ class FileSystemManager { return CreateFileImpl(file) == CreationStatus::Created; } + /// \brief Determine user home directory + [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { + char const* root{nullptr}; + +#ifdef __unix__ + root = std::getenv("HOME"); + if (root == nullptr) { + root = getpwuid(getuid())->pw_dir; + } +#endif + + if (root == nullptr) { + Logger::Log(LogLevel::Error, + "Cannot determine user home directory."); + std::exit(EXIT_FAILURE); + } + + return root; + } + /// \brief We are POSIX-compliant, therefore we only care about the string /// value the symlinks points to, whether it exists or not, not the target /// type. As such, we don't distinguish directory or file targets. However, diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index b5e4326f..c6e44f0f 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -5,7 +5,6 @@ , "deps": [ ["src/buildtool/execution_api/remote", "config"] , ["@", "gsl", "", "gsl"] - , ["@", "json", "", "json"] , ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/logging", "logging"] @@ -63,11 +62,11 @@ , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/common", "artifact_description"] , ["src/buildtool/compatibility", "compatibility"] + , ["src/buildtool/execution_api/remote", "config"] ] , "stage": ["src", "buildtool", "storage"] , "private-deps": - [ ["src/buildtool/execution_api/remote", "config"] - , ["src/buildtool/logging", "log_level"] + [ ["src/buildtool/logging", "log_level"] , ["src/buildtool/execution_api/common", "message_limits"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "path_hash"] diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp index cd05fa64..44ba6968 100644 --- a/src/buildtool/storage/config.hpp +++ b/src/buildtool/storage/config.hpp @@ -15,23 +15,12 @@ #ifndef INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP #define INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP -#ifdef __unix__ -#include <pwd.h> -#include <sys/types.h> -#include <unistd.h> -#else -#error "Non-unix is not supported yet" -#endif - #include <cstddef> -#include <exception> #include <filesystem> #include <string> #include "gsl/gsl" -#include "nlohmann/json.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -41,28 +30,8 @@ /// \brief Global storage configuration. class StorageConfig { public: - /// \brief Determine user home directory - [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path { - char const* root{nullptr}; - -#ifdef __unix__ - root = std::getenv("HOME"); - if (root == nullptr) { - root = getpwuid(getuid())->pw_dir; - } -#endif - - if (root == nullptr) { - Logger::Log(LogLevel::Error, - "Cannot determine user home directory."); - std::exit(EXIT_FAILURE); - } - - return root; - } - static inline auto const kDefaultBuildRoot = - GetUserHome() / ".cache" / "just"; + FileSystemManager::GetUserHome() / ".cache" / "just"; [[nodiscard]] static auto Instance() noexcept -> StorageConfig& { static StorageConfig config; @@ -125,47 +94,6 @@ class StorageConfig { is_compatible); } - /// \brief String representation of the used execution backend. - [[nodiscard]] static auto ExecutionBackendDescription() noexcept - -> std::string { - auto address = RemoteExecutionConfig::RemoteAddress(); - auto properties = RemoteExecutionConfig::PlatformProperties(); - auto dispatch = RemoteExecutionConfig::DispatchList(); - auto description = nlohmann::json{ - {"remote_address", address ? address->ToJson() : nlohmann::json{}}, - {"platform_properties", properties}}; - if (!dispatch.empty()) { - try { - // only add the dispatch list, if not empty, so that keys remain - // not only more readable, but also backwards compatible with - // earlier versions. - auto dispatch_list = nlohmann::json::array(); - for (auto const& [props, endpoint] : dispatch) { - auto entry = nlohmann::json::array(); - entry.push_back(nlohmann::json(props)); - entry.push_back(endpoint.ToJson()); - dispatch_list.push_back(entry); - } - description["endpoint dispatch list"] = dispatch_list; - } catch (std::exception const& e) { - Logger::Log(LogLevel::Error, - "Failed to serialize endpoint dispatch list: {}", - e.what()); - } - } - try { - // json::dump with json::error_handler_t::replace will not throw an - // exception if invalid UTF-8 sequences are detected in the input. - // Instead, it will replace them with the UTF-8 replacement - // character, but still it needs to be inside a try-catch clause to - // ensure the noexcept modifier of the enclosing function. - return description.dump( - 2, ' ', false, nlohmann::json::error_handler_t::replace); - } catch (...) { - return ""; - } - } - /// \brief Root directory for all ephemeral directories, i.e., directories /// that can (and should) be removed immediately by garbage collection. [[nodiscard]] static auto EphemeralRoot() noexcept diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp index 6bb5a234..2458b727 100644 --- a/src/buildtool/storage/target_cache.hpp +++ b/src/buildtool/storage/target_cache.hpp @@ -27,10 +27,10 @@ #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/local_cas.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" @@ -59,7 +59,7 @@ class TargetCache { if constexpr (kDoGlobalUplink) { // write backend description (shard) to CAS [[maybe_unused]] auto id = - cas_->StoreBlob(StorageConfig::ExecutionBackendDescription()); + cas_->StoreBlob(RemoteExecutionConfig::DescribeBackend()); EnsuresAudit(id and ArtifactDigest{*id}.hash() == ComputeShard()); } } @@ -147,7 +147,7 @@ class TargetCache { [[nodiscard]] static auto ComputeShard() noexcept -> std::string { return ArtifactDigest::Create<ObjectType::File>( - StorageConfig::ExecutionBackendDescription()) + RemoteExecutionConfig::DescribeBackend()) .hash(); } diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index a2878802..4f4dc20d 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -43,6 +43,7 @@ [ ["@", "gsl", "", "gsl"] , ["@", "json", "", "json"] , ["src/buildtool/storage", "config"] + , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/build_engine/expression", "expression"] ] , "stage": ["src", "other_tools", "just_mr"] diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index f470ee61..16f86614 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -25,17 +25,18 @@ #include "gsl/gsl" #include "nlohmann/json.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/config.hpp" /* Paths and constants required by just-mr */ auto const kDefaultJustPath = "just"; auto const kDefaultGitPath = "git"; -auto const kDefaultRCPath = StorageConfig::GetUserHome() / ".just-mrrc"; +auto const kDefaultRCPath = FileSystemManager::GetUserHome() / ".just-mrrc"; auto const kDefaultBuildRoot = StorageConfig::kDefaultBuildRoot; auto const kDefaultCheckoutLocationsFile = - StorageConfig::GetUserHome() / ".just-local.json"; -auto const kDefaultDistdirs = StorageConfig::GetUserHome() / ".distfiles"; + FileSystemManager::GetUserHome() / ".just-local.json"; +auto const kDefaultDistdirs = FileSystemManager::GetUserHome() / ".distfiles"; std::vector<std::string> const kTakeOver = {"bindings", "target_file_name", |