summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xbin/just-mr.py4
-rw-r--r--src/buildtool/common/TARGETS3
-rw-r--r--src/buildtool/common/cli.hpp10
-rw-r--r--src/buildtool/compatibility/TARGETS8
-rw-r--r--src/buildtool/compatibility/compatibility.hpp68
-rw-r--r--src/buildtool/crypto/hash_generator.hpp16
-rw-r--r--src/buildtool/execution_api/common/execution_common.hpp2
-rw-r--r--src/buildtool/execution_api/local/TARGETS3
-rw-r--r--src/buildtool/execution_api/local/config.hpp80
-rw-r--r--src/buildtool/execution_api/local/local_ac.hpp8
-rw-r--r--src/buildtool/execution_api/local/local_action.cpp2
-rw-r--r--src/buildtool/execution_api/local/local_cas.hpp11
-rw-r--r--src/buildtool/execution_api/local/local_storage.hpp8
-rw-r--r--src/buildtool/execution_engine/executor/TARGETS1
-rw-r--r--src/buildtool/execution_engine/executor/executor.hpp40
-rw-r--r--src/buildtool/file_system/TARGETS1
-rw-r--r--src/buildtool/file_system/file_root.hpp8
-rw-r--r--src/buildtool/main/TARGETS1
-rw-r--r--src/buildtool/main/main.cpp26
19 files changed, 228 insertions, 72 deletions
diff --git a/bin/just-mr.py b/bin/just-mr.py
index 2c2a08b3..7b68eb19 100755
--- a/bin/just-mr.py
+++ b/bin/just-mr.py
@@ -188,7 +188,7 @@ def git_hash(content):
def add_to_cas(data):
if isinstance(data, str):
data = data.encode('utf-8')
- cas_root = os.path.join(ROOT,"casf")
+ cas_root = os.path.join(ROOT,"protocol-dependent/git-sha1/casf")
basename = git_hash(data)
target = os.path.join(cas_root, basename)
tempname = os.path.join(cas_root, "%s.%d" % (basename, os.getpid()))
@@ -205,7 +205,7 @@ def add_to_cas(data):
return target
def cas_path(h):
- return os.path.join(ROOT, "casf", h)
+ return os.path.join(ROOT, "protocol-dependent/git-sha1/casf", h)
def is_in_cas(h):
return os.path.exists(cas_path(h))
diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS
index eec2b011..a9c1547f 100644
--- a/src/buildtool/common/TARGETS
+++ b/src/buildtool/common/TARGETS
@@ -3,7 +3,8 @@
, "name": ["cli"]
, "hdrs": ["cli.hpp"]
, "deps":
- [ ["src/buildtool/logging", "log_level"]
+ [ ["src/buildtool/compatibility", "compatibility"]
+ , ["src/buildtool/logging", "log_level"]
, ["@", "cli11", "", "cli11"]
, ["@", "json", "", "json"]
, ["@", "fmt", "", "fmt"]
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
index 8bb0ec89..c00b7b1b 100644
--- a/src/buildtool/common/cli.hpp
+++ b/src/buildtool/common/cli.hpp
@@ -12,6 +12,7 @@
#include "fmt/core.h"
#include "gsl-lite/gsl-lite.hpp"
#include "nlohmann/json.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/logging/log_level.hpp"
constexpr auto kDefaultLogLevel = LogLevel::Progress;
@@ -374,4 +375,13 @@ static inline auto SetupGraphArguments(
"missing KNOWN artifacts.");
}
+static inline auto SetupCompatibilityArguments(
+ gsl::not_null<CLI::App*> const& app) {
+ app->add_flag_function(
+ "--compatible",
+ [](auto /*unused*/) { Compatibility::SetCompatible(); },
+ "At increased computational effort, be compatible with the original "
+ "remote build execution protocol. As the change affects identifiers, "
+ "the flag must be used consistently for all related invocations.");
+}
#endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP
diff --git a/src/buildtool/compatibility/TARGETS b/src/buildtool/compatibility/TARGETS
new file mode 100644
index 00000000..277ca22b
--- /dev/null
+++ b/src/buildtool/compatibility/TARGETS
@@ -0,0 +1,8 @@
+{ "compatibility":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["compatibility"]
+ , "hdrs": ["compatibility.hpp"]
+ , "deps": [["src/buildtool/crypto", "hash_generator"]]
+ , "stage": ["src", "buildtool", "compatibility"]
+ }
+}
diff --git a/src/buildtool/compatibility/compatibility.hpp b/src/buildtool/compatibility/compatibility.hpp
new file mode 100644
index 00000000..271d590e
--- /dev/null
+++ b/src/buildtool/compatibility/compatibility.hpp
@@ -0,0 +1,68 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMPATIBILITY_COMPATIBILITY_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMPATIBILITY_COMPATIBILITY_HPP
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
+
+#include "src/buildtool/crypto/hash_generator.hpp"
+#include "src/buildtool/logging/logger.hpp"
+class Compatibility {
+ using git_hash = std::string;
+ using sha256_hash = std::string;
+ using git_repo = std::string;
+ using GitToComaptibleMap = std::unordered_map<git_hash, sha256_hash>;
+ using CompatibleToGitMap =
+ std::unordered_map<sha256_hash, std::pair<git_hash, git_repo>>;
+
+ public:
+ [[nodiscard]] static auto Instance() noexcept -> Compatibility& {
+ static Compatibility instance{};
+ return instance;
+ }
+ [[nodiscard]] static auto IsCompatible() noexcept -> bool {
+ return Instance().compatible_;
+ }
+ static void SetCompatible() noexcept { Instance().compatible_ = true; }
+
+ static auto RegisterGitEntry(std::string const& git_hash,
+ std::string const& data,
+ std::string const& repo) -> sha256_hash {
+
+ {
+ auto& git_to_compatible = Instance().git_to_compatible_;
+ std::shared_lock lock_{Instance().mutex_};
+ auto it = git_to_compatible.find(git_hash);
+ if (it != git_to_compatible.end()) {
+ return it->second;
+ }
+ }
+ auto compatible_hash = ComputeHash(data);
+ std::unique_lock lock_{Instance().mutex_};
+ Instance().git_to_compatible_[git_hash] = compatible_hash;
+ Instance().compatible_to_git_[compatible_hash] = {git_hash, repo};
+ return compatible_hash;
+ }
+
+ static auto GetGitEntry(std::string const& compatible_hash)
+ -> std::optional<std::pair<git_hash const&, git_repo const&>> {
+ auto const& compatible_to_git = Instance().compatible_to_git_;
+ std::shared_lock lock_{Instance().mutex_};
+ auto it = compatible_to_git.find(compatible_hash);
+ if (it != compatible_to_git.end()) {
+ return it->second;
+ }
+ Logger::Log(
+ LogLevel::Warning,
+ fmt::format("Unable to get the git-sha1 code associated to {}",
+ compatible_hash));
+ return std::nullopt;
+ }
+
+ private:
+ GitToComaptibleMap git_to_compatible_{};
+
+ CompatibleToGitMap compatible_to_git_{};
+ bool compatible_{false};
+ std::shared_mutex mutex_;
+};
+#endif // INCLUDED_SRC_BUILDTOOL_COMPATIBILITY_COMPATIBILITY_HPP
diff --git a/src/buildtool/crypto/hash_generator.hpp b/src/buildtool/crypto/hash_generator.hpp
index 7ff9e004..79d744fc 100644
--- a/src/buildtool/crypto/hash_generator.hpp
+++ b/src/buildtool/crypto/hash_generator.hpp
@@ -20,6 +20,19 @@ class HashGenerator {
/// \brief Types of hash implementations supported by generator.
enum class HashType { MD5, SHA1, SHA256, GIT };
+ // this function is called for the first time from the main to initialize
+ // the HashGenerator according to the command line flag --compatible
+ // note that, once the HashGenerator is initialized, it cannot be changed
+ [[maybe_unused]] static auto SetHashGenerator(
+ std::optional<HashType> const x = std::nullopt) -> HashGenerator& {
+ static HashGenerator gen{x.value_or(HashType::GIT)};
+ return gen;
+ }
+ [[nodiscard]] static auto Instance() -> HashGenerator& {
+ return SetHashGenerator(); // return the already initialized
+ // HashGenerator
+ }
+
/// \brief The universal hash digest.
/// The type of hash and the digest length depends on the hash
/// implementation used to generated this digest.
@@ -123,8 +136,7 @@ class HashGenerator {
/// \brief Hash function used for the entire buildtool
[[maybe_unused]] [[nodiscard]] static inline auto ComputeHash(
std::string const& data) noexcept -> std::string {
- static HashGenerator gen{HashGenerator::HashType::GIT};
- return gen.Run(data).HexString();
+ return HashGenerator::Instance().Run(data).HexString();
}
#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_GENERATOR_HPP
diff --git a/src/buildtool/execution_api/common/execution_common.hpp b/src/buildtool/execution_api/common/execution_common.hpp
index 1cf198fe..1208fc99 100644
--- a/src/buildtool/execution_api/common/execution_common.hpp
+++ b/src/buildtool/execution_api/common/execution_common.hpp
@@ -89,7 +89,7 @@ static void EncodeUUIDVariant1(std::string* uuid) {
constexpr auto kHexDashPos = std::array{8UL, 12UL, 16UL, 20UL};
auto value = fmt::format("{}-{}", std::to_string(kRandomConstant), seed);
- auto uuid = HashGenerator{HashGenerator::HashType::SHA1}.Run(value).Bytes();
+ auto uuid = HashGenerator::Instance().Run(value).Bytes();
EncodeUUIDVersion4(&uuid);
EncodeUUIDVariant1(&uuid);
gsl_Expects(uuid.size() >= kRawLength);
diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS
index b6d265a6..e395f7d2 100644
--- a/src/buildtool/execution_api/local/TARGETS
+++ b/src/buildtool/execution_api/local/TARGETS
@@ -3,7 +3,8 @@
, "name": ["config"]
, "hdrs": ["config.hpp"]
, "deps":
- [ ["src/buildtool/logging", "logging"]
+ [ ["src/buildtool/compatibility", "compatibility"]
+ , ["src/buildtool/logging", "logging"]
, ["src/buildtool/file_system", "file_system_manager"]
]
, "stage": ["src", "buildtool", "execution_api", "local"]
diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp
index 3f293b95..c65f7d17 100644
--- a/src/buildtool/execution_api/local/config.hpp
+++ b/src/buildtool/execution_api/local/config.hpp
@@ -14,20 +14,25 @@
#include <string>
#include <vector>
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
/// \brief Store global build system configuration.
class LocalExecutionConfig {
struct ConfigData {
- // User root directory (Unix default: /home/${USER})
- std::filesystem::path user_root{};
- // Build root directory (default: empty)
+ // Build root directory. All the cache dirs are subdirs of build_root.
+ // By default, build_root is set to $HOME/.cache/just.
+ // If the user uses --local_build_root PATH,
+ // then build_root will be set to PATH.
std::filesystem::path build_root{};
- // Disk cache directory (default: empty)
- std::filesystem::path disk_cache{};
+ // cache_root points to one of the following
+ // build_root/protocol-dependent/{git-sha1,compatible-sha256}
+ // git-sha1 is the current default. If the user passes the flag
+ // --compatible, then the subfolder compatible_sha256 is used
+ std::filesystem::path cache_root{};
// Launcher to be prepended to action's command before executed.
// Default: ["env", "--"]
@@ -37,6 +42,13 @@ class LocalExecutionConfig {
bool keep_build_dir{false};
};
+ // different folder for different caching protocol
+ [[nodiscard]] static auto UpdatePathForCompatibility(
+ std::filesystem::path const& dir) -> std::filesystem::path {
+ return dir / (Compatibility::IsCompatible() ? "compatible-sha256"
+ : "git-sha1");
+ }
+
public:
[[nodiscard]] static auto SetBuildRoot(
std::filesystem::path const& dir) noexcept -> bool {
@@ -47,18 +59,8 @@ class LocalExecutionConfig {
return false;
}
Data().build_root = dir;
- return true;
- }
-
- [[nodiscard]] static auto SetDiskCache(
- std::filesystem::path const& dir) noexcept -> bool {
- if (FileSystemManager::IsRelativePath(dir)) {
- Logger::Log(LogLevel::Error,
- "Disk cache must be absolute path but got '{}'.",
- dir.string());
- return false;
- }
- Data().disk_cache = dir;
+ Data().cache_root = ""; // in case we re-set build_root, we are sure
+ // that the cache path is recomputed as well
return true;
}
@@ -83,29 +85,42 @@ class LocalExecutionConfig {
/// \brief User directory.
[[nodiscard]] static auto GetUserDir() noexcept -> std::filesystem::path {
- auto& user_root = Data().user_root;
- if (user_root.empty()) {
- user_root = GetUserRoot() / ".cache" / "just";
- }
- return user_root;
+ return GetUserHome() / ".cache" / "just";
}
/// \brief Build directory, defaults to user directory if not set
- [[nodiscard]] static auto GetBuildDir() noexcept -> std::filesystem::path {
+ [[nodiscard]] static auto BuildRoot() noexcept -> std::filesystem::path {
auto& build_root = Data().build_root;
if (build_root.empty()) {
- return GetUserDir();
+ build_root = GetUserDir();
}
return build_root;
}
- /// \brief Cache directory, defaults to user directory if not set
- [[nodiscard]] static auto GetCacheDir() noexcept -> std::filesystem::path {
- auto& disk_cache = Data().disk_cache;
- if (disk_cache.empty()) {
- return GetBuildDir();
+ [[nodiscard]] static auto CacheRoot() noexcept -> std::filesystem::path {
+ auto& cache_root = Data().cache_root;
+ if (cache_root.empty()) {
+ cache_root =
+ UpdatePathForCompatibility(BuildRoot() / "protocol-dependent");
}
- return disk_cache;
+ return cache_root;
+ }
+
+ // CAS directory based on the type of the file.
+ // Specialized just for regular File or Exectuable
+ template <ObjectType kType,
+ typename = std::enable_if<kType == ObjectType::File or
+ kType == ObjectType::Executable>>
+ [[nodiscard]] static inline auto CASDir() noexcept
+ -> std::filesystem::path {
+ static const std::string kSuffix = std::string{"cas"} + ToChar(kType);
+ return CacheRoot() / kSuffix;
+ }
+
+ /// \brief Action cache directory
+ [[nodiscard]] static auto ActionCacheDir() noexcept
+ -> std::filesystem::path {
+ return CacheRoot() / "ac";
}
[[nodiscard]] static auto GetLauncher() noexcept
@@ -124,7 +139,7 @@ class LocalExecutionConfig {
}
/// \brief Determine user root directory
- [[nodiscard]] static auto GetUserRoot() noexcept -> std::filesystem::path {
+ [[nodiscard]] static auto GetUserHome() noexcept -> std::filesystem::path {
char const* root{nullptr};
#ifdef __unix__
@@ -135,7 +150,8 @@ class LocalExecutionConfig {
#endif
if (root == nullptr) {
- Logger::Log(LogLevel::Error, "Cannot determine user directory.");
+ Logger::Log(LogLevel::Error,
+ "Cannot determine user home directory.");
std::exit(EXIT_FAILURE);
}
diff --git a/src/buildtool/execution_api/local/local_ac.hpp b/src/buildtool/execution_api/local/local_ac.hpp
index 0378a74d..08f384d8 100644
--- a/src/buildtool/execution_api/local/local_ac.hpp
+++ b/src/buildtool/execution_api/local/local_ac.hpp
@@ -15,10 +15,6 @@ class LocalAC {
explicit LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas) noexcept
: cas_{std::move(cas)} {};
- LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas,
- std::filesystem::path cache_root) noexcept
- : cas_{std::move(cas)}, cache_root_{std::move(cache_root)} {}
-
LocalAC(LocalAC const&) = delete;
LocalAC(LocalAC&&) = delete;
auto operator=(LocalAC const&) -> LocalAC& = delete;
@@ -74,10 +70,8 @@ class LocalAC {
Logger logger_{"LocalAC"};
gsl::not_null<LocalCAS<ObjectType::File>*> cas_;
- std::filesystem::path const cache_root_{
- LocalExecutionConfig::GetCacheDir()};
FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false>
- file_store_{cache_root_ / "ac"};
+ file_store_{LocalExecutionConfig::ActionCacheDir()};
};
#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP
diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp
index d8ec8be3..93a00c0d 100644
--- a/src/buildtool/execution_api/local/local_action.cpp
+++ b/src/buildtool/execution_api/local/local_action.cpp
@@ -80,7 +80,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept
auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept
-> std::optional<Output> {
- auto exec_path = CreateUniquePath(LocalExecutionConfig::GetBuildDir() /
+ auto exec_path = CreateUniquePath(LocalExecutionConfig::CacheRoot() /
"exec_root" / action_id.hash());
if (not exec_path) {
diff --git a/src/buildtool/execution_api/local/local_cas.hpp b/src/buildtool/execution_api/local/local_cas.hpp
index 1b76b72f..ea10b638 100644
--- a/src/buildtool/execution_api/local/local_cas.hpp
+++ b/src/buildtool/execution_api/local/local_cas.hpp
@@ -17,9 +17,6 @@ class LocalCAS {
public:
LocalCAS() noexcept = default;
- explicit LocalCAS(std::filesystem::path cache_root) noexcept
- : cache_root_{std::move(cache_root)} {}
-
LocalCAS(LocalCAS const&) = delete;
LocalCAS(LocalCAS&&) = delete;
auto operator=(LocalCAS const&) -> LocalCAS& = delete;
@@ -48,12 +45,10 @@ class LocalCAS {
}
private:
- static constexpr char kSuffix = ToChar(kType);
- Logger logger_{std::string{"LocalCAS"} + kSuffix};
- std::filesystem::path const cache_root_{
- LocalExecutionConfig::GetCacheDir()};
+ Logger logger_{std::string{"LocalCAS"} + ToChar(kType)};
+
FileStorage<kType, StoreMode::FirstWins, /*kSetEpochTime=*/true>
- file_store_{cache_root_ / (std::string{"cas"} + kSuffix)};
+ file_store_{LocalExecutionConfig::CASDir<kType>()};
[[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept
-> std::optional<bazel_re::Digest> {
diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp
index 568d412d..4560a859 100644
--- a/src/buildtool/execution_api/local/local_storage.hpp
+++ b/src/buildtool/execution_api/local/local_storage.hpp
@@ -16,14 +16,6 @@ class LocalStorage {
std::shared_ptr<LocalTreeMap> tree_map = nullptr) noexcept
: tree_map_{std::move(tree_map)} {}
- explicit LocalStorage(
- std::filesystem::path const& cache_root,
- std::shared_ptr<LocalTreeMap> tree_map = nullptr) noexcept
- : cas_file_{cache_root},
- cas_exec_{cache_root},
- ac_{&cas_file_, cache_root},
- tree_map_{std::move(tree_map)} {}
-
/// \brief Store blob from file path with x-bit determined from file system.
template <bool kOwner = false>
[[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path)
diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS
index 6354665a..fa1e3e9f 100644
--- a/src/buildtool/execution_engine/executor/TARGETS
+++ b/src/buildtool/execution_engine/executor/TARGETS
@@ -6,6 +6,7 @@
[ ["src/buildtool/logging", "logging"]
, ["src/buildtool/common", "config"]
, ["src/buildtool/common", "tree"]
+ , ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/execution_engine/dag", "dag"]
, ["src/buildtool/execution_api/common", "common"]
diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp
index 9caa7bee..c08a6a2c 100644
--- a/src/buildtool/execution_engine/executor/executor.hpp
+++ b/src/buildtool/execution_engine/executor/executor.hpp
@@ -14,6 +14,7 @@
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/common/tree.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/execution_engine/dag/dag.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
@@ -112,10 +113,10 @@ class ExecutorImpl {
// is available to the execution API
if (object_info_opt) {
if (not api->IsAvailable(object_info_opt->digest) and
- not UploadGitBlob(api,
- artifact->Content().Repository(),
- object_info_opt->digest,
- /*skip_check=*/true)) {
+ not UploadKnownArtifact(api,
+ artifact->Content().Repository(),
+ object_info_opt->digest,
+ /*skip_check=*/true)) {
Logger::Log(
LogLevel::Error,
"artifact {} should be present in CAS but is missing.",
@@ -149,27 +150,52 @@ class ExecutorImpl {
/// \param repo The global repository name, the artifact belongs to
/// \param digest The digest of the object
/// \param skip_check Skip check for existence before upload
+ /// \param hash The git-sha1 hash of the object
/// \returns true on success
[[nodiscard]] static auto UploadGitBlob(
gsl::not_null<IExecutionApi*> const& api,
std::string const& repo,
ArtifactDigest const& digest,
- bool skip_check) noexcept -> bool {
+ bool skip_check,
+ std::string const& hash) noexcept -> bool {
auto const& repo_config = RepositoryConfig::Instance();
std::optional<std::string> blob{};
if (auto const* ws_root = repo_config.WorkspaceRoot(repo)) {
// try to obtain blob from local workspace's Git CAS, if any
- blob = ws_root->ReadBlob(digest.hash());
+ blob = ws_root->ReadBlob(hash);
}
if (not blob) {
// try to obtain blob from global Git CAS, if any
- blob = repo_config.ReadBlobFromGitCAS(digest.hash());
+ blob = repo_config.ReadBlobFromGitCAS(hash);
}
return blob and
api->Upload(BlobContainer{{BazelBlob{digest, std::move(*blob)}}},
skip_check);
}
+ /// \brief Lookup blob via digest in local git repositories and upload.
+ /// \param api The endpoint used for uploading
+ /// \param repo The global repository name, the artifact belongs to
+ /// \param digest The digest of the object
+ /// \param skip_check Skip check for existence before upload
+ /// \returns true on success
+ [[nodiscard]] static auto UploadKnownArtifact(
+ gsl::not_null<IExecutionApi*> const& api,
+ std::string const& repo,
+ ArtifactDigest const& digest,
+ bool skip_check) noexcept -> bool {
+ if (Compatibility::IsCompatible()) {
+ auto opt = Compatibility::GetGitEntry(digest.hash());
+ if (opt) {
+ auto const& [git_sha1_hash, comp_repo] = *opt;
+ return UploadGitBlob(
+ api, comp_repo, digest, skip_check, git_sha1_hash);
+ }
+ return false;
+ }
+ return UploadGitBlob(api, repo, digest, skip_check, digest.hash());
+ }
+
/// \brief Lookup file via path in local workspace root and upload.
/// \param api The endpoint used for uploading
/// \param repo The global repository name, the artifact belongs to
diff --git a/src/buildtool/file_system/TARGETS b/src/buildtool/file_system/TARGETS
index 929c372f..32dd96db 100644
--- a/src/buildtool/file_system/TARGETS
+++ b/src/buildtool/file_system/TARGETS
@@ -62,6 +62,7 @@
[ "git_tree"
, "file_system_manager"
, ["src/buildtool/common", "artifact_description"]
+ , ["src/buildtool/compatibility", "compatibility"]
, ["@", "gsl-lite", "", "gsl-lite"]
]
, "stage": ["src", "buildtool", "file_system"]
diff --git a/src/buildtool/file_system/file_root.hpp b/src/buildtool/file_system/file_root.hpp
index 58f521cd..c374e937 100644
--- a/src/buildtool/file_system/file_root.hpp
+++ b/src/buildtool/file_system/file_root.hpp
@@ -12,6 +12,7 @@
#include "gsl-lite/gsl-lite.hpp"
#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/file_system/git_tree.hpp"
#include "src/utils/cpp/concepts.hpp"
@@ -387,6 +388,13 @@ class FileRoot {
std::get<git_root_t>(root_).tree->LookupEntryByPath(
file_path)) {
if (entry->IsBlob()) {
+ if (Compatibility::IsCompatible()) {
+ auto compatible_hash = Compatibility::RegisterGitEntry(
+ entry->Hash(), *entry->Blob(), repository);
+ return ArtifactDescription{
+ ArtifactDigest{compatible_hash, *entry->Size()},
+ entry->Type()};
+ }
return ArtifactDescription{
ArtifactDigest{entry->Hash(), *entry->Size()},
entry->Type(),
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index 3151d9ae..d8d1f8f9 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -7,6 +7,7 @@
, "deps":
[ ["src/buildtool/common", "cli"]
, ["src/buildtool/common", "config"]
+ , ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/graph_traverser", "graph_traverser"]
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/progress_reporting", "base_progress_reporter"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index a608fb07..7cb26e45 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -18,6 +18,7 @@
#include "src/buildtool/common/artifact_description.hpp"
#include "src/buildtool/common/cli.hpp"
#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#ifndef BOOTSTRAP_BUILD_TOOL
#include "src/buildtool/graph_traverser/graph_traverser.hpp"
#include "src/buildtool/progress_reporting/base_progress_reporter.hpp"
@@ -84,6 +85,7 @@ auto SetupBuildCommandArguments(
SetupAnalysisArguments(app, &clargs->analysis);
SetupEndpointArguments(app, &clargs->endpoint);
SetupBuildArguments(app, &clargs->build);
+ SetupCompatibilityArguments(app);
}
/// \brief Setup arguments for sub command "just install".
@@ -106,6 +108,7 @@ auto SetupRebuildCommandArguments(
auto SetupInstallCasCommandArguments(
gsl::not_null<CLI::App*> const& app,
gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCompatibilityArguments(app);
SetupEndpointArguments(app, &clargs->endpoint);
SetupFetchArguments(app, &clargs->fetch);
}
@@ -119,6 +122,7 @@ auto SetupTraverseCommandArguments(
SetupGraphArguments(app, &clargs->graph); // instead of analysis
SetupBuildArguments(app, &clargs->build);
SetupStageArguments(app, &clargs->stage);
+ SetupCompatibilityArguments(app);
}
auto ParseCommandLineArguments(int argc, char const* const* argv)
@@ -198,13 +202,22 @@ void SetupLocalExecution(EndpointArguments const& eargs,
using LocalConfig = LocalExecutionConfig;
if (not LocalConfig::SetKeepBuildDir(bargs.persistent_build_dir) or
not(not eargs.local_root or
- (LocalConfig::SetBuildRoot(*eargs.local_root) and
- LocalConfig::SetDiskCache(*eargs.local_root))) or
+ (LocalConfig::SetBuildRoot(*eargs.local_root))) or
not(not bargs.local_launcher or
LocalConfig::SetLauncher(*bargs.local_launcher))) {
Logger::Log(LogLevel::Error, "failed to configure local execution.");
}
}
+
+void SetupHashGenerator() {
+ if (Compatibility::IsCompatible()) {
+ HashGenerator::SetHashGenerator(HashGenerator::HashType::SHA256);
+ }
+ else {
+ HashGenerator::SetHashGenerator(HashGenerator::HashType::GIT);
+ }
+}
+
#endif
// returns path relative to `root`.
@@ -1185,6 +1198,7 @@ auto main(int argc, char* argv[]) -> int {
SetupLogging(arguments.common);
#ifndef BOOTSTRAP_BUILD_TOOL
+ SetupHashGenerator();
SetupLocalExecution(arguments.endpoint, arguments.build);
#endif
@@ -1224,6 +1238,14 @@ auto main(int argc, char* argv[]) -> int {
#ifndef BOOTSTRAP_BUILD_TOOL
if (arguments.cmd == SubCommand::kTraverse) {
if (arguments.graph.git_cas) {
+ if (Compatibility::IsCompatible()) {
+ Logger::Log(LogLevel::Error,
+ "Command line options {} and {} cannot be used "
+ "together.",
+ "--git_cas",
+ "--compatible");
+ std::exit(EXIT_FAILURE);
+ }
if (not RepositoryConfig::Instance().SetGitCAS(
*arguments.graph.git_cas)) {
Logger::Log(LogLevel::Warning,
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-15 15:44:17 +0000