diff options
| -rwxr-xr-x | bin/just-mr.py | 7 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/file_storage.hpp | 20 |
2 files changed, 20 insertions, 7 deletions
diff --git a/bin/just-mr.py b/bin/just-mr.py index d48f18a0..634842ef 100755 --- a/bin/just-mr.py +++ b/bin/just-mr.py @@ -268,8 +268,9 @@ def git_hash(content): def add_to_cas(data): if isinstance(data, str): data = data.encode('utf-8') - cas_root = os.path.join(ROOT, "protocol-dependent/generation-0/git-sha1/casf") - basename = git_hash(data) + h = git_hash(data) + cas_root = os.path.join(ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}") + basename = h[2:] target = os.path.join(cas_root, basename) tempname = os.path.join(cas_root, "%s.%d" % (basename, os.getpid())) @@ -288,7 +289,7 @@ def add_to_cas(data): def cas_path(h): - return os.path.join(ROOT, "protocol-dependent/generation-0/git-sha1/casf", h) + return os.path.join(ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}", h[2:]) def is_in_cas(h): diff --git a/src/buildtool/execution_api/local/file_storage.hpp b/src/buildtool/execution_api/local/file_storage.hpp index 601dfb51..e562fdc8 100644 --- a/src/buildtool/execution_api/local/file_storage.hpp +++ b/src/buildtool/execution_api/local/file_storage.hpp @@ -54,9 +54,9 @@ class FileStorage { return AtomicAddFromBytes(id, bytes); } - [[nodiscard]] auto GetPath(std::string const& name) const noexcept + [[nodiscard]] auto GetPath(std::string const& id) const noexcept -> std::filesystem::path { - return storage_root_ / name; + return GetShardedPath(id); } private: @@ -72,7 +72,7 @@ class FileStorage { [[nodiscard]] auto AtomicAddFromFile(std::string const& id, std::filesystem::path const& path, bool is_owner) const noexcept -> bool { - auto file_path = storage_root_ / id; + auto file_path = GetShardedPath(id); if ((kMode == StoreMode::LastWins or not FileSystemManager::Exists(file_path)) and FileSystemManager::CreateDirectory(file_path.parent_path())) { @@ -115,7 +115,7 @@ class FileStorage { [[nodiscard]] auto AtomicAddFromBytes( std::string const& id, std::string const& bytes) const noexcept -> bool { - auto file_path = storage_root_ / id; + auto file_path = GetShardedPath(id); if (kMode == StoreMode::LastWins or not FileSystemManager::Exists(file_path)) { auto unique_path = CreateUniquePath(file_path); @@ -131,6 +131,18 @@ class FileStorage { return FileSystemManager::IsFile(file_path); } + /// \brief Determines the storage path of a blob identified by a hash value. + /// The same sharding technique as git is used, meaning, the hash value is + /// separated into a directory part and file part. Two characters are used + /// for the directory part, the rest for the file, which results in 256 + /// possible directories. + /// \param id The hash value. + /// \returns The sharded file path. + [[nodiscard]] auto GetShardedPath(std::string const& id) const noexcept + -> std::filesystem::path { + return storage_root_ / id.substr(0, 2) / id.substr(2, id.size() - 2); + } + /// \brief Create file from file path. [[nodiscard]] static auto CreateFileFromPath( std::filesystem::path const& file_path, |