From d43c11f551d5c72a099aae9e1bea66178419f17a Mon Sep 17 00:00:00 2001 From: Sascha Roloff Date: Wed, 19 Oct 2022 15:25:55 +0200 Subject: Apply sharding as used in git to files stored in the just cache directory. This commit also introduces an incompatible change, since it modifies the way how files are stored in the just cache directory. This modification reduces the number of files per directory and only introduces a maximum number new directories to avoid possible performance bottlenecks. --- bin/just-mr.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'bin/just-mr.py') diff --git a/bin/just-mr.py b/bin/just-mr.py index d48f18a0..634842ef 100755 --- a/bin/just-mr.py +++ b/bin/just-mr.py @@ -268,8 +268,9 @@ def git_hash(content): def add_to_cas(data): if isinstance(data, str): data = data.encode('utf-8') - cas_root = os.path.join(ROOT, "protocol-dependent/generation-0/git-sha1/casf") - basename = git_hash(data) + h = git_hash(data) + cas_root = os.path.join(ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}") + basename = h[2:] target = os.path.join(cas_root, basename) tempname = os.path.join(cas_root, "%s.%d" % (basename, os.getpid())) @@ -288,7 +289,7 @@ def add_to_cas(data): def cas_path(h): - return os.path.join(ROOT, "protocol-dependent/generation-0/git-sha1/casf", h) + return os.path.join(ROOT, f"protocol-dependent/generation-0/git-sha1/casf/{h[0:2]}", h[2:]) def is_in_cas(h): -- cgit v1.2.3