just-mr: more safely create directories

While the just binary itself was carefully created to work correctly
in the presence of several instances running simultaneously, the
just-mr.py script started as proof-of-concept out of the need to
conveniently generate a multi-repository configuration for just.
Nevertheless, it is now actually used, so parallel running instances
have to be taken seriously. Moreover, we have to ensure that in
case of failures, we don't leave the local build root in a state
giving a false impression on how an unpacked archive looks like.
Both can be avoided by always working on temporary directories and,
should a persistent location be needed, only renaming to the final
desitantion once everything is set up correclty.

1 files changed, 32 insertions, 16 deletions

diff --git a/bin/just-mr.py b/bin/just-mr.py
index bae39f8f..48b3ae92 100755
--- a/bin/just-mr.py
+++ b/bin/just-mr.py
@@ -65,6 +65,16 @@ def try_rmtree(tree):
           time.sleep(1.0)
     fail("Failed to remove %s" % (tree,))
 
+def move_to_place(src, dst):
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    try:
+        os.rename(src, dst)
+    except Exception as e:
+        if not os.path.exists(dst):
+            fail("Failed to move %s to %s: %s" % (src, dst, e))
+    if not os.path.exists(dst):
+        fail("%s not present after move" % (dst,))
+
 def run_cmd(cmd, *, env=None, stdout=subprocess.DEVNULL, stdin=None, cwd):
     result = subprocess.run(cmd, cwd=cwd, env=env, stdout=stdout, stdin=stdin)
     if result.returncode != 0:
@@ -268,7 +278,8 @@ def archive_checkout_dir(content, repo_type):
 
 
 def archive_tmp_checkout_dir(content, repo_type):
-    return os.path.join(ROOT, "tmp-workspaces", repo_type, content)
+    return os.path.join(ROOT, "tmp-workspaces", repo_type,
+                        "%d-%s" % (os.getpid(), content))
 
 
 def archive_tree_id_file(content, repo_type):
@@ -327,19 +338,19 @@ def archive_checkout(desc, repo_type="archive", *, fetch_only=False):
     archive_fetch(desc, content=content_id)
     if fetch_only:
         return
-    if not ALWAYS_FILE:
-        target = archive_tmp_checkout_dir(content_id, repo_type=repo_type)
-    if os.path.exists(target):
-        try_rmtree(target)
-    os.makedirs(target)
+    target_tmp = archive_tmp_checkout_dir(content_id, repo_type=repo_type)
+    if os.path.exists(target_tmp):
+        try_rmtree(target_tmp)
+    os.makedirs(target_tmp)
     if repo_type == "zip":
-        run_cmd(["unzip", "-d", ".", cas_path(content_id)], cwd=target)
+        run_cmd(["unzip", "-d", ".", cas_path(content_id)], cwd=target_tmp)
     else:
-        run_cmd(["tar", "xf", cas_path(content_id)], cwd=target)
+        run_cmd(["tar", "xf", cas_path(content_id)], cwd=target_tmp)
     if ALWAYS_FILE:
+        move_to_place(target_tmp, target)
         return ["file", subdir_path(target, desc)]
-    tree = import_to_git(target, repo_type, content_id)
-    try_rmtree(target)
+    tree = import_to_git(target_tmp, repo_type, content_id)
+    try_rmtree(target_tmp)
     os.makedirs(os.path.dirname(tree_id_file), exist_ok=True)
     with open(tree_id_file, "w") as f:
         f.write(tree)
@@ -428,6 +439,9 @@ def resolve_repo(desc, *, seen=None, repos):
 def distdir_repo_dir(content):
     return os.path.join(ROOT, "distdir", content)
 
+def distdir_tmp_dir(content):
+    return os.path.join(ROOT, "tmp-workspaces", "distdir",
+                        "%d-%s" % (os.getpid(), content))
 
 def distdir_tree_id_file(content):
     return os.path.join(ROOT, "distfiles-tree-map", content)
@@ -473,20 +487,22 @@ def distdir_checkout(desc, repos):
         ]
 
     # Create the dirstdir repo folder content
-    if os.path.exists(target_distdir_dir):
-        try_rmtree(target_distdir_dir)
-    os.makedirs(target_distdir_dir)
+    target_tmp_dir = distdir_tmp_dir(distdir_content_id)
+    if os.path.exists(target_tmp_dir):
+        try_rmtree(target_tmp_dir)
+    os.makedirs(target_tmp_dir)
     for name, content_id in content.items():
-        target = os.path.join(target_distdir_dir, name)
+        target = os.path.join(target_tmp_dir, name)
         os.link(cas_path(content_id), target)
 
     if (ALWAYS_FILE):
         # Return with path to distdir repo
+        move_to_place(target_tmp_dir, target_distdir_dir)
         return ["file", target_distdir_dir]
 
     # Gitify the distdir repo folder
-    tree = import_to_git(target_distdir_dir, "distdir", distdir_content_id)
-    try_rmtree(target_distdir_dir)
+    tree = import_to_git(target_tmp_dir, "distdir", distdir_content_id)
+    try_rmtree(target_tmp_dir)
 
     # Cache git info to tree id file
     os.makedirs(os.path.dirname(tree_id_file), exist_ok=True)