just-lock clone: Fix wrong order in resolving trees

Match correctly the behavior of just-mr, i.e., resolve the special
entries of the relevant subtree of the original tree instead of
taking the subtree of the resolved original tree. This also means
that the clones of archives can directly stage the relevant subdir
of the unpacked content and forward any 'special' pragma (same as
for other cloned repository types).

2 files changed, 23 insertions, 73 deletions

diff --git a/bin/just-lock.py b/bin/just-lock.py
index 4e747fb8..0310a090 100755
--- a/bin/just-lock.py
+++ b/bin/just-lock.py
@@ -70,12 +70,6 @@ GIT_NOBODY_ENV: Dict[str, str] = {
     "GIT_CONFIG_SYSTEM": "/dev/null",
 }
 
-SPECIAL_PRAGMA_TO_CAS_RESOLVE_MAP: Dict[str, str] = {
-    "ignore": "ignore",
-    "resolve-partially": "tree-upwards",
-    "resolve-completely": "tree-all"
-}
-
 
 class ObjectType(Enum):
     FILE = 1
@@ -2051,10 +2045,9 @@ def rewrite_cloned_repo(repos: Json, *, clone_to: str, target_repo: str,
     # Keep relevant pragmas from the workspace root repository
     pragma: Json = {}
     existing: Json = repos[ws_root_repo]["repository"].get("pragma", {})
-    if ws_root_desc["repository"]["type"] not in ["archive", "zip"]:
-        special = existing.get("special", None)
-        if special:
-            pragma["special"] = special
+    special = existing.get("special", None)
+    if special:
+        pragma["special"] = special
     to_git = existing.get("to_git", False)
     if to_git:
         pragma["to_git"] = True
@@ -2188,13 +2181,8 @@ def clone_repo(repos: Json, known_repo: str, deps_chain: List[str],
         # Fetch the archive
         content = archive_fetch_with_parse(repository,
                                            fail_context=fail_context)
-        # Stage the content; first resolve special entries, if needed, then keep
-        # the relevant subdir, if given
-        special_pragma: Optional[str] = repository.get("pragma",
-                                                       {}).get("special", None)
-
-        if (special_pragma not in SPECIAL_PRAGMA_TO_CAS_RESOLVE_MAP.keys()
-                and subdir is None):
+        # Stage the content of the relevant subdir
+        if subdir is None:
             # Unpack directly to clone location
             unpack_archive(content,
                            archive_type=repo_type,
@@ -2203,43 +2191,12 @@ def clone_repo(repos: Json, known_repo: str, deps_chain: List[str],
         else:
             # Unpack to a temporary dir
             workdir: str = create_tmp_dir(type="archive-unpack")
-            srcdir: str = os.path.join(workdir, "src")
             unpack_archive(content,
                            archive_type=repo_type,
-                           unpack_to=srcdir,
+                           unpack_to=workdir,
                            fail_context=fail_context)
-
-            move_from_dir: str = srcdir
-            if (special_pragma is not None and special_pragma
-                    in SPECIAL_PRAGMA_TO_CAS_RESOLVE_MAP.keys()):
-                # Resolve the tree according to the pragma
-                resolve_special_arg = SPECIAL_PRAGMA_TO_CAS_RESOLVE_MAP[
-                    special_pragma]
-                resolved_tree = run_cmd(
-                    g_LAUNCHER + [
-                        g_JUST, "add-to-cas", "--local-build-root", g_ROOT,
-                        "--resolve-special=%s" % resolve_special_arg, srcdir
-                    ],
-                    cwd=workdir,
-                    stdout=subprocess.PIPE,
-                    fail_context=fail_context)[0].decode('utf-8').strip()
-                # Stage the resolved tree
-                resolved_dir: str = os.path.join(workdir, "resolved")
-                subdir_args: List[str] = []
-                if subdir is not None:
-                    subdir_args = ["-P", subdir]
-                run_cmd(g_LAUNCHER + [
-                    g_JUST, "install-cas", "--local-build-root", g_ROOT,
-                    "%s::t" % resolved_tree, "-o", resolved_dir
-                ] + subdir_args,
-                        cwd=workdir,
-                        fail_context=fail_context)
-                move_from_dir = resolved_dir
-            else:
-                # Subdir is not None, so keep only that subdirectory
-                move_from_dir = os.path.join(move_from_dir, cast(str, subdir))
-
-            # Do the move into clone directory
+            # Keep relevant subdir
+            move_from_dir: str = os.path.join(workdir, subdir)
             os.makedirs(clone_to, exist_ok=True)
             for entry in os.listdir(move_from_dir):
                 # shutil.move uses os.rename if on same filesystem or
@@ -2250,7 +2207,6 @@ def clone_repo(repos: Json, known_repo: str, deps_chain: List[str],
                 except Exception as ex:
                     fail(fail_context + "Moving file path %s failed with:\n%r" %
                          (os.path.join(move_from_dir, entry), ex))
-
             # Clean up tmp dir
             try_rmtree(workdir)
 
diff --git a/test/end-to-end/just-lock/clone/archive-repos.sh b/test/end-to-end/just-lock/clone/archive-repos.sh
index 2d0c314e..c2c8a5b9 100644
--- a/test/end-to-end/just-lock/clone/archive-repos.sh
+++ b/test/end-to-end/just-lock/clone/archive-repos.sh
@@ -57,7 +57,7 @@ cat > repos.in.json <<EOF
       , "content": "${ZIP_REPO_CONTENT}"
       , "fetch": "http://non-existent.example.org/zip_repo.zip"
       , "subdir": "root"
-      , "pragma": {"special": "resolve-partially"}
+      , "pragma": {"special": "resolve-completely"}
       }
     }
   , "tgz_repo":
@@ -118,7 +118,9 @@ grep "${CLONE_TO}/zip" repos.json
 grep "${CLONE_TO}/tgz" repos.json
 grep "${CLONE_TO}/foreign" repos.json
 grep "${CLONE_TO}/distdir" repos.json
-[ ! $(grep pragma repos.json) ]  # special pragmas should have been resolved
+# check special pragmas are kept
+grep resolve-completely repos.json
+grep ignore repos.json
 echo
 
 # Check setup with local clones:
@@ -127,24 +129,16 @@ echo
 echo
 
 # Check that the clones have the expected content
-"${JUST}" install-cas --local-build-root "${LBR_ARCHIVES}" \
-    "$(jq '."repositories"."zip_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"')::t" \
-    -P root -o "${INSTALL_ZIP}"
-diff -ruN "${INSTALL_ZIP}" "${CLONE_TO}/zip"
-
-"${JUST}" install-cas --local-build-root "${LBR_ARCHIVES}" \
-    "$(jq '."repositories"."tgz_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"')::t" \
-    -P root/baz -o "${INSTALL_TGZ}"
-diff -ruN "${INSTALL_TGZ}" "${CLONE_TO}/tgz"
-
-"${JUST}" install-cas --local-build-root "${LBR_ARCHIVES}" \
-    "$(jq '."repositories"."foreign_file"."workspace_root" | .[1]' "${CONF}" | tr -d '"')::t" \
-    -o "${INSTALL_FOREIGN}"
-diff -ruN "${INSTALL_FOREIGN}" "${CLONE_TO}/foreign"
-
-"${JUST}" install-cas --local-build-root "${LBR_ARCHIVES}" \
-    "$(jq '."repositories"."distdir_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"')::t" \
-    -o "${INSTALL_DISTDIR}"
-diff -ruN "${INSTALL_DISTDIR}" "${CLONE_TO}/distdir"
+[ $(jq '."repositories"."zip_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \
+    = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" --resolve-special=tree-all "${CLONE_TO}/zip") ]
+
+[ $(jq '."repositories"."tgz_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \
+    = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" --resolve-special=ignore "${CLONE_TO}/tgz") ]
+
+[ $(jq '."repositories"."foreign_file"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \
+    = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" "${CLONE_TO}/foreign") ]
+
+[ $(jq '."repositories"."distdir_repo"."workspace_root" | .[1]' "${CONF}" | tr -d '"') \
+    = $("${JUST}" add-to-cas --local-build-root "${LBR_CLONES}" "${CLONE_TO}/distdir") ]
 
 echo "OK"