just-lock git-tree imports: Separate checkout logic into own method

This will allow it to be run mostly asynchronously with other
checkouts in the future.

1 files changed, 86 insertions, 80 deletions

diff --git a/bin/just-lock.py b/bin/just-lock.py
index 254ef229..2157f9aa 100755
--- a/bin/just-lock.py
+++ b/bin/just-lock.py
@@ -1819,16 +1819,67 @@ def import_from_archive(core_repos: Json, imports_entry: Json,
 ##
 
 
-def git_tree_checkout(command: List[str], do_generate: bool, *,
-                      command_env: Json, subdir: Optional[str],
-                      inherit_env: List[str],
-                      fail_context: str) -> Tuple[str, Dict[str, Any], str]:
+def git_tree_checkout(imports_entry: Json) -> Optional[CheckoutInfo]:
     """Run a given command or the command generated by the given command and
     import the obtained tree to Git cache. Return the checkout location, the
     repository description stub to use for rewriting 'file'-type dependencies,
     containing any additional needed data, and the temp dir to later clean up.
     """
-    fail_context += "While checking out Git-tree:\n"
+    # Set granular logging message
+    fail_context: str = "While checking out source \"git tree\":\n"
+
+    # Get the repositories list
+    repos: List[Any] = imports_entry.get("repos", [])
+    if not isinstance(repos, list):
+        fail(fail_context +
+             "Expected field \"repos\" to be a list, but found:\n%r" %
+             (json.dumps(repos, indent=2), ))
+
+    # Check if anything is to be done
+    if not repos:
+        return None
+
+    # Parse source config fields
+    command: Optional[List[str]] = imports_entry.get("cmd", None)
+    if command is not None and not isinstance(command, list):
+        fail(fail_context +
+             "Expected field \"cmd\" to be a list, but found:\n%r" %
+             (json.dumps(command, indent=2), ))
+    command_gen: Optional[List[str]] = imports_entry.get("cmd gen", None)
+    if command_gen is not None and not isinstance(command_gen, list):
+        fail(fail_context +
+             "Expected field \"cmd gen\" to be a list, but found:\n%r" %
+             (json.dumps(command_gen, indent=2), ))
+    if command is None == command_gen is None:
+        fail(fail_context +
+             "Only one of fields \"cmd\" and \"cmd gen\" must be provided!")
+
+    subdir: Optional[str] = imports_entry.get("subdir", None)
+    if subdir is not None:
+        if not isinstance(subdir, str):
+            fail(fail_context +
+                 "Expected field \"subdir\" to be a string, but found:\n%r" %
+                 (json.dumps(subdir, indent=2), ))
+        subdir = os.path.normpath(subdir)
+        if os.path.isabs(subdir) or subdir.startswith(".."):
+            fail(
+                fail_context +
+                "Expected field \"subdir\" to be a relative non-upward path, but found:\n%r"
+                % (json.dumps(subdir, indent=2), ))
+        if subdir == ".":
+            subdir = None  # treat as if missing
+
+    command_env: Json = imports_entry.get("env", {})
+    if not isinstance(command_env, dict):
+        fail(fail_context +
+             "Expected field \"env\" to be a map, but found:\n%r" %
+             (json.dumps(command_env, indent=2), ))
+
+    inherit_env: List[str] = imports_entry.get("inherit env", [])
+    if not isinstance(inherit_env, list):
+        fail(fail_context +
+             "Expected field \"inherit env\" to be a list, but found:\n%r" %
+             (json.dumps(inherit_env, indent=2), ))
 
     # Set the command environment
     curr_env = os.environ.copy()
@@ -1836,25 +1887,33 @@ def git_tree_checkout(command: List[str], do_generate: bool, *,
     for envar in inherit_env:
         if envar in curr_env:
             new_envs[envar] = curr_env[envar]
-    cmd_env = dict(command_env, **new_envs)
+    command_env = dict(command_env, **new_envs)
 
     # Generate the command to be run, if needed
-    if do_generate:
+    report("\tGenerating Git-tree content")
+    if command_gen is not None:
         tmpdir: str = create_tmp_dir(
             type="cmd-gen")  # to avoid polluting the current dir
-        data, _ = run_cmd(g_LAUNCHER + command,
+        data, _ = run_cmd(g_LAUNCHER + command_gen,
                           cwd=tmpdir,
-                          env=cmd_env,
+                          env=command_env,
                           stdout=subprocess.PIPE,
                           fail_context=fail_context)
-        command = json.loads(data)
+        tmp_cmd = json.loads(data)
+        if not isinstance(tmp_cmd, list):
+            fail(
+                fail_context +
+                "Generative command should have produced a list, but found:\n%r"
+                % (data, ))
+        command = tmp_cmd
         try_rmtree(tmpdir)
 
     # Generate the sources tree content; here we use the environment provided
+    command = cast(List[str], command)
     workdir: str = create_tmp_dir(type="git-tree-checkout")
     run_cmd(g_LAUNCHER + command,
             cwd=workdir,
-            env=cmd_env,
+            env=command_env,
             fail_context=fail_context)
 
     # Import root tree to Git cache; as we do not have the tree hash, identify
@@ -1881,68 +1940,21 @@ def git_tree_checkout(command: List[str], do_generate: bool, *,
     # computing the subtree identifier
     if subdir is not None:
         repo_stub = dict(repo_stub, **{"subdir": subdir})
-    return srcdir, repo_stub, workdir
+
+    return CheckoutInfo(srcdir, repo_stub, workdir)
 
 
-def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json:
+def import_from_git_tree(core_repos: Json, imports_entry: Json,
+                         checkout_info: CheckoutInfo) -> Json:
     """Handles imports from general Git trees obtained by running a command
     (explicitly given or generated by a given command)."""
     # Set granular logging message
     fail_context: str = "While importing from source \"git tree\":\n"
 
-    # Get the repositories list
-    repos: List[Any] = imports_entry.get("repos", [])
-    if not isinstance(repos, list):
-        fail(fail_context +
-             "Expected field \"repos\" to be a list, but found:\n%r" %
-             (json.dumps(repos, indent=2), ))
-
-    # Check if anything is to be done
-    if not repos:  # empty
-        return core_repos
-
-    # Parse source config fields
-    command: Optional[List[str]] = imports_entry.get("cmd", None)
-    if command is not None and not isinstance(command, list):
-        fail(fail_context +
-             "Expected field \"cmd\" to be a list, but found:\n%r" %
-             (json.dumps(command, indent=2), ))
-    command_gen: Optional[List[str]] = imports_entry.get("cmd gen", None)
-    if command_gen is not None and not isinstance(command_gen, list):
-        fail(fail_context +
-             "Expected field \"cmd gen\" to be a list, but found:\n%r" %
-             (json.dumps(command_gen, indent=2), ))
-    if command is None == command_gen is None:
-        fail(fail_context +
-             "Only one of fields \"cmd\" and \"cmd gen\" must be provided!")
-
-    subdir: Optional[str] = imports_entry.get("subdir", None)
-    if subdir is not None:
-        if not isinstance(subdir, str):
-            fail(fail_context +
-                 "Expected field \"subdir\" to be a string, but found:\n%r" %
-                 (json.dumps(subdir, indent=2), ))
-        subdir = os.path.normpath(subdir)
-        if os.path.isabs(subdir) or subdir.startswith(".."):
-            fail(
-                fail_context +
-                "Expected field \"subdir\" to be a relative non-upward path, but found:\n%r"
-                % (json.dumps(subdir, indent=2), ))
-        if subdir == ".":
-            subdir = None  # treat as if missing
-
-    command_env: Json = imports_entry.get("env", {})
-    if not isinstance(command_env, dict):
-        fail(fail_context +
-             "Expected field \"env\" to be a map, but found:\n%r" %
-             (json.dumps(command_env, indent=2), ))
-
-    inherit_env: List[str] = imports_entry.get("inherit env", [])
-    if not isinstance(inherit_env, list):
-        fail(fail_context +
-             "Expected field \"inherit env\" to be a list, but found:\n%r" %
-             (json.dumps(inherit_env, indent=2), ))
+    # Get needed known fields (validated during checkout)
+    repos: List[Any] = imports_entry["repos"]
 
+    # Parse remaining fields
     as_plain: Optional[bool] = imports_entry.get("as plain", False)
     if as_plain is not None and not isinstance(as_plain, bool):
         fail(fail_context +
@@ -1966,21 +1978,13 @@ def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json:
         # only enabled if as_plain is true
         pragma_special = None
 
-    # Fetch the Git tree
-    srcdir, remote_stub, to_clean_up = git_tree_checkout(
-        command=cast(List[str], command_gen if command is None else command),
-        do_generate=command is None,
-        command_env=command_env,
-        subdir=subdir,
-        inherit_env=inherit_env,
-        fail_context=fail_context)
-
     # Read in the foreign config file
     if foreign_config_file:
-        foreign_config_file = os.path.join(srcdir, foreign_config_file)
+        foreign_config_file = os.path.join(checkout_info.srcdir,
+                                           foreign_config_file)
     else:
         foreign_config_file = get_repository_config_file(
-            DEFAULT_JUSTMR_CONFIG_NAME, srcdir)
+            DEFAULT_JUSTMR_CONFIG_NAME, checkout_info.srcdir)
     foreign_config: Json = {}
     if as_plain:
         foreign_config = {"main": "", "repositories": DEFAULT_REPO}
@@ -2009,7 +2013,7 @@ def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json:
         repo_entry = cast(Json, repo_entry)
 
         new_repos = handle_import("git tree",
-                                  remote_stub,
+                                  checkout_info.remote_stub,
                                   repo_entry,
                                   new_repos,
                                   foreign_config,
@@ -2017,7 +2021,7 @@ def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json:
                                   fail_context=fail_context)
 
     # Clean up local fetch
-    try_rmtree(to_clean_up)
+    try_rmtree(checkout_info.to_clean_up)
     return new_repos
 
 
@@ -2902,8 +2906,10 @@ def lock_config(input_file: str) -> Json:
                 core_config["repositories"] = import_from_archive(
                     core_config["repositories"], entry, checkout_info)
         elif source == "git tree":
-            core_config["repositories"] = import_from_git_tree(
-                core_config["repositories"], entry)
+            checkout_info = git_tree_checkout(entry)
+            if checkout_info is not None:
+                core_config["repositories"] = import_from_git_tree(
+                    core_config["repositories"], entry, checkout_info)
         elif source == "generic":
             core_config = import_generic(core_config, entry)
         else: