Add script to import a git repos as dependency

... assuming this repo already uses just and has a multi-repository
configuration committed. In the import, transitive dependencies, as
well as repositories serving as layers, are taken into account, and
"file" repositories are rewritten to be subdirs of the repository
imported. The imported repositories are renamed to reflect the
repository pulling them in, extending the name appropriately to
avoid conflicts. This renaming is reflected in the bindings and
layer references of the imported repositories as well.

In this simple version, no automatic deduplication of imported
repositories to already existing repositories is made, but the
user can specify that certain foreign repositories should not be
imported and mapped to already present repositories.

1 files changed, 315 insertions, 0 deletions

diff --git a/bin/just-import-git.py b/bin/just-import-git.py
new file mode 100755
index 00000000..000d2f33
--- /dev/null
+++ b/bin/just-import-git.py
@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import subprocess
+import shutil
+import sys
+import tempfile
+
+from argparse import ArgumentParser
+from pathlib import Path
+
+
+def log(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def fail(s, exit_code=1):
+    log(f"Error: {s}")
+    sys.exit(exit_code)
+
+MARKERS = [".git", "ROOT", "WORKSPACE"]
+SYSTEM_ROOT = os.path.abspath(os.sep)
+DEFAULT_CONFIG_LOCATIONS = [{
+    "root": "workspace",
+    "path": "repos.json"
+}, {
+    "root": "workspace",
+    "path": "etc/repos.json"
+}, {
+    "root": "home",
+    "path": ".just-repos.json"
+}, {
+    "root": "system",
+    "path": "etc/just-repos.json"
+}]
+
+def run_cmd(cmd,
+            *,
+            env=None,
+            stdout=subprocess.DEVNULL,
+            stdin=None,
+            cwd):
+    result = subprocess.run(cmd,
+                            cwd=cwd,
+                            env=env,
+                            stdout=stdout,
+                            stdin=stdin)
+    if result.returncode != 0:
+        fail("Command %s in %s failed" % (cmd, cwd))
+    return result.stdout
+
+def find_workspace_root(path=None):
+    def is_workspace_root(path):
+        for m in MARKERS:
+            if os.path.exists(os.path.join(path, m)):
+                return True
+        return False
+
+    if not path:
+        path = os.getcwd()
+    while True:
+        if is_workspace_root(path):
+            return path
+        if path == SYSTEM_ROOT:
+            return None
+        path = os.path.dirname(path)
+
+
+def read_location(location, root=None):
+    search_root = location.get("root", None)
+    search_path = location.get("path", None)
+
+    fs_root = None
+    if search_root == "workspace":
+        if root:
+            fs_root = root
+        else:
+            fs_root = find_workspace_root()
+    if not root:
+        if search_root == "home":
+            fs_root = Path.home()
+        if search_root == "system":
+            fs_root = SYSTEM_ROOT
+
+    if fs_root:
+        return os.path.realpath(os.path.join(fs_root, search_path))
+    return "/" # certainly not a file
+
+def get_repository_config_file(root=None):
+    for location in DEFAULT_CONFIG_LOCATIONS:
+        path = read_location(location, root=root)
+        if path and os.path.isfile(path):
+            return path
+
+
+def get_base_config(repository_config):
+    if not repository_config:
+        repository_config = get_repository_config_file()
+    with open(repository_config) as f:
+        return json.load(f)
+
+def clone(url, branch):
+    # clone the given git repository, checkout the specified
+    # branch, and return the checkout location
+    workdir = tempfile.mkdtemp()
+    run_cmd(["git", "clone", url, "src"],
+            cwd=workdir)
+    srcdir = os.path.join(workdir, "src")
+    run_cmd(["git", "checkout", branch],
+            cwd = srcdir)
+    commit = run_cmd(["git", "log", "-n", "1", "--pretty=%H"],
+                     cwd = srcdir, stdout=subprocess.PIPE).decode('utf-8').strip()
+    log("Importing commit %s" % (commit,))
+    repo = { "type": "git",
+             "repository": url,
+             "branch": branch,
+             "commit": commit,
+            }
+    return srcdir, repo, workdir
+
+def get_repo_to_import(config):
+    """From a given repository config, take the main repository."""
+    if config.get("main") is not None:
+        return config.get("main")
+    repos = config.get("repositories", {}).keys()
+    if repos:
+        return sorted(repos)[0]
+    fail("Config does not contain any repositories; unsure what to import")
+
+def repos_to_import(repos_config, entry, known=None):
+    """Compute the set of transitively reachable repositories"""
+    visited = set()
+    to_import = []
+    if known:
+        visited = set(known)
+
+    def visit(name):
+        if name in visited:
+            return
+        to_import.append(name)
+        visited.add(name)
+        for n in repos_config.get(name, {}).get("bindings", {}).values():
+            visit(n)
+
+    visit(entry)
+    return to_import
+
+def extra_layers_to_import(repos_config, repos):
+    """Compute the collection of repositories additionally needed as they serve
+    as layers for the repositories to import."""
+    extra_imports = set()
+    for repo in repos:
+        for layer in ["target_root", "rule_root", "expression_root"]:
+            if layer in repos_config[repo]:
+                extra = repos_config[repo][layer]
+                if (extra not in repos) and (extra not in extra_imports):
+                    extra_imports.add(extra)
+    return list(extra_imports)
+
+def name_imports(to_import, existing, base_name, main=None):
+    """Assign names to the repositories to import in such a way that
+    no conflicts arise."""
+    assign = {}
+
+    def find_name(name):
+        base = "%s/%s" % (base_name, name)
+        if (base not in existing) and (base not in assign):
+            return base
+        count = 0
+        while True:
+            count += 1
+            candidate = base + " (%d)" % count
+            if (candidate not in existing) and (candidate not in assign):
+                return candidate
+
+    if main and (base_name not in existing):
+        assign[main] = base_name
+        to_import = [x for x in to_import if x != main]
+    for repo in to_import:
+        assign[repo] = find_name(repo)
+    return assign
+
+def rewrite_repo(repo_spec, *, remote, assign):
+    new_spec = {}
+    repo = repo_spec.get("repository", {})
+    if isinstance(repo, str):
+        repo = assign[repo]
+    elif repo.get("type") == "file":
+        changes = {}
+        subdir = repo.get("path", ".")
+        if subdir not in ["", "."]:
+            changes["subdir"] = subdir
+        repo = dict(remote, **changes)
+    new_spec["repository"] = repo
+    for key in ["target_root", "rule_root", "expression_root"]:
+        if key in repo_spec:
+            new_spec[key] = assign[repo_spec[key]]
+    for key in ["target_file_name", "rule_file_name", "expression_file_name"]:
+        if key in repo_spec:
+            new_spec[key] = repo_spec[key]
+    bindings = repo_spec.get("bindings", {})
+    new_bindings = {}
+    for k, v in bindings.items():
+        new_bindings[k] = assign[v]
+    if new_bindings:
+        new_spec["bindings"] = new_bindings
+    return new_spec
+
+def handle_import(args):
+    base_config = get_base_config(args.repository_config)
+    base_repos = base_config.get("repositories", {})
+    srcdir, remote, to_cleanup = clone(args.URL, args.branch)
+    if args.foreign_repository_config:
+        foreign_config_file = args.foreign_repository_config
+    else:
+        foreign_config_file = get_repository_config_file(srcdir)
+    with open(foreign_config_file) as f:
+        foreign_config = json.load(f)
+    foreign_repos = foreign_config.get("repositories", {})
+    if args.foreign_repository_name:
+        foreign_name = args.foreign_repository_name
+    else:
+        foreign_name = get_repo_to_import(foreign_config)
+    import_map = {}
+    for theirs, ours in args.import_map:
+        import_map[theirs] = ours
+    main_repos = repos_to_import(foreign_repos, foreign_name, import_map.keys())
+    extra_repos = sorted([x for x in main_repos if x != foreign_name])
+    extra_imports = sorted(extra_layers_to_import(foreign_repos, main_repos))
+    ordered_imports = [foreign_name] + extra_repos + extra_imports
+    import_name = foreign_name
+    if args.import_as is not None:
+        import_name = args.import_as
+    assign = name_imports(
+        ordered_imports,
+        set(base_repos.keys()),
+        import_name,
+        main = foreign_name,
+    )
+    log("Importing %r as %r" % (foreign_name, import_name))
+    log("Transitive dependencies to import: %r" % (extra_repos,))
+    log("Repositories imported as layers: %r" % (extra_imports,))
+    total_assign = dict(assign, **import_map)
+    for repo in ordered_imports:
+        base_repos[assign[repo]] = rewrite_repo(
+            foreign_repos[repo],
+            remote = remote,
+            assign = total_assign,
+        )
+    base_config["repositories"] = base_repos
+    shutil.rmtree(to_cleanup)
+    return base_config
+
+def main():
+    parser = ArgumentParser(
+        prog = "just-import-deps",
+        description =
+        "Import a dependency transitively into a given"
+        + " multi-repository configuration"
+    )
+    parser.add_argument(
+        "-C",
+        dest="repository_config",
+        help="Repository-description file to import into",
+        metavar="FILE"
+    )
+    parser.add_argument(
+        "-b",
+        dest="branch",
+        help="The branch of the remote repository to import and follow",
+        metavar="branch",
+        default="master"
+    )
+    parser.add_argument(
+        "-R",
+        dest="foreign_repository_config",
+        help="Repository-description file in the repository to import",
+        metavar="relative-path"
+    )
+    parser.add_argument(
+        "--as",
+        dest="import_as",
+        help="Name prefix to import the foreign repository as",
+        metavar="NAME",
+    )
+    parser.add_argument(
+        "--map",
+        nargs=2,
+        dest="import_map",
+        help="Map the specified foreign repository to the specified existing repository",
+        action="append",
+        default=[]
+    )
+    parser.add_argument('URL')
+    parser.add_argument('foreign_repository_name', nargs='?')
+    args = parser.parse_args()
+    new_config = handle_import(args)
+    print(json.dumps(new_config))
+
+
+if __name__ == "__main__":
+    main()