Add script to import a git repos as dependency

... assuming this repo already uses just and has a multi-repository
configuration committed. In the import, transitive dependencies, as
well as repositories serving as layers, are taken into account, and
"file" repositories are rewritten to be subdirs of the repository
imported. The imported repositories are renamed to reflect the
repository pulling them in, extending the name appropriately to
avoid conflicts. This renaming is reflected in the bindings and
layer references of the imported repositories as well.

In this simple version, no automatic deduplication of imported
repositories to already existing repositories is made, but the
user can specify that certain foreign repositories should not be
imported and mapped to already present repositories.

2 files changed, 394 insertions, 0 deletions

diff --git a/bin/just-import-git.py b/bin/just-import-git.py
new file mode 100755
index 00000000..000d2f33
--- /dev/null
+++ b/bin/just-import-git.py
@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import subprocess
+import shutil
+import sys
+import tempfile
+
+from argparse import ArgumentParser
+from pathlib import Path
+
+
+def log(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def fail(s, exit_code=1):
+    log(f"Error: {s}")
+    sys.exit(exit_code)
+
+MARKERS = [".git", "ROOT", "WORKSPACE"]
+SYSTEM_ROOT = os.path.abspath(os.sep)
+DEFAULT_CONFIG_LOCATIONS = [{
+    "root": "workspace",
+    "path": "repos.json"
+}, {
+    "root": "workspace",
+    "path": "etc/repos.json"
+}, {
+    "root": "home",
+    "path": ".just-repos.json"
+}, {
+    "root": "system",
+    "path": "etc/just-repos.json"
+}]
+
+def run_cmd(cmd,
+            *,
+            env=None,
+            stdout=subprocess.DEVNULL,
+            stdin=None,
+            cwd):
+    result = subprocess.run(cmd,
+                            cwd=cwd,
+                            env=env,
+                            stdout=stdout,
+                            stdin=stdin)
+    if result.returncode != 0:
+        fail("Command %s in %s failed" % (cmd, cwd))
+    return result.stdout
+
+def find_workspace_root(path=None):
+    def is_workspace_root(path):
+        for m in MARKERS:
+            if os.path.exists(os.path.join(path, m)):
+                return True
+        return False
+
+    if not path:
+        path = os.getcwd()
+    while True:
+        if is_workspace_root(path):
+            return path
+        if path == SYSTEM_ROOT:
+            return None
+        path = os.path.dirname(path)
+
+
+def read_location(location, root=None):
+    search_root = location.get("root", None)
+    search_path = location.get("path", None)
+
+    fs_root = None
+    if search_root == "workspace":
+        if root:
+            fs_root = root
+        else:
+            fs_root = find_workspace_root()
+    if not root:
+        if search_root == "home":
+            fs_root = Path.home()
+        if search_root == "system":
+            fs_root = SYSTEM_ROOT
+
+    if fs_root:
+        return os.path.realpath(os.path.join(fs_root, search_path))
+    return "/" # certainly not a file
+
+def get_repository_config_file(root=None):
+    for location in DEFAULT_CONFIG_LOCATIONS:
+        path = read_location(location, root=root)
+        if path and os.path.isfile(path):
+            return path
+
+
+def get_base_config(repository_config):
+    if not repository_config:
+        repository_config = get_repository_config_file()
+    with open(repository_config) as f:
+        return json.load(f)
+
+def clone(url, branch):
+    # clone the given git repository, checkout the specified
+    # branch, and return the checkout location
+    workdir = tempfile.mkdtemp()
+    run_cmd(["git", "clone", url, "src"],
+            cwd=workdir)
+    srcdir = os.path.join(workdir, "src")
+    run_cmd(["git", "checkout", branch],
+            cwd = srcdir)
+    commit = run_cmd(["git", "log", "-n", "1", "--pretty=%H"],
+                     cwd = srcdir, stdout=subprocess.PIPE).decode('utf-8').strip()
+    log("Importing commit %s" % (commit,))
+    repo = { "type": "git",
+             "repository": url,
+             "branch": branch,
+             "commit": commit,
+            }
+    return srcdir, repo, workdir
+
+def get_repo_to_import(config):
+    """From a given repository config, take the main repository."""
+    if config.get("main") is not None:
+        return config.get("main")
+    repos = config.get("repositories", {}).keys()
+    if repos:
+        return sorted(repos)[0]
+    fail("Config does not contain any repositories; unsure what to import")
+
+def repos_to_import(repos_config, entry, known=None):
+    """Compute the set of transitively reachable repositories"""
+    visited = set()
+    to_import = []
+    if known:
+        visited = set(known)
+
+    def visit(name):
+        if name in visited:
+            return
+        to_import.append(name)
+        visited.add(name)
+        for n in repos_config.get(name, {}).get("bindings", {}).values():
+            visit(n)
+
+    visit(entry)
+    return to_import
+
+def extra_layers_to_import(repos_config, repos):
+    """Compute the collection of repositories additionally needed as they serve
+    as layers for the repositories to import."""
+    extra_imports = set()
+    for repo in repos:
+        for layer in ["target_root", "rule_root", "expression_root"]:
+            if layer in repos_config[repo]:
+                extra = repos_config[repo][layer]
+                if (extra not in repos) and (extra not in extra_imports):
+                    extra_imports.add(extra)
+    return list(extra_imports)
+
+def name_imports(to_import, existing, base_name, main=None):
+    """Assign names to the repositories to import in such a way that
+    no conflicts arise."""
+    assign = {}
+
+    def find_name(name):
+        base = "%s/%s" % (base_name, name)
+        if (base not in existing) and (base not in assign):
+            return base
+        count = 0
+        while True:
+            count += 1
+            candidate = base + " (%d)" % count
+            if (candidate not in existing) and (candidate not in assign):
+                return candidate
+
+    if main and (base_name not in existing):
+        assign[main] = base_name
+        to_import = [x for x in to_import if x != main]
+    for repo in to_import:
+        assign[repo] = find_name(repo)
+    return assign
+
+def rewrite_repo(repo_spec, *, remote, assign):
+    new_spec = {}
+    repo = repo_spec.get("repository", {})
+    if isinstance(repo, str):
+        repo = assign[repo]
+    elif repo.get("type") == "file":
+        changes = {}
+        subdir = repo.get("path", ".")
+        if subdir not in ["", "."]:
+            changes["subdir"] = subdir
+        repo = dict(remote, **changes)
+    new_spec["repository"] = repo
+    for key in ["target_root", "rule_root", "expression_root"]:
+        if key in repo_spec:
+            new_spec[key] = assign[repo_spec[key]]
+    for key in ["target_file_name", "rule_file_name", "expression_file_name"]:
+        if key in repo_spec:
+            new_spec[key] = repo_spec[key]
+    bindings = repo_spec.get("bindings", {})
+    new_bindings = {}
+    for k, v in bindings.items():
+        new_bindings[k] = assign[v]
+    if new_bindings:
+        new_spec["bindings"] = new_bindings
+    return new_spec
+
+def handle_import(args):
+    base_config = get_base_config(args.repository_config)
+    base_repos = base_config.get("repositories", {})
+    srcdir, remote, to_cleanup = clone(args.URL, args.branch)
+    if args.foreign_repository_config:
+        foreign_config_file = args.foreign_repository_config
+    else:
+        foreign_config_file = get_repository_config_file(srcdir)
+    with open(foreign_config_file) as f:
+        foreign_config = json.load(f)
+    foreign_repos = foreign_config.get("repositories", {})
+    if args.foreign_repository_name:
+        foreign_name = args.foreign_repository_name
+    else:
+        foreign_name = get_repo_to_import(foreign_config)
+    import_map = {}
+    for theirs, ours in args.import_map:
+        import_map[theirs] = ours
+    main_repos = repos_to_import(foreign_repos, foreign_name, import_map.keys())
+    extra_repos = sorted([x for x in main_repos if x != foreign_name])
+    extra_imports = sorted(extra_layers_to_import(foreign_repos, main_repos))
+    ordered_imports = [foreign_name] + extra_repos + extra_imports
+    import_name = foreign_name
+    if args.import_as is not None:
+        import_name = args.import_as
+    assign = name_imports(
+        ordered_imports,
+        set(base_repos.keys()),
+        import_name,
+        main = foreign_name,
+    )
+    log("Importing %r as %r" % (foreign_name, import_name))
+    log("Transitive dependencies to import: %r" % (extra_repos,))
+    log("Repositories imported as layers: %r" % (extra_imports,))
+    total_assign = dict(assign, **import_map)
+    for repo in ordered_imports:
+        base_repos[assign[repo]] = rewrite_repo(
+            foreign_repos[repo],
+            remote = remote,
+            assign = total_assign,
+        )
+    base_config["repositories"] = base_repos
+    shutil.rmtree(to_cleanup)
+    return base_config
+
+def main():
+    parser = ArgumentParser(
+        prog = "just-import-deps",
+        description =
+        "Import a dependency transitively into a given"
+        + " multi-repository configuration"
+    )
+    parser.add_argument(
+        "-C",
+        dest="repository_config",
+        help="Repository-description file to import into",
+        metavar="FILE"
+    )
+    parser.add_argument(
+        "-b",
+        dest="branch",
+        help="The branch of the remote repository to import and follow",
+        metavar="branch",
+        default="master"
+    )
+    parser.add_argument(
+        "-R",
+        dest="foreign_repository_config",
+        help="Repository-description file in the repository to import",
+        metavar="relative-path"
+    )
+    parser.add_argument(
+        "--as",
+        dest="import_as",
+        help="Name prefix to import the foreign repository as",
+        metavar="NAME",
+    )
+    parser.add_argument(
+        "--map",
+        nargs=2,
+        dest="import_map",
+        help="Map the specified foreign repository to the specified existing repository",
+        action="append",
+        default=[]
+    )
+    parser.add_argument('URL')
+    parser.add_argument('foreign_repository_name', nargs='?')
+    args = parser.parse_args()
+    new_config = handle_import(args)
+    print(json.dumps(new_config))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/share/man/just-import-git.1.org b/share/man/just-import-git.1.org
new file mode 100644
index 00000000..354c2c2b
--- /dev/null
+++ b/share/man/just-import-git.1.org
@@ -0,0 +1,79 @@
+#+TITLE: JUST-import-git
+#+MAN_CLASS_OPTIONS: section-id=1
+
+* NAME
+
+just-import-git - import one git repository to a multi-repository configuration
+
+* SYNOPSIS
+
+just-import-deps [OPTIONS] URL [foreign repository name]
+
+* DESCRIPTION
+
+Extend an existing *just-mr-repository-config(5)* by adding one
+git repository. In doing so, the dependencies declared in the
+imported repository are added as well and ~"file"~ respositories
+are transformed to ~"subdir"~ parts of the imported repository.
+This solves the problem, that a repository can refer to itself only
+as ~"."~ in a portable way. The importing party, however, always
+knows the URL it is importing from.
+
+The imported repositories are renamed in a way that no conflicts
+with already present repositories arise. The repositories pulled in
+as dependencies are named in a way to remind for which repositories
+they came as a dependency. This renaming is taken into account at
+all places repositories are referred to, i.e., the ~"bindings"~
+field, as well as roots defined by reference to other repositories.
+
+Only the main parts of repositories are imported (~"repository"~,
+~"bindings"~, names, and roots). The ~"pragma"~ part, as well as
+unknown fields are dropped.
+
+The repository to import is specified by its URL
+
+The resulting configuration is printed on standard output.
+
+
+* OPTIONS
+
+   *--as* NAME\\
+   Specify the name the imported repository should have in the
+   final configuration. If not specified, default to the name
+   the repository has in the configuration file of the repository
+   imported. In any case, the name is amended if it conflicts
+   with an already existing name.
+
+   *-b* BRANCH\\
+   The branch in the imported repository to use; this branch is
+   also recorded as the branch to follow. Defaults to ~"master"~.
+
+   *-C* CONFIGFILE\\
+   Use the specified file as the configuration to import into.
+   If not specified, for a config file is searched in the same
+   way, as ~just-mr~ does, when invoved with ~--norc~.
+
+   *-h*, *--help* \\
+   Output a usage message and exit.
+
+   *--map* THEIRS OURS\\
+   Map repositories from the imported configuration to already
+   existing ones. Those repositories are not imported (and the
+   search for their transitive dependency is ommitted) and instead
+   the specified already existing repository is used. This is
+   useful, if a sufficiently compatible repository already exists
+   in the configuration.
+
+   *-R* RELPATH\\
+   Use the file, specified by path relative to the repository
+   root, as multi-repository specification in the imported
+   repository. If not specifed, for a config file is searched
+   in the same way as ~just-mr~ does, when invoced with ~--no-rc~,
+   however leaving out searches relative to global roots (~"home"~
+   and ~"system"~). In other words, ~repos.json~ and ~etc/repos.json~
+   are tried if this option is not given.
+
+
+* See also
+
+*just-mr-repository-config(5)*, *just-mr(1)*