diff options
| -rwxr-xr-x | bin/bootstrap-traverser.py | 14 | ||||
| -rwxr-xr-x | bin/just-deduplicate-repos.py | 79 | ||||
| -rwxr-xr-x | bin/just-import-git.py | 36 | ||||
| -rwxr-xr-x | bin/just-mr.py | 13 | ||||
| -rwxr-xr-x | bin/parallel-bootstrap-traverser.py | 8 |
5 files changed, 83 insertions, 67 deletions
diff --git a/bin/bootstrap-traverser.py b/bin/bootstrap-traverser.py index 9b82b338..1598c842 100755 --- a/bin/bootstrap-traverser.py +++ b/bin/bootstrap-traverser.py @@ -19,7 +19,7 @@ import os import shutil import subprocess import sys -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, NoReturn, cast from argparse import ArgumentParser @@ -32,7 +32,7 @@ def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) -def fail(s: str) -> None: +def fail(s: str) -> NoReturn: log(s) sys.exit(1) @@ -66,7 +66,7 @@ def link(src: str, dest: str) -> None: os.symlink(src, dest) -def build_local(desc: Json, *, root: str, config: Json) -> Optional[str]: +def build_local(desc: Json, *, root: str, config: Json) -> str: repo_name = desc["data"]["repository"] repo: List[str] = config["repositories"][repo_name]["workspace_root"] rel_path = desc["data"]["path"] @@ -83,7 +83,7 @@ def build_tree(desc: Json, *, config: Json, root: str, graph: Json) -> str: tree_dir_tmp = tree_dir + ".tmp" tree_desc = graph["trees"][tree_id] for location, desc in tree_desc.items(): - link(cast(str, build(desc, config=config, root=root, graph=graph)), + link(build(desc, config=config, root=root, graph=graph), os.path.join(tree_dir_tmp, location)) # correctly handle the empty tree os.makedirs(tree_dir_tmp, exist_ok=True) @@ -98,7 +98,7 @@ def run_action(action_id: str, *, config: Json, root: str, graph: Json) -> str: os.makedirs(action_dir) action_desc = graph["actions"][action_id] for location, desc in action_desc.get("input", {}).items(): - link(cast(str, build(desc, config=config, root=root, graph=graph)), + link(build(desc, config=config, root=root, graph=graph), os.path.join(action_dir, location)) cmd = action_desc["command"] env = action_desc.get("env") @@ -121,7 +121,7 @@ def build_action(desc: Json, *, config: Json, root: str, graph: Json) -> str: return os.path.join(action_dir, desc["data"]["path"]) -def build(desc: Json, *, config: Json, root: str, graph: Json) -> Optional[str]: +def build(desc: Json, *, config: Json, root: str, graph: Json) -> str: if desc["type"] == "TREE": return build_tree(desc, config=config, root=root, graph=graph) if desc["type"] == "ACTION": @@ -139,7 +139,7 @@ def traverse(*, graph: Json, to_build: Json, out: str, root: str, os.makedirs(root, exist_ok=True) create_blobs(graph["blobs"], root=root) for location, artifact in to_build.items(): - link(cast(str, build(artifact, config=config, root=root, graph=graph)), + link(build(artifact, config=config, root=root, graph=graph), os.path.join(out, location)) diff --git a/bin/just-deduplicate-repos.py b/bin/just-deduplicate-repos.py index ca4780b0..df64df8e 100755 --- a/bin/just-deduplicate-repos.py +++ b/bin/just-deduplicate-repos.py @@ -16,15 +16,17 @@ import json import sys -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union, cast # generic JSON type Json = Any + def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) -def fail(s: str, exit_code: int = 1): + +def fail(s: str, exit_code: int = 1) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) @@ -45,29 +47,39 @@ def roots_equal(a: Json, b: Json) -> bool: # for full equality return a == b -def get_root(repos: Json, name: str, *, root_name: str="repository", - default_root : Optional[Json]=None) -> Json: + +def get_root(repos: Json, + name: str, + *, + root_name: str = "repository", + default_root: Optional[Json] = None) -> Json: root = repos[name].get(root_name) if root is None: if default_root is not None: return default_root else: - fail("Did not find mandatory root %s" % (name,)) + fail("Did not find mandatory root %s" % (name, )) if isinstance(root, str): return get_root(repos, root) return root + def local_repos_equal(repos: Json, name_a: str, name_b: str) -> bool: if name_a == name_b: return True root_a = None root_b = None - for root_name in ["repository", - "target_root", "rule_root", "expression_root"]: - root_a = get_root(repos, name_a, root_name=root_name, - default_root = root_a) - root_b = get_root(repos, name_b, root_name=root_name, - default_root = root_b) + for root_name in [ + "repository", "target_root", "rule_root", "expression_root" + ]: + root_a = get_root(repos, + name_a, + root_name=root_name, + default_root=root_a) + root_b = get_root(repos, + name_b, + root_name=root_name, + default_root=root_b) if not roots_equal(root_a, root_b): return False for file_name, default_name in [("target_file_name", "TARGETS"), @@ -83,38 +95,38 @@ def local_repos_equal(repos: Json, name_a: str, name_b: str) -> bool: return False return True + def bisimilar_repos(repos: Json) -> List[List[str]]: """Compute the maximal bisimulation between the repositories and return the bisimilarity classes.""" - bisim = {} + bisim: Dict[Tuple[str, str], Json] = {} def is_different(name_a: str, name_b: str) -> bool: return bisim.get((name_a, name_b), {}).get("different", False) def mark_as_different(name_a: str, name_b: str): nonlocal bisim - entry = bisim.get((name_a, name_b),{}) + entry = bisim.get((name_a, name_b), {}) if entry.get("different"): return bisim[(name_a, name_b)] = dict(entry, **{"different": True}) - also_different = entry.get("different_if", []) + also_different: List[Tuple[str, str]] = entry.get("different_if", []) for a, b in also_different: mark_as_different(a, b) def register_dependency(name_a: str, name_b: str, dep_a: str, dep_b: str): pos = (name_a, name_b) if name_a < name_b else (name_b, name_a) entry = bisim.get(pos, {}) - deps = entry.get("different_if", []) + deps: List[Tuple[str, str]] = entry.get("different_if", []) deps.append((dep_a, dep_b)) bisim[pos] = dict(entry, **{"different_if": deps}) - names = sorted(repos.keys()) for j in range(len(names)): b = names[j] for i in range(j): a = names[i] - if is_different(a,b): + if is_different(a, b): continue if not local_repos_equal(repos, names[i], names[j]): mark_as_different(names[i], names[j]) @@ -126,12 +138,12 @@ def bisimilar_repos(repos: Json) -> List[List[str]]: next_b = links_b[link] if next_a != next_b: if is_different(next_a, next_b): - mark_as_different(a,b) + mark_as_different(a, b) continue else: register_dependency(next_a, next_b, a, b) - classes = [] - done = {} + classes: List[List[str]] = [] + done: Dict[str, bool] = {} for j in reversed(range(len(names))): name_j = names[j] if done.get(name_j): @@ -139,12 +151,13 @@ def bisimilar_repos(repos: Json) -> List[List[str]]: c = [name_j] for i in range(j): name_i = names[i] - if not bisim.get((name_i, name_j),{}).get("different"): + if not bisim.get((name_i, name_j), {}).get("different"): c.append(name_i) done[name_i] = True classes.append(c) return classes + def dedup(repos: Json, user_keep: List[str]) -> Json: keep = set(user_keep) @@ -157,9 +170,10 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: candidates = c # Keep a repository with a proper root, if any of those has a root. # In this way, we're not losing actual roots. - with_root = [ n for n in candidates - if isinstance(repos["repositories"][n]["repository"], - dict)] + with_root = [ + n for n in candidates + if isinstance(repos["repositories"][n]["repository"], dict) + ] if with_root: candidates = with_root @@ -168,8 +182,7 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: if keep_entries: candidates = list(keep_entries) - return sorted(candidates, - key=lambda s: (s.count("/"), len(s), s))[0] + return sorted(candidates, key=lambda s: (s.count("/"), len(s), s))[0] def merge_pragma(rep: str, merged: List[str]) -> Json: desc = cast(Union[str, Dict[str, Json]], @@ -179,7 +192,7 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: pragma = desc.get("pragma", {}) # Clear pragma absent unless all merged repos that are not references # have the pragma - absent = pragma.get("absent", False) + absent: bool = pragma.get("absent", False) for c in merged: alt_desc = cast(Union[str, Dict[str, Json]], repos["repositories"][c]["repository"]) @@ -207,8 +220,8 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: return desc bisim = bisimilar_repos(repos["repositories"]) - renaming = {} - updated_repos = {} + renaming: Dict[str, str] = {} + updated_repos: Json = {} for c in bisim: if len(c) == 1: continue @@ -226,12 +239,11 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: # actual root; can still be merged into a different once, but only # one with a proper root as well. return renaming.get(name, name) - elif isinstance(root, str): + if isinstance(root, str): return final_root_reference(root) - else: - fail("Invalid root found for %r: %r" % (name, root)) + fail("Invalid root found for %r: %r" % (name, root)) - new_repos = {} + new_repos: Json = {} for name in repos["repositories"].keys(): if name not in renaming: desc = repos["repositories"][name] @@ -255,6 +267,7 @@ def dedup(repos: Json, user_keep: List[str]) -> Json: new_repos[name] = desc return dict(repos, **{"repositories": new_repos}) + if __name__ == "__main__": orig = json.load(sys.stdin) final = dedup(orig, sys.argv[1:]) diff --git a/bin/just-import-git.py b/bin/just-import-git.py index 7416bb7b..eb3b690f 100755 --- a/bin/just-import-git.py +++ b/bin/just-import-git.py @@ -23,7 +23,7 @@ import tempfile from argparse import ArgumentParser, Namespace from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast +from typing import Any, Dict, Iterable, List, NoReturn, Optional, Set, Tuple, cast # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters @@ -34,7 +34,7 @@ def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) -def fail(s: str, exit_code: int = 1): +def fail(s: str, exit_code: int = 1) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) @@ -114,7 +114,7 @@ def get_repository_config_file(root: Optional[str] = None) -> Optional[str]: return path -def get_base_config(repository_config: Optional[str]) -> Optional[Json]: +def get_base_config(repository_config: Optional[str]) -> Json: if repository_config == "-": return json.load(sys.stdin) if not repository_config: @@ -125,9 +125,13 @@ def get_base_config(repository_config: Optional[str]) -> Optional[Json]: fail('Could not get base config') -def clone(url: str, branch: str, *, - mirrors: List[str], inherit_env: List[str], - ) -> Tuple[str, Dict[str, Any], str]: +def clone( + url: str, + branch: str, + *, + mirrors: List[str], + inherit_env: List[str], +) -> Tuple[str, Dict[str, Any], str]: # clone the given git repository, checkout the specified # branch, and return the checkout location workdir: str = tempfile.mkdtemp() @@ -157,7 +161,7 @@ def get_repo_to_import(config: Json) -> str: return cast(str, config.get("main")) repos = config.get("repositories", {}).keys() if repos: - return sorted(repos)[0] + return cast(str, sorted(repos)[0]) fail("Config does not contain any repositories; unsure what to import") @@ -226,8 +230,8 @@ def name_imports(to_import: List[str], return assign -def rewrite_repo(repo_spec: Json, *, remote: Dict[str, Any], - assign: Json, absent: bool) -> Json: +def rewrite_repo(repo_spec: Json, *, remote: Dict[str, Any], assign: Json, + absent: bool) -> Json: new_spec: Json = {} repo = repo_spec.get("repository", {}) if isinstance(repo, str): @@ -261,12 +265,13 @@ def rewrite_repo(repo_spec: Json, *, remote: Dict[str, Any], def handle_import(args: Namespace) -> Json: - base_config: Json = cast(Json, get_base_config(args.repository_config)) + base_config: Json = get_base_config(args.repository_config) base_repos: Json = base_config.get("repositories", {}) srcdir, remote, to_cleanup = clone( - args.URL, args.branch, - mirrors = args.mirrors, - inherit_env = args.inherit_env, + args.URL, + args.branch, + mirrors=args.mirrors, + inherit_env=args.inherit_env, ) if args.foreign_repository_config: foreign_config_file = args.foreign_repository_config @@ -293,7 +298,7 @@ def handle_import(args: Namespace) -> Json: fail('Could not get repository config file') foreign_repos: Json = foreign_config.get("repositories", {}) if args.foreign_repository_name: - foreign_name = args.foreign_repository_name + foreign_name = cast(str, args.foreign_repository_name) else: foreign_name = get_repo_to_import(foreign_config) import_map: Json = {} @@ -357,8 +362,7 @@ def main(): parser.add_argument( "--absent", action="store_true", - help="Import repository and all its dependencies as absent." - ) + help="Import repository and all its dependencies as absent.") parser.add_argument( "--as", dest="import_as", diff --git a/bin/just-mr.py b/bin/just-mr.py index 21413546..4b79c295 100755 --- a/bin/just-mr.py +++ b/bin/just-mr.py @@ -22,7 +22,7 @@ import sys import tempfile import time import zlib -from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast +from typing import Any, Dict, List, NoReturn, Optional, Set, Tuple, Union, cast from argparse import ArgumentParser from pathlib import Path @@ -140,7 +140,7 @@ def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) -def fail(s: str, exit_code: int = 65) -> None: +def fail(s: str, exit_code: int = 65) -> NoReturn: log(f"Error: {s}") sys.exit(exit_code) @@ -701,7 +701,7 @@ def distdir_tree_id_file(content: str) -> str: content) -def distdir_checkout(desc: Json, repos: Json): +def distdir_checkout(desc: Json, repos: Json) -> List[str]: """ Logic for processing the distdir repo type. """ content: Dict[str, str] = {} @@ -783,7 +783,7 @@ def distdir_checkout(desc: Json, repos: Json): ] -def checkout(desc: Json, *, name: str, repos: Json) -> Optional[List[str]]: +def checkout(desc: Json, *, name: str, repos: Json) -> List[str]: repo_desc = resolve_repo(desc, repos=repos) repo_type = repo_desc.get("type") if repo_type == "git": @@ -965,8 +965,7 @@ def fetch(*, return os.path.commonpath([path, base]) == base # warn if fetch_dir is in invocation workspace - if g_WORKSPACE_ROOT and is_subpath(cast(str, fetch_dir), - g_WORKSPACE_ROOT): + if g_WORKSPACE_ROOT and is_subpath(fetch_dir, g_WORKSPACE_ROOT): repo = repos.get(main, {}).get("repository", {}) repo_path = repo.get("path", None) if repo_path is not None and repo.get("type", None) == "file": @@ -993,7 +992,7 @@ Warning: Writing distribution files to workspace location '{fetch_dir}', print("%r --> %r (content: %s)" % (repo, distfile, content)) archive_fetch(repo_desc, content) shutil.copyfile(cas_path(content), - os.path.join(cast(str, fetch_dir), distfile)) + os.path.join(fetch_dir, distfile)) sys.exit(0) diff --git a/bin/parallel-bootstrap-traverser.py b/bin/parallel-bootstrap-traverser.py index ee5bb194..f27525d7 100755 --- a/bin/parallel-bootstrap-traverser.py +++ b/bin/parallel-bootstrap-traverser.py @@ -23,7 +23,7 @@ import sys import threading from enum import Enum from argparse import ArgumentParser -from typing import Any, Callable, Dict, List, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, cast # generic JSON type that avoids getter issues; proper use is being enforced by # return types of methods and typing vars holding return values of json getters @@ -195,11 +195,11 @@ class AtomicListMap: g_CALLBACKS_PER_ID = AtomicListMap() -def log(*args: str, **kwargs: Any): +def log(*args: str, **kwargs: Any) -> None: print(*args, file=sys.stderr, **kwargs) -def fail(s: str): +def fail(s: str) -> NoReturn: log(s) sys.exit(1) @@ -236,7 +236,7 @@ def link(src: str, dest: str) -> None: os.symlink(src, dest) -def build_local(desc: Json, *, root: str, config: Json) -> Optional[str]: +def build_local(desc: Json, *, root: str, config: Json) -> str: repo_name: str = desc["data"]["repository"] repo: List[str] = config["repositories"][repo_name]["workspace_root"] rel_path: str = desc["data"]["path"] |