diff options
| -rwxr-xr-x | bin/just-lock.py | 81 | ||||
| -rw-r--r-- | doc/future-designs/just-lock.md | 28 | ||||
| -rw-r--r-- | share/man/just-lock-config.5.md | 20 | ||||
| -rw-r--r-- | test/end-to-end/just-lock/TARGETS | 11 | ||||
| -rw-r--r-- | test/end-to-end/just-lock/plain-imports.sh | 137 |
5 files changed, 263 insertions, 14 deletions
diff --git a/bin/just-lock.py b/bin/just-lock.py index 1639a656..ddbbd21f 100755 --- a/bin/just-lock.py +++ b/bin/just-lock.py @@ -745,21 +745,27 @@ def rewrite_file_repo(repo: Json, remote_type: str, remote_stub: Dict[str, Any], fail("Unsupported remote type!") -def update_pragmas(repo: Json, pragma: Json) -> Json: - """Update the description with any input-provided "absent" and "to_git" - pragmas, as needed: - - for all repositories, merge the "absent" pragma - - for "file"-type repositories, merge the "to_git" pragma""" +def update_pragmas(repo: Json, import_pragma: Json, + pragma_special: Optional[str]) -> Json: + """Update the description with any input-provided pragmas: + - for all repositories, merge with import-level "absent" pragma + - for "file"-type repositories, merge with import-level "to_git" pragma + - for all repositories, overwrite with source-level "special" pragma.""" existing: Json = dict(repo.get("pragma", {})) # operate on copy # all repos support "absent pragma" - absent: bool = existing.get("absent", False) or pragma.get("absent", False) + absent: bool = existing.get("absent", False) or import_pragma.get( + "absent", False) if absent: existing["absent"] = True # support "to_git" pragma for "file"-type repos if repo.get("type") == "file": - to_git = existing.get("to_git", False) or pragma.get("to_git", False) + to_git = existing.get("to_git", False) or import_pragma.get( + "to_git", False) if to_git: existing["to_git"] = True + # all repos get the "special" pragma overwritten, if provided + if pragma_special is not None: + existing["special"] = pragma_special # all other pragmas as kept; if no pragma was set, do not set any if existing: repo = dict(repo, **{"pragma": existing}) @@ -767,8 +773,9 @@ def update_pragmas(repo: Json, pragma: Json) -> Json: def rewrite_repo(repo_spec: Json, *, remote_type: str, - remote_stub: Dict[str, Any], assign: Json, pragma: Json, - as_layer: bool, fail_context: str) -> Json: + remote_stub: Dict[str, Any], assign: Json, import_pragma: Json, + pragma_special: Optional[str], as_layer: bool, + fail_context: str) -> Json: """Rewrite description of imported repositories.""" new_spec: Json = {} repo = repo_spec.get("repository", {}) @@ -789,7 +796,7 @@ def rewrite_repo(repo_spec: Json, *, remote_type: str, repo = dict(repo, **{"repo": assign[target]}) # update pragmas, as needed if isinstance(repo, dict): - repo = update_pragmas(repo, pragma) + repo = update_pragmas(repo, import_pragma, pragma_special) new_spec["repository"] = repo # rewrite other roots and bindings, if actually needed to be imported if not as_layer: @@ -811,8 +818,8 @@ def rewrite_repo(repo_spec: Json, *, remote_type: str, def handle_import(remote_type: str, remote_stub: Dict[str, Any], - repo_desc: Json, core_repos: Json, foreign_config: Json, *, - fail_context: str) -> Json: + repo_desc: Json, core_repos: Json, foreign_config: Json, + pragma_special: Optional[str], *, fail_context: str) -> Json: """General handling of repository import from a foreign config.""" fail_context += "While handling import from remote type \"%s\"\n" % ( remote_type, ) @@ -885,7 +892,8 @@ def handle_import(remote_type: str, remote_stub: Dict[str, Any], remote_type=remote_type, remote_stub=remote_stub, assign=total_assign, - pragma=pragma, + import_pragma=pragma, + pragma_special=pragma_special, as_layer=False, fail_context=fail_context) for repo in extra_imports: @@ -893,7 +901,8 @@ def handle_import(remote_type: str, remote_stub: Dict[str, Any], remote_type=remote_type, remote_stub=remote_stub, assign=total_assign, - pragma=pragma, + import_pragma=pragma, + pragma_special=pragma_special, as_layer=True, fail_context=fail_context) @@ -1048,6 +1057,16 @@ def import_from_git(core_repos: Json, imports_entry: Json) -> Json: "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) + pragma_special: Optional[str] = imports_entry.get("pragma", + {}).get("special", None) + if pragma_special is not None and not isinstance(pragma_special, str): + fail(fail_context + + "Expected pragma \"special\" to be a string, but found:\n%r" % + (json.dumps(pragma_special, indent=2), )) + if not as_plain: + # only enabled if as_plain is true + pragma_special = None + # Fetch the source Git repository srcdir, remote_stub, to_clean_up = git_checkout(url, branch, @@ -1093,6 +1112,7 @@ def import_from_git(core_repos: Json, imports_entry: Json) -> Json: repo_entry, core_repos, foreign_config, + pragma_special, fail_context=fail_context) # Clean up local fetch @@ -1141,6 +1161,16 @@ def import_from_file(core_repos: Json, imports_entry: Json) -> Json: "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) + pragma_special: Optional[str] = imports_entry.get("pragma", + {}).get("special", None) + if pragma_special is not None and not isinstance(pragma_special, str): + fail(fail_context + + "Expected pragma \"special\" to be a string, but found:\n%r" % + (json.dumps(pragma_special, indent=2), )) + if not as_plain: + # only enabled if as_plain is true + pragma_special = None + # Read in the foreign config file if foreign_config_file: foreign_config_file = os.path.join(path, foreign_config_file) @@ -1184,6 +1214,7 @@ def import_from_file(core_repos: Json, imports_entry: Json) -> Json: repo_entry, core_repos, foreign_config, + pragma_special, fail_context=fail_context) return core_repos @@ -1418,6 +1449,16 @@ def import_from_archive(core_repos: Json, imports_entry: Json) -> Json: "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) + pragma_special: Optional[str] = imports_entry.get("pragma", + {}).get("special", None) + if pragma_special is not None and not isinstance(pragma_special, str): + fail(fail_context + + "Expected pragma \"special\" to be a string, but found:\n%r" % + (json.dumps(pragma_special, indent=2), )) + if not as_plain: + # only enabled if as_plain is true + pragma_special = None + # Fetch archive to local CAS and unpack srcdir, remote_stub, to_clean_up = archive_checkout( fetch, @@ -1466,6 +1507,7 @@ def import_from_archive(core_repos: Json, imports_entry: Json) -> Json: repo_entry, core_repos, foreign_config, + pragma_special, fail_context=fail_context) # Clean up local fetch @@ -1615,6 +1657,16 @@ def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json: "Expected field \"config\" to be a string, but found:\n%r" % (json.dumps(foreign_config_file, indent=2), )) + pragma_special: Optional[str] = imports_entry.get("pragma", + {}).get("special", None) + if pragma_special is not None and not isinstance(pragma_special, str): + fail(fail_context + + "Expected pragma \"special\" to be a string, but found:\n%r" % + (json.dumps(pragma_special, indent=2), )) + if not as_plain: + # only enabled if as_plain is true + pragma_special = None + # Fetch the Git tree srcdir, remote_stub, to_clean_up = git_tree_checkout( command=cast(List[str], command_gen if command is None else command), @@ -1661,6 +1713,7 @@ def import_from_git_tree(core_repos: Json, imports_entry: Json) -> Json: repo_entry, core_repos, foreign_config, + pragma_special, fail_context=fail_context) # Clean up local fetch diff --git a/doc/future-designs/just-lock.md b/doc/future-designs/just-lock.md index 151625c1..61e3c900 100644 --- a/doc/future-designs/just-lock.md +++ b/doc/future-designs/just-lock.md @@ -230,6 +230,12 @@ The type of a _source_ is defined by the string value of the mandatory subfield that will be used in the resulting repository description corresponding to any imported `"file"`-type repositories (see `just-import-git`). + If `"as plain": true`, any provided `"special"` key for the `"pragma"` field + in the source description is unconditionally set in the imported repositories, + superseding any other config- or import-level treatment of pragmas during the + import. Note that `"as plain": true` results in only one repository + (containing the whole source repository tree) being imported. + Proposed format: ``` jsonc { "source": "git" @@ -254,6 +260,7 @@ The type of a _source_ is defined by the string value of the mandatory subfield , "inherit env": [...] // optional; corresponds to `inherit_env` var (option --inherit-env) , "config": "<foreign_repos.json>" // optional; corresponds to `foreign_repository_config` var (option -R) , "as plain": false // optional; corresponds to `plain` var (option --plain) + , "pragma": {"special": "<value>"} // optional; only considered if `"as plain": true` } ``` @@ -268,6 +275,12 @@ The type of a _source_ is defined by the string value of the mandatory subfield one can also set the `"to_git": true` pragma with a corresponding entry in the usual `"pragma"` field. + If `"as plain": true`, any provided `"special"` key for the `"pragma"` field + in the source description is unconditionally set in the imported repositories, + superseding any other config- or import-level treatment of pragmas during the + import. Note that `"as plain": true` results in only one repository + (containing the whole source repository tree) being imported. + Proposed format: ``` jsonc { "source": "file" @@ -289,6 +302,7 @@ The type of a _source_ is defined by the string value of the mandatory subfield , "path": "<source/repo/path>" // mandatory , "config": "<foreign_repos.json>" // optional; corresponds to `foreign_repository_config` var (option -R) , "as plain": false // optional; corresponds to `plain` var (option --plain) + , "pragma": {"special": "<value>"} // optional; only considered if `"as plain": true` } ``` @@ -301,6 +315,12 @@ The type of a _source_ is defined by the string value of the mandatory subfield A field `"subdir"` is provided to account for the fact that source repository root often is not the root directory of the unpacked archive. + If `"as plain": true`, any provided `"special"` key for the `"pragma"` field + in the source description is unconditionally set in the imported repositories, + superseding any other config- or import-level treatment of pragmas during the + import. Note that `"as plain": true` results in only one repository + (containing the whole source repository tree) being imported. + Proposed format: ``` jsonc { "source": "archive" @@ -327,6 +347,7 @@ The type of a _source_ is defined by the string value of the mandatory subfield , "sha512": "<HASH>" // optional checksum; if given, will be checked , "config": "<foreign_repos.json>" // optional; corresponds to `foreign_repository_config` var (option -R) , "as plain": false // optional; corresponds to `plain` var (option --plain) + , "pragma": {"special": "<value>"} // optional; only considered if `"as plain": true` } ``` @@ -358,6 +379,12 @@ The type of a _source_ is defined by the string value of the mandatory subfield such repositories will be translated to appropriate `"git tree"`-type repositories in the output configuration. + If `"as plain": true`, any provided `"special"` key for the `"pragma"` field + in the source description is unconditionally set in the imported repositories, + superseding any other config- or import-level treatment of pragmas during the + import. Note that `"as plain": true` results in only one repository + (containing the whole source repository tree) being imported. + Proposed format: ``` jsonc { "source": "git tree" @@ -381,6 +408,7 @@ The type of a _source_ is defined by the string value of the mandatory subfield , "config": "<foreign_repos.json>" // optional; corresponds to `foreign_repository_config` var (option -R) // searched for in the "subdir" tree , "as plain": false // optional; corresponds to `plain` var (option --plain) + , "pragma": {"special": "<value>"} // optional; only considered if `"as plain": true` } ``` diff --git a/share/man/just-lock-config.5.md b/share/man/just-lock-config.5.md index 5ae901fb..3e71fd30 100644 --- a/share/man/just-lock-config.5.md +++ b/share/man/just-lock-config.5.md @@ -107,6 +107,11 @@ The following fields are supported: Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. + - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, + if a pragma object with key `"special"` is provided, it will unconditionally + be forwarded to the `"pragma"` object of the repository being imported for + this source. This entry is optional. + - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not @@ -136,6 +141,11 @@ The following fields are supported: Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. + - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, + if a pragma object with key `"special"` is provided, it will unconditionally + be forwarded to the `"pragma"` object of the repository being imported for + this source. This entry is optional. + - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not @@ -190,6 +200,11 @@ The following fields are supported: archived repository does not have a configuration file or should be imported as-is, without dependencies. This entry is optional. + - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, + if a pragma object with key `"special"` is provided, it will unconditionally + be forwarded to the `"pragma"` object of the repository being imported for + this source. This entry is optional. + - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the unpacked archive root. This entry is optional. If not provided and the `"as plain"` field does @@ -239,6 +254,11 @@ The following fields are supported: Git repository does not have a repository configuration or should be imported as-is, without dependencies. This entry is optional. + - *`"pragma"`* has as value a JSON object. If `"as plain"` evaluates to `true`, + if a pragma object with key `"special"` is provided, it will unconditionally + be forwarded to the `"pragma"` object of the repository being imported for + this source. This entry is optional. + - *`"config"`* has a string value defining the relative path of the foreign repository configuration file to be considered from the Git repository. This entry is optional. If not provided and the `"as plain"` field does not diff --git a/test/end-to-end/just-lock/TARGETS b/test/end-to-end/just-lock/TARGETS index 38c26ad3..ab3bc758 100644 --- a/test/end-to-end/just-lock/TARGETS +++ b/test/end-to-end/just-lock/TARGETS @@ -78,6 +78,16 @@ , ["end-to-end", "lock-tool-under-test"] ] } +, "plain-imports": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["plain-imports"] + , "test": ["plain-imports.sh"] + , "deps": + [ ["", "mr-tool-under-test"] + , ["", "tool-under-test"] + , ["end-to-end", "lock-tool-under-test"] + ] + } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] @@ -95,6 +105,7 @@ , "file-imports" , "archive-imports" , "git-tree-imports" + , "plain-imports" ] } ] diff --git a/test/end-to-end/just-lock/plain-imports.sh b/test/end-to-end/just-lock/plain-imports.sh new file mode 100644 index 00000000..7935f1df --- /dev/null +++ b/test/end-to-end/just-lock/plain-imports.sh @@ -0,0 +1,137 @@ +#!/bin/sh +# Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -eu + +readonly JUST_LOCK="${PWD}/bin/lock-tool-under-test" +readonly JUST="${PWD}/bin/tool-under-test" +readonly JUST_MR="${PWD}/bin/mr-tool-under-test" +readonly LBR="${TEST_TMPDIR}/local-build-root" +readonly LBR_PLAIN="${TEST_TMPDIR}/local-build-root-plain" +readonly OUT="${TEST_TMPDIR}/build-output" +readonly OUT_PLAIN="${TEST_TMPDIR}/build-output-plain" +readonly REPO_DIRS="${TEST_TMPDIR}/repos" +readonly WRKDIR="${PWD}/work" + +mkdir -p "${REPO_DIRS}/foo/inner" +cd "${REPO_DIRS}/foo" +touch ROOT +cat > repos.json <<'EOF' +{ "repositories": + { "": + { "repository": + { "type": "file" + , "path": "inner" + } + } + } +} +EOF +# add resolvable linked TARGETS file +ln -s inner/linked_TARGETS TARGETS +cat > inner/linked_TARGETS <<'EOF' +{ "": {"type": "file_gen", "name": "foo.txt", "data": "LINK"}} +EOF +# add inner TARGETS file shadowed by the linked one +cat > inner/TARGETS << 'EOF' +{ "": {"type": "file_gen", "name": "foo.txt", "data": "INNER"}} +EOF + +mkdir -p "${WRKDIR}" +cd "${WRKDIR}" +touch ROOT +cat > TARGETS <<'EOF' +{ "": + { "type": "generic" + , "cmds": ["cat foo.txt > out.txt"] + , "outs": ["out.txt"] + , "deps": [["@", "foo", "", ""]] + } +} +EOF + + +echo === Check normal import === + +cat > repos.in.json <<EOF +{ "repositories": + { "": + { "repository": {"type": "file", "path": "."} + , "bindings": {"foo": "foo"} + } + } + , "imports": + [ { "source": "file" + , "repos": [{"alias": "foo", "pragma": {"to_git": true}}] + , "path": "${REPO_DIRS}/foo" + } + ] +} +EOF +cat repos.in.json + +echo +"${JUST_LOCK}" -C repos.in.json -o repos.json --local-build-root "${LBR}" 2>&1 +cat repos.json +echo +# Check pragmas: "to_git" is kept +[ $(jq -r '.repositories.foo.repository.pragma.to_git' repos.json) = true ] +# Check that the subdir is taken as expected +"${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ + --local-build-root "${LBR}" install -o "${OUT}" 2>&1 +echo +cat "${OUT}/out.txt" +echo +grep -q INNER "${OUT}/out.txt" + + +echo == Check plain import === + +cat > repos.in.json <<EOF +{ "repositories": + { "": + { "repository": {"type": "file", "path": "."} + , "bindings": {"foo": "foo"} + } + } + , "imports": + [ { "source": "file" + , "repos": [{"alias": "foo", "pragma": {"to_git": true}}] + , "path": "${REPO_DIRS}/foo" + , "as plain": true + , "pragma": {"special": "resolve-completely"} + } + ] +} +EOF +cat repos.in.json + +echo +"${JUST_LOCK}" -C repos.in.json -o repos.json --local-build-root "${LBR_PLAIN}" 2>&1 +cat repos.json +echo +# Check pragmas: "to_git" is kept, "special" is unconditionally set +[ $(jq -r '.repositories.foo.repository.pragma.special' repos.json) = "resolve-completely" ] +[ $(jq -r '.repositories.foo.repository.pragma.to_git' repos.json) = true ] +# Check the symlink gets resolved as expected +"${JUST_MR}" -L '["env", "PATH='"${PATH}"'"]' --norc --just "${JUST}" \ + --local-build-root "${LBR_PLAIN}" install -o "${OUT_PLAIN}" 2>&1 +echo +cat "${OUT_PLAIN}/out.txt" +echo +grep -q LINK "${OUT_PLAIN}/out.txt" + +echo "OK" |