diff options
| -rw-r--r-- | test/end-to-end/TARGETS | 4 | ||||
| -rw-r--r-- | test/end-to-end/computed-roots/TARGETS | 19 | ||||
| -rw-r--r-- | test/end-to-end/computed-roots/remote.sh | 162 |
3 files changed, 172 insertions, 13 deletions
diff --git a/test/end-to-end/TARGETS b/test/end-to-end/TARGETS index 93f0a01c..c78d2c05 100644 --- a/test/end-to-end/TARGETS +++ b/test/end-to-end/TARGETS @@ -27,7 +27,8 @@ , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] - , "else": [["./", "serve-service", "TESTS"]] + , "else": + [["./", "serve-service", "TESTS"], ["./", "computed-roots", "TESTS"]] } ] } @@ -61,7 +62,6 @@ , ["./", "build-fails", "TESTS"] , ["./", "built-in-rules", "TESTS"] , ["./", "cli", "TESTS"] - , ["./", "computed-roots", "TESTS"] , ["./", "execution-service", "TESTS"] , ["./", "gc", "TESTS"] , ["./", "generated-binary", "TESTS"] diff --git a/test/end-to-end/computed-roots/TARGETS b/test/end-to-end/computed-roots/TARGETS index 5a5535cf..0bf9c685 100644 --- a/test/end-to-end/computed-roots/TARGETS +++ b/test/end-to-end/computed-roots/TARGETS @@ -26,20 +26,17 @@ , "keep": ["out/not-export.log", "out/not-content-fixed.log", "out/cycle.log"] } +, "basic remote build": + { "type": ["end-to-end", "with remote"] + , "name": ["remote"] + , "test": ["remote.sh"] + , "deps": [["", "tool-under-test"]] + , "keep": ["out/log", "out/log.root"] + } , "TESTS": { "type": ["@", "rules", "test", "suite"] , "stage": ["computed-roots"] - , "arguments_config": ["TEST_BOOTSTRAP_JUST_MR"] , "deps": - { "type": "++" - , "$1": - [ ["basic", "error-reporting"] - , { "type": "if" - , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} - , "then": [] - , "else": ["mr_computed_setup"] - } - ] - } + ["basic", "basic remote build", "error-reporting", "mr_computed_setup"] } } diff --git a/test/end-to-end/computed-roots/remote.sh b/test/end-to-end/computed-roots/remote.sh new file mode 100644 index 00000000..59a0f123 --- /dev/null +++ b/test/end-to-end/computed-roots/remote.sh @@ -0,0 +1,162 @@ +#!/bin/sh +# Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +readonly ROOT="$(pwd)" +readonly LBRDIR="$TMPDIR/local-build-root" +readonly JUST="${ROOT}/bin/tool-under-test" +readonly OUT="${ROOT}/out" + +REMOTE_ARGS="-r ${REMOTE_EXECUTION_ADDRESS}" +if [ -n "${COMPATIBLE:-}" ] +then + REMOTE_ARGS="${REMOTE_ARGS} --compatible" +fi +echo Using ${REMOTE_ARGS} + +readonly BASE_ROOT="${ROOT}/base" +mkdir -p "${BASE_ROOT}" +cd "${BASE_ROOT}" +cat > generate.py <<'EOF' +import json +import sys + +COUNT = int(sys.argv[1]) +targets = {} +for i in range(COUNT): + targets["%d" % i] = {"type": "generic", "outs": ["%d.txt" %i], + "cmds": ["sh bin/seqfile.sh %d" % (i,)], + "deps": ["bin/seqfile.sh"]} +targets[""] = {"type": "generic", + "deps": ["%d" % i for i in range(COUNT)], + "cmds": [" ".join(["cat"] + ["%d.txt" % i for i in range(COUNT)] + + ["> out"])], + "outs": ["out"]} +print (json.dumps(targets, indent=2)) +EOF +cat > TARGETS <<'EOF' +{ "": {"type": "export", "flexible_config": ["COUNT"], "target": "root"} +, "root": + { "type": "install" + , "files": {"TARGETS": "targets", "bin/seqfile.sh": "seqfile"} + } +, "targets": + { "type": "generic" + , "arguments_config": ["COUNT"] + , "outs": ["TARGETS"] + , "deps": ["generate.py"] + , "cmds": + [ { "type": "join" + , "separator": " " + , "$1": + [ "python3" + , "generate.py" + , {"type": "var", "name": "COUNT"} + , ">" + , "TARGETS" + ] + } + ] + } +, "seqfile": + {"type": "file_gen", "name": "bin/seqfile.sh", "data": "seq 0 $1 > $1.txt"} +} +EOF +git init 2>&1 +git branch -m stable-1.0 2>&1 +git config user.email "nobody@example.org" 2>&1 +git config user.name "Nobody" 2>&1 +git add . 2>&1 +git commit -m "Initial commit" 2>&1 +GIT_TREE=$(git log -n 1 --format="%T") + + +mkdir -p "${ROOT}/main" +cd "${ROOT}/main" + +cat > repo-config.json <<EOF +{ "repositories": + { "base": + {"workspace_root": ["git tree", "${GIT_TREE}", "${BASE_ROOT}/.git"]} + , "derived": + {"workspace_root": ["computed", "base", "", "", {"COUNT": "10"}]} + , "other derived": + {"workspace_root": ["computed", "base", "", "", {"COUNT": "12"}]} + } +} +EOF +cat repo-config.json + +echo +echo Building base, putting the export target to cache +echo +"${JUST}" install ${REMOTE_ARGS} \ + --local-build-root "${LBRDIR}" -C repo-config.json \ + --main base -D '{"COUNT": "10"}' -o "${OUT}/base" 2>&1 +echo + +echo +echo Building computed +echo +"${JUST}" install ${REMOTE_ARGS} \ + --local-build-root "${LBRDIR}" -C repo-config.json \ + --log-limit 4 -f "${OUT}/computed.log" \ + --main derived -o "${OUT}/derived" 2>&1 +echo +[ "$(cat "${OUT}/derived/out" | wc -l)" -eq 55 ] + +# Given the previous build, some form of cache hit is expected +grep 'from cache' "${OUT}/computed.log" + +echo +echo Building a different computed root, without previous build +echo +"${JUST}" install ${REMOTE_ARGS} \ + --local-build-root "${LBRDIR}" -C repo-config.json \ + --log-limit 4 -f "${OUT}/log" \ + --main 'other derived' -o "${OUT}/other-derived" 2>&1 +echo + +[ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ] + +echo +echo Sanity-check of the log +echo +# As this target was never built before, there can't be a cache hit, +# hence we expect a reference to an evaluation log. +grep '[Rr]oot.*base.*evaluted.*' "${OUT}/log" > "${TMPDIR}/log_line" +cat "${TMPDIR}/log_line" +sed -i 's/.*log //' "${TMPDIR}/log_line" +"${JUST}" install-cas --local-build-root "${LBRDIR}" \ + ${REMOTE_ARGS} \ + -o "${OUT}/log.root" $(cat "${TMPDIR}/log_line") +echo +cat "${OUT}/log.root" +echo +grep '[Dd]iscovered.*1 action' "${OUT}/log.root" +grep '0 cache hit' "${OUT}/log.root" + +echo +echo Building computed root again, expecting a hit +echo +"${JUST}" build ${REMOTE_ARGS} \ + --local-build-root "${LBRDIR}" -C repo-config.json \ + --log-limit 4 -f "${OUT}/log2" \ + --main 'other derived' 2>&1 +echo +grep 'from cache' "${OUT}/log2" + +echo OK |