diff options
Diffstat (limited to 'rules/shell')
| -rw-r--r-- | rules/shell/test/EXPRESSIONS | 236 | ||||
| -rw-r--r-- | rules/shell/test/RULES | 245 | ||||
| -rw-r--r-- | rules/shell/test/TARGETS | 1 | ||||
| -rwxr-xr-x | rules/shell/test/runner | 59 | ||||
| -rwxr-xr-x | rules/shell/test/summarizer | 67 |
5 files changed, 608 insertions, 0 deletions
diff --git a/rules/shell/test/EXPRESSIONS b/rules/shell/test/EXPRESSIONS new file mode 100644 index 0000000..2692c31 --- /dev/null +++ b/rules/shell/test/EXPRESSIONS @@ -0,0 +1,236 @@ +{ "test-action": + { "vars": + [ "TEST_ENV" + , "TIMEOUT_SCALE" + , "ATTEMPT" + , "name" + , "test.sh" + , "keep" + , "runner" + , "deps-fieldname" + , "deps-transition" + , "target properties" + ] + , "imports": + { "artifacts_list": ["./", "../..", "field_artifacts_list"] + , "runfiles_list": ["./", "../..", "field_runfiles_list"] + , "map_provider": ["./", "../..", "field_map_provider"] + } + , "expression": + { "type": "let*" + , "bindings": + [ [ "runner" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "runner" + , "range": {"type": "var", "name": "runner"} + , "body": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "runner" + , "range": + { "type": "values" + , "$1": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "runner"} + } + } + , "body": {"type": "env", "vars": ["runner"]} + } + } + } + } + ] + , [ "deps" + , { "type": "TREE" + , "$1": + { "type": "disjoint_map_union" + , "msg": + [ "Field" + , {"type": "var", "name": "deps-fieldname"} + , "has to stage in a conflict free way" + ] + , "$1": + { "type": "++" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": + [ {"type": "CALL_EXPRESSION", "name": "runfiles_list"} + , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} + ] + } + } + } + } + ] + , [ "run-libs" + , { "type": "TREE" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] + , ["provider", "run-libs"] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} + } + } + ] + , [ "attempt marker" + , { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "ATTEMPT"} + , "$2": null + } + , "then": {"type": "empty_map"} + , "else": + { "type": "singleton_map" + , "key": "ATTEMPT" + , "value": + {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}} + } + } + ] + , [ "outs" + , { "type": "++" + , "$1": + [ ["result", "stdout", "stderr", "time-start", "time-stop"] + , { "type": "foreach" + , "var": "filename" + , "range": {"type": "var", "name": "keep"} + , "body": + { "type": "join" + , "$1": ["work/", {"type": "var", "name": "filename"}] + } + } + ] + } + ] + , [ "inputs" + , { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "work" + , "value": {"type": "var", "name": "deps"} + } + , { "type": "singleton_map" + , "key": "libs" + , "value": {"type": "var", "name": "run-libs"} + } + , {"type": "var", "name": "runner"} + , {"type": "var", "name": "test.sh"} + , {"type": "var", "name": "attempt marker"} + ] + } + ] + , [ "cmd" + , {"type": "++", "$1": [["./runner"], {"type": "var", "name": "keep"}]} + ] + , [ "test_env" + , { "type": "map_union" + , "$1": + [ { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "ATTEMPT"} + , "$2": null + } + , "then": {"type": "empty_map"} + , "else": + { "type": "singelton_map" + , "key": "TEST_RUN_NUMBER" + , "value": {"type": "var", "name": "ATTEMPT"} + } + } + , { "type": "var" + , "name": "TEST_ENV" + , "default": {"type": "empty_map"} + } + ] + } + ] + ] + , "body": + { "type": "if" + , "cond": + {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null} + , "then": + { "type": "ACTION" + , "outs": {"type": "var", "name": "outs"} + , "inputs": {"type": "var", "name": "inputs"} + , "cmd": {"type": "var", "name": "cmd"} + , "env": {"type": "var", "name": "test_env"} + , "may_fail": ["test"] + , "fail_message": + { "type": "join" + , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} + } + , "else": + { "type": "ACTION" + , "outs": {"type": "var", "name": "outs"} + , "inputs": {"type": "var", "name": "inputs"} + , "cmd": {"type": "var", "name": "cmd"} + , "env": {"type": "var", "name": "test_env"} + , "may_fail": ["test"] + , "no_cache": ["test"] + , "fail_message": + { "type": "join" + , "$1": + [ "shell test " + , {"type": "var", "name": "name"} + , " failed (Run " + , {"type": "var", "name": "ATTEMPT"} + , ")" + ] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} + } + } + } + } +, "test-result": + { "vars": + [ "TEST_ENV" + , "TIMEOUT_SCALE" + , "name" + , "test.sh" + , "keep" + , "runner" + , "deps-fieldname" + , "deps-transition" + , "target properties" + ] + , "imports": {"action": "test-action"} + , "expression": + { "type": "let*" + , "bindings": + [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}] + , [ "runfiles" + , { "type": "singleton_map" + , "key": {"type": "var", "name": "name"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": {"type": "var", "name": "test-results"} + , "runfiles": {"type": "var", "name": "runfiles"} + } + } + } +} diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES new file mode 100644 index 0000000..ec26609 --- /dev/null +++ b/rules/shell/test/RULES @@ -0,0 +1,245 @@ +{ "script": + { "doc": ["Shell test, given by a test script"] + , "target_fields": ["deps", "test"] + , "string_fields": ["keep", "name"] + , "config_vars": + [ "ARCH" + , "HOST_ARCH" + , "RUNS_PER_TEST" + , "TEST_ENV" + , "TIMEOUT_SCALE" + , "TARGET_ARCH" + , "ARCH_DISPATCH" + ] + , "field_doc": + { "test": + [ "The shell script for the test, launched with sh." + , "" + , "An empty directory is created to store any temporary files needed" + , "by the test, and it is made available in the environment variable" + , "TEST_TMPDIR. The test should not assume write permissions" + , "outside the working directory and the TEST_TMPDIR." + , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "If the configuration variable RUNS_PER_TEST is set, the environment" + , "variable TEST_RUN_NUMBER will also be set to the number of the attempt," + , "counting from 0." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." + ] + , "name": + [ "A name for the test, used in reporting, as well as for staging" + , "the test result tree in the runfiles" + ] + , "keep": + [ "List of names (relative to the test working directory) of files that" + , "the test might generate that should be kept as part of the output." + , "This might be useful for further analysis of the test" + ] + , "deps": + [ "Any targets that should be staged (with artifacts and runfiles) into" + , "the tests working directory" + ] + , "runner": + [ "The test runner which starts the actual test script after providing" + , "the respective environment. The runner also takes care of capturing" + , "stdout/stderr, timing information, and ensure the presence of the" + , "files to keep even if the script failed to produce them." + ] + , "summarizer": + [ "Tool to aggregate the results of individual test runs (for flakyness" + , "detection) to an overall test result." + ] + } + , "config_doc": + { "RUNS_PER_TEST": + [ "The number of times the test should be run in order to detect flakyness." + , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." + ] + , "TEST_ENV": ["Additional environment for executing the test runner."] + , "TIMEOUT_SCALE": + ["Factor on how to scale the timeout for this test. Defaults to 1.0."] + , "TARGET_ARCH": + [ "The architecture to build the test for." + , "" + , "Will only be honored, if that architecture is available in the" + , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" + , "on the host architecture." + ] + , "ARCH_DISPATCH": + [ "Map of architectures to execution properties that ensure execution" + , "on that architecture. Only the actual test binary will be run with" + , "the specified execution properties (i.e., on the target architecture);" + , "all building will be done on the host architecture." + ] + } + , "tainted": ["test"] + , "artifacts_doc": + [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" + , " generating test reports." + , "stdout/stderr: Any output the invocation of the test binary produced on" + , " the respective file descriptor" + , "work: In this directory, all the files specified to \"keep\" are staged" + , "time-start/time-stop: The time (decimally coded) in seconds since the" + , " epoch when the test invocation started and ended." + ] + , "runfiles_doc": + [ "A tree consisting of the artifacts staged at the name of the test." + , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" + , "argument, this gives an easy way of defining test suites." + ] + , "implicit": {"runner": ["runner"], "summarizer": ["summarizer"]} + , "imports": + { "test-result": "test-result" + , "action": "test-action" + , "stage": ["./", "../..", "stage_singleton_field"] + , "host transition": ["transitions", "maybe for host"] + , "target properties": ["transitions", "target properties"] + } + , "config_transitions": + { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + } + , "expression": + { "type": "let*" + , "bindings": + [ [ "test.sh" + , { "type": "context" + , "msg": "Expecting 'test' to specify precisely one file containing a shell script" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", "test"] + , ["location", "test.sh"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + } + ] + , [ "name" + , { "type": "assert_non_empty" + , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" + , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} + } + ] + , ["keep", {"type": "FIELD", "name": "keep"}] + , ["runner", {"type": "FIELD", "name": "runner"}] + , ["deps-fieldname", "deps"] + , [ "deps-transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + , [ "target properties" + , {"type": "CALL_EXPRESSION", "name": "target properties"} + ] + ] + , "body": + { "type": "if" + , "cond": {"type": "var", "name": "RUNS_PER_TEST"} + , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} + , "then": + { "type": "let*" + , "bindings": + [ [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "ATTEMPT" + , "range": + { "type": "range" + , "$1": {"type": "var", "name": "RUNS_PER_TEST"} + } + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "ATTEMPT"} + , "value": + { "type": "TREE" + , "$1": {"type": "CALL_EXPRESSION", "name": "action"} + } + } + } + } + ] + , [ "summarizer" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "x" + , "range": {"type": "FIELD", "name": "summarizer"} + , "body": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "x" + , "range": + { "type": "values" + , "$1": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "x"} + } + } + , "body": + { "type": "singleton_map" + , "key": "summarizer" + , "value": {"type": "var", "name": "x"} + } + } + } + } + } + ] + , [ "summary" + , { "type": "ACTION" + , "inputs": + { "type": "map_union" + , "$1": + [ {"type": "var", "name": "attempts"} + , {"type": "var", "name": "summarizer"} + ] + } + , "outs": + ["stdout", "stderr", "result", "time-start", "time-stop"] + , "cmd": ["./summarizer"] + } + ] + , [ "artifacts" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "summary"} + , { "type": "singleton_map" + , "key": "work" + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} + } + ] + } + ] + , [ "runfiles" + , { "type": "singleton_map" + , "key": {"type": "var", "name": "name"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": {"type": "var", "name": "artifacts"} + , "runfiles": {"type": "var", "name": "runfiles"} + } + } + } + } + } +} diff --git a/rules/shell/test/TARGETS b/rules/shell/test/TARGETS new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/rules/shell/test/TARGETS @@ -0,0 +1 @@ +{} diff --git a/rules/shell/test/runner b/rules/shell/test/runner new file mode 100755 index 0000000..97c75ef --- /dev/null +++ b/rules/shell/test/runner @@ -0,0 +1,59 @@ +#!/bin/sh +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ensure all required outputs are present +touch stdout +touch stderr +RESULT=UNKNOWN +echo "${RESULT}" > result +echo UNKNOWN > time-start +echo UNKNOWN > time-stop + +mkdir scratch +export TEST_TMPDIR=$(realpath scratch) +export TMPDIR="${TEST_TMPDIR}" +export LD_LIBRARY_PATH=$(realpath ./libs):$LD_LIBRARY_PATH + +# Change to the working directory; note: while unlikely, the test +# might not have test data, so we have to ensure the presence of +# the work directory. +mkdir -p work +cd work + +date +%s > ../time-start +# TODO: +# - proper wrapping with timeout +if sh ../test.sh > ../stdout 2> ../stderr +then + RESULT=PASS +else + RESULT=FAIL +fi +date +%s > ../time-stop + +# Ensure all the promissed output files in the work directory +# are present, even if the test failed to create them. +for f in "$@" +do + touch "./${f}" +done + +echo "${RESULT}" > ../result + +if [ "${RESULT}" '!=' PASS ] +then + exit 1; +fi diff --git a/rules/shell/test/summarizer b/rules/shell/test/summarizer new file mode 100755 index 0000000..0b5e656 --- /dev/null +++ b/rules/shell/test/summarizer @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import time + +RESULTS = {} + +time_start = time.time() +time_stop = 0 + +for attempt in os.listdir("."): + if os.path.isdir(attempt): + with open(os.path.join(attempt, "result")) as f: + result = f.read().strip() + RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] + try: + with open(os.path.join(attempt, "time-start")) as f: + time_start = min(time_start, float(f.read().strip())) + except: + pass + try: + with open(os.path.join(attempt, "time-stop")) as f: + time_stop = max(time_start, float(f.read().strip())) + except: + pass + +result = "UNKNOWN" +if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): + if not RESULTS.get("FAIL"): + result = "PASS" + elif not RESULTS.get("PASS"): + result = "FAIL" + else: + result = "FLAKY" +with open("result", "w") as f: + f.write("%s\n" % (result,)) + +with open("time-start", "w") as f: + f.write("%d\n" % (time_start,)) +with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop,)) + +with open("stdout", "w") as f: + f.write("Summary: %s\n\n" % (result,)) + f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) + f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) + RESULTS.pop("PASS", None) + RESULTS.pop("FAIL", None) + if RESULTS: + f.write("\nother results: %r\n" % (RESULTS,)) + +with open("stderr", "w") as f: + pass |