diff options
Diffstat (limited to 'shell/test')
| -rw-r--r-- | shell/test/EXPRESSIONS | 226 | ||||
| -rw-r--r-- | shell/test/RULES | 190 | ||||
| -rw-r--r-- | shell/test/TARGETS | 1 | ||||
| -rwxr-xr-x | shell/test/test_runner.sh | 59 | ||||
| -rwxr-xr-x | shell/test/test_summary.py | 67 |
5 files changed, 0 insertions, 543 deletions
diff --git a/shell/test/EXPRESSIONS b/shell/test/EXPRESSIONS deleted file mode 100644 index ea36b32..0000000 --- a/shell/test/EXPRESSIONS +++ /dev/null @@ -1,226 +0,0 @@ -{ "test-action": - { "vars": - [ "TEST_ENV" - , "ATTEMPT" - , "name" - , "test.sh" - , "keep" - , "runner" - , "deps-fieldname" - , "deps-transition" - ] - , "imports": - { "artifacts_list": ["./", "../..", "field_artifacts_list"] - , "runfiles_list": ["./", "../..", "field_runfiles_list"] - , "map_provider": ["./", "../..", "field_map_provider"] - } - , "expression": - { "type": "let*" - , "bindings": - [ [ "runner" - , { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "runner" - , "range": {"type": "var", "name": "runner"} - , "body": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "runner" - , "range": - { "type": "values" - , "$1": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "runner"} - } - } - , "body": {"type": "env", "vars": ["runner"]} - } - } - } - } - ] - , [ "deps" - , { "type": "TREE" - , "$1": - { "type": "disjoint_map_union" - , "msg": - [ "Field" - , {"type": "var", "name": "deps-fieldname"} - , "has to stage in a conflict free way" - ] - , "$1": - { "type": "++" - , "$1": - { "type": "let*" - , "bindings": - [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] - , ["transition", {"type": "var", "name": "deps-transition"}] - ] - , "body": - [ {"type": "CALL_EXPRESSION", "name": "runfiles_list"} - , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} - ] - } - } - } - } - ] - , [ "run-libs" - , { "type": "TREE" - , "$1": - { "type": "let*" - , "bindings": - [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] - , ["provider", "run-libs"] - , ["transition", {"type": "var", "name": "deps-transition"}] - ] - , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} - } - } - ] - , [ "attempt marker" - , { "type": "if" - , "cond": - { "type": "==" - , "$1": {"type": "var", "name": "ATTEMPT"} - , "$2": null - } - , "then": {"type": "empty_map"} - , "else": - { "type": "singleton_map" - , "key": "ATTEMPT" - , "value": - {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}} - } - } - ] - , [ "outs" - , { "type": "++" - , "$1": - [ ["result", "stdout", "stderr", "time-start", "time-stop"] - , { "type": "foreach" - , "var": "filename" - , "range": {"type": "var", "name": "keep"} - , "body": - { "type": "join" - , "$1": ["work/", {"type": "var", "name": "filename"}] - } - } - ] - } - ] - , [ "inputs" - , { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "work" - , "value": {"type": "var", "name": "deps"} - } - , { "type": "singleton_map" - , "key": "libs" - , "value": {"type": "var", "name": "run-libs"} - } - , {"type": "var", "name": "runner"} - , {"type": "var", "name": "test.sh"} - , {"type": "var", "name": "attempt marker"} - ] - } - ] - , [ "cmd" - , {"type": "++", "$1": [["./runner"], {"type": "var", "name": "keep"}]} - ] - , [ "test_env" - , { "type": "map_union" - , "$1": - [ { "type": "if" - , "cond": - { "type": "==" - , "$1": {"type": "var", "name": "ATTEMPT"} - , "$2": null - } - , "then": {"type": "empty_map"} - , "else": - { "type": "singelton_map" - , "key": "TEST_RUN_NUMBER" - , "value": {"type": "var", "name": "ATTEMPT"} - } - } - , { "type": "var" - , "name": "TEST_ENV" - , "default": {"type": "empty_map"} - } - ] - } - ] - ] - , "body": - { "type": "if" - , "cond": - {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null} - , "then": - { "type": "ACTION" - , "outs": {"type": "var", "name": "outs"} - , "inputs": {"type": "var", "name": "inputs"} - , "cmd": {"type": "var", "name": "cmd"} - , "env": {"type": "var", "name": "test_env"} - , "may_fail": ["test"] - , "fail_message": - { "type": "join" - , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"] - } - } - , "else": - { "type": "ACTION" - , "outs": {"type": "var", "name": "outs"} - , "inputs": {"type": "var", "name": "inputs"} - , "cmd": {"type": "var", "name": "cmd"} - , "env": {"type": "var", "name": "test_env"} - , "may_fail": ["test"] - , "no_cache": ["test"] - , "fail_message": - { "type": "join" - , "$1": - [ "shell test " - , {"type": "var", "name": "name"} - , " failed (Run " - , {"type": "var", "name": "ATTEMPT"} - , ")" - ] - } - } - } - } - } -, "test-result": - { "vars": - [ "TEST_ENV" - , "name" - , "test.sh" - , "keep" - , "runner" - , "deps-fieldname" - , "deps-transition" - ] - , "imports": {"action": "test-action"} - , "expression": - { "type": "let*" - , "bindings": - [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}] - , [ "runfiles" - , { "type": "singleton_map" - , "key": {"type": "var", "name": "name"} - , "value": - {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} - } - ] - ] - , "body": - { "type": "RESULT" - , "artifacts": {"type": "var", "name": "test-results"} - , "runfiles": {"type": "var", "name": "runfiles"} - } - } - } -} diff --git a/shell/test/RULES b/shell/test/RULES deleted file mode 100644 index 19f859f..0000000 --- a/shell/test/RULES +++ /dev/null @@ -1,190 +0,0 @@ -{ "script": - { "doc": ["Shell test, given by a test script"] - , "target_fields": ["deps", "test"] - , "string_fields": ["keep", "name"] - , "config_vars": ["ARCH", "HOST_ARCH", "RUNS_PER_TEST", "TEST_ENV"] - , "field_doc": - { "test": - [ "The shell script for the test, launched with sh." - , "" - , "An empty directory is created to store any temporary files needed" - , "by the test, and it is made available in the environment variable" - , "TEST_TMPDIR. The test should not assume write permissions" - , "outside the working directory and the TEST_TMPDIR." - , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." - , "" - , "If the configuration variable RUNS_PER_TEST is set, the environment" - , "variable TEST_RUN_NUMBER will also be set to the number of the attempt," - , "counting from 0." - ] - , "name": - [ "A name for the test, used in reporting, as well as for staging" - , "the test result tree in the runfiles" - ] - , "keep": - [ "List of names (relative to the test working directory) of files that" - , "the test might generate that should be kept as part of the output." - , "This might be useful for further analysis of the test" - ] - , "deps": - [ "Any targets that should be staged (with artifacts and runfiles) into" - , "the tests working directory" - ] - } - , "config_doc": - { "RUNS_PER_TEST": - [ "The number of times the test should be run in order to detect flakyness." - , "If set, no test action will be taken from cache." - ] - , "TEST_ENV": ["Additional environment for executing the test runner."] - } - , "tainted": ["test"] - , "artifacts_doc": - [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" - , " generating test reports." - , "stdout/stderr: Any output the invocation of the test binary produced on" - , " the respective file descriptor" - , "work: In this directory, all the files specified to \"keep\" are staged" - , "time-start/time-stop: The time (decimally coded) in seconds since the" - , " epoch when the test invocation started and ended." - ] - , "runfiles_doc": - [ "A tree consisting of the artifacts staged at the name of the test." - , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" - , "argument, this gives an easy way of defining test suites." - ] - , "implicit": - {"runner": ["test_runner.sh"], "summarizer": ["test_summary.py"]} - , "imports": - { "test-result": "test-result" - , "action": "test-action" - , "stage": ["./", "../..", "stage_singleton_field"] - , "host transition": ["transitions", "for host"] - } - , "config_transitions": - {"deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}]} - , "expression": - { "type": "let*" - , "bindings": - [ [ "test.sh" - , { "type": "context" - , "msg": "Expecting 'test' to specify precisely one file containing a shell script" - , "$1": - { "type": "let*" - , "bindings": [["fieldname", "test"], ["location", "test.sh"]] - , "body": {"type": "CALL_EXPRESSION", "name": "stage"} - } - } - ] - , [ "name" - , { "type": "assert_non_empty" - , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" - , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} - } - ] - , ["keep", {"type": "FIELD", "name": "keep"}] - , ["runner", {"type": "FIELD", "name": "runner"}] - , ["deps-fieldname", "deps"] - , [ "deps-transition" - , {"type": "CALL_EXPRESSION", "name": "host transition"} - ] - ] - , "body": - { "type": "if" - , "cond": {"type": "var", "name": "RUNS_PER_TEST"} - , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} - , "then": - { "type": "let*" - , "bindings": - [ [ "attempts" - , { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "ATTEMPT" - , "range": - { "type": "range" - , "$1": {"type": "var", "name": "RUNS_PER_TEST"} - } - , "body": - { "type": "singleton_map" - , "key": {"type": "var", "name": "ATTEMPT"} - , "value": - { "type": "TREE" - , "$1": {"type": "CALL_EXPRESSION", "name": "action"} - } - } - } - } - ] - , [ "summarizer" - , { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "x" - , "range": {"type": "FIELD", "name": "summarizer"} - , "body": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "x" - , "range": - { "type": "values" - , "$1": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "x"} - } - } - , "body": - { "type": "singleton_map" - , "key": "summarizer" - , "value": {"type": "var", "name": "x"} - } - } - } - } - } - ] - , [ "summary" - , { "type": "ACTION" - , "inputs": - { "type": "map_union" - , "$1": - [ {"type": "var", "name": "attempts"} - , {"type": "var", "name": "summarizer"} - ] - } - , "outs": - ["stdout", "stderr", "result", "time-start", "time-stop"] - , "cmd": ["./summarizer"] - } - ] - , [ "artifacts" - , { "type": "map_union" - , "$1": - [ {"type": "var", "name": "summary"} - , { "type": "singleton_map" - , "key": "work" - , "value": - {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} - } - ] - } - ] - , [ "runfiles" - , { "type": "singleton_map" - , "key": {"type": "var", "name": "name"} - , "value": - {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} - } - ] - ] - , "body": - { "type": "RESULT" - , "artifacts": {"type": "var", "name": "artifacts"} - , "runfiles": {"type": "var", "name": "runfiles"} - } - } - } - } - } -} diff --git a/shell/test/TARGETS b/shell/test/TARGETS deleted file mode 100644 index 0967ef4..0000000 --- a/shell/test/TARGETS +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/shell/test/test_runner.sh b/shell/test/test_runner.sh deleted file mode 100755 index 97c75ef..0000000 --- a/shell/test/test_runner.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# ensure all required outputs are present -touch stdout -touch stderr -RESULT=UNKNOWN -echo "${RESULT}" > result -echo UNKNOWN > time-start -echo UNKNOWN > time-stop - -mkdir scratch -export TEST_TMPDIR=$(realpath scratch) -export TMPDIR="${TEST_TMPDIR}" -export LD_LIBRARY_PATH=$(realpath ./libs):$LD_LIBRARY_PATH - -# Change to the working directory; note: while unlikely, the test -# might not have test data, so we have to ensure the presence of -# the work directory. -mkdir -p work -cd work - -date +%s > ../time-start -# TODO: -# - proper wrapping with timeout -if sh ../test.sh > ../stdout 2> ../stderr -then - RESULT=PASS -else - RESULT=FAIL -fi -date +%s > ../time-stop - -# Ensure all the promissed output files in the work directory -# are present, even if the test failed to create them. -for f in "$@" -do - touch "./${f}" -done - -echo "${RESULT}" > ../result - -if [ "${RESULT}" '!=' PASS ] -then - exit 1; -fi diff --git a/shell/test/test_summary.py b/shell/test/test_summary.py deleted file mode 100755 index 0b5e656..0000000 --- a/shell/test/test_summary.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import time - -RESULTS = {} - -time_start = time.time() -time_stop = 0 - -for attempt in os.listdir("."): - if os.path.isdir(attempt): - with open(os.path.join(attempt, "result")) as f: - result = f.read().strip() - RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] - try: - with open(os.path.join(attempt, "time-start")) as f: - time_start = min(time_start, float(f.read().strip())) - except: - pass - try: - with open(os.path.join(attempt, "time-stop")) as f: - time_stop = max(time_start, float(f.read().strip())) - except: - pass - -result = "UNKNOWN" -if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): - if not RESULTS.get("FAIL"): - result = "PASS" - elif not RESULTS.get("PASS"): - result = "FAIL" - else: - result = "FLAKY" -with open("result", "w") as f: - f.write("%s\n" % (result,)) - -with open("time-start", "w") as f: - f.write("%d\n" % (time_start,)) -with open("time-stop", "w") as f: - f.write("%d\n" % (time_stop,)) - -with open("stdout", "w") as f: - f.write("Summary: %s\n\n" % (result,)) - f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) - f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) - RESULTS.pop("PASS", None) - RESULTS.pop("FAIL", None) - if RESULTS: - f.write("\nother results: %r\n" % (RESULTS,)) - -with open("stderr", "w") as f: - pass |