diff options
Diffstat (limited to 'rules/shell/test')
| -rw-r--r-- | rules/shell/test/EXPRESSIONS | 380 | ||||
| -rw-r--r-- | rules/shell/test/RULES | 367 | ||||
| -rw-r--r-- | rules/shell/test/TARGETS | 6 | ||||
| -rwxr-xr-x | rules/shell/test/runner | 60 | ||||
| -rwxr-xr-x | rules/shell/test/summarizer | 84 |
5 files changed, 897 insertions, 0 deletions
diff --git a/rules/shell/test/EXPRESSIONS b/rules/shell/test/EXPRESSIONS new file mode 100644 index 0000000..9791911 --- /dev/null +++ b/rules/shell/test/EXPRESSIONS @@ -0,0 +1,380 @@ +{ "test-action": + { "vars": + [ "TEST_ENV" + , "TIMEOUT_SCALE" + , "ATTEMPT" + , "name" + , "test.sh" + , "keep" + , "keep-dirs" + , "runner" + , "deps-fieldname" + , "deps-transition" + , "target properties" + ] + , "imports": + { "artifacts_list": ["./", "../..", "field_artifacts_list"] + , "runfiles_list": ["./", "../..", "field_runfiles_list"] + , "map_provider": ["./", "../..", "field_map_provider"] + , "default-TOOLCHAIN": ["./", "../../CC", "default-TOOLCHAIN"] + , "default-NON_SYSTEM_TOOLS": + ["./", "../../CC", "default-NON_SYSTEM_TOOLS"] + , "default-PATH": ["./", "../../CC", "default-PATH"] + , "default-sh": ["./", "..", "default-sh"] + , "sh prolog": ["shell", "prolog"] + } + , "expression": + { "type": "let*" + , "bindings": + [ [ "runner" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "runner" + , "range": {"type": "var", "name": "runner"} + , "body": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "runner" + , "range": + { "type": "values" + , "$1": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "runner"} + } + } + , "body": {"type": "env", "vars": ["runner"]} + } + } + } + } + ] + , ["toolchain dirname", "toolchain"] + , [ "toolchain" + , { "type": "to_subdir" + , "subdir": {"type": "var", "name": "toolchain dirname"} + , "$1": {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"} + } + ] + , ["sh", {"type": "CALL_EXPRESSION", "name": "default-sh"}] + , [ "NON_SYSTEM_TOOLS" + , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} + ] + , [ "sh from outside" + , { "type": "if" + , "cond": + { "type": "lookup" + , "key": "sh" + , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} + } + , "then": + { "type": "join" + , "$1": ["./toolchain/", {"type": "var", "name": "sh"}] + } + , "else": {"type": "var", "name": "sh"} + } + ] + , [ "sh from workdir" + , { "type": "if" + , "cond": + { "type": "lookup" + , "key": "sh" + , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} + } + , "then": + { "type": "join" + , "$1": ["../toolchain/", {"type": "var", "name": "sh"}] + } + , "else": {"type": "var", "name": "sh"} + } + ] + , [ "with-env" + , { "type": "singleton_map" + , "key": "with-env" + , "value": + { "type": "BLOB" + , "data": + { "type": "join" + , "separator": "\n" + , "$1": + { "type": "++" + , "$1": + [ { "type": "let*" + , "bindings": [["fieldname", "defaults"]] + , "body": {"type": "CALL_EXPRESSION", "name": "sh prolog"} + } + , [ "" + , { "type": "join_cmd" + , "$1": + { "type": "++" + , "$1": [["./runner"], {"type": "var", "name": "keep"}] + } + } + , "" + ] + ] + } + } + } + } + ] + , [ "invocation cmd" + , [{"type": "var", "name": "sh from workdir"}, "../test.sh"] + ] + , [ "invocation" + , { "type": "singleton_map" + , "key": "invocation" + , "value": + { "type": "BLOB" + , "data": + { "type": "join_cmd" + , "$1": {"type": "var", "name": "invocation cmd"} + } + } + } + ] + , [ "test_env" + , {"type": "var", "name": "TEST_ENV", "default": {"type": "empty_map"}} + ] + , [ "test_env PATH" + , { "type": "lookup" + , "key": "PATH" + , "map": {"type": "var", "name": "test_env"} + } + ] + , [ "PATH" + , { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ { "type": "if" + , "cond": {"type": "var", "name": "test_env PATH"} + , "then": [{"type": "var", "name": "test_env PATH"}] + } + , {"type": "CALL_EXPRESSION", "name": "default-PATH"} + ] + } + } + ] + , [ "test_env" + , { "type": "if" + , "cond": {"type": "var", "name": "PATH"} + , "then": + { "type": "map_union" + , "$1": + [ {"type": "var", "name": "test_env"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": {"type": "var", "name": "PATH"} + } + ] + } + , "else": {"type": "var", "name": "test_env"} + } + ] + , [ "deps" + , { "type": "TREE" + , "$1": + { "type": "disjoint_map_union" + , "msg": + [ "Field" + , {"type": "var", "name": "deps-fieldname"} + , "has to stage in a conflict free way" + ] + , "$1": + { "type": "++" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": + [ {"type": "CALL_EXPRESSION", "name": "runfiles_list"} + , {"type": "CALL_EXPRESSION", "name": "artifacts_list"} + ] + } + } + } + } + ] + , [ "run-libs" + , { "type": "TREE" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", {"type": "var", "name": "deps-fieldname"}] + , ["provider", "run-libs"] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} + } + } + ] + , [ "attempt marker" + , { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "ATTEMPT"} + , "$2": null + } + , "then": {"type": "empty_map"} + , "else": + { "type": "singleton_map" + , "key": "ATTEMPT" + , "value": + {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}} + } + } + ] + , [ "outs" + , { "type": "++" + , "$1": + [ ["result", "stdout", "stderr", "time-start", "time-stop", "pwd"] + , { "type": "foreach" + , "var": "filename" + , "range": {"type": "var", "name": "keep"} + , "body": + { "type": "join" + , "$1": ["work/", {"type": "var", "name": "filename"}] + } + } + ] + } + ] + , [ "out_dirs" + , { "type": "foreach" + , "var": "dir_path" + , "range": {"type": "var", "name": "keep-dirs"} + , "body": + { "type": "join" + , "$1": ["work/", {"type": "var", "name": "dir_path"}] + } + } + ] + , [ "inputs" + , { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "work" + , "value": {"type": "var", "name": "deps"} + } + , { "type": "singleton_map" + , "key": "libs" + , "value": {"type": "var", "name": "run-libs"} + } + , {"type": "var", "name": "toolchain"} + , {"type": "var", "name": "with-env"} + , {"type": "var", "name": "runner"} + , {"type": "var", "name": "invocation"} + , {"type": "var", "name": "test.sh"} + , {"type": "var", "name": "attempt marker"} + ] + } + ] + , ["cmd", [{"type": "var", "name": "sh from outside"}, "with-env"]] + , [ "test_env" + , { "type": "map_union" + , "$1": + [ { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "ATTEMPT"} + , "$2": null + } + , "then": {"type": "empty_map"} + , "else": + { "type": "singleton_map" + , "key": "TEST_RUN_NUMBER" + , "value": {"type": "var", "name": "ATTEMPT"} + } + } + , {"type": "var", "name": "test_env"} + ] + } + ] + ] + , "body": + { "type": "if" + , "cond": + {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null} + , "then": + { "type": "ACTION" + , "outs": {"type": "var", "name": "outs"} + , "out_dirs": {"type": "var", "name": "out_dirs"} + , "inputs": {"type": "var", "name": "inputs"} + , "cmd": {"type": "var", "name": "cmd"} + , "env": {"type": "var", "name": "test_env"} + , "may_fail": ["test"] + , "fail_message": + { "type": "join" + , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} + } + , "else": + { "type": "ACTION" + , "outs": {"type": "var", "name": "outs"} + , "out_dirs": {"type": "var", "name": "out_dirs"} + , "inputs": {"type": "var", "name": "inputs"} + , "cmd": {"type": "var", "name": "cmd"} + , "env": {"type": "var", "name": "test_env"} + , "may_fail": ["test"] + , "no_cache": ["test"] + , "fail_message": + { "type": "join" + , "$1": + [ "shell test " + , {"type": "var", "name": "name"} + , " failed (Run " + , {"type": "var", "name": "ATTEMPT"} + , ")" + ] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} + } + } + } + } +, "test-result": + { "vars": + [ "TEST_ENV" + , "TIMEOUT_SCALE" + , "name" + , "test.sh" + , "keep" + , "keep-dirs" + , "runner" + , "deps-fieldname" + , "deps-transition" + , "target properties" + , "lint" + ] + , "imports": {"action": "test-action"} + , "expression": + { "type": "let*" + , "bindings": + [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}] + , [ "runfiles" + , { "type": "singleton_map" + , "key": {"type": "var", "name": "name"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "test-results"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": {"type": "var", "name": "test-results"} + , "runfiles": {"type": "var", "name": "runfiles"} + , "provides": {"type": "env", "vars": ["lint"]} + } + } + } +} diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES new file mode 100644 index 0000000..1c069dc --- /dev/null +++ b/rules/shell/test/RULES @@ -0,0 +1,367 @@ +{ "summarizer": + { "doc": + ["Specify a test summarizer together with the required additional fields"] + , "target_fields": ["summarizer"] + , "string_fields": ["artifacts"] + , "imports": {"stage": ["./", "../..", "stage_singleton_field"]} + , "field_doc": + { "summarizer": ["The single artifact acting as summarizer"] + , "artifacts": + [ "Any additional artifacts, besides \"result\", the summaries needs from" + , "the individual test results" + ] + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "let*" + , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + , "provides": + { "type": "singleton_map" + , "key": "artifacts" + , "value": {"type": "FIELD", "name": "artifacts"} + } + } + } +, "script": + { "doc": ["Shell test, given by a test script"] + , "target_fields": ["deps", "test"] + , "string_fields": ["keep", "keep-dirs", "name"] + , "config_vars": + [ "ARCH" + , "HOST_ARCH" + , "RUNS_PER_TEST" + , "TEST_ENV" + , "TIMEOUT_SCALE" + , "TARGET_ARCH" + , "ARCH_DISPATCH" + , "TEST_SUMMARY_EXECUTION_PROPERTIES" + , "LINT" + ] + , "field_doc": + { "test": + [ "The shell script for the test, launched with sh." + , "" + , "An empty directory is created to store any temporary files needed" + , "by the test, and it is made available in the environment variable" + , "TEST_TMPDIR. The test should not assume write permissions" + , "outside the working directory and the TEST_TMPDIR." + , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "If the configuration variable RUNS_PER_TEST is set, the environment" + , "variable TEST_RUN_NUMBER will also be set to the number of the attempt," + , "counting from 0." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." + ] + , "name": + [ "A name for the test, used in reporting, as well as for staging" + , "the test result tree in the runfiles" + ] + , "keep": + [ "List of names (relative to the test working directory) of files that" + , "the test might generate that should be kept as part of the output." + , "This might be useful for further analysis of the test" + ] + , "keep-dirs": + [ "List of names (relative to the test working directory) of directories" + , "that the test might generate that should be kept as part of the" + , "output. This might be useful for further analysis of the test" + ] + , "deps": + [ "Any targets that should be staged (with artifacts and runfiles) into" + , "the tests working directory" + ] + , "runner": + [ "The test runner which starts the actual test script after providing" + , "the respective environment. The runner also takes care of capturing" + , "stdout/stderr, timing information, and ensure the presence of the" + , "files to keep even if the script failed to produce them." + ] + , "summarizer": + [ "Tool to aggregate the results of individual test runs (for flakyness" + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." + ] + , "defaults": ["The shell toolcahin to use."] + } + , "config_doc": + { "RUNS_PER_TEST": + [ "The number of times the test should be run in order to detect flakyness." + , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." + ] + , "TEST_ENV": ["Additional environment for executing the test runner."] + , "TIMEOUT_SCALE": + ["Factor on how to scale the timeout for this test. Defaults to 1.0."] + , "TARGET_ARCH": + [ "The architecture to build the test for." + , "" + , "Will only be honored, if that architecture is available in the" + , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" + , "on the host architecture." + ] + , "ARCH_DISPATCH": + [ "Map of architectures to execution properties that ensure execution" + , "on that architecture. Only the actual test binary will be run with" + , "the specified execution properties (i.e., on the target architecture);" + , "all building will be done on the host architecture." + ] + , "TEST_SUMMARY_EXECUTION_PROPERTIES": + [ "Additional remote-execution properties for the test-summarizing action" + , "in case RUNS_PER_TEST is set; defaults to the empty map." + ] + } + , "tainted": ["test"] + , "artifacts_doc": + [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" + , " generating test reports." + , "stdout/stderr: Any output the invocation of the test binary produced on" + , " the respective file descriptor" + , "work: In this directory, all the files specified to \"keep\" and" + , " \"keep-dirs\" are staged" + , "time-start/time-stop: The time (decimally coded) in seconds since the" + , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" + ] + , "runfiles_doc": + [ "A tree consisting of the artifacts staged at the name of the test." + , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" + , "argument, this gives an easy way of defining test suites." + ] + , "implicit": + { "runner": ["runner"] + , "summarizer": ["summarizer"] + , "defaults": [["./", "..", "defaults"]] + } + , "imports": + { "test-result": "test-result" + , "action": "test-action" + , "stage": ["./", "../..", "stage_singleton_field"] + , "host transition": ["transitions", "maybe for host"] + , "target properties": ["transitions", "target properties"] + , "default-PATH": ["./", "../../CC", "default-PATH"] + , "field_list": ["", "field_list_provider"] + } + , "config_transitions": + { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + } + , "expression": + { "type": "let*" + , "bindings": + [ [ "test.sh" + , { "type": "context" + , "msg": "Expecting 'test' to specify precisely one file containing a shell script" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", "test"] + , ["location", "test.sh"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + } + ] + , [ "name" + , { "type": "assert_non_empty" + , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" + , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} + } + ] + , ["keep", {"type": "FIELD", "name": "keep"}] + , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}] + , ["runner", {"type": "FIELD", "name": "runner"}] + , ["deps-fieldname", "deps"] + , [ "deps-transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + , [ "target properties" + , {"type": "CALL_EXPRESSION", "name": "target properties"} + ] + , [ "lint" + , { "type": "if" + , "cond": {"type": "var", "name": "LINT"} + , "then": + { "type": "let*" + , "bindings": + [ ["fieldname", "deps"] + , ["provider", "lint"] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + } + ] + ] + , "body": + { "type": "if" + , "cond": {"type": "var", "name": "RUNS_PER_TEST"} + , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} + , "then": + { "type": "let*" + , "bindings": + [ [ "attempts (plain)" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "ATTEMPT" + , "range": + { "type": "range" + , "$1": {"type": "var", "name": "RUNS_PER_TEST"} + } + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "ATTEMPT"} + , "value": {"type": "CALL_EXPRESSION", "name": "action"} + } + } + } + ] + , [ "summarizer" + , { "type": "let*" + , "bindings": + [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] + , [ "attempts (for summary)" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "TREE" + , "$1": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} + } + } + } + } + } + } + } + } + ] + , [ "summary PATH" + , { "type": "join" + , "separator": ":" + , "$1": {"type": "CALL_EXPRESSION", "name": "default-PATH"} + } + ] + , [ "summary" + , { "type": "ACTION" + , "inputs": + { "type": "map_union" + , "$1": + [ {"type": "var", "name": "attempts (for summary)"} + , {"type": "var", "name": "summarizer"} + ] + } + , "outs": + ["stdout", "stderr", "result", "time-start", "time-stop"] + , "cmd": ["./summarizer"] + , "execution properties": + { "type": "var" + , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" + , "default": {"type": "empty_map"} + } + , "env": + { "type": "if" + , "cond": {"type": "var", "name": "summary PATH"} + , "then": + { "type": "singleton_map" + , "key": "PATH" + , "value": {"type": "var", "name": "summary PATH"} + } + , "else": {"type": "empty_map"} + } + } + ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] + , [ "artifacts" + , { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} + , { "type": "singleton_map" + , "key": "work" + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} + } + ] + } + ] + , [ "runfiles" + , { "type": "singleton_map" + , "key": {"type": "var", "name": "name"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": {"type": "var", "name": "artifacts"} + , "runfiles": {"type": "var", "name": "runfiles"} + , "provides": {"type": "env", "vars": ["lint"]} + } + } + } + } + } +} diff --git a/rules/shell/test/TARGETS b/rules/shell/test/TARGETS new file mode 100644 index 0000000..5a4b849 --- /dev/null +++ b/rules/shell/test/TARGETS @@ -0,0 +1,6 @@ +{ "summarizer": + { "type": "summarizer" + , "summarizer": [["FILE", null, "summarizer"]] + , "artifacts": ["time-start", "time-stop"] + } +} diff --git a/rules/shell/test/runner b/rules/shell/test/runner new file mode 100755 index 0000000..f2f5766 --- /dev/null +++ b/rules/shell/test/runner @@ -0,0 +1,60 @@ +#!/bin/sh +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ensure all required outputs are present +touch stdout +touch stderr +RESULT=UNKNOWN +echo "${RESULT}" > result +echo UNKNOWN > time-start +echo UNKNOWN > time-stop +pwd > pwd + +mkdir scratch +export TEST_TMPDIR=$(realpath scratch) +export TMPDIR="${TEST_TMPDIR}" +export LD_LIBRARY_PATH=$(realpath ./libs):$LD_LIBRARY_PATH + +# Change to the working directory; note: while unlikely, the test +# might not have test data, so we have to ensure the presence of +# the work directory. +mkdir -p work +cd work + +date +%s > ../time-start +# TODO: +# - proper wrapping with timeout +if . ../invocation > ../stdout 2> ../stderr +then + RESULT=PASS +else + RESULT=FAIL +fi +date +%s > ../time-stop + +# Ensure all the promissed output files in the work directory +# are present, even if the test failed to create them. +for f in "$@" +do + touch "./${f}" +done + +echo "${RESULT}" > ../result + +if [ "${RESULT}" '!=' PASS ] +then + exit 1; +fi diff --git a/rules/shell/test/summarizer b/rules/shell/test/summarizer new file mode 100755 index 0000000..39b66be --- /dev/null +++ b/rules/shell/test/summarizer @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from typing import Any, Dict, List + +g_RESULTS: Dict[str, List[Any]] = {} +g_COUNT: float = 0 + +PASS_count: float = 0 +PASS_time: float = 0 + +time_start: float = time.time() +time_stop: float = 0 + +for attempt in os.listdir("."): + if os.path.isdir(attempt): + g_COUNT += 1 + with open(os.path.join(attempt, "result")) as f: + result = f.read().strip() + g_RESULTS[result] = g_RESULTS.get(result, []) + [int(attempt)] + try: + with open(os.path.join(attempt, "time-start")) as f: + start = float(f.read().strip()) + time_start = min(time_start, start) + except: + pass + try: + with open(os.path.join(attempt, "time-stop")) as f: + stop = float(f.read().strip()) + time_stop = max(time_start, stop) + except: + pass + if (start > 0) and (stop >= start) and result == "PASS": + PASS_count += 1 + PASS_time += stop - start + +result: str = "UNKNOWN" +if set(g_RESULTS.keys()) <= set(["PASS", "FAIL"]): + if not g_RESULTS.get("FAIL"): + result = "PASS" + elif not g_RESULTS.get("PASS"): + result = "FAIL" + else: + result = "FLAKY" +with open("result", "w") as f: + f.write("%s\n" % (result, )) + +with open("time-start", "w") as f: + f.write("%d\n" % (time_start, )) +with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop, )) + +with open("stdout", "w") as f: + f.write("Summary: %s\n\n" % (result, )) + f.write("PASS: %s\n" % (sorted(g_RESULTS.get("PASS", [])), )) + failures =sorted(g_RESULTS.get("FAIL", [])) + f.write("FAIL: %s\n" % (failures, )) + g_RESULTS.pop("PASS", None) + g_RESULTS.pop("FAIL", None) + if g_RESULTS: + f.write("\nother results: %r\n" % (g_RESULTS, )) + if result == "FLAKY": + f.write("\nFailure rate %5.2f%%\n" % (100.0 * len(failures) / g_COUNT)) + if PASS_count >= 2: + f.write("\nAverage time of a passed test instance: %.1fs\n" + % (PASS_time / PASS_count)) + +with open("stderr", "w") as f: + pass |