diff options
Diffstat (limited to 'rules/shell/test/RULES')
| -rw-r--r-- | rules/shell/test/RULES | 367 |
1 files changed, 367 insertions, 0 deletions
diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES new file mode 100644 index 0000000..1c069dc --- /dev/null +++ b/rules/shell/test/RULES @@ -0,0 +1,367 @@ +{ "summarizer": + { "doc": + ["Specify a test summarizer together with the required additional fields"] + , "target_fields": ["summarizer"] + , "string_fields": ["artifacts"] + , "imports": {"stage": ["./", "../..", "stage_singleton_field"]} + , "field_doc": + { "summarizer": ["The single artifact acting as summarizer"] + , "artifacts": + [ "Any additional artifacts, besides \"result\", the summaries needs from" + , "the individual test results" + ] + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "let*" + , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + , "provides": + { "type": "singleton_map" + , "key": "artifacts" + , "value": {"type": "FIELD", "name": "artifacts"} + } + } + } +, "script": + { "doc": ["Shell test, given by a test script"] + , "target_fields": ["deps", "test"] + , "string_fields": ["keep", "keep-dirs", "name"] + , "config_vars": + [ "ARCH" + , "HOST_ARCH" + , "RUNS_PER_TEST" + , "TEST_ENV" + , "TIMEOUT_SCALE" + , "TARGET_ARCH" + , "ARCH_DISPATCH" + , "TEST_SUMMARY_EXECUTION_PROPERTIES" + , "LINT" + ] + , "field_doc": + { "test": + [ "The shell script for the test, launched with sh." + , "" + , "An empty directory is created to store any temporary files needed" + , "by the test, and it is made available in the environment variable" + , "TEST_TMPDIR. The test should not assume write permissions" + , "outside the working directory and the TEST_TMPDIR." + , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "If the configuration variable RUNS_PER_TEST is set, the environment" + , "variable TEST_RUN_NUMBER will also be set to the number of the attempt," + , "counting from 0." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." + ] + , "name": + [ "A name for the test, used in reporting, as well as for staging" + , "the test result tree in the runfiles" + ] + , "keep": + [ "List of names (relative to the test working directory) of files that" + , "the test might generate that should be kept as part of the output." + , "This might be useful for further analysis of the test" + ] + , "keep-dirs": + [ "List of names (relative to the test working directory) of directories" + , "that the test might generate that should be kept as part of the" + , "output. This might be useful for further analysis of the test" + ] + , "deps": + [ "Any targets that should be staged (with artifacts and runfiles) into" + , "the tests working directory" + ] + , "runner": + [ "The test runner which starts the actual test script after providing" + , "the respective environment. The runner also takes care of capturing" + , "stdout/stderr, timing information, and ensure the presence of the" + , "files to keep even if the script failed to produce them." + ] + , "summarizer": + [ "Tool to aggregate the results of individual test runs (for flakyness" + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." + ] + , "defaults": ["The shell toolcahin to use."] + } + , "config_doc": + { "RUNS_PER_TEST": + [ "The number of times the test should be run in order to detect flakyness." + , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." + ] + , "TEST_ENV": ["Additional environment for executing the test runner."] + , "TIMEOUT_SCALE": + ["Factor on how to scale the timeout for this test. Defaults to 1.0."] + , "TARGET_ARCH": + [ "The architecture to build the test for." + , "" + , "Will only be honored, if that architecture is available in the" + , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" + , "on the host architecture." + ] + , "ARCH_DISPATCH": + [ "Map of architectures to execution properties that ensure execution" + , "on that architecture. Only the actual test binary will be run with" + , "the specified execution properties (i.e., on the target architecture);" + , "all building will be done on the host architecture." + ] + , "TEST_SUMMARY_EXECUTION_PROPERTIES": + [ "Additional remote-execution properties for the test-summarizing action" + , "in case RUNS_PER_TEST is set; defaults to the empty map." + ] + } + , "tainted": ["test"] + , "artifacts_doc": + [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for" + , " generating test reports." + , "stdout/stderr: Any output the invocation of the test binary produced on" + , " the respective file descriptor" + , "work: In this directory, all the files specified to \"keep\" and" + , " \"keep-dirs\" are staged" + , "time-start/time-stop: The time (decimally coded) in seconds since the" + , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" + ] + , "runfiles_doc": + [ "A tree consisting of the artifacts staged at the name of the test." + , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" + , "argument, this gives an easy way of defining test suites." + ] + , "implicit": + { "runner": ["runner"] + , "summarizer": ["summarizer"] + , "defaults": [["./", "..", "defaults"]] + } + , "imports": + { "test-result": "test-result" + , "action": "test-action" + , "stage": ["./", "../..", "stage_singleton_field"] + , "host transition": ["transitions", "maybe for host"] + , "target properties": ["transitions", "target properties"] + , "default-PATH": ["./", "../../CC", "default-PATH"] + , "field_list": ["", "field_list_provider"] + } + , "config_transitions": + { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + } + , "expression": + { "type": "let*" + , "bindings": + [ [ "test.sh" + , { "type": "context" + , "msg": "Expecting 'test' to specify precisely one file containing a shell script" + , "$1": + { "type": "let*" + , "bindings": + [ ["fieldname", "test"] + , ["location", "test.sh"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + } + ] + , [ "name" + , { "type": "assert_non_empty" + , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)" + , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}} + } + ] + , ["keep", {"type": "FIELD", "name": "keep"}] + , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}] + , ["runner", {"type": "FIELD", "name": "runner"}] + , ["deps-fieldname", "deps"] + , [ "deps-transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + , [ "target properties" + , {"type": "CALL_EXPRESSION", "name": "target properties"} + ] + , [ "lint" + , { "type": "if" + , "cond": {"type": "var", "name": "LINT"} + , "then": + { "type": "let*" + , "bindings": + [ ["fieldname", "deps"] + , ["provider", "lint"] + , ["transition", {"type": "var", "name": "deps-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + } + ] + ] + , "body": + { "type": "if" + , "cond": {"type": "var", "name": "RUNS_PER_TEST"} + , "else": {"type": "CALL_EXPRESSION", "name": "test-result"} + , "then": + { "type": "let*" + , "bindings": + [ [ "attempts (plain)" + , { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "ATTEMPT" + , "range": + { "type": "range" + , "$1": {"type": "var", "name": "RUNS_PER_TEST"} + } + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "ATTEMPT"} + , "value": {"type": "CALL_EXPRESSION", "name": "action"} + } + } + } + ] + , [ "summarizer" + , { "type": "let*" + , "bindings": + [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] + , [ "attempts (for summary)" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "TREE" + , "$1": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} + } + } + } + } + } + } + } + } + ] + , [ "summary PATH" + , { "type": "join" + , "separator": ":" + , "$1": {"type": "CALL_EXPRESSION", "name": "default-PATH"} + } + ] + , [ "summary" + , { "type": "ACTION" + , "inputs": + { "type": "map_union" + , "$1": + [ {"type": "var", "name": "attempts (for summary)"} + , {"type": "var", "name": "summarizer"} + ] + } + , "outs": + ["stdout", "stderr", "result", "time-start", "time-stop"] + , "cmd": ["./summarizer"] + , "execution properties": + { "type": "var" + , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES" + , "default": {"type": "empty_map"} + } + , "env": + { "type": "if" + , "cond": {"type": "var", "name": "summary PATH"} + , "then": + { "type": "singleton_map" + , "key": "PATH" + , "value": {"type": "var", "name": "summary PATH"} + } + , "else": {"type": "empty_map"} + } + } + ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] + , [ "artifacts" + , { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} + , { "type": "singleton_map" + , "key": "work" + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "attempts"}} + } + ] + } + ] + , [ "runfiles" + , { "type": "singleton_map" + , "key": {"type": "var", "name": "name"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": {"type": "var", "name": "artifacts"} + , "runfiles": {"type": "var", "name": "runfiles"} + , "provides": {"type": "env", "vars": ["lint"]} + } + } + } + } + } +} |