diff options
| -rw-r--r-- | rules/CC/test/EXPRESSIONS | 141 | ||||
| -rw-r--r-- | rules/CC/test/RULES | 17 | ||||
| -rwxr-xr-x | rules/CC/test/runner | 2 | ||||
| -rw-r--r-- | rules/shell/test/EXPRESSIONS | 2 | ||||
| -rw-r--r-- | rules/shell/test/RULES | 117 | ||||
| -rw-r--r-- | rules/shell/test/TARGETS | 7 | ||||
| -rwxr-xr-x | rules/shell/test/runner | 1 | ||||
| -rwxr-xr-x | rules/shell/test/summarizer | 15 |
8 files changed, 224 insertions, 78 deletions
diff --git a/rules/CC/test/EXPRESSIONS b/rules/CC/test/EXPRESSIONS index fe4918e..27d92f3 100644 --- a/rules/CC/test/EXPRESSIONS +++ b/rules/CC/test/EXPRESSIONS @@ -38,6 +38,7 @@ , "test-args" , "test-data" , "summarizer" + , "summary artifacts" ] , "imports": { "artifacts": ["./", "../..", "field_artifacts"] @@ -173,7 +174,13 @@ [ [ "test-results" , { "type": "ACTION" , "outs": - ["result", "stdout", "stderr", "time-start", "time-stop"] + [ "result" + , "stdout" + , "stderr" + , "time-start" + , "time-stop" + , "pwd" + ] , "inputs": {"type": "var", "name": "test input"} , "cmd": ["./runner"] , "env": @@ -210,7 +217,7 @@ , "then": { "type": "let*" , "bindings": - [ [ "attempts" + [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" @@ -223,54 +230,74 @@ { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} , "value": + { "type": "ACTION" + , "outs": + ["result", "stdout", "stderr", "time-start", "time-stop"] + , "inputs": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "ATTEMPT" + , "value": + { "type": "BLOB" + , "data": {"type": "var", "name": "ATTEMPT"} + } + } + , {"type": "var", "name": "test input"} + ] + } + , "cmd": ["./runner"] + , "env": + { "type": "var" + , "name": "TEST_ENV" + , "default": {"type": "empty_map"} + } + , "may_fail": ["test"] + , "no_cache": ["test"] + , "fail_message": + { "type": "join" + , "$1": + [ "CC test " + , {"type": "var", "name": "test-name"} + , " failed (Run" + , {"type": "var", "name": "ATTEMPT"} + , ")" + ] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": + {"type": "var", "name": "target properties"} + } + } + } + } + ] + , [ "attempts (for summary)" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": { "type": "TREE" , "$1": - { "type": "ACTION" - , "outs": - [ "result" - , "stdout" - , "stderr" - , "time-start" - , "time-stop" - ] - , "inputs": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "ATTEMPT" - , "value": - { "type": "BLOB" - , "data": {"type": "var", "name": "ATTEMPT"} - } + { "type": "map_union" + , "$1": + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} } - , {"type": "var", "name": "test input"} - ] - } - , "cmd": ["./runner"] - , "env": - { "type": "var" - , "name": "TEST_ENV" - , "default": {"type": "empty_map"} - } - , "may_fail": ["test"] - , "no_cache": ["test"] - , "fail_message": - { "type": "join" - , "$1": - [ "CC test " - , {"type": "var", "name": "test-name"} - , " failed (Run" - , {"type": "var", "name": "ATTEMPT"} - , ")" - ] + } } - , "timeout scaling": - { "type": "var" - , "name": "TIMEOUT_SCALE" - , "default": 1.0 - } - , "execution properties": - {"type": "var", "name": "target properties"} } } } @@ -293,7 +320,7 @@ , "inputs": { "type": "map_union" , "$1": - [ {"type": "var", "name": "attempts"} + [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } @@ -317,10 +344,28 @@ } } ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] , [ "artifacts" , { "type": "map_union" , "$1": - [ {"type": "var", "name": "summary"} + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": diff --git a/rules/CC/test/RULES b/rules/CC/test/RULES index 9f3313d..244bca0 100644 --- a/rules/CC/test/RULES +++ b/rules/CC/test/RULES @@ -87,7 +87,8 @@ ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" - , "detection) to an overall test result." + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." ] } , "config_doc": @@ -158,6 +159,7 @@ , " the respective file descriptor" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." @@ -171,6 +173,7 @@ , "host transition": ["transitions", "maybe for host"] , "stage": ["./", "../..", "stage_singleton_field"] , "run_test": "run_test" + , "field_list": ["", "field_list_provider"] } , "config_transitions": { "defaults": [{"type": "CALL_EXPRESSION", "name": "host transition"}] @@ -251,6 +254,18 @@ , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] ] , "body": {"type": "CALL_EXPRESSION", "name": "run_test"} } diff --git a/rules/CC/test/runner b/rules/CC/test/runner index 45a1bee..4984b17 100755 --- a/rules/CC/test/runner +++ b/rules/CC/test/runner @@ -36,6 +36,8 @@ def dump_results() -> None: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) + with open("pwd", "w") as f: + f.write("%s\n" % (os.getcwd(), )) dump_results() diff --git a/rules/shell/test/EXPRESSIONS b/rules/shell/test/EXPRESSIONS index 635a407..fee558a 100644 --- a/rules/shell/test/EXPRESSIONS +++ b/rules/shell/test/EXPRESSIONS @@ -231,7 +231,7 @@ , [ "outs" , { "type": "++" , "$1": - [ ["result", "stdout", "stderr", "time-start", "time-stop"] + [ ["result", "stdout", "stderr", "time-start", "time-stop", "pwd"] , { "type": "foreach" , "var": "filename" , "range": {"type": "var", "name": "keep"} diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES index 91b7637..d9d8587 100644 --- a/rules/shell/test/RULES +++ b/rules/shell/test/RULES @@ -1,4 +1,31 @@ -{ "script": +{ "summarizer": + { "doc": + ["Specify a test summarizer together with the required additional fields"] + , "target_fields": ["summarizer"] + , "string_fields": ["artifacts"] + , "imports": {"stage": ["./", "../..", "stage_singleton_field"]} + , "field_doc": + { "summarizer": ["The single artifact acting as summarizer"] + , "artifacts": + [ "Any additional artifacts, besides \"result\", the summaries needs from" + , "the individual test results" + ] + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "let*" + , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + , "provides": + { "type": "singleton_map" + , "key": "artifacts" + , "value": {"type": "FIELD", "name": "artifacts"} + } + } + } +, "script": { "doc": ["Shell test, given by a test script"] , "target_fields": ["deps", "test"] , "string_fields": ["keep", "name"] @@ -52,7 +79,8 @@ ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" - , "detection) to an overall test result." + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." ] , "defaults": ["The shell toolcahin to use."] } @@ -97,6 +125,7 @@ , "work: In this directory, all the files specified to \"keep\" are staged" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." @@ -115,6 +144,7 @@ , "host transition": ["transitions", "maybe for host"] , "target properties": ["transitions", "target properties"] , "default-PATH": ["./", "../../CC", "default-PATH"] + , "field_list": ["", "field_list_provider"] } , "config_transitions": { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] @@ -162,7 +192,7 @@ , "then": { "type": "let*" , "bindings": - [ [ "attempts" + [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" @@ -174,37 +204,56 @@ , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} - , "value": - { "type": "TREE" - , "$1": {"type": "CALL_EXPRESSION", "name": "action"} - } + , "value": {"type": "CALL_EXPRESSION", "name": "action"} } } } ] , [ "summarizer" + , { "type": "let*" + , "bindings": + [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] + , [ "attempts (for summary)" , { "type": "map_union" , "$1": - { "type": "foreach" - , "var": "x" - , "range": {"type": "FIELD", "name": "summarizer"} + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} , "body": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "x" - , "range": - { "type": "values" + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "TREE" + , "$1": + { "type": "map_union" , "$1": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "x"} + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} + } + } } } - , "body": - { "type": "singleton_map" - , "key": "summarizer" - , "value": {"type": "var", "name": "x"} - } } } } @@ -221,7 +270,7 @@ , "inputs": { "type": "map_union" , "$1": - [ {"type": "var", "name": "attempts"} + [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } @@ -245,10 +294,28 @@ } } ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] , [ "artifacts" , { "type": "map_union" , "$1": - [ {"type": "var", "name": "summary"} + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": diff --git a/rules/shell/test/TARGETS b/rules/shell/test/TARGETS index 0967ef4..5a4b849 100644 --- a/rules/shell/test/TARGETS +++ b/rules/shell/test/TARGETS @@ -1 +1,6 @@ -{} +{ "summarizer": + { "type": "summarizer" + , "summarizer": [["FILE", null, "summarizer"]] + , "artifacts": ["time-start", "time-stop"] + } +} diff --git a/rules/shell/test/runner b/rules/shell/test/runner index 2f97005..f2f5766 100755 --- a/rules/shell/test/runner +++ b/rules/shell/test/runner @@ -21,6 +21,7 @@ RESULT=UNKNOWN echo "${RESULT}" > result echo UNKNOWN > time-start echo UNKNOWN > time-stop +pwd > pwd mkdir scratch export TEST_TMPDIR=$(realpath scratch) diff --git a/rules/shell/test/summarizer b/rules/shell/test/summarizer index 72cd1b9..39b66be 100755 --- a/rules/shell/test/summarizer +++ b/rules/shell/test/summarizer @@ -21,6 +21,9 @@ from typing import Any, Dict, List g_RESULTS: Dict[str, List[Any]] = {} g_COUNT: float = 0 +PASS_count: float = 0 +PASS_time: float = 0 + time_start: float = time.time() time_stop: float = 0 @@ -32,14 +35,19 @@ for attempt in os.listdir("."): g_RESULTS[result] = g_RESULTS.get(result, []) + [int(attempt)] try: with open(os.path.join(attempt, "time-start")) as f: - time_start = min(time_start, float(f.read().strip())) + start = float(f.read().strip()) + time_start = min(time_start, start) except: pass try: with open(os.path.join(attempt, "time-stop")) as f: - time_stop = max(time_start, float(f.read().strip())) + stop = float(f.read().strip()) + time_stop = max(time_start, stop) except: pass + if (start > 0) and (stop >= start) and result == "PASS": + PASS_count += 1 + PASS_time += stop - start result: str = "UNKNOWN" if set(g_RESULTS.keys()) <= set(["PASS", "FAIL"]): @@ -68,6 +76,9 @@ with open("stdout", "w") as f: f.write("\nother results: %r\n" % (g_RESULTS, )) if result == "FLAKY": f.write("\nFailure rate %5.2f%%\n" % (100.0 * len(failures) / g_COUNT)) + if PASS_count >= 2: + f.write("\nAverage time of a passed test instance: %.1fs\n" + % (PASS_time / PASS_count)) with open("stderr", "w") as f: pass |