From befb472b1b9650d485f46ae54227caea8bf81cdd Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 10 Jul 2024 17:25:23 +0200 Subject: Tests: also record `pwd` ... as test meta data. Tests are executed in an unspecified directory, assuming pass or fail is independent of the location where the test is run. While this generally is true, test logs often contain the working directory. So, in order to more easily compare different execution orders of a potential race condition, it can be desirable to compare logs "up to the execution directory". This, however, requires that this directory is recored in the first place. Do so. For consistency of the output format, also have a (fixed) artifact pwd in the summary report. --- CC/test/EXPRESSIONS | 14 ++++++++++++-- CC/test/RULES | 1 + CC/test/runner | 2 ++ shell/test/EXPRESSIONS | 2 +- shell/test/RULES | 7 ++++++- shell/test/runner | 1 + 6 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CC/test/EXPRESSIONS b/CC/test/EXPRESSIONS index b0a7f2b..1646dd7 100644 --- a/CC/test/EXPRESSIONS +++ b/CC/test/EXPRESSIONS @@ -165,7 +165,13 @@ [ [ "test-results" , { "type": "ACTION" , "outs": - ["result", "stdout", "stderr", "time-start", "time-stop"] + [ "result" + , "stdout" + , "stderr" + , "time-start" + , "time-stop" + , "pwd" + ] , "inputs": {"type": "var", "name": "test input"} , "cmd": ["./runner"] , "env": @@ -291,7 +297,11 @@ , [ "artifacts" , { "type": "map_union" , "$1": - [ {"type": "var", "name": "summary"} + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": diff --git a/CC/test/RULES b/CC/test/RULES index 7444551..e23359e 100644 --- a/CC/test/RULES +++ b/CC/test/RULES @@ -143,6 +143,7 @@ , " the respective file descriptor" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." diff --git a/CC/test/runner b/CC/test/runner index 45a1bee..4984b17 100755 --- a/CC/test/runner +++ b/CC/test/runner @@ -36,6 +36,8 @@ def dump_results() -> None: f.write("%s\n" % (stdout, )) with open("stderr", "w") as f: f.write("%s\n" % (stderr, )) + with open("pwd", "w") as f: + f.write("%s\n" % (os.getcwd(), )) dump_results() diff --git a/shell/test/EXPRESSIONS b/shell/test/EXPRESSIONS index e158a4c..e445c2a 100644 --- a/shell/test/EXPRESSIONS +++ b/shell/test/EXPRESSIONS @@ -172,7 +172,7 @@ , [ "outs" , { "type": "++" , "$1": - [ ["result", "stdout", "stderr", "time-start", "time-stop"] + [ ["result", "stdout", "stderr", "time-start", "time-stop", "pwd"] , { "type": "foreach" , "var": "filename" , "range": {"type": "var", "name": "keep"} diff --git a/shell/test/RULES b/shell/test/RULES index af18828..ef67708 100644 --- a/shell/test/RULES +++ b/shell/test/RULES @@ -93,6 +93,7 @@ , "work: In this directory, all the files specified to \"keep\" are staged" , "time-start/time-stop: The time (decimally coded) in seconds since the" , " epoch when the test invocation started and ended." + , "pwd: the directory in which the test was carried out" ] , "runfiles_doc": [ "A tree consisting of the artifacts staged at the name of the test." @@ -227,7 +228,11 @@ , [ "artifacts" , { "type": "map_union" , "$1": - [ {"type": "var", "name": "summary"} + [ { "type": "singleton_map" + , "key": "pwd" + , "value": {"type": "BLOB", "data": "/summary"} + } + , {"type": "var", "name": "summary"} , { "type": "singleton_map" , "key": "work" , "value": diff --git a/shell/test/runner b/shell/test/runner index f762355..624610b 100755 --- a/shell/test/runner +++ b/shell/test/runner @@ -21,6 +21,7 @@ RESULT=UNKNOWN echo "${RESULT}" > result echo UNKNOWN > time-start echo UNKNOWN > time-stop +pwd > pwd mkdir scratch export TEST_TMPDIR=$(realpath scratch) -- cgit v1.2.3 From 70cfd4babc0e03f9da6cff5e02ee4b0a1cf0d7a5 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 11 Jul 2024 12:13:25 +0200 Subject: Test rules: only provide to the summary action what is needed ... and add a rule allowing the summarizer to specify what it needs. --- CC/test/EXPRESSIONS | 127 +++++++++++++++++++++++++++++++++------------------- CC/test/RULES | 16 ++++++- shell/test/RULES | 110 +++++++++++++++++++++++++++++++++++---------- shell/test/TARGETS | 7 ++- 4 files changed, 188 insertions(+), 72 deletions(-) diff --git a/CC/test/EXPRESSIONS b/CC/test/EXPRESSIONS index 1646dd7..f3e4b04 100644 --- a/CC/test/EXPRESSIONS +++ b/CC/test/EXPRESSIONS @@ -38,6 +38,7 @@ , "test-args" , "test-data" , "summarizer" + , "summary artifacts" ] , "imports": { "artifacts": ["./", "../..", "field_artifacts"] @@ -208,7 +209,7 @@ , "then": { "type": "let*" , "bindings": - [ [ "attempts" + [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" @@ -220,55 +221,75 @@ , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} + , "value": + { "type": "ACTION" + , "outs": + ["result", "stdout", "stderr", "time-start", "time-stop"] + , "inputs": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "ATTEMPT" + , "value": + { "type": "BLOB" + , "data": {"type": "var", "name": "ATTEMPT"} + } + } + , {"type": "var", "name": "test input"} + ] + } + , "cmd": ["./runner"] + , "env": + { "type": "var" + , "name": "TEST_ENV" + , "default": {"type": "empty_map"} + } + , "may_fail": ["test"] + , "no_cache": ["test"] + , "fail_message": + { "type": "join" + , "$1": + [ "CC test " + , {"type": "var", "name": "test-name"} + , " failed (Run" + , {"type": "var", "name": "ATTEMPT"} + , ")" + ] + } + , "timeout scaling": + {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": + {"type": "var", "name": "target properties"} + } + } + } + } + ] + , [ "attempts (for summary)" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} , "value": { "type": "TREE" , "$1": - { "type": "ACTION" - , "outs": - [ "result" - , "stdout" - , "stderr" - , "time-start" - , "time-stop" - ] - , "inputs": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "ATTEMPT" - , "value": - { "type": "BLOB" - , "data": {"type": "var", "name": "ATTEMPT"} - } + { "type": "map_union" + , "$1": + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} } - , {"type": "var", "name": "test input"} - ] - } - , "cmd": ["./runner"] - , "env": - { "type": "var" - , "name": "TEST_ENV" - , "default": {"type": "empty_map"} - } - , "may_fail": ["test"] - , "no_cache": ["test"] - , "fail_message": - { "type": "join" - , "$1": - [ "CC test " - , {"type": "var", "name": "test-name"} - , " failed (Run" - , {"type": "var", "name": "ATTEMPT"} - , ")" - ] + } } - , "timeout scaling": - { "type": "var" - , "name": "TIMEOUT_SCALE" - , "default": 1.0 - } - , "execution properties": - {"type": "var", "name": "target properties"} } } } @@ -280,7 +301,7 @@ , "inputs": { "type": "map_union" , "$1": - [ {"type": "var", "name": "attempts"} + [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } @@ -294,6 +315,20 @@ } } ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] , [ "artifacts" , { "type": "map_union" , "$1": diff --git a/CC/test/RULES b/CC/test/RULES index e23359e..f61d9f7 100644 --- a/CC/test/RULES +++ b/CC/test/RULES @@ -81,7 +81,8 @@ ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" - , "detection) to an overall test result." + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." ] } , "config_doc": @@ -157,6 +158,7 @@ , "host transition": ["transitions", "maybe for host"] , "stage": ["./", "../..", "stage_singleton_field"] , "run_test": "run_test" + , "field_list": ["", "field_list_provider"] } , "config_transitions": { "defaults": [{"type": "CALL_EXPRESSION", "name": "host transition"}] @@ -237,6 +239,18 @@ , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] ] , "body": {"type": "CALL_EXPRESSION", "name": "run_test"} } diff --git a/shell/test/RULES b/shell/test/RULES index ef67708..e3ee4a2 100644 --- a/shell/test/RULES +++ b/shell/test/RULES @@ -1,4 +1,31 @@ -{ "script": +{ "summarizer": + { "doc": + ["Specify a test summarizer together with the required additional fields"] + , "target_fields": ["summarizer"] + , "string_fields": ["artifacts"] + , "imports": {"stage": ["./", "../..", "stage_singleton_field"]} + , "field_doc": + { "summarizer": ["The single artifact acting as summarizer"] + , "artifacts": + [ "Any additional artifacts, besides \"result\", the summaries needs from" + , "the individual test results" + ] + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "let*" + , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + , "provides": + { "type": "singleton_map" + , "key": "artifacts" + , "value": {"type": "FIELD", "name": "artifacts"} + } + } + } +, "script": { "doc": ["Shell test, given by a test script"] , "target_fields": ["deps", "test"] , "string_fields": ["keep", "name"] @@ -48,7 +75,8 @@ ] , "summarizer": [ "Tool to aggregate the results of individual test runs (for flakyness" - , "detection) to an overall test result." + , "detection) to an overall test result. If more fields than the result" + , "itself is needed, those can be specified using the \"summarizer\" rule." ] , "defaults": ["The shell toolcahin to use."] } @@ -111,6 +139,7 @@ , "stage": ["./", "../..", "stage_singleton_field"] , "host transition": ["transitions", "maybe for host"] , "target properties": ["transitions", "target properties"] + , "field_list": ["", "field_list_provider"] } , "config_transitions": { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] @@ -158,7 +187,7 @@ , "then": { "type": "let*" , "bindings": - [ [ "attempts" + [ [ "attempts (plain)" , { "type": "map_union" , "$1": { "type": "foreach" @@ -170,37 +199,56 @@ , "body": { "type": "singleton_map" , "key": {"type": "var", "name": "ATTEMPT"} - , "value": - { "type": "TREE" - , "$1": {"type": "CALL_EXPRESSION", "name": "action"} - } + , "value": {"type": "CALL_EXPRESSION", "name": "action"} } } } ] , [ "summarizer" + , { "type": "let*" + , "bindings": + [["fieldname", "summarizer"], ["location", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "stage"} + } + ] + , [ "summary artifacts" + , { "type": "++" + , "$1": + [ ["result"] + , { "type": "let*" + , "bindings": + [["provider", "artifacts"], ["fieldname", "summarizer"]] + , "body": {"type": "CALL_EXPRESSION", "name": "field_list"} + } + ] + } + ] + , [ "attempts (for summary)" , { "type": "map_union" , "$1": - { "type": "foreach" - , "var": "x" - , "range": {"type": "FIELD", "name": "summarizer"} + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} , "body": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "x" - , "range": - { "type": "values" + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "TREE" + , "$1": + { "type": "map_union" , "$1": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "x"} + { "type": "foreach" + , "range": {"type": "var", "name": "summary artifacts"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + { "type": "lookup" + , "map": {"type": "var", "name": "$_"} + , "key": {"type": "var", "name": "_"} + } + } } } - , "body": - { "type": "singleton_map" - , "key": "summarizer" - , "value": {"type": "var", "name": "x"} - } } } } @@ -211,7 +259,7 @@ , "inputs": { "type": "map_union" , "$1": - [ {"type": "var", "name": "attempts"} + [ {"type": "var", "name": "attempts (for summary)"} , {"type": "var", "name": "summarizer"} ] } @@ -225,6 +273,20 @@ } } ] + , [ "attempts" + , { "type": "map_union" + , "$1": + { "type": "foreach_map" + , "range": {"type": "var", "name": "attempts (plain)"} + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "_"} + , "value": + {"type": "TREE", "$1": {"type": "var", "name": "$_"}} + } + } + } + ] , [ "artifacts" , { "type": "map_union" , "$1": diff --git a/shell/test/TARGETS b/shell/test/TARGETS index 0967ef4..5a4b849 100644 --- a/shell/test/TARGETS +++ b/shell/test/TARGETS @@ -1 +1,6 @@ -{} +{ "summarizer": + { "type": "summarizer" + , "summarizer": [["FILE", null, "summarizer"]] + , "artifacts": ["time-start", "time-stop"] + } +} -- cgit v1.2.3 From 965d97835da8dd2cb71a6444537003577b60a0ad Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Fri, 12 Jul 2024 09:51:56 +0200 Subject: Test summarizer: also report on average time ... of passed tests (as only for those, the time is meaningful). Given that we read the timing information anyway, if available, we can as well report more useful information. --- shell/test/summarizer | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shell/test/summarizer b/shell/test/summarizer index 72cd1b9..39b66be 100755 --- a/shell/test/summarizer +++ b/shell/test/summarizer @@ -21,6 +21,9 @@ from typing import Any, Dict, List g_RESULTS: Dict[str, List[Any]] = {} g_COUNT: float = 0 +PASS_count: float = 0 +PASS_time: float = 0 + time_start: float = time.time() time_stop: float = 0 @@ -32,14 +35,19 @@ for attempt in os.listdir("."): g_RESULTS[result] = g_RESULTS.get(result, []) + [int(attempt)] try: with open(os.path.join(attempt, "time-start")) as f: - time_start = min(time_start, float(f.read().strip())) + start = float(f.read().strip()) + time_start = min(time_start, start) except: pass try: with open(os.path.join(attempt, "time-stop")) as f: - time_stop = max(time_start, float(f.read().strip())) + stop = float(f.read().strip()) + time_stop = max(time_start, stop) except: pass + if (start > 0) and (stop >= start) and result == "PASS": + PASS_count += 1 + PASS_time += stop - start result: str = "UNKNOWN" if set(g_RESULTS.keys()) <= set(["PASS", "FAIL"]): @@ -68,6 +76,9 @@ with open("stdout", "w") as f: f.write("\nother results: %r\n" % (g_RESULTS, )) if result == "FLAKY": f.write("\nFailure rate %5.2f%%\n" % (100.0 * len(failures) / g_COUNT)) + if PASS_count >= 2: + f.write("\nAverage time of a passed test instance: %.1fs\n" + % (PASS_time / PASS_count)) with open("stderr", "w") as f: pass -- cgit v1.2.3