summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--shell/test/EXPRESSIONS127
-rw-r--r--shell/test/RULES108
-rwxr-xr-xshell/test/test_summary.py53
3 files changed, 250 insertions, 38 deletions
diff --git a/shell/test/EXPRESSIONS b/shell/test/EXPRESSIONS
index 429cd47..2b9a62e 100644
--- a/shell/test/EXPRESSIONS
+++ b/shell/test/EXPRESSIONS
@@ -1,5 +1,5 @@
-{ "test-result":
- { "vars": ["name", "test.sh"]
+{ "test-action":
+ { "vars": ["name", "test.sh", "ATTEMPT"]
, "expression":
{ "type": "let*"
, "bindings":
@@ -60,44 +60,99 @@
}
}
]
- , [ "test-results"
- , { "type": "ACTION"
- , "outs":
- { "type": "++"
- , "$1":
- [ ["result", "stdout", "stderr", "time-start", "time-stop"]
- , { "type": "foreach"
- , "var": "filename"
- , "range": {"type": "FIELD", "name": "keep"}
- , "body":
- { "type": "join"
- , "$1": ["work/", {"type": "var", "name": "filename"}]
- }
- }
- ]
- }
- , "inputs":
- { "type": "map_union"
- , "$1":
- [ { "type": "singleton_map"
- , "key": "work"
- , "value": {"type": "var", "name": "deps"}
- }
- , {"type": "var", "name": "runner"}
- , {"type": "var", "name": "test.sh"}
- ]
- }
- , "cmd":
- { "type": "++"
- , "$1": [["./runner"], {"type": "FIELD", "name": "keep"}]
+ , [ "attempt marker"
+ , { "type": "if"
+ , "cond":
+ { "type": "=="
+ , "$1": {"type": "var", "name": "ATTEMPT"}
+ , "$2": null
}
- , "may_fail": ["test"]
- , "fail_message":
- { "type": "join"
- , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"]
+ , "then": {"type": "empty_map"}
+ , "else":
+ { "type": "singleton_map"
+ , "key": "ATTEMPT"
+ , "value":
+ {"type": "BLOB", "data": {"type": "var", "name": "ATTEMPT"}}
}
}
]
+ , [ "outs"
+ , { "type": "++"
+ , "$1":
+ [ ["result", "stdout", "stderr", "time-start", "time-stop"]
+ , { "type": "foreach"
+ , "var": "filename"
+ , "range": {"type": "FIELD", "name": "keep"}
+ , "body":
+ { "type": "join"
+ , "$1": ["work/", {"type": "var", "name": "filename"}]
+ }
+ }
+ ]
+ }
+ ]
+ , [ "inputs"
+ , { "type": "map_union"
+ , "$1":
+ [ { "type": "singleton_map"
+ , "key": "work"
+ , "value": {"type": "var", "name": "deps"}
+ }
+ , {"type": "var", "name": "runner"}
+ , {"type": "var", "name": "test.sh"}
+ , {"type": "var", "name": "attempt marker"}
+ ]
+ }
+ ]
+ , [ "cmd"
+ , { "type": "++"
+ , "$1": [["./runner"], {"type": "FIELD", "name": "keep"}]
+ }
+ ]
+ ]
+ , "body":
+ { "type": "if"
+ , "cond":
+ {"type": "==", "$1": {"type": "var", "name": "ATTEMPT"}, "$2": null}
+ , "then":
+ { "type": "ACTION"
+ , "outs": {"type": "var", "name": "outs"}
+ , "inputs": {"type": "var", "name": "inputs"}
+ , "cmd": {"type": "var", "name": "cmd"}
+ , "may_fail": ["test"]
+ , "fail_message":
+ { "type": "join"
+ , "$1": ["shell test ", {"type": "var", "name": "name"}, " failed"]
+ }
+ }
+ , "else":
+ { "type": "ACTION"
+ , "outs": {"type": "var", "name": "outs"}
+ , "inputs": {"type": "var", "name": "inputs"}
+ , "cmd": {"type": "var", "name": "cmd"}
+ , "may_fail": ["test"]
+ , "no_cache": ["test"]
+ , "fail_message":
+ { "type": "join"
+ , "$1":
+ [ "shell test "
+ , {"type": "var", "name": "name"}
+ , " failed (Run "
+ , {"type": "var", "name": "ATTEMPT"}
+ , ")"
+ ]
+ }
+ }
+ }
+ }
+ }
+, "test-result":
+ { "vars": ["name", "test.sh"]
+ , "imports": {"action": "test-action"}
+ , "expression":
+ { "type": "let*"
+ , "bindings":
+ [ ["test-results", {"type": "CALL_EXPRESSION", "name": "action"}]
, [ "runfiles"
, { "type": "singleton_map"
, "key": {"type": "var", "name": "name"}
diff --git a/shell/test/RULES b/shell/test/RULES
index ca5ddbc..cf9713a 100644
--- a/shell/test/RULES
+++ b/shell/test/RULES
@@ -2,6 +2,7 @@
{ "doc": ["Shell test, given by a test script"]
, "target_fields": ["deps", "test"]
, "string_fields": ["keep", "name"]
+ , "config_vars": ["RUNS_PER_TEST"]
, "field_doc":
{ "test": ["The shell script for the test, launched with sh"]
, "name":
@@ -18,10 +19,18 @@
, "the tests working directory"
]
}
+ , "config_doc":
+ { "RUNS_PER_TEST":
+ [ "The number of times the test should be run in order to detect flakyness."
+ , "If set, no test action will be taken from cache."
+ ]
+ }
, "tainted": ["test"]
- , "implicit": {"runner": ["test_runner.sh"]}
+ , "implicit":
+ {"runner": ["test_runner.sh"], "summarizer": ["test_summary.py"]}
, "imports":
{ "test-result": "test-result"
+ , "action": "test-action"
, "stage": ["./", "../..", "stage_singleton_field"]
}
, "expression":
@@ -44,7 +53,102 @@
}
]
]
- , "body": {"type": "CALL_EXPRESSION", "name": "test-result"}
+ , "body":
+ { "type": "if"
+ , "cond": {"type": "var", "name": "RUNS_PER_TEST"}
+ , "else": {"type": "CALL_EXPRESSION", "name": "test-result"}
+ , "then":
+ { "type": "let*"
+ , "bindings":
+ [ [ "attempts"
+ , { "type": "map_union"
+ , "$1":
+ { "type": "foreach"
+ , "var": "ATTEMPT"
+ , "range":
+ { "type": "range"
+ , "$1": {"type": "var", "name": "RUNS_PER_TEST"}
+ }
+ , "body":
+ { "type": "singleton_map"
+ , "key": {"type": "var", "name": "ATTEMPT"}
+ , "value":
+ { "type": "TREE"
+ , "$1": {"type": "CALL_EXPRESSION", "name": "action"}
+ }
+ }
+ }
+ }
+ ]
+ , [ "summarizer"
+ , { "type": "map_union"
+ , "$1":
+ { "type": "foreach"
+ , "var": "x"
+ , "range": {"type": "FIELD", "name": "summarizer"}
+ , "body":
+ { "type": "map_union"
+ , "$1":
+ { "type": "foreach"
+ , "var": "x"
+ , "range":
+ { "type": "values"
+ , "$1":
+ { "type": "DEP_ARTIFACTS"
+ , "dep": {"type": "var", "name": "x"}
+ }
+ }
+ , "body":
+ { "type": "singleton_map"
+ , "key": "summarizer"
+ , "value": {"type": "var", "name": "x"}
+ }
+ }
+ }
+ }
+ }
+ ]
+ , [ "summary"
+ , { "type": "ACTION"
+ , "inputs":
+ { "type": "map_union"
+ , "$1":
+ [ {"type": "var", "name": "attempts"}
+ , {"type": "var", "name": "summarizer"}
+ ]
+ }
+ , "outs":
+ ["stdout", "stderr", "result", "time-start", "time-stop"]
+ , "cmd": ["./summarizer"]
+ }
+ ]
+ , [ "artifacts"
+ , { "type": "map_union"
+ , "$1":
+ [ {"type": "var", "name": "summary"}
+ , { "type": "singleton_map"
+ , "key": "work"
+ , "value":
+ {"type": "TREE", "$1": {"type": "var", "name": "attempts"}}
+ }
+ ]
+ }
+ ]
+ , [ "runfiles"
+ , { "type": "singleton_map"
+ , "key": {"type": "var", "name": "name"}
+ , "value":
+ {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}}
+ }
+ ]
+ ]
+ , "body":
+ { "type": "RESULT"
+ , "artifacts": {"type": "var", "name": "artifacts"}
+ , "runfiles": {"type": "var", "name": "runfiles"}
+ }
+ }
+ }
}
}
}
diff --git a/shell/test/test_summary.py b/shell/test/test_summary.py
new file mode 100755
index 0000000..cba8d50
--- /dev/null
+++ b/shell/test/test_summary.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import os
+import time
+
+RESULTS = {}
+
+time_start = time.time()
+time_stop = 0
+
+for attempt in os.listdir("."):
+ if os.path.isdir(attempt):
+ with open(os.path.join(attempt, "result")) as f:
+ result = f.read().strip()
+ RESULTS[result] = RESULTS.get(result, []) + [int(attempt)]
+ try:
+ with open(os.path.join(attempt, "time-start")) as f:
+ time_start = min(time_start, float(f.read().strip()))
+ except:
+ pass
+ try:
+ with open(os.path.join(attempt, "time-stop")) as f:
+ time_stop = max(time_start, float(f.read().strip()))
+ except:
+ pass
+
+result = "UNKNOWN"
+if set(RESULTS.keys()) <= set(["PASS", "FAIL"]):
+ if not RESULTS.get("FAIL"):
+ result = "PASS"
+ elif not RESULTS.get("PASS"):
+ result = "FAIL"
+ else:
+ result = "FLAKY"
+with open("result", "w") as f:
+ f.write("%s\n" % (result,))
+
+with open("time-start", "w") as f:
+ f.write("%d\n" % (time_start,))
+with open("time-stop", "w") as f:
+ f.write("%d\n" % (time_stop,))
+
+with open("stdout", "w") as f:
+ f.write("Summary: %s\n\n" % (result,))
+ f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),))
+ f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),))
+ RESULTS.pop("PASS", None)
+ RESULTS.pop("FAIL", None)
+ if RESULTS:
+ f.write("\nother results: %r\n" % (RESULTS,))
+
+with open("stderr", "w") as f:
+ pass
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 20:50:31 +0000