Add infrastructure for end-to-end tests using just serve

1 files changed, 230 insertions, 1 deletions

diff --git a/test/end-to-end/RULES b/test/end-to-end/RULES
index aef5a668..13efd588 100644
--- a/test/end-to-end/RULES
+++ b/test/end-to-end/RULES
@@ -87,7 +87,10 @@
   , "expression":
     { "type": "let*"
     , "bindings":
-      [ [ "test.sh"
+      [ ["test type name", "remote execution"]
+      , ["extra_infra", []]
+      , ["data", {"type": "empty_map"}]
+      , [ "test.sh"
         , { "type": "context"
           , "msg": "Expecting 'test' to specify precisely one file containing a shell script"
           , "$1":
@@ -205,4 +208,230 @@
       }
     }
   }
+, "with serve":
+  { "doc":
+    [ "Shell test, given by a test script,"
+    , "implictly assuming a remote execution and a just serve instance"
+    , "in the background."
+    ]
+  , "target_fields": ["deps", "test", "repos"]
+  , "string_fields": ["keep", "name"]
+  , "config_vars": ["ARCH", "HOST_ARCH", "RUNS_PER_TEST", "TEST_ENV"]
+  , "field_doc":
+    { "test":
+      [ "The shell script for the test, launched with sh."
+      , ""
+      , "An empty directory is created to store any temporary files needed"
+      , "by the test, and it is made available in the environment variable"
+      , "TEST_TMPDIR. The test should not assume write permissions"
+      , "outside the working directory and the TEST_TMPDIR."
+      , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR."
+      ]
+    , "name":
+      [ "A name for the test, used in reporting, as well as for staging"
+      , "the test result tree in the runfiles"
+      ]
+    , "keep":
+      [ "List of names (relative to the test working directory) of files that"
+      , "the test might generate that should be kept as part of the output."
+      , "This might be useful for further analysis of the test"
+      ]
+    , "deps":
+      [ "Any targets that should be staged (with artifacts and runfiles) into"
+      , "the tests working directory"
+      ]
+    , "repos":
+      [ "The trees, one per entry, that the just serve instance should have"
+      , "available. The respective commits will be generated and passed to the"
+      , "test script as COMMIT_0, COMMIT_1, etc."
+      ]
+    }
+  , "config_doc":
+    { "RUNS_PER_TEST":
+      [ "The number of times the test should be run in order to detect flakyness."
+      , "If set, no test action will be taken from cache."
+      ]
+    , "TEST_ENV": ["The environment for executing the test runner."]
+    }
+  , "tainted": ["test"]
+  , "artifacts_doc":
+    [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for"
+    , "        generating test reports."
+    , "stdout/stderr: Any output the invocation of the test binary produced on"
+    , "        the respective file descriptor"
+    , "remotestdout/remotestderr: Any output of the remote-execution server"
+    , "        implicit to that test"
+    , "servestdout/servestderr: Any output of the serve-execution server"
+    , "        implicit to that test"
+    , "work:   In this directory, all the files specified to \"keep\" are staged"
+    , "time-start/time-stop: The time (decimally coded) in seconds since the"
+    , "        epoch when the test invocation started and ended."
+    ]
+  , "runfiles_doc":
+    [ "A tree consisting of the artifacts staged at the name of the test."
+    , "As the built-in \"install\" rule only takes the runfiles of its \"deps\""
+    , "argument, this gives an easy way of defining test suites."
+    ]
+  , "implicit":
+    { "runner": ["with_serve_test_runner.py"]
+    , "summarizer": [["@", "rules", "shell/test", "summarizer"]]
+    , "just": [["@", "src", "", "installed just"]]
+    }
+  , "imports":
+    { "test-result": "test-result"
+    , "action": "test-action"
+    , "stage": ["@", "rules", "", "stage_singleton_field"]
+    , "host transition": ["@", "rules", "transitions", "for host"]
+    }
+  , "config_transitions":
+    { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
+    , "repos": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
+    , "just": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
+    , "runner": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
+    }
+  , "expression":
+    { "type": "let*"
+    , "bindings":
+      [ ["test type name", "target-level cache serving"]
+      , ["extra_infra", ["servestdout", "servestderr"]]
+      , [ "test.sh"
+        , { "type": "context"
+          , "msg": "Expecting 'test' to specify precisely one file containing a shell script"
+          , "$1":
+            { "type": "let*"
+            , "bindings": [["fieldname", "test"], ["location", "test.sh"]]
+            , "body": {"type": "CALL_EXPRESSION", "name": "stage"}
+            }
+          }
+        ]
+      , [ "name"
+        , { "type": "assert_non_empty"
+          , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)"
+          , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}}
+          }
+        ]
+      , ["keep", {"type": "FIELD", "name": "keep"}]
+      , ["deps-fieldname", "deps"]
+      , ["transition", {"type": "CALL_EXPRESSION", "name": "host transition"}]
+      , [ "data"
+        , { "type": "to_subdir"
+          , "subdir": "data"
+          , "$1":
+            { "type": "map_union"
+            , "$1":
+              { "type": "foreach_map"
+              , "range":
+                {"type": "enumerate", "$1": {"type": "FIELD", "name": "repos"}}
+              , "body":
+                { "type": "to_subdir"
+                , "subdir": {"type": "var", "name": "_"}
+                , "$1":
+                  { "type": "DEP_ARTIFACTS"
+                  , "dep": {"type": "var", "name": "$_"}
+                  , "transition": {"type": "var", "name": "transition"}
+                  }
+                }
+              }
+            }
+          }
+        ]
+      ]
+    , "body":
+      { "type": "if"
+      , "cond": {"type": "var", "name": "RUNS_PER_TEST"}
+      , "else": {"type": "CALL_EXPRESSION", "name": "test-result"}
+      , "then":
+        { "type": "let*"
+        , "bindings":
+          [ [ "attempts"
+            , { "type": "map_union"
+              , "$1":
+                { "type": "foreach"
+                , "var": "ATTEMPT"
+                , "range":
+                  { "type": "range"
+                  , "$1": {"type": "var", "name": "RUNS_PER_TEST"}
+                  }
+                , "body":
+                  { "type": "singleton_map"
+                  , "key": {"type": "var", "name": "ATTEMPT"}
+                  , "value":
+                    { "type": "TREE"
+                    , "$1": {"type": "CALL_EXPRESSION", "name": "action"}
+                    }
+                  }
+                }
+              }
+            ]
+          , [ "summarizer"
+            , { "type": "map_union"
+              , "$1":
+                { "type": "foreach"
+                , "var": "x"
+                , "range": {"type": "FIELD", "name": "summarizer"}
+                , "body":
+                  { "type": "map_union"
+                  , "$1":
+                    { "type": "foreach"
+                    , "var": "x"
+                    , "range":
+                      { "type": "values"
+                      , "$1":
+                        { "type": "DEP_ARTIFACTS"
+                        , "dep": {"type": "var", "name": "x"}
+                        }
+                      }
+                    , "body":
+                      { "type": "singleton_map"
+                      , "key": "summarizer"
+                      , "value": {"type": "var", "name": "x"}
+                      }
+                    }
+                  }
+                }
+              }
+            ]
+          , [ "summary"
+            , { "type": "ACTION"
+              , "inputs":
+                { "type": "map_union"
+                , "$1":
+                  [ {"type": "var", "name": "attempts"}
+                  , {"type": "var", "name": "summarizer"}
+                  ]
+                }
+              , "outs":
+                ["stdout", "stderr", "result", "time-start", "time-stop"]
+              , "cmd": ["./summarizer"]
+              }
+            ]
+          , [ "artifacts"
+            , { "type": "map_union"
+              , "$1":
+                [ {"type": "var", "name": "summary"}
+                , { "type": "singleton_map"
+                  , "key": "work"
+                  , "value":
+                    {"type": "TREE", "$1": {"type": "var", "name": "attempts"}}
+                  }
+                ]
+              }
+            ]
+          , [ "runfiles"
+            , { "type": "singleton_map"
+              , "key": {"type": "var", "name": "name"}
+              , "value":
+                {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}}
+              }
+            ]
+          ]
+        , "body":
+          { "type": "RESULT"
+          , "artifacts": {"type": "var", "name": "artifacts"}
+          , "runfiles": {"type": "var", "name": "runfiles"}
+          }
+        }
+      }
+    }
+  }
 }