path: root/rules/shell/test/RULES

{ "summarizer":
  { "doc":
    ["Specify a test summarizer together with the required additional fields"]
  , "target_fields": ["summarizer"]
  , "string_fields": ["artifacts"]
  , "imports": {"stage": ["./", "../..", "stage_singleton_field"]}
  , "field_doc":
    { "summarizer": ["The single artifact acting as summarizer"]
    , "artifacts":
      [ "Any additional artifacts, besides \"result\", the summaries needs from"
      , "the individual test results"
      ]
    }
  , "expression":
    { "type": "RESULT"
    , "artifacts":
      { "type": "let*"
      , "bindings": [["fieldname", "summarizer"], ["location", "summarizer"]]
      , "body": {"type": "CALL_EXPRESSION", "name": "stage"}
      }
    , "provides":
      { "type": "singleton_map"
      , "key": "artifacts"
      , "value": {"type": "FIELD", "name": "artifacts"}
      }
    }
  }
, "script":
  { "doc": ["Shell test, given by a test script"]
  , "target_fields": ["deps", "test"]
  , "string_fields": ["keep", "keep-dirs", "name"]
  , "config_vars":
    [ "ARCH"
    , "HOST_ARCH"
    , "RUNS_PER_TEST"
    , "TEST_ENV"
    , "TIMEOUT_SCALE"
    , "TARGET_ARCH"
    , "ARCH_DISPATCH"
    , "TEST_SUMMARY_EXECUTION_PROPERTIES"
    , "LINT"
    ]
  , "field_doc":
    { "test":
      [ "The shell script for the test, launched with sh."
      , ""
      , "An empty directory is created to store any temporary files needed"
      , "by the test, and it is made available in the environment variable"
      , "TEST_TMPDIR. The test should not assume write permissions"
      , "outside the working directory and the TEST_TMPDIR."
      , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR."
      , ""
      , "This running of the test is carried out by the implicit dependency"
      , "on the target \"runner\". By setting this target in the target layer"
      , "of this rues repository (instead of letting it default to the"
      , "respective file), the shell test environment can be modified globally."
      ]
    , "name":
      [ "A name for the test, used in reporting, as well as for staging"
      , "the test result tree in the runfiles"
      ]
    , "keep":
      [ "List of names (relative to the test working directory) of files that"
      , "the test might generate that should be kept as part of the output."
      , "This might be useful for further analysis of the test"
      ]
    , "keep-dirs":
      [ "List of names (relative to the test working directory) of directories"
      , "that the test might generate that should be kept as part of the"
      , "output. This might be useful for further analysis of the test"
      ]
    , "deps":
      [ "Any targets that should be staged (with artifacts and runfiles) into"
      , "the tests working directory"
      ]
    , "runner":
      [ "The test runner which starts the actual test script after providing"
      , "the respective environment. The runner also takes care of capturing"
      , "stdout/stderr, timing information, and ensure the presence of the"
      , "files to keep even if the script failed to produce them."
      ]
    , "summarizer":
      [ "Tool to aggregate the results of individual test runs (for flakyness"
      , "detection) to an overall test result. If more fields than the result"
      , "itself is needed, those can be specified using the \"summarizer\" rule."
      ]
    , "defaults": ["The shell toolcahin to use."]
    }
  , "config_doc":
    { "RUNS_PER_TEST":
      [ "The number of times the test should be run in order to detect flakyness."
      , "If set, no test action will be taken from cache."
      , ""
      , "The individual test runs will be summarized by the implict dependency"
      , "on the target \"summarizer\". By setting this target in the target"
      , "in the target layer of this rues repository (instead of letting it"
      , "default to the respective file) the layout of the summary can be"
      , "changed globally."
      ]
    , "TEST_ENV": ["The environment for executing the test runner."]
    , "TIMEOUT_SCALE":
      ["Factor on how to scale the timeout for this test. Defaults to 1.0."]
    , "TARGET_ARCH":
      [ "The architecture to build the test for."
      , ""
      , "Will only be honored, if that architecture is available in the"
      , "ARCH_DISPATCH map. Otherwise, the test will be built for and run"
      , "on the host architecture."
      ]
    , "ARCH_DISPATCH":
      [ "Map of architectures to execution properties that ensure execution"
      , "on that architecture. Only the actual test binary will be run with"
      , "the specified execution properties (i.e., on the target architecture);"
      , "all building will be done on the host architecture."
      ]
    , "TEST_SUMMARY_EXECUTION_PROPERTIES":
      [ "Additional remote-execution properties for the test-summarizing action"
      , "in case RUNS_PER_TEST is set; defaults to the empty map."
      ]
    }
  , "tainted": ["test"]
  , "artifacts_doc":
    [ "result: the result of this test (\"PASS\" or \"FAIL\"); useful for"
    , "        generating test reports."
    , "stdout/stderr: Any output the invocation of the test binary produced on"
    , "        the respective file descriptor"
    , "work:   In this directory, all the files specified to \"keep\" and"
    , "        \"keep-dirs\" are staged"
    , "time-start/time-stop: The time (decimally coded) in seconds since the"
    , "        epoch when the test invocation started and ended."
    , "pwd:    the directory in which the test was carried out"
    ]
  , "runfiles_doc":
    [ "A tree consisting of the artifacts staged at the name of the test."
    , "As the built-in \"install\" rule only takes the runfiles of its \"deps\""
    , "argument, this gives an easy way of defining test suites."
    ]
  , "implicit":
    { "runner": ["runner"]
    , "summarizer": ["summarizer"]
    , "defaults": [["./", "..", "defaults"]]
    }
  , "imports":
    { "test-result": "test-result"
    , "action": "test-action"
    , "stage": ["./", "../..", "stage_singleton_field"]
    , "host transition": ["transitions", "maybe for host"]
    , "target properties": ["transitions", "target properties"]
    , "field_list": ["", "field_list_provider"]
    }
  , "config_transitions":
    { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
    , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}]
    }
  , "expression":
    { "type": "let*"
    , "bindings":
      [ [ "test.sh"
        , { "type": "context"
          , "msg": "Expecting 'test' to specify precisely one file containing a shell script"
          , "$1":
            { "type": "let*"
            , "bindings":
              [ ["fieldname", "test"]
              , ["location", "test.sh"]
              , [ "transition"
                , {"type": "CALL_EXPRESSION", "name": "host transition"}
                ]
              ]
            , "body": {"type": "CALL_EXPRESSION", "name": "stage"}
            }
          }
        ]
      , [ "name"
        , { "type": "assert_non_empty"
          , "msg": "Have to provide a non-empty name for the test (e.g., for result staging)"
          , "$1": {"type": "join", "$1": {"type": "FIELD", "name": "name"}}
          }
        ]
      , ["keep", {"type": "FIELD", "name": "keep"}]
      , ["keep-dirs", {"type": "FIELD", "name": "keep-dirs"}]
      , ["runner", {"type": "FIELD", "name": "runner"}]
      , ["deps-fieldname", "deps"]
      , [ "deps-transition"
        , {"type": "CALL_EXPRESSION", "name": "host transition"}
        ]
      , [ "target properties"
        , {"type": "CALL_EXPRESSION", "name": "target properties"}
        ]
      , [ "lint"
        , { "type": "if"
          , "cond": {"type": "var", "name": "LINT"}
          , "then":
            { "type": "let*"
            , "bindings":
              [ ["fieldname", "deps"]
              , ["provider", "lint"]
              , ["transition", {"type": "var", "name": "deps-transition"}]
              ]
            , "body": {"type": "CALL_EXPRESSION", "name": "field_list"}
            }
          }
        ]
      ]
    , "body":
      { "type": "if"
      , "cond": {"type": "var", "name": "RUNS_PER_TEST"}
      , "else": {"type": "CALL_EXPRESSION", "name": "test-result"}
      , "then":
        { "type": "let*"
        , "bindings":
          [ [ "attempts (plain)"
            , { "type": "map_union"
              , "$1":
                { "type": "foreach"
                , "var": "ATTEMPT"
                , "range":
                  { "type": "range"
                  , "$1": {"type": "var", "name": "RUNS_PER_TEST"}
                  }
                , "body":
                  { "type": "singleton_map"
                  , "key": {"type": "var", "name": "ATTEMPT"}
                  , "value": {"type": "CALL_EXPRESSION", "name": "action"}
                  }
                }
              }
            ]
          , [ "summarizer"
            , { "type": "let*"
              , "bindings":
                [["fieldname", "summarizer"], ["location", "summarizer"]]
              , "body": {"type": "CALL_EXPRESSION", "name": "stage"}
              }
            ]
          , [ "summary artifacts"
            , { "type": "++"
              , "$1":
                [ ["result"]
                , { "type": "let*"
                  , "bindings":
                    [["provider", "artifacts"], ["fieldname", "summarizer"]]
                  , "body": {"type": "CALL_EXPRESSION", "name": "field_list"}
                  }
                ]
              }
            ]
          , [ "attempts (for summary)"
            , { "type": "map_union"
              , "$1":
                { "type": "foreach_map"
                , "range": {"type": "var", "name": "attempts (plain)"}
                , "body":
                  { "type": "singleton_map"
                  , "key": {"type": "var", "name": "_"}
                  , "value":
                    { "type": "TREE"
                    , "$1":
                      { "type": "map_union"
                      , "$1":
                        { "type": "foreach"
                        , "range": {"type": "var", "name": "summary artifacts"}
                        , "body":
                          { "type": "singleton_map"
                          , "key": {"type": "var", "name": "_"}
                          , "value":
                            { "type": "lookup"
                            , "map": {"type": "var", "name": "$_"}
                            , "key": {"type": "var", "name": "_"}
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            ]
          , [ "summary"
            , { "type": "ACTION"
              , "inputs":
                { "type": "map_union"
                , "$1":
                  [ {"type": "var", "name": "attempts (for summary)"}
                  , {"type": "var", "name": "summarizer"}
                  ]
                }
              , "outs":
                ["stdout", "stderr", "result", "time-start", "time-stop"]
              , "cmd": ["./summarizer"]
              , "execution properties":
                { "type": "var"
                , "name": "TEST_SUMMARY_EXECUTION_PROPERTIES"
                , "default": {"type": "empty_map"}
                }
              }
            ]
          , [ "attempts"
            , { "type": "map_union"
              , "$1":
                { "type": "foreach_map"
                , "range": {"type": "var", "name": "attempts (plain)"}
                , "body":
                  { "type": "singleton_map"
                  , "key": {"type": "var", "name": "_"}
                  , "value":
                    {"type": "TREE", "$1": {"type": "var", "name": "$_"}}
                  }
                }
              }
            ]
          , [ "artifacts"
            , { "type": "`"
              , "$1":
                { "pwd":
                  {"type": ",", "$1": {"type": "BLOB", "data": "/summary"}}
                , "work":
                  { "type": ","
                  , "$1":
                    {"type": "TREE", "$1": {"type": "var", "name": "attempts"}}
                  }
                }
              }
            ]
          , [ "runfiles"
            , { "type": "singleton_map"
              , "key": {"type": "var", "name": "name"}
              , "value":
                {"type": "TREE", "$1": {"type": "var", "name": "artifacts"}}
              }
            ]
          ]
        , "body":
          { "type": "RESULT"
          , "artifacts": {"type": "var", "name": "artifacts"}
          , "runfiles": {"type": "var", "name": "runfiles"}
          , "provides": {"type": "env", "vars": ["lint"]}
          }
        }
      }
    }
  }
}