Add simple http server allowing to browse an invocation-log directory

Being able to browse through past invocations of the build tool can
actually be useful and doing so in the browser is a way many users
prefer. Therefore, add a small WSGI application (written in python,
using werkzeug and jinja) serving a directory of invocation logs
via http.

7 files changed, 535 insertions, 0 deletions

diff --git a/doc/invocations-http-server/server.py b/doc/invocations-http-server/server.py
new file mode 100755
index 00000000..940e03a1
--- /dev/null
+++ b/doc/invocations-http-server/server.py
@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+# Copyright 2025 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jinja2
+import json
+import os
+import subprocess
+import werkzeug.exceptions
+import werkzeug.routing
+import werkzeug.utils
+
+from werkzeug.wrappers import Request, Response
+
+def core_config(conf):
+    new_conf = {}
+    for k, v in conf.items():
+        if v is not None:
+            new_conf[k] = v
+    return new_conf
+
+class HashIdentifierConverter(werkzeug.routing.BaseConverter):
+    regex = '[a-zA-Z0-9]{40,51}'
+
+class InvocationIdentifierConverter(werkzeug.routing.BaseConverter):
+    regex = '[-:_a-zA-Z0-9]{1,200}'
+
+class InvocationServer:
+    def __init__(self, logsdir, *,
+                 just_mr = None,
+                 graph = "graph.json",
+                 profile = "profile.json",
+                 meta = "meta.json"):
+        self.logsdir = logsdir
+        if just_mr is None:
+            self.just_mr = ["just-mr"]
+        else:
+            self.just_mr = just_mr
+        self.profile = profile
+        self.graph = graph
+        self.meta = meta
+        self.templatepath = os.path.join(os.path.dirname(__file__), "templates")
+        self.jinjaenv = jinja2.Environment(
+                loader=jinja2.FileSystemLoader(self.templatepath))
+        rule = werkzeug.routing.Rule
+        self.routingmap = werkzeug.routing.Map([
+            rule("/", methods=("GET",), endpoint="list_invocations"),
+            rule("/blob/<hashidentifier:blob>",
+                 methods=("GET",),
+                 endpoint="get_blob"),
+            rule("/tree/<hashidentifier:tree>",
+                 methods=("GET",),
+                 endpoint="get_tree"),
+            rule("/invocations/<invocationidentifier:invocation>",
+                 methods=("GET",),
+                 endpoint="get_invocation"),
+        ], converters=dict(
+            invocationidentifier=InvocationIdentifierConverter,
+            hashidentifier=HashIdentifierConverter,
+        ))
+
+    @Request.application
+    def __call__(self, request):
+        mapadapter = self.routingmap.bind_to_environ(request.environ)
+        try:
+            endpoint, args = mapadapter.match()
+            return getattr(self, "do_%s" % (endpoint,))(**args)
+        except werkzeug.exceptions.HTTPException as e:
+            return e
+
+    def render(self, templatename, params):
+        response = Response()
+        response.content_type = "text/html; charset=utf8"
+        template = self.jinjaenv.get_template(templatename)
+        response.data = template.render(params).encode("utf8")
+        return response
+
+    def do_list_invocations(self):
+        invocations = []
+        count = 0
+        entries = sorted(os.listdir(self.logsdir), reverse=True)
+        for e in entries:
+            profile = os.path.join(self.logsdir, e, self.profile)
+            if not os.path.exists(profile):
+                # only consider invocations that provide a profile; unfinished
+                # invocations as well as not build related ones (like
+                # install-cas) are not relevant.
+                continue
+            try:
+                with open(profile) as f:
+                    profile_data = json.load(f)
+            except:
+                profile_data = {}
+            count += 1
+            target = profile_data.get("target")
+            config = profile_data.get("config", {})
+            invocation = {
+                "name": e,
+                "subcommand": profile_data.get("subcommand"),
+                "target": json.dumps(target) if target else None,
+                "config": json.dumps(core_config(config)),
+                "exit_code": profile_data.get('exit code', 0),
+            }
+            invocations.append(invocation)
+            if count >= 50:
+                break
+        return self.render("invocations.html",
+                           {"invocations": invocations})
+
+    def process_failure(self, cmd, procobj, *, failure_kind=None):
+        params = {"stdout": None, "stderr": None, "failure_kind": failure_kind}
+        params["cmd"] = json.dumps(cmd)
+        params["exit_code"] = procobj.returncode
+        try:
+            params["stdout"] = procobj.stdout.decode('utf-8')
+        except:
+            pass
+        try:
+            params["stderr"] = procobj.stderr.decode('utf-8')
+        except:
+            pass
+        return self.render("failure.html", params)
+
+    def do_get_blob(self, blob):
+        cmd = self.just_mr + ["install-cas", "--remember", blob]
+        blob_data = subprocess.run(cmd,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        if blob_data.returncode !=0:
+            return self.process_failure(cmd, blob_data)
+        try:
+            blob_content = blob_data.stdout.decode('utf-8')
+        except:
+            # Not utf-8, so return as binary file
+            response = Response()
+            response.content_type = "application/octet-stream"
+            response.data = blob_data.stdout
+            response.headers['Content-Disposition'] = \
+                "attachement; filename=%s" %(blob,)
+            return response
+        return self.render("blob.html",
+                           {"name": blob,
+                            "data": blob_content})
+
+    def do_get_tree(self, tree):
+        cmd = self.just_mr + ["install-cas", "%s::t" % (tree,)]
+        tree_data = subprocess.run(cmd,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        if tree_data.returncode !=0:
+            return self.process_failure(cmd, tree_data)
+        try:
+            tree_info = json.loads(tree_data.stdout.decode('utf-8'))
+        except Exception as e:
+            return self.process_failure(
+                cmd, tree_data,
+                failure_kind = "Malformed output: %s" % (e,))
+
+        entries = []
+        for k, v in tree_info.items():
+            h, s, t = v.strip('[]').split(':')
+            entries.append({"path": k,
+                            "hash": h,
+                            "type": "tree" if t == 't' else "blob"})
+        return self.render("tree.html", {"name": tree,
+                                         "entries": entries})
+
+    def do_get_invocation(self, invocation):
+        params = {"invocation": invocation}
+
+        try:
+            with open(os.path.join(
+                    self.logsdir, invocation, self.profile)) as f:
+                profile = json.load(f)
+        except:
+            profile = {}
+
+        try:
+            with open(os.path.join(
+                    self.logsdir, invocation, self.meta)) as f:
+                meta = json.load(f)
+        except:
+            meta = {}
+
+        try:
+            with open(os.path.join(
+                    self.logsdir, invocation, self.graph)) as f:
+                graph = json.load(f)
+        except:
+            graph = {}
+
+        params["repo_config"] = meta.get('configuration')
+        params["exit_code"] = profile.get('exit code')
+        # For complex conditional data fill with None as default
+        for k in ["cmdline", "cmd", "target", "config"]:
+            params[k] = None
+        # Fill this data, if available
+        if meta.get('cmdline'):
+            params["cmdline"] = json.dumps(meta.get('cmdline'))
+        if profile.get('subcommand'):
+            params["cmd"] = json.dumps(
+                [profile.get('subcommand')] + profile.get('subcommand args', []))
+        if profile.get('target'):
+            params["target"] = json.dumps(profile.get('target'))
+        if profile.get('configuration') is not None:
+            params["config"] = json.dumps(core_config(
+                profile.get('configuration')))
+
+        def action_data(name, profile_value):
+            data = { "name_prefix": "",
+                     "name": name,
+                     "exit_code": profile_value.get('exit code', 0)}
+            desc = graph.get('actions', {}).get(name, {})
+            data["may_fail"] = desc.get("may_fail")
+            data["stdout"] = profile_value.get('stdout')
+            data["stderr"] = profile_value.get('stderr')
+            data["output"] = desc.get('output', [])
+            data["output_dirs"] = desc.get('output_dirs', [])
+            if len(data["output"]) == 1:
+                data["primary_output"] = data["output"][0]
+            elif len(data["output"]) == 0 and len(data["output_dirs"]) == 1:
+                data["primary_output"] = data["output_dirs"][0]
+            else:
+                data["primary_output"] = None
+            data["artifacts"] = profile_value.get('artifacts', {})
+            origins = []
+            for origin in desc.get('origins', []):
+                origins.append("%s#%d@%s" % (
+                    json.dumps(origin.get('target', [])),
+                    origin.get('subtask', 0),
+                    core_config(origin.get('config', {})),))
+            data["origins"] = origins
+            return data
+
+        failed_build_actions = []
+        failed_test_actions = []
+        for k, v in profile.get('actions', {}).items():
+            if v.get('exit code', 0) != 0:
+                if graph.get('actions', {}).get(k, {}).get('may_fail') != None:
+                    failed_test_actions.append(action_data(k, v))
+                else:
+                    failed_build_actions.append(action_data(k, v))
+        params["failed_actions"] = failed_build_actions + failed_test_actions
+
+        # longest running non-cached non-failed actions
+        candidates = []
+        for k, v in profile.get('actions', {}).items():
+            if not v.get('cached'):
+                if v.get('exit code', 0) == 0:
+                    candidates.append((v.get('duration', 0.0), k, v))
+        candidates.sort(reverse=True)
+        non_cached = []
+        params["more_noncached"] = None
+        if len(candidates) > 30:
+            params["more_non_cached"] = len(candidates) - 30
+            candidates = candidates[:30]
+        for t, k, v in candidates:
+            action = action_data(k, v)
+            action["name_prefix"] = "%5.1fs" % (t,)
+            non_cached.append(action)
+        params["non_cached"] = non_cached
+
+        return self.render("invocation.html", params)
+
+if __name__ == '__main__':
+    import sys
+    from argparse import ArgumentParser
+    from wsgiref.simple_server import make_server
+    parser = ArgumentParser(
+        description="Serve invocations of a given directory")
+    parser.add_argument("--meta", dest="meta", default="meta.json",
+                        help="Name of the logged metadata file")
+    parser.add_argument("--graph", dest="graph", default="graph.json",
+                        help="Name of the logged action-graph file")
+    parser.add_argument("--profile", dest="profile", default="profile.json",
+                        help="Name of the logged profile file")
+    parser.add_argument(
+        "--just-mr", dest="just_mr", default='["just-mr"]',
+        help="The correct way of invoking just-mr, as JSON vector of strings")
+    parser.add_argument("-i", dest="interface", default='127.0.0.1',
+                        help="The interface to listen on")
+    parser.add_argument("-p", dest="port", default=9000, type=int,
+                        help="The port to listen on")
+    parser.add_argument("DIR",
+                        help="Directory (for a single project id) to serve")
+    args = parser.parse_args()
+    try:
+        just_mr = json.loads(args.just_mr)
+    except Exception as e:
+        print("just-mr argument should be valid json, but %r is not: %s"
+              % (args.just_mr, e), file=sys.stderr)
+        sys.exit(1)
+    app = InvocationServer(args.DIR,
+                           just_mr=just_mr,
+                           graph=args.graph,
+                           meta=args.meta,
+                           profile=args.profile)
+    make_server(args.interface, args.port, app).serve_forever()
diff --git a/doc/invocations-http-server/templates/base.html b/doc/invocations-http-server/templates/base.html
new file mode 100644
index 00000000..cf591652
--- /dev/null
+++ b/doc/invocations-http-server/templates/base.html
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+  <head>
+    <title>
+      {% block heading %}Invocation Browser{% endblock %}
+    </title>
+  </head>
+  <body>
+    {% block content %}
+    <i>(no content)</i>
+    {% endblock %}
+  </body>
+</html>
diff --git a/doc/invocations-http-server/templates/blob.html b/doc/invocations-http-server/templates/blob.html
new file mode 100644
index 00000000..2a64d064
--- /dev/null
+++ b/doc/invocations-http-server/templates/blob.html
@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block heading %}
+Blob {{ name | e }}
+{% endblock %}
+
+{% block content %}
+<h1>Blob {{ name | e }}</h1>
+
+<pre>{{ data | e}}</pre>
+{% endblock %}
diff --git a/doc/invocations-http-server/templates/failure.html b/doc/invocations-http-server/templates/failure.html
new file mode 100644
index 00000000..30892281
--- /dev/null
+++ b/doc/invocations-http-server/templates/failure.html
@@ -0,0 +1,24 @@
+{% extends "base.html" %}
+{% block heading %}
+Failure calling an external process
+{% endblock %}
+
+{% block content %}
+<h1>Failure calling an external process</h1>
+
+<ul>
+  {% if failure_kind %}
+  <li> <b>{{ failure_kind | e }}</b></li>
+  {% endif %}
+  <li> Command: <tt>{{ cmd|e }}</tt></li>
+  <li> exit code: <tt>{{ exit_code|e }}</tt></li>
+  {% if stdout %}
+  <li> stdout:
+    <pre>{{ stdout | e }}</pre></li>
+  {% endif %}
+  {% if stderr %}
+  <li> stderr:
+    <pre>{{ stderr | e }}</pre></li>
+  {% endif %}
+</ul>
+{% endblock %}
diff --git a/doc/invocations-http-server/templates/invocation.html b/doc/invocations-http-server/templates/invocation.html
new file mode 100644
index 00000000..7fec7e6e
--- /dev/null
+++ b/doc/invocations-http-server/templates/invocation.html
@@ -0,0 +1,130 @@
+{% extends "base.html" %}
+{% block heading %}
+Inocations {{invocation | e}}
+{% endblock %}
+
+{% macro show_action(action) %}
+<li>
+  <details>
+    <summary>
+      {{ action["name_prefix"] | e }} <tt>{{ action["name"] | e}}</tt>
+      {% if action["may_fail"] %}
+      {% if action["exit_code"] != 0 %}
+      failure: <b>{{ action["may_fail"] }}</b>
+      {% else %}
+      tainted (<tt>{{ action["may_fail"] }}</tt>)
+      {% endif %}
+      {% elif action["primary_output"] %}
+      {% if action["exit_code"] != 0 %}
+      failed to build: <b>{{ action["primary_output"] | e }}</b>
+      {% else %}
+      build: <tt>{{ action["primary_output"] | e }}</tt>
+      {% endif %}
+      {% endif %}
+    </summary>
+    <ul>
+      {% if action["stdout"] %}
+      <li> stdout: <a href="/blob/{{ action["stdout"] | e }}">{{action["stdout"] | e }}</a></li>
+      {% endif %}
+      {% if action["stderr"] %}
+      <li> stderr: <a href="/blob/{{ action["stderr"] | e }}">{{action["stderr"] | e }}</a></li>
+      {% endif %}
+      {% if action["origins"] %}
+      <li> origins
+        <ul>
+          {% for origin in action["origins"] %}
+          <li> <tt>{{ origin | e }}</tt></li>
+          {% endfor %}
+        </ul>
+      </li>
+      {% endif %}
+      {% if action["exit_code"] != 0 %}
+      <li> exit code: {{ action["exit_code"] }}</li>
+      {% endif %}
+      {% if action["output"] %}
+      <li> output
+        <ul>
+          {% for out in action["output"] %}
+          {% if action["artifacts"].get(out) %}
+          <li> <a href="/blob/{{ action["artifacts"].get(out) | e}}"><tt>{{ out | e }}</tt></a></li>
+          {% else %}
+          <li> <tt>{{ out | e }}</tt></li>
+          {% endif %}
+          {% endfor %}
+        </ul>
+      </li>
+      {% endif %}
+      {% if action["output_dirs"] %}
+      <li> output directories
+        <ul>
+          {% for out in action["output_dirs"] %}
+          {% if action["artifacts"].get(out) %}
+          <li> <a href="/tree/{{ action["artifacts"].get(out) | e}}"><tt>{{ out | e }}</tt></a></li>
+          {% else %}
+          <li> <tt>{{ out | e }}</tt></li>
+          {% endif %}
+          {% endfor %}
+        </ul>
+      </li>
+      {% endif %}
+    </ul>
+  </details>
+</li>
+{% endmacro %}
+
+{% block content %}
+<h1>Invocation {{invocation | e}}</h1>
+
+<h2>Overview</h2>
+<ul>
+  {% if cmd %}
+  <li> Subcommand and positional arguments: <tt>{{ cmd | e }}</tt></li>
+  {% endif %}
+  {% if cmdline %}
+  <li> Full command line: <tt>{{ cmdline | e }}</tt></li>
+  {% endif %}
+  {% if repo_config %}
+  <li> Repository configuration:
+    <a href="/blob/{{ repo_config | e }}">{{ repo_config | e}}</a></li>
+  {% endif %}
+  {% if target %}
+  <li> Target: <tt>{{ target | e }}</tt></li>
+  {% endif %}
+  {% if config %}
+  <li> Target configuration: <tt>{{ config | e }}</tt></li>
+  {% endif %}
+  {% if exit_code != None %}
+  <li> exit code: {{ exit_code | e }}</li>
+  {% endif %}
+</ul>
+
+<h2>Failed actions</h2>
+{% if failed_actions %}
+<ul>
+  {% for action in failed_actions %}
+  {{ show_action(action) }}
+  {% endfor %}
+</ul>
+{% else %}
+<i>(none)</i>
+{% endif %}
+
+
+<h2>Remaining non-cached actions</h2>
+{% if non_cached %}
+In order of decreasing run time.
+<ul>
+  {% for action in non_cached %}
+  {{ show_action(action) }}
+  {% endfor %}
+  {% if more_non_cached %}
+  <li> &hellip; and {{ more_non_cached | e }} actions</li>
+  {% endif %}
+</ul>
+{% else %}
+<i>(none)</i>
+{% endif %}
+
+
+
+{% endblock %}
diff --git a/doc/invocations-http-server/templates/invocations.html b/doc/invocations-http-server/templates/invocations.html
new file mode 100644
index 00000000..bab67cd3
--- /dev/null
+++ b/doc/invocations-http-server/templates/invocations.html
@@ -0,0 +1,35 @@
+{% extends "base.html" %}
+{% block heading %}
+Recent Invocations
+{% endblock %}
+
+{% block content %}
+<h1>Recent Invocations</h1>
+{% if invocations %}
+<ul>
+  {% for invocation in invocations %}
+  <li> <a href="/invocations/{{invocation.name |e}}">{{ invocation.name | e }}</a>
+    {% if invocation.subcommand %}
+    {{ invocation.subcommand | e }}
+    {% endif %}
+    {% if invocation.target %}
+    <tt>{{ invocation.target | e}}@{{ invocation.config | e}}</tt>
+    {% endif %}
+    {% if invocation.exit_code != 0 %}
+    <b>
+      {% if invocation.exit_code == 1 %}
+      build failed
+      {% elif invocation.exit_code == 2 %}
+      failed
+      {% else %}
+      exit code {{ invocation.exit_code | e }}
+      {% endif %}
+    </b>
+    {% endif %}
+  </li>
+  {% endfor %}
+</ul>
+{% else %}
+<i>(none)</i>
+{% endif %}
+{% endblock %}
diff --git a/doc/invocations-http-server/templates/tree.html b/doc/invocations-http-server/templates/tree.html
new file mode 100644
index 00000000..765be73e
--- /dev/null
+++ b/doc/invocations-http-server/templates/tree.html
@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block heading %}
+Tree {{ name | e }}
+{% endblock %}
+
+{% block content %}
+<h1>Tree {{ name | e }}</h1>
+
+<ul>
+  {% for entry in entries %}
+  <li><a href="/{{ entry["type"] | e}}/{{ entry["hash"] | e}}">{{ entry["path"] | e}}</a></li>
+  {% endfor %}
+</ul>
+{% endblock %}