Display additional run information on data points on charts (#148)

This allows arbitrary run information to be displayed on the charts
when clicking the data point. The content of the run information is
controled by the submitter of that run, which allows a great deal of
flexibility.

Folks will want to avoid attaching too much information to the runs
to avoid making graphs too slow, but that's on them.

Fixes #75

Author: ldionne

docs/developer_guide.rst

@@ -19,7 +19,8 @@ virtual environment and installing the development dependencies::
     pip install ".[dev]"
 
 This will install the current version of the package, along with the dependencies
-required for development (``lit``, ``filecheck``, etc).
+required for development (``lit``, ``filecheck``, etc). Note that ``curl`` and
+``jq`` are also required for running the tests.
 
 Running LNT's Regression Tests
 ------------------------------

docs/importing_data.rst

@@ -58,16 +58,23 @@ First, make sure you've understood the underlying :ref:`concepts` used by LNT.
       ]
   }
 
+Any optional fields provided in the ``run`` section will be associated to that run and visible
+in the UI when looking at that run. This allows annotating runs with useful additional information,
+such as the commit information related to this run or similar. This data will also be visible on
+charts when viewing historical results. Arbitrary data can be provided, however including too much
+run-related information can cause very large amounts of data to have to be transfered to display
+graphs.
+
 A concrete small example is
 
 .. literalinclude:: report-example.json
     :language: json
 
 
-Given how simple it is to make your own results and send them to LNT,
-it is common to not use the LNT client application at all, and just have a
-custom script run your tests and submit the data to the LNT server. Details
-on how to do this are in :mod:`lnt.testing`
+Given how simple it is to make your own results and send them to LNT, it is common to
+not use the LNT client application at all, and just have a custom script run your tests
+and submit the data to the LNT server in JSON format.
+
 
 .. _nts_suite:
 

lnt/server/ui/static/lnt_graph.js

@@ -101,7 +101,15 @@ function plotly_show_tooltip(data) {
             get_run_url(db_name, test_suite_name, point.meta.runID) +
             "\">" + point.meta.runID + "</a><br>";
     }
-    
+
+    // Display any additional run parameters
+    var known_fields = ['order', 'orderID', 'date', 'runID', 'state'];
+    for (var key in point.meta) {
+        if (point.meta.hasOwnProperty(key) && known_fields.indexOf(key) === -1) {
+            tip_body += "<b>" + key + ":</b> " + point.meta[key] + "<br>";
+        }
+    }
+
     if (point.meta.runID && point.data.url) { // url = machine.id/test.id/field_index
         tip_body += "<a href=\"" +
             get_manual_regression_url(db_name, test_suite_name, point.data.url, point.meta.runID) +

lnt/server/ui/views.py

@@ -899,7 +899,7 @@ def load_graph_data(plot_parameter, show_failures, limit, xaxis_date, revision_c
     # FIXME: Don't join to Order here, aggregate this across all the tests
     # we want to load. Actually, we should just make this a single query.
     values = session.query(plot_parameter.field.column, ts.Order,
-                           ts.Run.start_time, ts.Run.id) \
+                           ts.Run.start_time, ts.Run.id, ts.Run.parameters_data) \
                     .select_from(ts.Sample) \
                     .join(ts.Run).join(ts.Order) \
                     .filter(ts.Run.machine_id == plot_parameter.machine.id) \
@@ -916,15 +916,15 @@ def load_graph_data(plot_parameter, show_failures, limit, xaxis_date, revision_c
     if xaxis_date:
         # Aggregate by date.
         data = list(multidict.multidict(
-            (date, (val, order, date, run_id))
-            for val, order, date, run_id in values).items())
+            (date, (val, order, date, run_id, parameters_data))
+            for val, order, date, run_id, parameters_data in values).items())
         # Sort data points according to date.
         data.sort(key=lambda sample: sample[0])
     else:
         # Aggregate by order (revision).
         data = list(multidict.multidict(
-            (order.llvm_project_revision, (val, order, date, run_id))
-            for val, order, date, run_id in values).items())
+            (order.llvm_project_revision, (val, order, date, run_id, parameters_data))
+            for val, order, date, run_id, parameters_data in values).items())
         # Sort data points according to order (revision).
         data.sort(key=lambda sample: convert_revision(sample[0], cache=revision_cache))
 
@@ -1194,7 +1194,9 @@ def trace_name(name, test_name, field_name):
             # And the date on which they were taken.
             dates = [data_array[2] for data_array in datapoints]
             # Run ID where this point was collected.
-            run_ids = [data_array[3] for data_array in datapoints if len(data_array) == 4]
+            run_ids = [data_array[3] for data_array in datapoints if len(data_array) >= 4]
+            # Run parameters (JSON-encoded binary data)
+            run_parameters_list = [data_array[4] for data_array in datapoints if len(data_array) >= 5]
 
             values = [v * normalize_by for v in values]
 
@@ -1224,6 +1226,13 @@ def trace_name(name, test_name, field_name):
                               "date": str(dates[agg_index])}
             if run_ids:
                 point_metadata["runID"] = str(run_ids[agg_index])
+
+            # Add run parameters to metadata if available
+            if run_parameters_list:
+                run_params = run_parameters_list[agg_index]
+                if run_params:
+                    point_metadata.update(json.loads(run_params.decode('utf-8')))
+
             meta.append(point_metadata)
 
             # Add the multisample points, if requested.
@@ -1235,6 +1244,13 @@ def trace_name(name, test_name, field_name):
                                             "date": str(dates[i])}
                     if run_ids:
                         multisample_metadata["runID"] = str(run_ids[i])
+
+                    # Add run parameters to metadata if available
+                    if run_parameters_list:
+                        run_params = run_parameters_list[i]
+                        if run_params:
+                            multisample_metadata.update(json.loads(run_params.decode('utf-8')))
+
                     multisample_points_data["x"].append(point_label)
                     multisample_points_data["y"].append(v)
                     multisample_points_data["meta"].append(multisample_metadata)

tests/lnttool/submit.extra-run-information.shtest

@@ -0,0 +1,70 @@
+#!/bin/bash
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: lnt create %t/instance
+# RUN: %{shared_inputs}/server_wrapper.sh --fail-on-error %t/instance 9094 \
+# RUN:    /bin/sh %s %t
+
+#
+# Make sure that we can have extra information in the run, and that such information
+# is saved into the DB.
+#
+
+set -eux
+
+t="$1"
+HOST="http://localhost:9094"
+
+# Simple case
+cat <<EOF > "$t/report1.json"
+{
+    "format_version": "2",
+    "machine": {
+       "name": "machine-foo"
+    },
+    "run": {
+       "start_time": "2026-01-01T11:28:33.00000",
+       "end_time": "2026-01-01T11:28:23.991076",
+       "llvm_project_revision": "1",
+       "some_extra_information1": "some tokens",
+       "some_extra_information2": "more tokens"
+    },
+    "tests": [
+       {
+           "name": "benchmark1",
+           "execution_time": [ 0.1056, 0.1055 ]
+       }
+    ]
+}
+EOF
+
+lnt submit "${HOST}/db_default/v4/nts/submitRun" "$t/report1.json"
+curl -sS "${HOST}/api/db_default/v4/nts/runs/1" > "$t/run1.json"
+jq .run.some_extra_information1 "$t/run1.json" | grep "some tokens"
+jq .run.some_extra_information2 "$t/run1.json" | grep "more tokens"
+
+
+# Add different fields with another run
+cat <<EOF > "$t/report2.json"
+{
+    "format_version": "2",
+    "machine": {
+       "name": "machine-foo"
+    },
+    "run": {
+       "start_time": "2026-01-02T11:28:33.00000",
+       "end_time": "2026-01-02T11:28:23.991076",
+       "llvm_project_revision": "2",
+       "some_extra_information3": "some tokens 3"
+    },
+    "tests": [
+       {
+           "name": "benchmark1",
+           "execution_time": [ 0.1056, 0.1055 ]
+       }
+    ]
+}
+EOF
+
+lnt submit "${HOST}/db_default/v4/nts/submitRun" "$t/report2.json"
+curl -sS "${HOST}/api/db_default/v4/nts/runs/2" > "$t/run2.json"
+jq .run.some_extra_information3 "$t/run2.json" | grep "some tokens 3"