From c15f8a734e30003b6821bad79157ecfb10045881 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 25 Mar 2026 18:07:40 -0400 Subject: [PATCH 1/4] add documentation for viz Signed-off-by: ArthurChenCoding --- docs/usage.md | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 85f854a2..b5371672 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -114,13 +114,13 @@ Control where and how results are saved: ```bash # Custom output file location -orion --output /path/to/results.csv --hunter-analyze +orion --save-output-path /path/to/results.txt --hunter-analyze # JSON output format -orion cmd o json --hunter-analyze +orion -o json --hunter-analyze # JUnit XML format -orion cmd o junit --hunter-analyze +orion -o junit --hunter-analyze # Collapse text output (print only regression summary, full table saved to file) orion --collapse --hunter-analyze @@ -129,6 +129,36 @@ orion --collapse --hunter-analyze orion --collapse -o json --hunter-analyze ``` +### Interactive Visualizations +Generate interactive HTML visualizations alongside the standard Orion output: + +```bash +# Generate text output plus an interactive HTML visualization +orion --config performance-config.yaml --hunter-analyze --viz + +# configure the path and name of the saved output and generated HTML +orion \ + --config performance-config.yaml \ + --hunter-analyze \ + --viz \ + --save-output-path ./outputs/results.txt + +# Generate visualizations for both periodic and PR runs +orion \ + --config performance-config.yaml \ + --hunter-analyze \ + --pr-analysis \ + --viz \ + --save-output-path ./outputs/pr-analysis.txt \ + --input-vars='{"jobtype": "pull", "pull_number": "2790", "organization": "openshift", "repository": "test"}' +``` + +- `--viz` keeps the output style and adds HTML files for each analyzed run +- HTML file name uses the argument from `--save-output-path` with the Orion test name +- Each visualization includes one subplot per metric, changepoint markers, percentage annotations, ACK markers, and hover details for timestamp, version, UUID, and build URL +- Clicking a point opens its `buildUrl` in a new browser tab +- The generated HTML loads Plotly from a CDN when opened in a browser + ### Display Metadata Fields Add custom metadata fields as columns in the output table: @@ -486,7 +516,7 @@ orion \ --hunter-analyze \ --lookback 7d \ --threshold 15 \ - --output results.csv \ + --save-output-path results.txt \ --debug ``` @@ -506,7 +536,7 @@ orion \ --config quick-check.yaml \ --anomaly-detection \ --lookback 24h \ - --output anomalies.json + --save-output-path anomalies.txt ``` ## Acknowledgment Examples @@ -554,6 +584,8 @@ To achieve this the following input_vars should be provided `--input-vars='{"jobtype": "pull","pull_number": "2790", "organization": "openshift", "repository": "test"}'` +Add `--viz` to this workflow to generate interactive HTML visualizations for each analyzed dataset alongside the standard PR report output. + ### Example ``` payload-cluster-density-v2 From e75010fa35e79299d969e3d74bcb0ca084e7381c Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 25 Mar 2026 18:55:53 -0400 Subject: [PATCH 2/4] add file name checker Signed-off-by: ArthurChenCoding --- orion/tests/test_visualization.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 orion/tests/test_visualization.py diff --git a/orion/tests/test_visualization.py b/orion/tests/test_visualization.py new file mode 100644 index 00000000..afb0b470 --- /dev/null +++ b/orion/tests/test_visualization.py @@ -0,0 +1,57 @@ +""" +Unit test file for visualization functionality +""" + +# pylint: disable = missing-function-docstring +import logging + +import pandas as pd + +from orion.logger import SingletonLogger +from orion.visualization import VizData, generate_test_html + + +def test_generate_test_html_writes_expected_file_and_injects_click_handler( + tmp_path, +): + SingletonLogger(debug=logging.INFO, name="Orion") + + dataframe = pd.DataFrame( + { + "timestamp": [ + "2026-03-01T00:00:00Z", + "2026-03-02T00:00:00Z", + ], + "uuid": ["uuid-1", "uuid-2"], + "ocpVersion": [ + "4.22.0-0.nightly-2026-03-01-000000", + "4.22.0-0.nightly-2026-03-02-000000", + ], + "buildUrl": [ + "https://example.com/build/1", + "https://example.com/build/2", + ], + "latency": [10.0, 12.5], + } + ) + viz_data = VizData( + test_name="node-density", + dataframe=dataframe, + metrics_config={"latency": {"direction": 1}}, + change_points_by_metric={}, + uuid_field="uuid", + version_field="ocpVersion", + ) + + output_base_path = str(tmp_path / "output_payload") + output_file = generate_test_html(viz_data, output_base_path) + expected_path = tmp_path / "output_payload_node-density_viz.html" + + assert output_file == str(expected_path) + assert expected_path.is_file() + + html = expected_path.read_text(encoding="utf-8") + assert "Orion: node-density" in html + assert ".plotly-graph-div { width: 100% !important; }" in html + assert "attachClickHandlers" in html + assert "window.open(pt.customdata[0], '_blank');" in html From 8690872b0d4893c6c7458d2ed05a752ffe4a8f63 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 25 Mar 2026 19:31:43 -0400 Subject: [PATCH 3/4] add marker check Signed-off-by: ArthurChenCoding --- orion/tests/test_visualization.py | 67 ++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/orion/tests/test_visualization.py b/orion/tests/test_visualization.py index afb0b470..6ab7e2e5 100644 --- a/orion/tests/test_visualization.py +++ b/orion/tests/test_visualization.py @@ -4,11 +4,19 @@ # pylint: disable = missing-function-docstring import logging +from types import SimpleNamespace import pandas as pd from orion.logger import SingletonLogger -from orion.visualization import VizData, generate_test_html +from orion.visualization import VizData, _build_test_figure, generate_test_html + + +def _make_changepoint(index, mean_1, mean_2): + return SimpleNamespace( + index=index, + stats=SimpleNamespace(mean_1=mean_1, mean_2=mean_2), + ) def test_generate_test_html_writes_expected_file_and_injects_click_handler( @@ -55,3 +63,60 @@ def test_generate_test_html_writes_expected_file_and_injects_click_handler( assert ".plotly-graph-div { width: 100% !important; }" in html assert "attachClickHandlers" in html assert "window.open(pt.customdata[0], '_blank');" in html + + +def test_build_test_figure_renders_changepoints_and_skips_out_of_range(): + dataframe = pd.DataFrame( + { + "timestamp": [ + "2026-03-01T00:00:00Z", + "2026-03-02T00:00:00Z", + "2026-03-03T00:00:00Z", + ], + "uuid": ["uuid-1", "uuid-2", "uuid-3"], + "ocpVersion": [ + "4.22.0-0.nightly-2026-03-01-000000", + "4.22.0-0.nightly-2026-03-02-000000", + "4.22.0-0.nightly-2026-03-03-000000", + ], + "buildUrl": [ + "https://example.com/build/1", + "https://example.com/build/2", + "https://example.com/build/3", + ], + "latency": [10.0, 20.0, 30.0], + "cpu": [30.0, 20.0, 10.0], + } + ) + viz_data = VizData( + test_name="node-density", + dataframe=dataframe, + metrics_config={ + "latency": {"direction": 1}, + "cpu": {"direction": 1}, + }, + change_points_by_metric={ + "latency": [ + _make_changepoint(index=1, mean_1=10.0, mean_2=20.0), + _make_changepoint(index=10, mean_1=20.0, mean_2=40.0), + ], + "cpu": [ + _make_changepoint(index=2, mean_1=20.0, mean_2=10.0), + ], + }, + uuid_field="uuid", + version_field="ocpVersion", + ) + + fig = _build_test_figure(viz_data) + changepoint_traces = [ + trace for trace in fig.data + if isinstance(trace.hovertext, str) and "CHANGEPOINT" in trace.hovertext + ] + + assert len(changepoint_traces) == 2 + assert {trace.x[0] for trace in changepoint_traces} == {1, 2} + assert {trace.marker.color for trace in changepoint_traces} == { + "#ff4444", + "#39ff14", + } From 782b0dbeb18a4ab707e1d3efc20ad838dc370495 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 25 Mar 2026 19:50:46 -0400 Subject: [PATCH 4/4] add ack test Signed-off-by: ArthurChenCoding --- orion/tests/test_visualization.py | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/orion/tests/test_visualization.py b/orion/tests/test_visualization.py index 6ab7e2e5..67a641ec 100644 --- a/orion/tests/test_visualization.py +++ b/orion/tests/test_visualization.py @@ -120,3 +120,51 @@ def test_build_test_figure_renders_changepoints_and_skips_out_of_range(): "#ff4444", "#39ff14", } + + +def test_build_test_figure_renders_only_matching_ack_markers(): + dataframe = pd.DataFrame( + { + "timestamp": [ + "2026-03-01T00:00:00Z", + "2026-03-02T00:00:00Z", + "2026-03-03T00:00:00Z", + ], + "uuid": ["uuid-1", "uuid-2", "uuid-3"], + "ocpVersion": [ + "4.22.0-0.nightly-2026-03-01-000000", + "4.22.0-0.nightly-2026-03-02-000000", + "4.22.0-0.nightly-2026-03-03-000000", + ], + "buildUrl": [ + "https://example.com/build/1", + "https://example.com/build/2", + "https://example.com/build/3", + ], + "latency": [10.0, 20.0, 30.0], + } + ) + viz_data = VizData( + test_name="node-density", + dataframe=dataframe, + metrics_config={"latency": {"direction": 1}}, + change_points_by_metric={}, + uuid_field="uuid", + version_field="ocpVersion", + acked_entries=[ + {"metric": "latency", "uuid": "uuid-2", "reason": "known issue"}, + {"metric": "cpu", "uuid": "uuid-2", "reason": "wrong metric"}, + {"metric": "latency", "uuid": "missing-uuid", "reason": "missing row"}, + ], + ) + + fig = _build_test_figure(viz_data) + ack_traces = [ + trace for trace in fig.data + if isinstance(trace.hovertext, str) and "ACKed" in trace.hovertext + ] + + assert len(ack_traces) == 1 + assert ack_traces[0].x[0] == 1 + assert ack_traces[0].marker.color == "#39ff14" + assert ack_traces[0].customdata[0][0] == "https://example.com/build/2"