|
| 1 | +{#- |
| 2 | + Condensed regression-detector report, sized for a PR comment. |
| 3 | +
|
| 4 | + A terser alternative to the built-in `report.md`: it drops the `Δ mean % CI` |
| 5 | + and `trials` columns and folds the `perf` column into the `Δ mean %` cell as a |
| 6 | + colored dot. Only regressions are shown up front; everything else is collapsed. |
| 7 | +
|
| 8 | + Render with: |
| 9 | + smp report render --template-file condensed-report.md.j2 \ |
| 10 | + --report-json outputs/report.json \ |
| 11 | + --target-config-dir ./test/smp/regression/adp/ \ |
| 12 | + --output-file outputs/condensed-report.md |
| 13 | +
|
| 14 | + Bounds-check figures cover the comparison variant only. |
| 15 | +-#} |
| 16 | +{#- Terse goal label, and which sign of Δ mean % is the improving direction. -#} |
| 17 | +{%- set GOALS = { |
| 18 | + "% cpu utilization": ["cpu", "down"], |
| 19 | + "memory utilization": ["memory", "down"], |
| 20 | + "ingress throughput": ["throughput", "up"], |
| 21 | + "egress throughput": ["egress throughput", "up"], |
| 22 | + } -%} |
| 23 | +{#- Decorate each goal row with its label, dot, tag and sort key. The regression |
| 24 | + verdict itself comes from the report: `change_detected` already encodes the |
| 25 | + effect-size, confidence-interval and configured-erratic criteria, so there is |
| 26 | + no statistics to re-derive here. -#} |
| 27 | +{%- set ns = namespace(rows=[]) -%} |
| 28 | +{%- for g in optimization_goals -%} |
| 29 | +{%- set info = GOALS[g.goal] or [g.goal, "down"] -%} |
| 30 | +{%- set tag = " **(ignored)**" if g.ignores_regressions else (" **(erratic)**" if g.is_erratic else "") -%} |
| 31 | +{#- `rank` is the negated goal-relative change, so ascending order puts the |
| 32 | + worst-for-this-goal first and a throughput drop sorts alongside a CPU rise |
| 33 | + rather than at the opposite end of the table. -#} |
| 34 | +{%- set ns.rows = ns.rows + [dict( |
| 35 | + experiment = g.experiment, |
| 36 | + name = g.experiment ~ tag, |
| 37 | + label = info[0], |
| 38 | + delta = g.delta_mean_percent_display, |
| 39 | + regressed = g.is_ci_fail, |
| 40 | + dot = ("🟢" if (g.change_detected and g.is_improvement) else ("🔴" if g.is_ci_fail else "⚪")), |
| 41 | + rank = ((0 - g.delta_mean_percent) if info[1] == "down" else g.delta_mean_percent), |
| 42 | + )] -%} |
| 43 | +{%- endfor -%} |
| 44 | +{%- set rows = ns.rows | sort(attribute="rank,experiment") -%} |
| 45 | +{%- set regressions = rows | selectattr("regressed") | list -%} |
| 46 | +{%- set others = rows | rejectattr("regressed") | list -%} |
| 47 | +{%- set bounds = checks | selectattr("kind", "equalto", "bounds") | list -%} |
| 48 | +{#- Links come from each experiment's `report_links`; absent a target config dir |
| 49 | + the cell renders empty rather than failing the whole report. -#} |
| 50 | +{%- macro links(experiment) -%} |
| 51 | +{%- set nsl = namespace(out="", sep="") -%} |
| 52 | +{%- if experiments and experiment in experiments -%} |
| 53 | +{%- for l in experiments[experiment].report_links -%} |
| 54 | +{%- set nsl.out = nsl.out ~ nsl.sep ~ "[" ~ l.text ~ "](" ~ (l.link | interpolate(experiment=experiment)) ~ ")" -%} |
| 55 | +{%- set nsl.sep = " " -%} |
| 56 | +{%- endfor -%} |
| 57 | +{%- endif -%} |
| 58 | +{{- nsl.out -}} |
| 59 | +{%- endmacro -%} |
| 60 | +{%- macro goal_table(rows) -%} |
| 61 | +| experiment | goal | Δ mean % | links | |
| 62 | +|---|---|---|---| |
| 63 | +{% for r in rows -%} |
| 64 | +| {{ r.name }} | {{ r.label }} | {{ r.dot }} {{ r.delta }} | {{ links(r.experiment) }} | |
| 65 | +{% endfor -%} |
| 66 | +{%- endmacro -%} |
| 67 | +## Regression Detector (Agent Data Plane) |
| 68 | + |
| 69 | +**Run ID:** `{{ job_id }}` |
| 70 | +**Baseline:** `{{ baseline_sha[:8] or "(unknown)" }}` · **Comparison:** `{{ comparison_sha[:8] or "(unknown)" }}` · {% if shas_are_shas %}[diff](../../compare/{{ baseline_sha }}..{{ comparison_sha }}){% else %}(diff unavailable){% endif %} |
| 71 | +{#- Data the report carries but the tables cannot show. One line each, and |
| 72 | + nothing at all on a clean run, so the common case stays terse. -#} |
| 73 | +{%- set no_analysis = failed_replicates.experiments_with_no_analysis %} |
| 74 | +{%- set retried = failed_replicates.experiments_with_retried_replicates %} |
| 75 | +{%- if no_analysis %} |
| 76 | +⚠ **{{ no_analysis | length }} experiment{{ "s" if no_analysis | length != 1 }} lost every replicate to target crashes** (no analysis): {{ no_analysis | join(", ") }} |
| 77 | +{%- endif %} |
| 78 | +{%- if missing_data %} |
| 79 | +⚠ **{{ missing_data | length }} experiment{{ "s" if missing_data | length != 1 }} produced no optimization-goal data**: {{ missing_data | map(attribute="experiment") | join(", ") }} |
| 80 | +{%- endif %} |
| 81 | +{%- if retried %} |
| 82 | +⚠ **{{ retried | length }} experiment{{ "s" if retried | length != 1 }} had replicates retried after a crash**: {{ retried | join(", ") }} |
| 83 | +{%- endif %} |
| 84 | +{%- if errors %} |
| 85 | +⚠ **{{ errors | length }} analysis error{{ "s" if errors | length != 1 }}**: {% for e in errors %}{{ e.experiment }} ({{ e.check_name }}){{ ", " if not loop.last }}{% endfor %} |
| 86 | +{%- endif %} |
| 87 | + |
| 88 | +{% if regressions %}## Optimization Goals: ❌ {{ regressions | length }} regression{{ "s" if regressions | length != 1 }} detected |
| 89 | + |
| 90 | +{{ goal_table(regressions) }} |
| 91 | +{%- else %}## Optimization Goals: ✅ No significant changes detected |
| 92 | +{% endif %} |
| 93 | +{%- if others %} |
| 94 | +<details><summary><b>Fine details of change detection per experiment</b> ({{ others | length }})</summary> |
| 95 | + |
| 96 | +Experiments configured `erratic: true` are tagged <code>(ignored)</code> and skipped when determining which experiments regressed or improved. Experiments which are _detected_ as erratic at runtime are tagged <code>(erratic)</code> to flag that the run's sample dispersion was high, but their regression / improvement signal still counts. |
| 97 | + |
| 98 | +{{ goal_table(others) }} |
| 99 | +</details> |
| 100 | +{% endif %} |
| 101 | +{%- if bounds %} |
| 102 | +<details><summary><b>Bounds Checks:</b> {{ "✅ Passed" if bounds_checks_passed else "❌ Failed" }} ({{ bounds | length }})</summary> |
| 103 | + |
| 104 | +| experiment | check | replicates | observed | links | |
| 105 | +|---|---|---|---|---| |
| 106 | +{% for c in bounds -%} |
| 107 | +| {{ c.experiment }} | {{ c.check_name }} | {{ c.replicates_passed }} | {{ "✅" if c.passed else "❌" }} {{ c.observed_value }} | {{ links(c.experiment) }} | |
| 108 | +{% endfor -%} |
| 109 | +</details> |
| 110 | +{% endif %} |
| 111 | +<details><summary><b>Explanation</b></summary> |
| 112 | + |
| 113 | +A change is flagged as a regression when |Δ mean %| > **{{ effect_size | fmt2 }}%** in the regressing direction for its optimization goal AND SMP marks the experiment as a regression (`is_regression: true`). Improvements use the matching criteria for the improving direction. Experiments configured `erratic: true` (tagged <code>(ignored)</code>) are skipped outright; experiments detected as erratic at runtime (tagged <code>(erratic)</code>) still count, since that flag describes sample dispersion rather than directional certainty. The Δ mean % cell is colored accordingly: 🟢 = improvement, 🔴 = regression, ⚪ = neutral. Reduction in CPU or memory is an improvement; reduction in ingress throughput is a regression. |
| 114 | + |
| 115 | +</details> |
| 116 | + |
0 commit comments