compile-perf: Slack notification on scheduled runs + 10% regression threshold

.github/workflows/compile-perf-nightly.yml

@@ -229,8 +229,51 @@ jobs:
       # job. trend.py exits non-zero (job red + ::error:: annotations) when a
       # primary timer drifts past threshold vs the trailing median.
       - name: Check trend (fail on regression)
+        id: trend
         shell: bash
         run: |
           set -euo pipefail
           cd "$GITHUB_WORKSPACE/tools/compile-perf"
           python trend.py --results "$GITHUB_WORKSPACE/perf-results"
+
+      - name: Notify Slack
+        if: github.event_name == 'schedule' && always()
+        shell: bash
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_COMPILE_PERF }}
+          TREND_OUTCOME: ${{ steps.trend.outcome }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          if [ -z "$SLACK_WEBHOOK" ]; then
+            echo "SLACK_WEBHOOK_COMPILE_PERF not set; skipping Slack notification"
+            exit 0
+          fi
+          DATE=$(date -u +%Y-%m-%d)
+          if [ "$TREND_OUTCOME" = "success" ]; then
+            ICON=":white_check_mark:"
+            STATUS="No regressions detected"
+          elif [ "$TREND_OUTCOME" = "failure" ]; then
+            ICON=":warning:"
+            STATUS="Regression detected — see CI run for details"
+          else
+            ICON=":x:"
+            STATUS="Nightly job failed — see CI run for details"
+          fi
+          python -c "
+          import json, os, urllib.request
+          payload = {
+              'text': (
+                  f'{os.environ[\"ICON\"]} *Compile-perf nightly — {os.environ[\"DATE\"]}*\n'
+                  f'{os.environ[\"STATUS\"]}\n'
+                  f'• Run: {os.environ[\"RUN_URL\"]}\n'
+                  f'• Report: https://shader-slang.org/slang-compile-perf/ '
+                  f'(may take a few minutes to update after the run)'
+              )
+          }
+          req = urllib.request.Request(
+              os.environ['SLACK_WEBHOOK'],
+              data=json.dumps(payload).encode(),
+              headers={'Content-Type': 'application/json'})
+          urllib.request.urlopen(req, timeout=10)
+          print('Slack notification sent')
+          "

tools/compile-perf/DESIGN.md

@@ -77,7 +77,7 @@ via `analyze.canonical_runs`, so history and daily points compare like-with-like
 After each nightly rebuild, `trend.py` compares the latest point's primary timers
 (per workload, always including `compileInner`) against the trailing-N-point
 median (default 7), restricted to the **same runner fingerprint**. A metric past
-both a relative (`--rel`, default 1.25×) and absolute (`--abs`, default 2 ms)
+both a relative (`--rel`, default 1.10×) and absolute (`--abs`, default 2 ms)
 threshold is flagged — printed, emitted as a GitHub `::error::` annotation +
 step-summary row, and the job exits non-zero (after the push, so the data is still
 stored). If the latest point's runner differs from the history's, it warns and

tools/compile-perf/trend.py

@@ -16,7 +16,7 @@
 compares only against same-runner points.
 
     python3 trend.py --results <perf-results>        # after track.py rebuild
-    python3 trend.py --results <dir> --window 7 --rel 1.25 --abs 2.0
+    python3 trend.py --results <dir> --window 7 --rel 1.10 --abs 2.0
 """
 import argparse
 import json
@@ -58,9 +58,11 @@ def main():
                                  formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("--results", default=os.path.join(HERE, "results"))
     # Threshold rationale:
-    # --rel 1.25: flag a 25% rise vs trailing median. The per-PR gate uses 15%;
-    #   25% here catches gradual drift that accumulates across many PRs without
-    #   any single one tripping the per-PR gate.
+    # --rel 1.10: flag a 10% rise vs trailing median. The runner is a dedicated
+    #   quiesced machine with 5-sample medians, giving a noise floor of ~1-3%,
+    #   so 10% catches real medium regressions while avoiding noise false positives.
+    #   The --abs guard (2 ms) prevents alerting on tiny absolute deltas even when
+    #   the relative ratio exceeds 10%.
     # --abs 2.0: ignore sub-2 ms absolute deltas regardless of ratio — a 50%
     #   rise in a 3 ms timer is within measurement noise, not a real regression.
     # --window 7: trailing-7-point median spans ~one week of nightly runs,
@@ -69,7 +71,7 @@ def main():
     # --min-baseline 3: require at least 3 prior same-runner points before judging,
     #   so the first few nights after a new runner don't produce false positives.
     ap.add_argument("--window", type=int, default=7, help="trailing points for the median")
-    ap.add_argument("--rel", type=float, default=1.25, help="relative regression threshold")
+    ap.add_argument("--rel", type=float, default=1.10, help="relative regression threshold")
     ap.add_argument("--abs", type=float, default=2.0, help="min absolute ms delta to flag")
     ap.add_argument("--min-baseline", type=int, default=3,
                     help="min trailing points required to judge a metric")