autorevert metrics: third FN category for killswitch-active windows (#8089)

Human reverts that landed while autorevert was disabled via the `ci:
disable-autorevert` label on an open issue are misclassified as False
Negatives — autorevert wasn't running, it didn't miss. Surface them as a
third category and exclude from the recall denominator.

## What changed

- New saved query `autorevert_killswitch_windows` returns raw label
add/remove events from `default.issues_label_event` plus each issue's
`closed_at`.
- `lib/autorevert/killswitchWindows.ts` folds the events per-issue into
`[on, off]` intervals, treating a close as an implicit `unlabeled` (the
lambda only honors the label on **open** issues).
- `pages/api/autorevert/metrics.ts` partitions `falseNegatives` into the
lambda-up subset (counted in recall) and a new
`falseNegativesKillswitch` subset, linked to the killswitch issue.
Weekly recall uses the lambda-up subset only.
- `pages/metrics/autorevert.tsx` adds a `FN (Disabled)` StatCard, chart
series (purple, stacked next to FN), legend entry, and renders FN chips
that fall in a killswitch window with the `FN (disabled)` label linking
to the issue.

## API additions

```
summary.false_negatives_killswitch: number
weeklyMetrics[].false_negatives_killswitch: number
killswitchWindows: [{ issue_number, on, off }]
falseNegativesKillswitch: [{ recovery_sha, recovery_time, signals_fixed, reverted_pr_numbers, killswitch_issue }]
```

`recall` (overall and weekly) now uses `tp / (tp + fn_lambda_up)`.

## Validation

Historical signal verified: issue #183016 was `labeled` 2026-05-09
01:36:48Z and `unlabeled` 2026-05-12 16:47:26Z, matching the ~79h
`misc.autorevert_state` row gap. Unit test
(`test/autorevertKillswitchWindows.test.ts`) covers labeled/unlabeled
pairing, `closed_at` fallback, still-active intervals, multi-cycle
issues, and cross-issue ordering.

`yarn tsc --noEmit` and `eslint` clean on the changed files. New +
existing autorevert jest tests pass.

Author: izaitsevfb

torchci/clickhouse_queries/autorevert_killswitch_windows/params.json

@@ -0,0 +1,15 @@
+{
+  "params": {
+    "stopTime": "DateTime64(3)"
+  },
+  "tests": [
+    {
+      "stopTime": "2026-05-15 00:00:00.000"
+    },
+    {
+      "stopTime": {
+        "from_now": 0
+      }
+    }
+  ]
+}

torchci/clickhouse_queries/autorevert_killswitch_windows/query.sql

@@ -0,0 +1,29 @@
+-- All add/remove events of the global autorevert killswitch label
+-- (`ci: disable-autorevert`) on pytorch/pytorch issues, plus each
+-- issue's `closed_at` (empty if still open). The caller folds the
+-- event stream per-issue into [on_ts, off_ts] active intervals,
+-- treating a `closed_at` as an implicit `unlabeled` (the autorevert
+-- lambda only honors the label on OPEN issues). Events outside the
+-- metrics window are needed to resolve intervals that span it.
+WITH issue_close AS (
+    SELECT
+        number AS issue_number,
+        argMax(closed_at, updated_at) AS closed_at_str
+    FROM default.issues
+    WHERE
+        repository_url = 'https://api.github.com/repos/pytorch/pytorch'
+    GROUP BY number
+)
+
+SELECT
+    e.event_time AS event_time,
+    e.action AS action,
+    e.issue_number AS issue_number,
+    coalesce(c.closed_at_str, '') AS issue_closed_at
+FROM default.issues_label_event AS e
+LEFT JOIN issue_close AS c ON c.issue_number = e.issue_number
+WHERE
+    e.repo_name = 'pytorch/pytorch'
+    AND e.label_name = 'ci: disable-autorevert'
+    AND e.event_time <= {stopTime: DateTime64(3)} + INTERVAL 1 DAY
+ORDER BY e.issue_number, e.event_time

torchci/lib/autorevert/killswitchWindows.ts

@@ -0,0 +1,67 @@
+// Fold `ci: disable-autorevert` label add/remove events on pytorch/pytorch
+// issues into per-issue active intervals. The autorevert lambda only honors
+// the label on OPEN issues, so an issue close acts as an implicit
+// `unlabeled` event if no explicit one preceded it.
+
+export interface KillswitchLabelEvent {
+  event_time: string;
+  action: "labeled" | "unlabeled" | string;
+  issue_number: number;
+  issue_closed_at: string;
+}
+
+export interface KillswitchWindow {
+  issue_number: number;
+  on: string;
+  off: string | null;
+}
+
+export function foldKillswitchWindows(
+  events: KillswitchLabelEvent[]
+): KillswitchWindow[] {
+  const byIssue = new Map<number, KillswitchLabelEvent[]>();
+  for (const e of events) {
+    if (!byIssue.has(e.issue_number)) byIssue.set(e.issue_number, []);
+    byIssue.get(e.issue_number)!.push(e);
+  }
+  const windows: KillswitchWindow[] = [];
+  for (const [issueNumber, evs] of byIssue.entries()) {
+    evs.sort((a, b) => a.event_time.localeCompare(b.event_time));
+    let onTs: string | null = null;
+    for (const e of evs) {
+      if (e.action === "labeled" && onTs === null) {
+        onTs = e.event_time;
+      } else if (e.action === "unlabeled" && onTs !== null) {
+        windows.push({
+          issue_number: issueNumber,
+          on: onTs,
+          off: e.event_time,
+        });
+        onTs = null;
+      }
+    }
+    if (onTs !== null) {
+      const closedAt = evs[0].issue_closed_at;
+      windows.push({
+        issue_number: issueNumber,
+        on: onTs,
+        off: closedAt && closedAt !== "" ? closedAt : null,
+      });
+    }
+  }
+  windows.sort((a, b) => a.on.localeCompare(b.on));
+  return windows;
+}
+
+export function killswitchWindowAt(
+  windows: KillswitchWindow[],
+  t: string
+): KillswitchWindow | null {
+  const ts = new Date(t).getTime();
+  for (const w of windows) {
+    const on = new Date(w.on).getTime();
+    const off = w.off === null ? Infinity : new Date(w.off).getTime();
+    if (ts >= on && ts <= off) return w;
+  }
+  return null;
+}

torchci/pages/api/autorevert/metrics.ts

@@ -1,4 +1,9 @@
 import { verifyFpForPr } from "lib/autorevert/fpVerification";
+import {
+  foldKillswitchWindows,
+  KillswitchLabelEvent,
+  killswitchWindowAt,
+} from "lib/autorevert/killswitchWindows";
 import { queryClickhouseSaved } from "lib/clickhouse";
 import { getOctokit } from "lib/github";
 import type { NextApiRequest, NextApiResponse } from "next";
@@ -81,6 +86,7 @@ interface WeeklyMetric {
   human_revert_rate: number;
   // New metrics
   false_positives: number;
+  false_negatives_killswitch: number;
   precision: number;
   recall: number;
 }
@@ -148,21 +154,28 @@ export default async function handler(
     };
 
     // Run queries in parallel
-    const [significantReverts, autorevertEvents, weeklyMetricsRaw] =
-      await Promise.all([
-        queryClickhouseSaved(
-          "autorevert_significant_reverts",
-          queryParams
-        ) as Promise<SignificantRevert[]>,
-        queryClickhouseSaved(
-          "autorevert_events_with_commits",
-          queryParams
-        ) as Promise<AutorevertEvent[]>,
-        queryClickhouseSaved(
-          "autorevert_weekly_metrics",
-          queryParams
-        ) as Promise<any[]>,
-      ]);
+    const [
+      significantReverts,
+      autorevertEvents,
+      weeklyMetricsRaw,
+      killswitchEvents,
+    ] = await Promise.all([
+      queryClickhouseSaved(
+        "autorevert_significant_reverts",
+        queryParams
+      ) as Promise<SignificantRevert[]>,
+      queryClickhouseSaved(
+        "autorevert_events_with_commits",
+        queryParams
+      ) as Promise<AutorevertEvent[]>,
+      queryClickhouseSaved("autorevert_weekly_metrics", queryParams) as Promise<
+        any[]
+      >,
+      queryClickhouseSaved("autorevert_killswitch_windows", {
+        stopTime: queryParams.stopTime,
+      }) as Promise<KillswitchLabelEvent[]>,
+    ]);
+    const killswitchWindows = foldKillswitchWindows(killswitchEvents);
 
     // Build set of recovery SHAs (reverts that fixed signals)
     const recoveryShaSet = new Set(
@@ -191,10 +204,30 @@ export default async function handler(
       }
     }
 
-    // Count False Negatives: human reverts with signal recovery
-    const falseNegatives = significantReverts.filter(
+    // Count False Negatives: human reverts with signal recovery. Partition
+    // out the ones where the autorevert killswitch was active at recovery
+    // time — those are not a lambda miss; they're an upstream_infra class
+    // (`ci: disable-autorevert` label on an open issue, see
+    // `default.issues_label_event`). Killswitch FNs are surfaced as a
+    // third category and excluded from the recall denominator.
+    const allFalseNegatives = significantReverts.filter(
       (r) => !r.is_autorevert && r.recovery_type === "human_revert_recovery"
     );
+    const falseNegatives: SignificantRevert[] = [];
+    const falseNegativesKillswitch: Array<
+      SignificantRevert & { killswitch_issue: number }
+    > = [];
+    for (const r of allFalseNegatives) {
+      const w = killswitchWindowAt(killswitchWindows, r.recovery_time);
+      if (w) {
+        falseNegativesKillswitch.push({
+          ...r,
+          killswitch_issue: w.issue_number,
+        });
+      } else {
+        falseNegatives.push(r);
+      }
+    }
 
     // Verify false positive candidates via GitHub API
     let verifiedFPs: VerifiedFalsePositive[] = [];
@@ -236,22 +269,32 @@ export default async function handler(
     const precision = tp + fp > 0 ? (tp / (tp + fp)) * 100 : 100;
     const recall = tp + fn > 0 ? (tp / (tp + fn)) * 100 : 100;
 
-    // Enhance weekly metrics with precision/recall
-    // Group FPs by week for weekly precision calculation
+    // Enhance weekly metrics with precision/recall.
+    // Group FPs and killswitch-FNs by week for the weekly aggregation.
     const fpByWeek = new Map<string, number>();
     for (const fp of confirmedFPs) {
       const week = getWeekStart(new Date(fp.autorevert_time));
       fpByWeek.set(week, (fpByWeek.get(week) || 0) + 1);
     }
+    const fnKsByWeek = new Map<string, number>();
+    for (const r of falseNegativesKillswitch) {
+      const week = getWeekStart(new Date(r.recovery_time));
+      fnKsByWeek.set(week, (fnKsByWeek.get(week) || 0) + 1);
+    }
 
     const weeklyMetrics: WeeklyMetric[] = weeklyMetricsRaw.map((w) => {
       const weekFPs = fpByWeek.get(w.week) || 0;
+      const weekFNKs = fnKsByWeek.get(w.week) || 0;
       const weekTP = w.autorevert_recoveries;
-      const weekFN = w.human_revert_recoveries;
+      // human_revert_recoveries from the saved query counts all human
+      // reverts with signal recovery; subtract the killswitch-attributed
+      // ones so weekly recall matches the overall recall semantics.
+      const weekFN = Math.max(0, w.human_revert_recoveries - weekFNKs);
 
       return {
         ...w,
         false_positives: weekFPs,
+        false_negatives_killswitch: weekFNKs,
         precision:
           weekTP + weekFPs > 0
             ? Math.round((weekTP / (weekTP + weekFPs)) * 1000) / 10
@@ -272,6 +315,7 @@ export default async function handler(
         tp_without_signal_recovery: tpWithoutSignalRecovery,
         confirmed_false_positives: fp,
         false_negatives: fn,
+        false_negatives_killswitch: falseNegativesKillswitch.length,
         // Rates
         precision: Math.round(precision * 10) / 10,
         recall: Math.round(recall * 10) / 10,
@@ -282,6 +326,7 @@ export default async function handler(
       },
       weeklyMetrics,
       significantReverts,
+      killswitchWindows,
       falsePositives: {
         candidates_checked: falsePositiveCandidates.length,
         confirmed: confirmedFPs,
@@ -294,6 +339,13 @@ export default async function handler(
         signals_fixed: r.signals_fixed,
         reverted_pr_numbers: r.reverted_pr_numbers,
       })),
+      falseNegativesKillswitch: falseNegativesKillswitch.map((r) => ({
+        recovery_sha: r.recovery_sha,
+        recovery_time: r.recovery_time,
+        signals_fixed: r.signals_fixed,
+        reverted_pr_numbers: r.reverted_pr_numbers,
+        killswitch_issue: r.killswitch_issue,
+      })),
     };
 
     setCache(cacheKey, result);