Skip to content

Commit ccc82f1

Browse files
authored
Add AI advisor manual dispatch to HUD PR page (#7980)
## Summary Adds the ability to trigger AI advisor analysis for individual failed jobs on the HUD PR page (hud.pytorch.org). ### New features: - **Verdict display**: Each failed job on a PR page shows existing AI advisor verdicts as colored chips (red=revert, green=not_related, yellow=unsure, grey=garbage) - **Manual dispatch**: An "AI Analyze" button appears next to failed jobs with no existing verdict, dispatching the `claude-autorevert-advisor.yml` workflow - **Signal pattern**: The dispatch builds a JSON payload with job status across PR head, merge base, and recent trunk commits from ClickHouse ### Files: - `clickhouse_queries/advisor_verdicts_for_pr/` — CH query for existing verdicts - `pages/api/.../pull/advisor-runs.ts` — API to fetch verdicts for a PR - `pages/api/.../pull/dispatch-advisor.ts` — API to dispatch advisor workflow - `components/job/AiAdvisorIndicator.tsx` — Verdict chip + dispatch button component - `components/job/FilteredJobList.tsx` — Integration into failed job list ## Test plan - [x] Verify TypeScript compiles - [x] Test on a PR page with known failed jobs - [x] Verify dispatch creates workflow run - [x] Verify verdict display after workflow completes tested locally on the PRs with failures: http://localhost:3000/pr/pytorch/pytorch/181449#72961893295 <details> <summary>screenshot</summary> <img width="1486" height="1365" alt="image" src="https://github.com/user-attachments/assets/9b1217b7-117d-465b-b196-5c11f3dafceb" /> </details>
1 parent 30f6c3d commit ccc82f1

6 files changed

Lines changed: 748 additions & 1 deletion

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"prNumber": "Int64"
5+
},
6+
"tests": []
7+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-- Fetch AI advisor verdicts for a given PR number
2+
SELECT
3+
suspect_commit AS sha,
4+
signal_key,
5+
signal_source,
6+
workflow_name,
7+
verdict,
8+
confidence,
9+
summary,
10+
causal_reasoning,
11+
run_id,
12+
pr_number,
13+
timestamp
14+
FROM
15+
misc.autorevert_advisor_verdicts
16+
WHERE
17+
repo = {repo: String}
18+
AND pr_number = {prNumber: Int64}
19+
ORDER BY
20+
timestamp DESC
Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
import { Button, Chip, CircularProgress, Tooltip } from "@mui/material";
2+
import { fetcher } from "lib/GeneralUtils";
3+
import { AdvisorVerdict, AdvisorVerdictType } from "lib/advisorVerdictUtils";
4+
import { useSession } from "next-auth/react";
5+
import { useRouter } from "next/router";
6+
import { useCallback, useState } from "react";
7+
import useSWR from "swr";
8+
import AdvisorSection from "./AdvisorSection";
9+
10+
const DISPATCH_STORAGE_KEY = "ai_advisor_dispatches";
11+
const DISPATCH_TTL_MS = 10 * 60 * 1000; // 10 minutes
12+
13+
interface DispatchEntry {
14+
timestamp: number;
15+
}
16+
17+
function getDispatchKey(
18+
prNumber: number,
19+
sha: string,
20+
signalKey: string
21+
): string {
22+
return `${prNumber}:${sha}:${signalKey}`;
23+
}
24+
25+
function getDispatches(): Record<string, DispatchEntry> {
26+
try {
27+
const raw = localStorage.getItem(DISPATCH_STORAGE_KEY);
28+
if (!raw) return {};
29+
const entries = JSON.parse(raw) as Record<string, DispatchEntry>;
30+
const now = Date.now();
31+
const valid: Record<string, DispatchEntry> = {};
32+
for (const [k, v] of Object.entries(entries)) {
33+
if (now - v.timestamp < DISPATCH_TTL_MS) {
34+
valid[k] = v;
35+
}
36+
}
37+
if (Object.keys(valid).length !== Object.keys(entries).length) {
38+
localStorage.setItem(DISPATCH_STORAGE_KEY, JSON.stringify(valid));
39+
}
40+
return valid;
41+
} catch {
42+
return {};
43+
}
44+
}
45+
46+
function markDispatched(
47+
prNumber: number,
48+
sha: string,
49+
signalKey: string
50+
): void {
51+
const dispatches = getDispatches();
52+
dispatches[getDispatchKey(prNumber, sha, signalKey)] = {
53+
timestamp: Date.now(),
54+
};
55+
localStorage.setItem(DISPATCH_STORAGE_KEY, JSON.stringify(dispatches));
56+
}
57+
58+
function isDispatched(
59+
prNumber: number,
60+
sha: string,
61+
signalKey: string
62+
): boolean {
63+
const dispatches = getDispatches();
64+
const entry = dispatches[getDispatchKey(prNumber, sha, signalKey)];
65+
if (!entry) return false;
66+
return Date.now() - entry.timestamp < DISPATCH_TTL_MS;
67+
}
68+
69+
const VERDICT_CHIP_COLORS: Record<
70+
string,
71+
"error" | "warning" | "success" | "default"
72+
> = {
73+
revert: "error",
74+
unsure: "warning",
75+
not_related: "success",
76+
garbage: "default",
77+
};
78+
79+
const VERDICT_LABELS: Record<string, string> = {
80+
revert: "Revert",
81+
unsure: "Unsure",
82+
not_related: "Not Related",
83+
garbage: "Garbage Signal",
84+
};
85+
86+
export default function AiAdvisorIndicator({
87+
jobName,
88+
sha,
89+
prNumber,
90+
conclusion,
91+
mergeBaseSha,
92+
workflowName,
93+
}: {
94+
jobName: string;
95+
sha: string;
96+
prNumber: number;
97+
conclusion?: string;
98+
mergeBaseSha?: string;
99+
workflowName?: string;
100+
}) {
101+
const isFailed =
102+
conclusion === "failure" ||
103+
conclusion === "cancelled" ||
104+
conclusion === "timed_out";
105+
const router = useRouter();
106+
const { repoOwner, repoName } = router.query;
107+
const session = useSession();
108+
const [dispatching, setDispatching] = useState(false);
109+
const [error, setError] = useState("");
110+
111+
// dr_ci_ prefix separates HUD-dispatched verdicts from autorevert-system ones
112+
const signalKey = `dr_ci_${jobName}`;
113+
114+
const { data: verdicts } = useSWR<AdvisorVerdict[]>(
115+
prNumber
116+
? `/api/${repoOwner}/${repoName}/pull/advisor-runs?prNumber=${prNumber}`
117+
: null,
118+
fetcher,
119+
{ refreshInterval: 60_000 }
120+
);
121+
122+
const matchingVerdict = verdicts?.find(
123+
(v) => v.signalKey === signalKey && v.sha === sha
124+
);
125+
126+
const dispatched = !matchingVerdict && isDispatched(prNumber, sha, signalKey);
127+
128+
const isAuthenticated =
129+
session?.data &&
130+
session.data["accessToken"] !== undefined &&
131+
session.data["user"] !== undefined;
132+
133+
const [, setTick] = useState(0);
134+
135+
const handleDispatch = useCallback(async () => {
136+
if (!isAuthenticated || dispatching || dispatched) return;
137+
138+
setDispatching(true);
139+
setError("");
140+
141+
try {
142+
const res = await fetch(
143+
`/api/${repoOwner}/${repoName}/pull/dispatch-advisor`,
144+
{
145+
method: "POST",
146+
headers: {
147+
"Content-Type": "application/json",
148+
Authorization: session.data!["accessToken"] as string,
149+
},
150+
body: JSON.stringify({
151+
prNumber,
152+
headSha: sha,
153+
mergeBaseSha: mergeBaseSha || "",
154+
jobName,
155+
workflowName: workflowName || "",
156+
}),
157+
}
158+
);
159+
160+
if (res.status === 409) {
161+
// Already dispatched — treat as success so we show "Dispatched"
162+
markDispatched(prNumber, sha, signalKey);
163+
setTick((t) => t + 1);
164+
return;
165+
}
166+
167+
if (!res.ok) {
168+
const data = await res.json();
169+
throw new Error(data.error || `HTTP ${res.status}`);
170+
}
171+
172+
markDispatched(prNumber, sha, signalKey);
173+
setTick((t) => t + 1);
174+
} catch (e: any) {
175+
setError(e.message || "Failed to dispatch");
176+
} finally {
177+
setDispatching(false);
178+
}
179+
}, [
180+
isAuthenticated,
181+
dispatching,
182+
dispatched,
183+
repoOwner,
184+
repoName,
185+
prNumber,
186+
sha,
187+
mergeBaseSha,
188+
jobName,
189+
signalKey,
190+
workflowName,
191+
session.data,
192+
]);
193+
194+
const chipColor =
195+
VERDICT_CHIP_COLORS[matchingVerdict?.verdict as AdvisorVerdictType] ||
196+
"default";
197+
const chipLabel =
198+
VERDICT_LABELS[matchingVerdict?.verdict as AdvisorVerdictType] ||
199+
matchingVerdict?.verdict;
200+
201+
return (
202+
<span
203+
style={{
204+
display: "inline-flex",
205+
alignItems: "center",
206+
gap: 4,
207+
}}
208+
>
209+
{matchingVerdict && (
210+
<Tooltip
211+
title={
212+
<AdvisorSection
213+
verdict={matchingVerdict}
214+
repoOwner={repoOwner as string}
215+
repoName={repoName as string}
216+
/>
217+
}
218+
arrow
219+
placement="bottom-start"
220+
slotProps={{
221+
tooltip: {
222+
sx: { maxWidth: 500, fontSize: "inherit", p: 0.5 },
223+
},
224+
}}
225+
>
226+
<Chip
227+
label={`AI: ${chipLabel}`}
228+
color={chipColor}
229+
size="small"
230+
variant="outlined"
231+
sx={{ ml: 1, cursor: "pointer" }}
232+
/>
233+
</Tooltip>
234+
)}
235+
{!matchingVerdict && !dispatched && isFailed && isAuthenticated && (
236+
<Tooltip title="Run AI advisor to analyze this failure">
237+
<Button
238+
size="small"
239+
variant="outlined"
240+
onClick={handleDispatch}
241+
disabled={dispatching}
242+
sx={{
243+
ml: 1,
244+
textTransform: "none",
245+
fontSize: "0.75rem",
246+
py: 0,
247+
minHeight: 24,
248+
}}
249+
>
250+
{dispatching ? (
251+
<CircularProgress size={14} sx={{ mr: 0.5 }} />
252+
) : (
253+
"🤖"
254+
)}{" "}
255+
AI Analyze
256+
</Button>
257+
</Tooltip>
258+
)}
259+
{dispatched && (
260+
<Chip
261+
label="AI: Dispatched"
262+
size="small"
263+
variant="outlined"
264+
color="info"
265+
sx={{ ml: 1 }}
266+
/>
267+
)}
268+
{error && (
269+
<Chip
270+
label={`Error: ${error}`}
271+
size="small"
272+
variant="outlined"
273+
color="error"
274+
sx={{ ml: 1 }}
275+
/>
276+
)}
277+
</span>
278+
);
279+
}

torchci/components/job/FilteredJobList.tsx

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import useScrollTo from "lib/useScrollTo";
44
import { useRouter } from "next/router";
55
import useSWR from "swr";
66
import LogViewer from "../common/log/LogViewer";
7+
import AiAdvisorIndicator from "./AiAdvisorIndicator";
78
import JobAnnotationToggle from "./JobAnnotationToggle";
89
import JobLinks from "./JobLinks";
910
import JobSummary from "./JobSummary";
@@ -21,10 +22,20 @@ function FailedJobInfo({
2122
}) {
2223
const router = useRouter();
2324
useScrollTo();
24-
const { repoOwner, repoName } = router.query;
25+
const { repoOwner, repoName, prNumber } = router.query;
26+
const prNum = prNumber ? parseInt(prNumber as string, 10) : 0;
2527
return (
2628
<li key={job.id} id={job.id}>
2729
<JobSummary job={job} unstableIssues={unstableIssues} />
30+
{prNum > 0 && job.name && job.sha && (
31+
<AiAdvisorIndicator
32+
jobName={job.name}
33+
sha={job.sha}
34+
prNumber={prNum}
35+
conclusion={job.conclusion}
36+
workflowName={job.workflowName}
37+
/>
38+
)}
2839
<div>
2940
<JobLinks job={job} />
3041
</div>
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import {
2+
AdvisorVerdictRow,
3+
deduplicateVerdicts,
4+
} from "lib/advisorVerdictUtils";
5+
import { queryClickhouseSaved } from "lib/clickhouse";
6+
import type { NextApiRequest, NextApiResponse } from "next";
7+
8+
export default async function handler(
9+
req: NextApiRequest,
10+
res: NextApiResponse
11+
) {
12+
const { repoOwner, repoName, prNumber } = req.query;
13+
14+
if (!repoOwner || !repoName || !prNumber) {
15+
res.status(400).json({ error: "Missing required parameters" });
16+
return;
17+
}
18+
19+
try {
20+
const rows = (await queryClickhouseSaved("advisor_verdicts_for_pr", {
21+
repo: `${repoOwner}/${repoName}`,
22+
prNumber: parseInt(prNumber as string, 10),
23+
})) as AdvisorVerdictRow[];
24+
25+
res.status(200).json(deduplicateVerdicts(rows));
26+
} catch (error: any) {
27+
res.status(500).json({
28+
error: "Internal server error",
29+
details:
30+
process.env.NODE_ENV === "development" ? error.message : undefined,
31+
});
32+
}
33+
}

0 commit comments

Comments
 (0)