Skip to content

Commit b199540

Browse files
authored
[CRCR] Add nightly health card to CI summary page (#8525)
## Summary - Adds a "CRCR Relay Health - Nightly" card to Row 1 of the Cross-Repository CI Summary page - Moves "Probe Failures" from Row 1 to Row 2 - Nightly health is determined by the last 5 nightly SHAs from pytorch/crcr-test — if all jobs passed, it shows "Healthy" - Handles x-prefixed probe jobs (xfail, xcancel, xtimeout) as expected outcomes, consistent with #8376 - Clicking the card links to the crcr-test nightly view ## Layout **Row 1:** CRCR Relay Health (PR) | CRCR Relay Health - Nightly | Total Probe Runs **Row 2:** Probe Failures | Registered Backends | Timed Out ## Dependencies - pytorch/crcr-test#21 should be merged first (adds nightly health probe workflow with x-prefixed jobs) ## Test plan - [ ] Verify nightly health card renders correctly on https://hud.pytorch.org/crcr - [ ] Confirm card shows "Healthy" when last 5 nightly SHAs all pass (including x-prefixed expected outcomes) - [ ] Clicking card navigates to https://hud.pytorch.org/crcr/pytorch/crcr-test?event=nightly
1 parent 20f8996 commit b199540

1 file changed

Lines changed: 183 additions & 3 deletions

File tree

torchci/pages/crcr/index.tsx

Lines changed: 183 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,172 @@ function CrcrTestHealthCard({
489489
);
490490
}
491491

492+
interface NightlyJobRow {
493+
pytorch_head_sha: string;
494+
job_name: string;
495+
status: string;
496+
conclusion: string;
497+
started_at: string;
498+
}
499+
500+
const NIGHTLY_HEALTH_COUNT = 5;
501+
502+
function isExpectedNightlyOutcome(job: NightlyJobRow): boolean {
503+
const name = job.job_name ?? "";
504+
return (
505+
(name.includes("xfail") && job.conclusion === "failure") ||
506+
(name.includes("xcancel") && job.conclusion === "cancelled") ||
507+
(name.includes("xtimeout") && job.conclusion === "timed_out")
508+
);
509+
}
510+
511+
function isNightlyJobPassing(job: NightlyJobRow): boolean {
512+
if (job.status !== "completed") return false;
513+
return job.conclusion === "success" || isExpectedNightlyOutcome(job);
514+
}
515+
516+
function CrcrNightlyHealthCard({
517+
nightlyJobs,
518+
}: {
519+
nightlyJobs: NightlyJobRow[] | undefined;
520+
}) {
521+
if (!nightlyJobs || nightlyJobs.length === 0) {
522+
const staleColor = "#9e9e9e";
523+
return (
524+
<NextLink
525+
href="/crcr/pytorch/crcr-test?event=nightly"
526+
passHref
527+
legacyBehavior
528+
>
529+
<Paper
530+
component="a"
531+
elevation={2}
532+
sx={{
533+
p: 2,
534+
flex: 1,
535+
minWidth: 160,
536+
textAlign: "center",
537+
borderLeft: `4px solid ${staleColor}`,
538+
textDecoration: "none",
539+
color: "inherit",
540+
cursor: "pointer",
541+
"&:hover": { bgcolor: "action.hover" },
542+
}}
543+
>
544+
<Typography variant="caption" color="text.secondary">
545+
CRCR Relay Health - Nightly
546+
</Typography>
547+
<Typography variant="h5" sx={{ fontWeight: 600, color: staleColor }}>
548+
No Data
549+
</Typography>
550+
<Typography variant="caption" color="text.secondary">
551+
no nightly results in last 14 days
552+
</Typography>
553+
</Paper>
554+
</NextLink>
555+
);
556+
}
557+
558+
const shaMap = new Map<string, NightlyJobRow[]>();
559+
for (const job of nightlyJobs) {
560+
if (!job.pytorch_head_sha) continue;
561+
const jobs = shaMap.get(job.pytorch_head_sha) ?? [];
562+
jobs.push(job);
563+
shaMap.set(job.pytorch_head_sha, jobs);
564+
}
565+
566+
const shasByTime = Array.from(shaMap.entries())
567+
.map(([sha, jobs]) => ({
568+
sha,
569+
jobs,
570+
latestTime: Math.max(
571+
...jobs.map((j) => new Date(j.started_at).getTime())
572+
),
573+
}))
574+
.sort((a, b) => b.latestTime - a.latestTime)
575+
.slice(0, NIGHTLY_HEALTH_COUNT);
576+
577+
if (shasByTime.length === 0) {
578+
const staleColor = "#9e9e9e";
579+
return (
580+
<NextLink
581+
href="/crcr/pytorch/crcr-test?event=nightly"
582+
passHref
583+
legacyBehavior
584+
>
585+
<Paper
586+
component="a"
587+
elevation={2}
588+
sx={{
589+
p: 2,
590+
flex: 1,
591+
minWidth: 160,
592+
textAlign: "center",
593+
borderLeft: `4px solid ${staleColor}`,
594+
textDecoration: "none",
595+
color: "inherit",
596+
cursor: "pointer",
597+
"&:hover": { bgcolor: "action.hover" },
598+
}}
599+
>
600+
<Typography variant="caption" color="text.secondary">
601+
CRCR Relay Health - Nightly
602+
</Typography>
603+
<Typography variant="h5" sx={{ fontWeight: 600, color: staleColor }}>
604+
No Data
605+
</Typography>
606+
<Typography variant="caption" color="text.secondary">
607+
no nightly results in last 14 days
608+
</Typography>
609+
</Paper>
610+
</NextLink>
611+
);
612+
}
613+
614+
const allPassed = shasByTime.every((entry) =>
615+
entry.jobs.every(isNightlyJobPassing)
616+
);
617+
const passedCount = shasByTime.filter((entry) =>
618+
entry.jobs.every(isNightlyJobPassing)
619+
).length;
620+
const borderColor = allPassed ? "#2e7d32" : "#ed6c02";
621+
const label = allPassed ? "Healthy" : "Degraded";
622+
623+
return (
624+
<NextLink
625+
href="/crcr/pytorch/crcr-test?event=nightly"
626+
passHref
627+
legacyBehavior
628+
>
629+
<Paper
630+
component="a"
631+
elevation={2}
632+
sx={{
633+
p: 2,
634+
flex: 1,
635+
minWidth: 160,
636+
textAlign: "center",
637+
borderLeft: `4px solid ${borderColor}`,
638+
textDecoration: "none",
639+
color: "inherit",
640+
cursor: "pointer",
641+
"&:hover": { bgcolor: "action.hover" },
642+
}}
643+
>
644+
<Typography variant="caption" color="text.secondary">
645+
CRCR Relay Health - Nightly
646+
</Typography>
647+
<Typography variant="h5" sx={{ fontWeight: 600, color: borderColor }}>
648+
{label}
649+
</Typography>
650+
<Typography variant="caption" color="text.secondary">
651+
{passedCount}/{shasByTime.length} recent nightlies passed
652+
</Typography>
653+
</Paper>
654+
</NextLink>
655+
);
656+
}
657+
492658
export default function CrcrSummaryPage() {
493659
const [days, setDays] = useState(7);
494660
const [activeTab, setActiveTab] = useState<EventTab>("pr");
@@ -523,6 +689,17 @@ export default function CrcrSummaryPage() {
523689
{ refreshInterval: 60_000 }
524690
);
525691

692+
const nightlyHealthUrl =
693+
`/api/clickhouse/crcr_nightly_dashboard?parameters=` +
694+
encodeURIComponent(
695+
JSON.stringify({ repo: "pytorch/crcr-test", days: "14" })
696+
);
697+
const { data: nightlyHealthJobs, error: nightlyHealthError } = useSWR<
698+
NightlyJobRow[]
699+
>(nightlyHealthUrl, fetcherHandleError, {
700+
refreshInterval: 60_000,
701+
});
702+
526703
const nightlyRepoCount = nightlyData?.length ?? 0;
527704

528705
const metricsMap = useMemo(() => {
@@ -613,7 +790,8 @@ export default function CrcrSummaryPage() {
613790
}, [ciData, metricsMap, allowlist, days]);
614791

615792
const isLoading = !ciData && !ciError && !allowlist && !alError;
616-
const hasError = ciError || alError || nightlyError || healthError;
793+
const hasError =
794+
ciError || alError || nightlyError || healthError || nightlyHealthError;
617795

618796
return (
619797
<>
@@ -669,6 +847,7 @@ export default function CrcrSummaryPage() {
669847
alError?.message ||
670848
nightlyError?.message ||
671849
healthError?.message ||
850+
nightlyHealthError?.message ||
672851
"Failed to load data"}
673852
</Typography>
674853
)}
@@ -679,18 +858,19 @@ export default function CrcrSummaryPage() {
679858
<Stack spacing={2}>
680859
<Box sx={{ display: "flex", gap: 2, flexWrap: "wrap" }}>
681860
<CrcrTestHealthCard healthPrs={healthPrs} />
861+
<CrcrNightlyHealthCard nightlyJobs={nightlyHealthJobs} />
682862
<StatCard
683863
label="Total Probe Runs"
684864
value={stats.totalRuns}
685865
sub={stats.runsSub}
686866
/>
867+
</Box>
868+
<Box sx={{ display: "flex", gap: 2, flexWrap: "wrap" }}>
687869
<StatCard
688870
label="Probe Failures"
689871
value={stats.failures}
690872
sub="pytorch/crcr-test failures"
691873
/>
692-
</Box>
693-
<Box sx={{ display: "flex", gap: 2, flexWrap: "wrap" }}>
694874
<StatCard
695875
label="Registered Backends"
696876
value={stats.totalRepos}

0 commit comments

Comments
 (0)