diff --git a/docs/pr-assets/pr-compare-default-before-after.png b/docs/pr-assets/pr-compare-default-before-after.png new file mode 100644 index 0000000..5ab2951 Binary files /dev/null and b/docs/pr-assets/pr-compare-default-before-after.png differ diff --git a/src/app/compare/compare-content.tsx b/src/app/compare/compare-content.tsx index 680c652..a361085 100644 --- a/src/app/compare/compare-content.tsx +++ b/src/app/compare/compare-content.tsx @@ -1,11 +1,18 @@ "use client"; -import { Fragment, useEffect, useMemo, useState } from "react"; +import { Fragment, useEffect, useMemo, useRef, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; import useSWR from "swr"; import { SearchableSelect } from "@/components/searchable-select"; import { DateRangePicker } from "@/components/date-range-picker"; import { commitFromImage } from "@/lib/commit-from-image"; +import { + compareImageLabel, + latestNightlyPair, + latestReleasePair, + type CompareImagePair, + type NightlyImageEntry, +} from "@/lib/compare-images"; type Area = "perf" | "eval"; type DeltaStatus = "regression" | "improvement" | "unchanged" | "noisy"; @@ -95,6 +102,12 @@ interface CompareFilters { device: string; task: string; perfThresholdPct: string; + startDate: string; + endDate: string; +} + +interface NightlyResponse { + nightlies: NightlyImageEntry[]; } const DEFAULT_EVAL_SIGMA = 2; @@ -1210,6 +1223,10 @@ function CoverageCard({ export default function ComparePage() { const router = useRouter(); const searchParams = useSearchParams(); + const hadExplicitImageParams = useRef( + searchParams.has("baseline") || searchParams.has("candidate") + ); + const didApplyDefault = useRef(false); const [baseline, setBaseline] = useState(searchParams.get("baseline") ?? ""); const [candidate, setCandidate] = useState(searchParams.get("candidate") ?? ""); const [model, setModel] = useState(searchParams.get("model") ?? ""); @@ -1241,6 +1258,8 @@ export default function ComparePage() { device, task, perfThresholdPct, + startDate, + endDate, ...updates, }; const params = new URLSearchParams(); @@ -1251,6 +1270,8 @@ export default function ComparePage() { setQueryParam(params, "model", next.model); setQueryParam(params, "device", next.device); setQueryParam(params, "task", next.task); + setQueryParam(params, "start", next.startDate); + setQueryParam(params, "end", next.endDate); if ( Number.isFinite(perfThresholdParam) && perfThresholdParam >= 0 && @@ -1295,6 +1316,18 @@ export default function ComparePage() { updateCompareUrl({ perfThresholdPct: value }); }; + const updateDateRange = (start: string, end: string) => { + setStartDate(start); + setEndDate(end); + updateCompareUrl({ startDate: start, endDate: end }); + }; + + const applyImagePair = (pair: CompareImagePair) => { + setBaseline(pair.baseline); + setCandidate(pair.candidate); + updateCompareUrl(pair); + }; + const filtersQuery = useMemo(() => { const p = new URLSearchParams(); if (startDate) p.set("start", startDate); @@ -1311,12 +1344,47 @@ export default function ComparePage() { `/api/eval/filters${filtersQuery}`, fetcher ); + const { data: nightlyData, error: nightlyError } = useSWR( + "/api/nightly?limit=2", + fetcher, + { revalidateOnFocus: false } + ); const imageOptions = useMemo( () => uniqueSorted([...(perfFilters?.images ?? []), ...(evalFilters?.images ?? [])]), [perfFilters?.images, evalFilters?.images] ); + const nightlyPair = useMemo( + () => latestNightlyPair(nightlyData?.nightlies ?? [], imageOptions), + [nightlyData?.nightlies, imageOptions] + ); + const releasePair = useMemo( + () => latestReleasePair(imageOptions), + [imageOptions] + ); + + useEffect(() => { + if ( + didApplyDefault.current || + hadExplicitImageParams.current || + imageOptions.length === 0 || + (!nightlyData && !nightlyError) + ) { + return; + } + + didApplyDefault.current = true; + const pair = nightlyPair ?? releasePair; + if (!pair) return; + + setBaseline(pair.baseline); + setCandidate(pair.candidate); + updateCompareUrl(pair); + // Only seed the initial blank state. Clearing a selection remains a user choice. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [imageOptions, nightlyData, nightlyError, nightlyPair, releasePair]); + useEffect(() => { if (imageOptions.length === 0) return; if (baseline && !imageOptions.includes(baseline)) { @@ -1430,6 +1498,12 @@ export default function ComparePage() { const baselineShort = data ? shortImage(data.baseline) : null; const candidateShort = data ? shortImage(data.candidate) : null; + const selectedPreset = + nightlyPair?.baseline === baseline && nightlyPair.candidate === candidate + ? "nightly" + : releasePair?.baseline === baseline && releasePair.candidate === candidate + ? "release" + : null; return (
@@ -1450,6 +1524,7 @@ export default function ComparePage() { onChange={updateBaseline} options={imageOptions} allLabel="Select baseline" + formatOption={compareImageLabel} />
@@ -1459,6 +1534,7 @@ export default function ComparePage() { onChange={updateCandidate} options={imageOptions} allLabel="Select candidate" + formatOption={compareImageLabel} />
+ )} + {releasePair && ( + + )} + + + )}
Advanced filters @@ -1542,10 +1660,7 @@ export default function ComparePage() { { - setStartDate(s); - setEndDate(e); - }} + onChange={updateDateRange} /> @@ -1593,8 +1708,8 @@ export default function ComparePage() { />
-
-
+
+
Baseline image @@ -1610,7 +1725,7 @@ export default function ComparePage() {
-
+
-
+
Candidate image diff --git a/src/components/searchable-select.tsx b/src/components/searchable-select.tsx index aa8429b..48a5187 100644 --- a/src/components/searchable-select.tsx +++ b/src/components/searchable-select.tsx @@ -10,6 +10,8 @@ interface SearchableSelectProps { allLabel?: string; /** Optional per-option count, shown muted on the right (e.g. datapoints). */ counts?: Record; + /** Optional concise label for long machine-oriented option values. */ + formatOption?: (option: string) => string; } export function SearchableSelect({ @@ -19,6 +21,7 @@ export function SearchableSelect({ options, allLabel, counts, + formatOption = (option) => option, }: SearchableSelectProps) { const [open, setOpen] = useState(false); const [search, setSearch] = useState(""); @@ -47,7 +50,9 @@ export function SearchableSelect({ const filtered = !search ? options : options - .filter((o) => o.toLowerCase().includes(search.toLowerCase())) + .filter((o) => + `${o} ${formatOption(o)}`.toLowerCase().includes(search.toLowerCase()) + ) .sort((a, b) => { const al = a.toLowerCase(); const bl = b.toLowerCase(); @@ -93,7 +98,7 @@ export function SearchableSelect({ className="dashboard-control flex min-h-11 w-full items-center justify-between rounded-md border border-zinc-200 bg-white px-3 text-left text-sm shadow-sm hover:border-zinc-300 dark:border-zinc-700 dark:bg-zinc-900 dark:hover:border-zinc-600 sm:min-h-10 sm:w-52" > - {value || allLabel} + {value ? formatOption(value) : allLabel} - {option} + + {formatOption(option)} + {counts?.[option] !== undefined && ( {counts[option]} diff --git a/src/lib/compare-images.test.ts b/src/lib/compare-images.test.ts new file mode 100644 index 0000000..4191220 --- /dev/null +++ b/src/lib/compare-images.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + compareImageLabel, + latestNightlyPair, + latestReleasePair, +} from "./compare-images"; + +test("latestReleasePair selects the two newest official releases", () => { + assert.deepEqual( + latestReleasePair([ + "vllm/vllm-openai:v0.25.1", + "vllm/vllm-openai-rocm:v0.30.0", + "vllm/vllm-openai:v0.9.0", + "vllm/vllm-openai:v0.26.0", + ]), + { + baseline: "vllm/vllm-openai:v0.25.1", + candidate: "vllm/vllm-openai:v0.26.0", + } + ); +}); + +test("latestNightlyPair prefers the canonical previous image", () => { + const previous = "public.ecr.aws/vllm-release-repo:bbbbbbbb-x86_64"; + const latest = "public.ecr.aws/vllm-release-repo:aaaaaaaa-x86_64"; + assert.deepEqual( + latestNightlyPair( + [ + { + sourceImage: latest, + deltaVsPrev: { prevSourceImage: previous }, + }, + ], + [latest, previous] + ), + { baseline: previous, candidate: latest } + ); +}); + +test("latestNightlyPair ignores pairs missing from the compare filters", () => { + assert.equal( + latestNightlyPair( + [ + { + sourceImage: "nightly-current", + deltaVsPrev: { prevSourceImage: "nightly-previous" }, + }, + ], + ["nightly-current"] + ), + null + ); +}); + +test("compareImageLabel makes release and nightly images scannable", () => { + assert.equal( + compareImageLabel("vllm/vllm-openai:v0.26.0"), + "Release v0.26.0" + ); + assert.equal( + compareImageLabel( + "public.ecr.aws/q9t5s3a7/vllm-release-repo:aa9903490c616dc6871e5acc62cec7bb1e5e9434-x86_64" + ), + "Nightly aa99034 · x86 64" + ); +}); diff --git a/src/lib/compare-images.ts b/src/lib/compare-images.ts new file mode 100644 index 0000000..8f57510 --- /dev/null +++ b/src/lib/compare-images.ts @@ -0,0 +1,86 @@ +export interface CompareImagePair { + baseline: string; + candidate: string; +} + +export interface NightlyImageEntry { + sourceImage: string; + deltaVsPrev?: { + prevSourceImage?: string | null; + }; +} + +interface ReleaseImage { + image: string; + version: [number, number, number]; +} + +const OFFICIAL_RELEASE_RE = + /^vllm\/vllm-openai:v(\d+)\.(\d+)\.(\d+)$/i; +const NIGHTLY_COMMIT_RE = + /(?:nightly-|vllm-release-repo:)([0-9a-f]{7,40})(?:[-_.](.+))?$/i; + +function compareVersion( + a: [number, number, number], + b: [number, number, number] +) { + return a[0] - b[0] || a[1] - b[1] || a[2] - b[2]; +} + +export function latestReleasePair(images: string[]): CompareImagePair | null { + const releases: ReleaseImage[] = images.flatMap((image) => { + const match = image.match(OFFICIAL_RELEASE_RE); + if (!match) return []; + return [ + { + image, + version: [Number(match[1]), Number(match[2]), Number(match[3])], + }, + ]; + }); + + releases.sort((a, b) => compareVersion(a.version, b.version)); + if (releases.length < 2) return null; + + return { + baseline: releases[releases.length - 2].image, + candidate: releases[releases.length - 1].image, + }; +} + +export function latestNightlyPair( + nightlies: NightlyImageEntry[], + availableImages: string[] +): CompareImagePair | null { + const available = new Set(availableImages); + const candidate = nightlies[0]?.sourceImage; + const baseline = + nightlies[0]?.deltaVsPrev?.prevSourceImage ?? nightlies[1]?.sourceImage; + + if ( + !baseline || + !candidate || + baseline === candidate || + !available.has(baseline) || + !available.has(candidate) + ) { + return null; + } + + return { baseline, candidate }; +} + +export function compareImageLabel(image: string): string { + const release = image.match(OFFICIAL_RELEASE_RE); + if (release) return `Release v${release[1]}.${release[2]}.${release[3]}`; + + const nightly = image.match(NIGHTLY_COMMIT_RE); + if (nightly) { + const suffix = nightly[2]?.replace(/[-_.]+/g, " "); + return `Nightly ${nightly[1].slice(0, 7)}${suffix ? ` · ${suffix}` : ""}`; + } + + const slash = image.lastIndexOf("/"); + const colon = image.lastIndexOf(":"); + return colon > slash ? image.slice(colon + 1) : image; +}