Skip to content

Commit ba3041c

Browse files
malfetmalfet
authored
Add [cuda/mac]_test_stats HUD page (#8127)
Add two new HUD pages under **Dev Infra** — `/mac_test_stats` and `/cuda_test_stats` — that show per-commit **total/skip/flaky/fail** counts for `pytorch/pytorch` trunk macOS / linux-jammy-cuda test jobs. Both are thin wrappers around a shared `TestStatsPage` component. - URL params: `?count=N` (default 30, cap 200), `?sha=<hex>` to anchor the window at a specific commit. Prev/Next pagination via `?sha`. - Δ vs the previous commit shown, color-coded via `useTheme()` for dark/light mode. - Rows whose `trunk` run is still queued/in_progress (or has running jobs) are tinted amber with a ⏳ marker — counts may still rise. - Time rendered using `LocalTimeHuman` widget, used by the main HUD page. ## Query `test_stats_per_commit` query taking `repo / ref / workflow / jobFilter / count / sha`. Returns one row per recent push with `sha, message, author, time, workflow_id, run_status, pending_jobs` and `success / skipped / flaky / failure`. Earliest `workflow_run.id` per `head_sha` is picked so reruns don't shift totals. ## Test plan Go to https://torchci-git-malfet-mac-test-stats-page-fbopensource.vercel.app/cuda_test_stats?sha=9661ae6e5416b89d274c2a348889264ac5816d68 and enjoy 14K skipped test spike
1 parent bc4e46b commit ba3041c

6 files changed

Lines changed: 515 additions & 0 deletions

File tree

‎torchci/clickhouse_queries/test_stats_per_commit/params.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"ref": "String",
5+
"workflow": "String",
6+
"jobFilter": "String",
7+
"count": "UInt32",
8+
"sha": "String"
9+
},
10+
"tests": [
11+
{
12+
"repo": "pytorch/pytorch",
13+
"ref": "refs/heads/main",
14+
"workflow": "trunk",
15+
"jobFilter": "(?i)macos",
16+
"count": 5,
17+
"sha": ""
18+
}
19+
]
20+
}

‎torchci/clickhouse_queries/test_stats_per_commit/query.sql‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
-- Per-commit aggregated test pass/skip/flaky/fail counts for the last N pushes
2+
-- on a branch, restricted to a single workflow and a job-name regex.
3+
--
4+
-- Mirrors the per-commit table produced by skip_delta.py: the "earliest"
5+
-- workflow_run (min id) is picked per commit so reruns don't shift the totals.
6+
WITH anchor_time AS (
7+
-- If a sha prefix is supplied, window ends at that commit's timestamp;
8+
-- otherwise the window ends at the latest push (sentinel far-future date).
9+
SELECT
10+
if(
11+
{sha: String } = '',
12+
toDateTime64('2099-01-01 00:00:00', 3),
13+
(
14+
SELECT p.head_commit.timestamp
15+
FROM default.push p
16+
WHERE
17+
p.repository.full_name = {repo: String }
18+
AND p.ref = {ref: String }
19+
AND startsWith(p.head_commit.id, {sha: String })
20+
ORDER BY p.head_commit.timestamp DESC
21+
LIMIT 1
22+
)
23+
) AS ts
24+
),
25+
26+
recent_commits AS (
27+
SELECT
28+
p.head_commit.id AS sha,
29+
p.head_commit.message AS message,
30+
p.head_commit.author.name AS author,
31+
p.head_commit.timestamp AS time
32+
FROM default.push p
33+
WHERE
34+
p.repository.full_name = {repo: String }
35+
AND p.ref = {ref: String }
36+
AND p.head_commit.timestamp <= (SELECT ts FROM anchor_time)
37+
ORDER BY p.head_commit.timestamp DESC
38+
LIMIT {count: UInt32 }
39+
),
40+
41+
matched_runs AS (
42+
-- argMin grabs the status of the SAME row that min(id) picks, so run_status
43+
-- describes the original push run that we're aggregating jobs from.
44+
SELECT
45+
wr.head_sha AS sha,
46+
min(wr.id) AS workflow_id,
47+
argMin(wr.status, wr.id) AS run_status
48+
FROM default.workflow_run wr FINAL
49+
WHERE
50+
wr.id IN (
51+
SELECT id FROM materialized_views.workflow_run_by_head_sha
52+
WHERE head_sha IN (SELECT sha FROM recent_commits)
53+
)
54+
AND wr.name = {workflow: String }
55+
AND wr.repository.full_name = {repo: String }
56+
GROUP BY wr.head_sha
57+
),
58+
59+
matched_jobs AS (
60+
-- FINAL is needed: workflow_job is a ReplacingMergeTree and unmerged parts
61+
-- can carry stale duplicate rows for the same job_id with differing
62+
-- status/conclusion_kg, which would inflate pending_jobs and could even
63+
-- misclassify a completed job as pending.
64+
--
65+
-- The id IN (materialized_views.workflow_job_by_head_sha ...) predicate is
66+
-- the same trick commit_jobs_batch_query uses: it bounds the workflow_job
67+
-- scan to just the rows for our 30 commits via the materialized view's
68+
-- sort key, instead of letting the JOIN stream all rows of the wide table.
69+
SELECT
70+
wj.id AS job_id,
71+
mr.sha AS sha,
72+
wj.status AS status,
73+
wj.conclusion_kg AS conclusion
74+
FROM default.workflow_job wj FINAL
75+
JOIN matched_runs mr ON wj.run_id = mr.workflow_id
76+
WHERE wj.id IN (
77+
SELECT id FROM materialized_views.workflow_job_by_head_sha
78+
WHERE head_sha IN (SELECT sha FROM recent_commits)
79+
)
80+
AND match(wj.name, {jobFilter: String })
81+
),
82+
83+
per_sha_pending AS (
84+
-- A job is "in progress" until it reaches a final conclusion.
85+
-- conclusion_kg is empty while queued / in_progress and gets filled when
86+
-- the job completes (mirrors the logic in hud_query / commit_jobs_query).
87+
SELECT
88+
sha,
89+
countIf(conclusion = '' OR status != 'completed') AS pending_jobs
90+
FROM matched_jobs
91+
GROUP BY sha
92+
),
93+
94+
test_statuses AS (
95+
-- Mirror the join+IN pattern from tests/test_status_counts_on_commits_by_file:
96+
-- the WHERE...IN gives ClickHouse a sargable predicate to skip data parts
97+
-- before the JOIN materializes the (sha) column.
98+
SELECT
99+
mj.sha AS sha,
100+
atr.invoking_file AS invoking_file,
101+
atr.name AS test_name,
102+
atr.classname AS classname,
103+
multiIf(
104+
countIf(
105+
atr.failure_count = 0
106+
AND atr.error_count = 0
107+
AND atr.skipped_count = 0
108+
AND atr.rerun_count = 0
109+
) = count(*),
110+
'success',
111+
sum(atr.skipped_count) > 0,
112+
'skipped',
113+
countIf(
114+
atr.failure_count = 0
115+
AND atr.error_count = 0
116+
) > 0,
117+
'flaky',
118+
'failure'
119+
) AS status
120+
FROM tests.all_test_runs atr
121+
JOIN matched_jobs mj ON mj.job_id = atr.job_id
122+
WHERE atr.job_id IN (SELECT job_id FROM matched_jobs)
123+
GROUP BY mj.sha, atr.invoking_file, atr.name, atr.classname
124+
),
125+
126+
per_sha_counts AS (
127+
SELECT
128+
sha,
129+
countIf(status = 'success') AS success,
130+
countIf(status = 'skipped') AS skipped,
131+
countIf(status = 'flaky') AS flaky,
132+
countIf(status = 'failure') AS failure
133+
FROM test_statuses
134+
GROUP BY sha
135+
)
136+
137+
SELECT
138+
rc.sha AS sha,
139+
rc.message AS message,
140+
rc.author AS author,
141+
rc.time AS time,
142+
mr.workflow_id AS workflow_id,
143+
coalesce(s.success, 0) AS success,
144+
coalesce(s.skipped, 0) AS skipped,
145+
coalesce(s.flaky, 0) AS flaky,
146+
coalesce(s.failure, 0) AS failure,
147+
coalesce(p.pending_jobs, 0) AS pending_jobs,
148+
mr.run_status AS run_status
149+
FROM recent_commits rc
150+
LEFT JOIN matched_runs mr ON mr.sha = rc.sha
151+
LEFT JOIN per_sha_counts s ON s.sha = rc.sha
152+
LEFT JOIN per_sha_pending p ON p.sha = rc.sha
153+
ORDER BY rc.time DESC

‎torchci/components/layout/NavBar.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ function NavBar() {
6767
name: "Test File Reports",
6868
href: "/tests/fileReport",
6969
},
70+
{
71+
name: "CUDA Trunk test stats",
72+
href: "/cuda_test_stats",
73+
},
74+
{
75+
name: "MacOS test stats",
76+
href: "/mac_test_stats",
77+
},
7078
].map((item) => ({
7179
label: item.name,
7280
route: item.href,

0 commit comments

Comments
 (0)