Skip to content

Commit 37275da

Browse files
jeanschmidtjeanschmidt
authored
Adds a drill down page to hud to help troubleshoot CI flakiness (#8496)
Adds a page "/flaky_trunk" in torchci from "Dev Infra" -> "Flaky Trunk" to help diagnose source of flakiness on PyTorch trunk. The page includes a break down of infra and job flakiness and lists the most impactful ones --------- Signed-off-by: Jean Schmidt <contato@jschmidt.me>
1 parent f24ded7 commit 37275da

18 files changed

Lines changed: 2279 additions & 0 deletions

File tree

‎torchci/clickhouse_queries/flaky_trunk_entity_runs/params.json‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"params": {
3+
"startTime": "DateTime64(3)",
4+
"stopTime": "DateTime64(3)",
5+
"repo": "String",
6+
"entityType": "String",
7+
"entityValue": "String",
8+
"viableStrictOnly": "Bool"
9+
},
10+
"defaults": {
11+
"viableStrictOnly": false
12+
},
13+
"tests": [
14+
{
15+
"startTime": "2026-07-12T00:00:00.000",
16+
"stopTime": "2026-08-11T00:00:00.000",
17+
"repo": "pytorch/pytorch",
18+
"entityType": "job",
19+
"entityValue": "trunk / macos-py3-arm64 / test (default)",
20+
"viableStrictOnly": true
21+
},
22+
{
23+
"startTime": "2026-07-12T00:00:00.000",
24+
"stopTime": "2026-08-11T00:00:00.000",
25+
"repo": "pytorch/pytorch",
26+
"entityType": "label",
27+
"entityValue": "linux.rocm.gpu.gfx942.1",
28+
"viableStrictOnly": false
29+
}
30+
]
31+
}

‎torchci/clickhouse_queries/flaky_trunk_entity_runs/query.sql‎

Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
-- Flaky Trunk row-click drill-down: the individual FAILED runs behind ONE clicked entity.
2+
--
3+
-- Given a job (entityType='job') or a runner/instance label (entityType='label'), returns one row per
4+
-- FAILED default.workflow_job run that belongs to a logical outcome classified as a flake. A "logical
5+
-- outcome" is one (commit, workflow, job-shard) collapsed across every run_attempt / restart run_id;
6+
-- flake = infra_flake (advisor infra_issue, persistent or not) OR test_flake (advisor
7+
-- not_related/garbage; advisor-only). real_regression (advisor related/revert, or a persistent
8+
-- fallback) and unclassified reds (non-persistent, no advisor verdict) are excluded.
9+
--
10+
-- The classification chain (trunk_commits .. final_jobs) is the same logic used by flaky_trunk_jobs and
11+
-- flaky_trunk_runner_labels, so categories are IDENTICAL to those aggregate tables; this query
12+
-- additionally carries head_sha through job_signals/final_jobs so each classified logical outcome can be
13+
-- joined back to its raw failed runs. Trunk filter: head_sha is a real push to refs/heads/main for repo.
14+
--
15+
-- entityType='job' -> runs of the job whose displayed name ("<workflow> / <norm_name>", the same
16+
-- expression flaky_trunk_jobs SELECTs) equals entityValue; BOTH infra_flake and
17+
-- test_flake outcomes.
18+
-- entityType='label' -> failed runs whose labels array contains entityValue; infra_flake outcomes only
19+
-- (the runner-labels view is infra-flakiness), matching flaky_trunk_runner_labels.
20+
--
21+
-- html_url is the GitHub Actions job page: the native workflow_job.html_url is populated and already
22+
-- reflects the true repo, so it is used directly; the {repo}-based construction is only a fallback for
23+
-- the rare empty value. LIMIT 2000 caps a single entity's flake runs over the window.
24+
WITH
25+
trunk_commits AS (
26+
SELECT
27+
tupleElement(head_commit, 'id') AS head_sha,
28+
min(tupleElement(head_commit, 'timestamp')) AS commit_time
29+
FROM default.push
30+
WHERE
31+
ref = 'refs/heads/main'
32+
AND tupleElement(repository, 'full_name') = {repo: String}
33+
AND tupleElement(head_commit, 'timestamp')
34+
>= {startTime: DateTime64(3)}
35+
AND tupleElement(head_commit, 'timestamp') < {stopTime: DateTime64(3)}
36+
GROUP BY head_sha
37+
),
38+
39+
-- Latest AI advisor verdict per (trunk commit, normalized job), for signal_source = 'job'.
40+
-- Dr.CI/PR-side verdicts ('dr_ci_' prefix) carry a PR-head suspect_commit that never equals a
41+
-- trunk sha, so they drop out of the advisor join below and need no handling here; the
42+
-- surviving bare trunk-autorevert keys are reduced to the normalized job name.
43+
advisor_agg AS (
44+
SELECT
45+
head_sha,
46+
adv_norm,
47+
maxIf(1, verdict IN ('related', 'revert')) AS advisor_real,
48+
maxIf(1, verdict = 'infra_issue') AS advisor_infra,
49+
maxIf(1, verdict IN ('not_related', 'garbage')) AS advisor_testflake
50+
FROM (
51+
SELECT
52+
toString(suspect_commit) AS head_sha,
53+
replaceRegexpOne(
54+
replaceRegexpOne(
55+
signal_key,
56+
' \\[[^\\]]+\\]$', ''
57+
),
58+
', [0-9]+, [0-9]+, .+\\)', ')'
59+
) AS adv_norm,
60+
argMax(verdict, timestamp) AS verdict
61+
FROM misc.autorevert_advisor_verdicts
62+
WHERE
63+
repo = {repo: String}
64+
AND signal_source = 'job'
65+
AND timestamp >= {startTime: DateTime64(3)}
66+
GROUP BY suspect_commit, signal_key
67+
)
68+
GROUP BY head_sha, adv_norm
69+
),
70+
71+
raw_jobs AS (
72+
SELECT
73+
j.head_sha AS head_sha,
74+
tc.commit_time AS commit_time,
75+
j.workflow_name AS workflow_name,
76+
-- cons_name keeps the shard (config, i, n) but drops the trailing runner, so a retry on a
77+
-- different runner/fleet collapses into the same shard; norm_name additionally drops the shard.
78+
replaceRegexpOne(
79+
j.name, ', ([0-9]+), ([0-9]+), [^)]+\\)$', ', \\1, \\2)'
80+
) AS cons_name,
81+
j.conclusion AS conclusion
82+
FROM default.workflow_job j
83+
INNER JOIN trunk_commits tc ON j.head_sha = tc.head_sha
84+
WHERE
85+
-- Bind jobs to trunk-commit membership rather than a created_at window: a commit's retries are
86+
-- separate rows created arbitrarily later, so over historical/backfill windows every attempt must
87+
-- be counted or a late retry-green is misclassified as a hard failure. Slower than the created_at
88+
-- skip-index scan, but the join above is the exact filter and correctness wins here.
89+
j.id IN (
90+
SELECT id
91+
FROM materialized_views.workflow_job_by_head_sha
92+
WHERE head_sha IN (SELECT head_sha FROM trunk_commits)
93+
)
94+
AND j.conclusion IN ('success', 'failure')
95+
AND j.name LIKE '%/%'
96+
AND j.name NOT LIKE '%rerun_disabled_tests%'
97+
AND j.name NOT LIKE '%mem_leak_check%'
98+
AND j.name NOT LIKE '%unstable%'
99+
-- Restrict to viable/strict blocking workflows (keep in sync with
100+
-- pytorch/pytorch .github/workflows/update-viablestrict.yml `requires`).
101+
AND (
102+
{viableStrictOnly: Bool} = false
103+
OR lower(j.workflow_name) IN ('pull', 'trunk', 'lint', 'docs-build')
104+
)
105+
),
106+
107+
consolidated AS (
108+
SELECT
109+
head_sha,
110+
workflow_name,
111+
cons_name,
112+
any(commit_time) AS commit_time,
113+
replaceRegexpOne(cons_name, ', [0-9]+, [0-9]+\\)$', ')') AS norm_name,
114+
countIf(conclusion = 'success') > 0 AS has_success,
115+
countIf(conclusion = 'failure') > 0 AS has_failure
116+
FROM raw_jobs
117+
GROUP BY head_sha, workflow_name, cons_name
118+
),
119+
120+
job_signals AS (
121+
SELECT
122+
c.head_sha AS head_sha,
123+
c.workflow_name AS workflow_name,
124+
c.cons_name AS cons_name,
125+
c.norm_name AS norm_name,
126+
c.commit_time AS commit_time,
127+
toUInt8(c.has_success AND NOT c.has_failure) AS is_green,
128+
toUInt8(c.has_failure) AS is_red,
129+
toUInt8(c.has_failure AND NOT c.has_success) AS hard_red,
130+
COALESCE(aa.advisor_real, 0) AS adv_real,
131+
COALESCE(aa.advisor_infra, 0) AS adv_infra,
132+
COALESCE(aa.advisor_testflake, 0) AS adv_testflake
133+
FROM consolidated c
134+
LEFT JOIN
135+
advisor_agg aa
136+
ON aa.head_sha = c.head_sha AND aa.adv_norm = c.norm_name
137+
),
138+
139+
final_jobs AS (
140+
SELECT
141+
head_sha,
142+
workflow_name,
143+
cons_name,
144+
norm_name,
145+
commit_time,
146+
is_green,
147+
is_red,
148+
-- Exactly one category per logical red, advisor-verdict-primary with a structural fallback:
149+
-- 1 real_regression, 3 test_flake, 4 infra_flake, 5 unclassified.
150+
multiIf(
151+
is_red = 0, 0,
152+
adv_real = 1, 1,
153+
adv_infra = 1, 4,
154+
adv_testflake = 1, 3,
155+
persistent = 1, 1,
156+
5
157+
) AS category
158+
FROM (
159+
SELECT
160+
head_sha,
161+
workflow_name,
162+
cons_name,
163+
norm_name,
164+
commit_time,
165+
is_green,
166+
is_red,
167+
adv_real,
168+
adv_infra,
169+
adv_testflake,
170+
-- persistent: this hard-red has an adjacent hard-red on trunk (run of >= 2 consecutive reds).
171+
toUInt8(
172+
hard_red = 1
173+
AND (
174+
lagInFrame(hard_red) OVER w = 1
175+
OR leadInFrame(hard_red) OVER w = 1
176+
)
177+
) AS persistent
178+
FROM job_signals
179+
WINDOW w AS (
180+
PARTITION BY workflow_name, cons_name
181+
ORDER BY commit_time
182+
ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING
183+
)
184+
)
185+
),
186+
187+
-- Only the logical outcomes classified as flake (test_flake=3, infra_flake=4), keyed for the join-back.
188+
flake_outcomes AS (
189+
SELECT
190+
head_sha,
191+
workflow_name,
192+
cons_name,
193+
category
194+
FROM final_jobs
195+
WHERE category IN (3, 4)
196+
),
197+
198+
-- Raw FAILED runs for the clicked entity only, deduped to one row per job id (SharedReplacingMergeTree).
199+
failed_runs AS (
200+
SELECT
201+
j.head_sha AS head_sha,
202+
j.name AS job_name,
203+
j.workflow_name AS workflow_name,
204+
replaceRegexpOne(
205+
j.name, ', ([0-9]+), ([0-9]+), [^)]+\\)$', ', \\1, \\2)'
206+
) AS cons_name,
207+
j.labels AS labels,
208+
j.runner_group_name AS runner_group_name,
209+
toDateTime(j.started_at) AS started_at,
210+
j.html_url AS html_url,
211+
j.run_id AS run_id,
212+
j.id AS id
213+
FROM default.workflow_job j
214+
INNER JOIN trunk_commits tc ON j.head_sha = tc.head_sha
215+
WHERE
216+
-- Bind jobs to trunk-commit membership rather than a created_at window: a commit's retries are
217+
-- separate rows created arbitrarily later, so over historical/backfill windows every attempt must
218+
-- be counted or a late retry-green is misclassified as a hard failure. Slower than the created_at
219+
-- skip-index scan, but the join above is the exact filter and correctness wins here.
220+
j.id IN (
221+
SELECT id
222+
FROM materialized_views.workflow_job_by_head_sha
223+
WHERE head_sha IN (SELECT head_sha FROM trunk_commits)
224+
)
225+
AND j.conclusion = 'failure'
226+
AND j.name LIKE '%/%'
227+
AND j.name NOT LIKE '%rerun_disabled_tests%'
228+
AND j.name NOT LIKE '%mem_leak_check%'
229+
AND j.name NOT LIKE '%unstable%'
230+
-- Restrict to viable/strict blocking workflows (keep in sync with
231+
-- pytorch/pytorch .github/workflows/update-viablestrict.yml `requires`).
232+
AND (
233+
{viableStrictOnly: Bool} = false
234+
OR lower(j.workflow_name) IN ('pull', 'trunk', 'lint', 'docs-build')
235+
)
236+
AND (
237+
(
238+
{entityType: String} = 'job'
239+
AND concat(
240+
j.workflow_name,
241+
' / ',
242+
replaceRegexpOne(
243+
replaceRegexpOne(
244+
j.name,
245+
', ([0-9]+), ([0-9]+), [^)]+\\)$',
246+
', \\1, \\2)'
247+
),
248+
', [0-9]+, [0-9]+\\)$', ')'
249+
)
250+
) = {entityValue: String}
251+
)
252+
OR
253+
(
254+
{entityType: String} = 'label'
255+
AND has(
256+
if(empty(j.labels), [''], j.labels), {entityValue: String}
257+
)
258+
)
259+
)
260+
ORDER BY j._inserted_at DESC
261+
LIMIT 1 BY id
262+
)
263+
264+
SELECT
265+
r.head_sha AS head_sha,
266+
r.job_name AS job_name,
267+
CAST(r.workflow_name AS String) AS workflow_name,
268+
if(f.category = 4, 'Infra flake', 'Job flake') AS category,
269+
if(
270+
empty(r.labels), r.runner_group_name, arrayStringConcat(r.labels, ', ')
271+
) AS runner_label,
272+
r.started_at AS started_at,
273+
if(
274+
r.html_url != '',
275+
r.html_url,
276+
concat(
277+
'https://github.com/',
278+
{repo: String},
279+
'/actions/runs/',
280+
toString(r.run_id),
281+
'/job/',
282+
toString(r.id)
283+
)
284+
) AS html_url,
285+
toInt64(r.run_id) AS run_id,
286+
toInt64(r.id) AS id
287+
FROM failed_runs r
288+
INNER JOIN flake_outcomes f
289+
ON
290+
r.head_sha = f.head_sha
291+
AND r.workflow_name = f.workflow_name
292+
AND r.cons_name = f.cons_name
293+
WHERE {entityType: String} != 'label' OR f.category = 4
294+
ORDER BY started_at DESC
295+
LIMIT 2000

‎torchci/clickhouse_queries/flaky_trunk_jobs/params.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"params": {
3+
"startTime": "DateTime64(3)",
4+
"stopTime": "DateTime64(3)",
5+
"repo": "String",
6+
"minRuns": "Int32",
7+
"viableStrictOnly": "Bool"
8+
},
9+
"defaults": {
10+
"viableStrictOnly": false
11+
},
12+
"tests": [
13+
{
14+
"startTime": "2026-07-12T00:00:00.000",
15+
"stopTime": "2026-08-11T00:00:00.000",
16+
"repo": "pytorch/pytorch",
17+
"minRuns": 20,
18+
"viableStrictOnly": true
19+
}
20+
]
21+
}

0 commit comments

Comments
 (0)