Skip to content

Commit 9ca5018

Browse files
huydhnhuydhn
authored
Update queued jobs queries to account for ARC runner container init (#7941)
ARC (OSDC) runners have a container initialization phase after a job is picked up but before actual work starts. This change splits the queued jobs queries into EC2/LF and ARC paths so that ARC jobs still initializing containers (<=2 steps) are counted as effectively queued. The possible_queued_jobs CTE is expanded to pre-filter both queued jobs (all runners) and in_progress ARC jobs. Each downstream CTE then applies runner-specific filters. ARC detection uses the `l-` label pattern from arc.yaml. Getting ClickHouse step data for in-progress jobs can be stale, so a job running for >10 min with <=2 steps likely has outdated step data rather than being genuinely in container init. Gate Phase 2 on job.created_at being within the last 10 minutes. ### Testing https://torchci-git-arc-queue-time-metrics-fbopensource.vercel.app/metrics --------- Signed-off-by: Huy Do <huydhn@gmail.com>
1 parent 7f983ca commit 9ca5018

2 files changed

Lines changed: 193 additions & 64 deletions

File tree

‎torchci/clickhouse_queries/queued_jobs/query.sql‎

Lines changed: 132 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,148 @@
11
--- This query is used by HUD metrics page to get the list of queued jobs
2+
---
3+
--- For EC2/LF runners: only jobs in 'queued' status
4+
--- For ARC runners (labels containing l-): jobs in 'queued' status + jobs in 'in_progress'
5+
--- still initializing containers (<=2 steps completed). Jobs with a recorded
6+
--- conclusion are excluded to avoid counting stale entries.
27
WITH possible_queued_jobs AS (
38
SELECT
49
id,
510
run_id
611
FROM default.workflow_job -- FINAL not needed since we just use this to filter a table that has already been FINALed
712
WHERE
8-
status = 'queued'
9-
AND created_at < (CURRENT_TIMESTAMP() - INTERVAL 5 MINUTE)
13+
created_at < (CURRENT_TIMESTAMP() - INTERVAL 5 MINUTE)
1014
AND created_at > (CURRENT_TIMESTAMP() - INTERVAL 1 WEEK)
11-
)
12-
13-
SELECT
14-
DATE_DIFF(
15-
'second',
16-
job.created_at,
17-
CURRENT_TIMESTAMP()
18-
) AS queue_s,
19-
CONCAT(workflow.name, ' / ', job.name) AS name,
20-
job.html_url,
21-
IF(
22-
LENGTH(job.labels) = 0,
23-
'N/A',
15+
AND (
16+
--- EC2/LF: jobs still in queued status
17+
status = 'queued'
18+
OR
19+
--- ARC: jobs in_progress but possibly still initializing containers
20+
(status = 'in_progress'
21+
AND conclusion = ''
22+
AND arrayExists(x -> x LIKE '%l-%', labels))
23+
)
24+
),
25+
--- EC2/LF runners: existing logic, only jobs in queued status
26+
ec2_queued_jobs AS (
27+
SELECT
28+
DATE_DIFF(
29+
'second',
30+
job.created_at,
31+
CURRENT_TIMESTAMP()
32+
) AS queue_s,
33+
CONCAT(workflow.name, ' / ', job.name) AS name,
34+
job.html_url,
35+
IF(
36+
LENGTH(job.labels) = 0,
37+
'N/A',
38+
IF(
39+
LENGTH(job.labels) > 1,
40+
job.labels[2],
41+
job.labels[1]
42+
)
43+
) AS machine_type,
44+
workflow.head_sha AS head_sha,
45+
workflow.head_branch AS head_branch,
46+
workflow.event AS event,
47+
CASE
48+
WHEN
49+
workflow.head_branch LIKE 'trunk/%'
50+
AND workflow.event = 'workflow_dispatch'
51+
THEN 'autorevert'
52+
WHEN workflow.head_branch LIKE 'ciflow/%' THEN 'ciflow'
53+
WHEN
54+
workflow.head_branch = 'main' OR workflow.event = 'push'
55+
THEN 'main'
56+
ELSE 'other'
57+
END AS source_type,
58+
CASE
59+
WHEN workflow.head_branch LIKE 'ciflow/trunk/%'
60+
THEN
61+
replaceRegexpOne(workflow.head_branch, '^ciflow/trunk/', '')
62+
WHEN workflow.head_branch LIKE 'ciflow/%' THEN
63+
replaceRegexpOne(workflow.head_branch, '^ciflow/[^/]+/', '')
64+
END AS ciflow_id
65+
FROM
66+
default.workflow_job job FINAL
67+
JOIN default.workflow_run workflow FINAL ON workflow.id = job.run_id
68+
WHERE
69+
job.id IN (SELECT id FROM possible_queued_jobs)
70+
AND workflow.id IN (SELECT run_id FROM possible_queued_jobs)
71+
AND workflow.repository.'full_name' = 'pytorch/pytorch'
72+
AND job.status = 'queued'
73+
AND LENGTH(job.steps) = 0
74+
AND workflow.status != 'completed'
75+
--- Exclude ARC runners from this path
76+
AND NOT arrayExists(x -> x LIKE '%l-%', job.labels)
77+
),
78+
--- ARC runners: queued OR in_progress but still initializing containers
79+
arc_queued_jobs AS (
80+
SELECT
81+
DATE_DIFF(
82+
'second',
83+
job.created_at,
84+
CURRENT_TIMESTAMP()
85+
) AS queue_s,
86+
CONCAT(workflow.name, ' / ', job.name) AS name,
87+
job.html_url,
2488
IF(
2589
LENGTH(job.labels) > 1,
2690
job.labels[2],
2791
job.labels[1]
92+
) AS machine_type,
93+
workflow.head_sha AS head_sha,
94+
workflow.head_branch AS head_branch,
95+
workflow.event AS event,
96+
CASE
97+
WHEN
98+
workflow.head_branch LIKE 'trunk/%'
99+
AND workflow.event = 'workflow_dispatch'
100+
THEN 'autorevert'
101+
WHEN workflow.head_branch LIKE 'ciflow/%' THEN 'ciflow'
102+
WHEN
103+
workflow.head_branch = 'main' OR workflow.event = 'push'
104+
THEN 'main'
105+
ELSE 'other'
106+
END AS source_type,
107+
CASE
108+
WHEN workflow.head_branch LIKE 'ciflow/trunk/%'
109+
THEN
110+
replaceRegexpOne(workflow.head_branch, '^ciflow/trunk/', '')
111+
WHEN workflow.head_branch LIKE 'ciflow/%' THEN
112+
replaceRegexpOne(workflow.head_branch, '^ciflow/[^/]+/', '')
113+
END AS ciflow_id
114+
FROM
115+
default.workflow_job job FINAL
116+
JOIN default.workflow_run workflow FINAL ON workflow.id = job.run_id
117+
WHERE
118+
job.id in (select id from possible_queued_jobs)
119+
and workflow.id in (select run_id from possible_queued_jobs)
120+
and workflow.repository. 'full_name' = 'pytorch/pytorch'
121+
--- ARC runner detection: labels contain l- pattern
122+
AND arrayExists(x -> x LIKE '%l-%', job.labels)
123+
AND workflow.status != 'completed'
124+
AND job.conclusion = ''
125+
AND (
126+
--- Phase 1: still in queued status
127+
(job.status = 'queued' AND LENGTH(job.steps) = 0)
128+
OR
129+
--- Phase 2: picked up by runner but still initializing containers.
130+
--- Container init is always the first 2 steps (Set up job +
131+
--- Initialize containers). If only those steps exist, actual
132+
--- work hasn't started yet. The 10 min cap guards against stale
133+
--- step data in ClickHouse — the job is either running ok or fails
134+
--- already if it has less than 2 steps after 10 minutes
135+
(job.status = 'in_progress'
136+
AND LENGTH(job.steps) > 0
137+
AND LENGTH(job.steps) <= 2
138+
AND job.created_at > (CURRENT_TIMESTAMP() - INTERVAL 10 MINUTE))
28139
)
29-
) AS machine_type,
30-
workflow.head_sha AS head_sha,
31-
workflow.head_branch AS head_branch,
32-
workflow.event AS event,
33-
CASE
34-
WHEN
35-
workflow.head_branch LIKE 'trunk/%'
36-
AND workflow.event = 'workflow_dispatch'
37-
THEN 'autorevert'
38-
WHEN workflow.head_branch LIKE 'ciflow/%' THEN 'ciflow'
39-
WHEN
40-
workflow.head_branch = 'main' OR workflow.event = 'push'
41-
THEN 'main'
42-
ELSE 'other'
43-
END AS source_type,
44-
CASE
45-
WHEN workflow.head_branch LIKE 'ciflow/trunk/%'
46-
THEN
47-
replaceRegexpOne(workflow.head_branch, '^ciflow/trunk/', '')
48-
WHEN workflow.head_branch LIKE 'ciflow/%' THEN
49-
replaceRegexpOne(workflow.head_branch, '^ciflow/[^/]+/', '')
50-
END AS ciflow_id
51-
FROM
52-
default.workflow_job job FINAL
53-
JOIN default.workflow_run workflow FINAL ON workflow.id = job.run_id
54-
WHERE
55-
job.id IN (SELECT id FROM possible_queued_jobs)
56-
AND workflow.id IN (SELECT run_id FROM possible_queued_jobs)
57-
AND workflow.repository.'full_name' = 'pytorch/pytorch'
58-
AND job.status = 'queued'
59-
/* These two conditions are workarounds for GitHub's broken API. Sometimes */
60-
/* jobs get stuck in a permanently "queued" state but definitely ran. We can */
61-
/* detect this by looking at whether any steps executed (if there were, */
62-
/* obviously the job started running), and whether the workflow was marked as */
63-
/* complete (somehow more reliable than the job-level API) */
64-
AND LENGTH(job.steps) = 0
65-
AND workflow.status != 'completed'
140+
)
141+
SELECT * FROM (
142+
SELECT * FROM ec2_queued_jobs
143+
UNION ALL
144+
SELECT * FROM arc_queued_jobs
145+
)
66146
ORDER BY
67147
queue_s DESC
68148
SETTINGS allow_experimental_analyzer = 1;

‎torchci/clickhouse_queries/queued_jobs_by_label/query.sql‎

Lines changed: 61 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,26 @@
11
--- This query is used by HUD metrics page to get the list of queued jobs grouped by their labels
2+
---
3+
--- For EC2/LF runners: queue time = time in 'queued' status (created_at to now)
4+
--- For ARC runners (labels containing l-): queue time = time in 'queued' status + container
5+
--- initialization time (before actual work starts). Phase 2 captures jobs that
6+
--- are in_progress but still initializing containers (<=2 steps completed).
7+
--- Jobs with a recorded conclusion are excluded to avoid counting stale entries.
28
WITH possible_queued_jobs as (
3-
select id, run_id from default.workflow_job where status = 'queued'
4-
AND created_at < (CURRENT_TIMESTAMP() - INTERVAL 5 MINUTE)
9+
select id, run_id from default.workflow_job where
10+
created_at < (CURRENT_TIMESTAMP() - INTERVAL 5 MINUTE)
511
AND created_at > (CURRENT_TIMESTAMP() - INTERVAL 1 WEEK)
6-
), queued_jobs AS (
12+
AND (
13+
--- EC2/LF: jobs still in queued status
14+
status = 'queued'
15+
OR
16+
--- ARC: jobs in_progress but possibly still initializing containers
17+
(status = 'in_progress'
18+
AND conclusion = ''
19+
AND arrayExists(x -> x LIKE '%l-%', labels))
20+
)
21+
),
22+
--- EC2/LF runners: existing logic, only jobs in queued status
23+
ec2_queued_jobs AS (
724
SELECT
825
DATE_DIFF('second', job.created_at, CURRENT_TIMESTAMP()) AS queue_s,
926
CONCAT(workflow.name, ' / ', job.name) AS name,
@@ -30,23 +47,55 @@ WITH possible_queued_jobs as (
3047
and workflow.repository. 'full_name' = 'pytorch/pytorch'
3148
AND job.status = 'queued'
3249
AND job.created_at < (CURRENT_TIMESTAMP() - INTERVAL 5 MINUTE)
33-
/* These two conditions are workarounds for GitHub's broken API. Sometimes */
34-
/* jobs get stuck in a permanently "queued" state but definitely ran. We can */
35-
/* detect this by looking at whether any steps executed (if there were, */
36-
/* obviously the job started running), and whether the workflow was marked as */
37-
/* complete (somehow more reliable than the job-level API) */
3850
AND LENGTH(job.steps) = 0
3951
AND workflow.status != 'completed'
40-
ORDER BY
41-
queue_s DESC
52+
--- Exclude ARC runners from this path
53+
AND NOT arrayExists(x -> x LIKE '%l-%', job.labels)
54+
),
55+
--- ARC runners: queued OR in_progress but still initializing containers
56+
arc_queued_jobs AS (
57+
SELECT
58+
DATE_DIFF('second', job.created_at, CURRENT_TIMESTAMP()) AS queue_s,
59+
CONCAT(workflow.name, ' / ', job.name) AS name,
60+
job.html_url,
61+
IF(LENGTH(job.labels) > 1, job.labels [ 2 ], job.labels [ 1 ]) AS machine_type
62+
FROM
63+
default.workflow_job job final
64+
JOIN default.workflow_run workflow final ON workflow.id = job.run_id
65+
WHERE
66+
job.id in (select id from possible_queued_jobs)
67+
and workflow.id in (select run_id from possible_queued_jobs)
68+
and workflow.repository. 'full_name' = 'pytorch/pytorch'
69+
--- ARC runner detection: labels contain l- pattern
70+
AND arrayExists(x -> x LIKE '%l-%', job.labels)
71+
AND workflow.status != 'completed'
72+
AND job.conclusion = ''
73+
AND (
74+
--- Phase 1: still in queued status
75+
(job.status = 'queued' AND LENGTH(job.steps) = 0)
76+
OR
77+
--- Phase 2: picked up by runner but still initializing containers.
78+
--- Container init is always the first 2 steps (Set up job +
79+
--- Initialize containers). If only those steps exist, actual
80+
--- work hasn't started yet. The 10 min cap guards against stale
81+
--- step data in ClickHouse — the job is either running ok or fails
82+
--- already if it has less than 2 steps after 10 minutes
83+
(job.status = 'in_progress'
84+
AND LENGTH(job.steps) > 0
85+
AND LENGTH(job.steps) <= 2
86+
AND job.created_at > (CURRENT_TIMESTAMP() - INTERVAL 10 MINUTE))
87+
)
4288
)
4389
SELECT
4490
COUNT(*) AS count,
4591
MAX(queue_s) AS avg_queue_s,
4692
machine_type,
4793
CURRENT_TIMESTAMP() AS time
48-
FROM
49-
queued_jobs
94+
FROM (
95+
SELECT queue_s, machine_type FROM ec2_queued_jobs
96+
UNION ALL
97+
SELECT queue_s, machine_type FROM arc_queued_jobs
98+
)
5099
GROUP BY
51100
machine_type
52101
ORDER BY

0 commit comments

Comments
 (0)