Skip to content

Commit 0c006ce

Browse files
authored
Follow release reusables whose runner comes from _select-release-runner outputs (#8530)
`_build-triton-wheel-linux.yml` never made it into the release runner-group allow-list, so all 40 `build-wheel-{cuda,aarch64,rocm,xpu}` jobs in "Build Triton wheels" sat `queued` with `runner_group_name: ""` — see [pytorch/pytorch run 31811475863](https://github.com/pytorch/pytorch/actions/runs/31811475863). `collect_release_workflow_paths` used two disagreeing notions of "release related": - **Entry detection** propagates the `rel-` signal through a local reusable, so a workflow calling `_select-release-runner.yml` is discovered. `build-triton-wheel.yml` was. - **Edge traversal** only followed a `uses:` when the *calling job body* held an inline `rel-` label. pytorch/pytorch#193378 split the triton build into `_build-triton-wheel-linux.yml`, invoked as: ```yaml build-wheel-cuda: needs: select-runner uses: ./.github/workflows/_build-triton-wheel-linux.yml with: runs_on: ${{ needs.select-runner.outputs.x86 }} ``` No inline label, so the edge was never taken and the reusable that actually consumes the release runner never reached the allow-list. `_binary-build-linux.yml` survived only incidentally: its generated caller still inlines the label in `runs_on:`. ### Testing Run locally and https://github.com/pytorch/pytorch/actions/runs/31811475863/job/94802925779 is working now
1 parent b199540 commit 0c006ce

2 files changed

Lines changed: 145 additions & 7 deletions

File tree

tools/scripts/release_manage_runner_groups.py

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515
from).
1616
1717
Release workflows are discovered, not hardcoded, by the release runner label
18-
they run on (the ``rel-`` marker, e.g. ``rel-l-x86iavx512-44-340``). An entry
19-
workflow references such a label directly; reusable workflows are then included
20-
only when a release entry invokes them via ``uses:`` in a job that runs on a
21-
release label - this pulls in the build reusable (``_binary-build-linux.yml``)
22-
while leaving test/upload reusables (which run on other runners) out.
18+
they run on (the ``rel-`` marker, e.g. ``rel-l-x86iavx512-44-340``). A job counts
19+
as running on a release runner when it names such a label inline, or when it
20+
takes its runner from a ``_select-release-runner.yml`` caller's outputs. An entry
21+
workflow has such a job; reusable workflows are then included only when a release
22+
entry invokes them via ``uses:`` from one - this pulls in the build reusables
23+
(``_binary-build-linux.yml``, ``_build-triton-wheel-linux.yml``) while leaving
24+
test/upload reusables (which run on other runners) out.
2325
2426
Reading and updating runner groups requires a token that can manage them.
2527
Defaults to a dry-run; pass ``--apply`` to write changes.
@@ -257,6 +259,53 @@ def local_uses_paths(wf: "WorkflowFile") -> Set[str]:
257259
return {local for local in map(local_uses, jobs.values()) if local is not None}
258260

259261

262+
def selector_job_names(wf: "WorkflowFile", label_files: Set[str]) -> Set[str]:
263+
"""Names of ``wf``'s jobs that invoke a reusable carrying release labels.
264+
265+
These are the ``select-runner`` style jobs: they hold no label themselves,
266+
they call ``_select-release-runner.yml`` and re-export its labels as outputs.
267+
"""
268+
jobs = wf.doc.get("jobs")
269+
if not isinstance(jobs, dict):
270+
return set()
271+
names = set()
272+
for name, job in jobs.items():
273+
local = local_uses(job)
274+
if local is not None and local in label_files:
275+
names.add(str(name))
276+
return names
277+
278+
279+
def selector_output_re(job_name: str) -> "re.Pattern[str]":
280+
"""Matches a reference to ``job_name``'s outputs, in either accessor form
281+
(``needs.select-runner.outputs.x86`` / ``needs['select-runner'].outputs``)."""
282+
name = re.escape(job_name)
283+
return re.compile(rf"needs(?:\.{name}|\[['\"]{name}['\"]\])\.outputs\.")
284+
285+
286+
def runs_on_release_runner(job: Any, selectors: Set[str]) -> bool:
287+
"""Whether ``job`` runs on a release runner.
288+
289+
Either it names a release label inline (the generated binary workflows still
290+
inline ``rel-l-x86iavx512-44-340`` in ``runs_on:``), or it takes its runner
291+
from a selector job's outputs -- which is how pytorch/pytorch#193378's
292+
``build-triton-wheel.yml`` build jobs get theirs, with no label of their own:
293+
294+
build-wheel-cuda:
295+
needs: select-runner
296+
uses: ./.github/workflows/_build-triton-wheel-linux.yml
297+
with:
298+
runs_on: ${{ needs.select-runner.outputs.x86 }}
299+
300+
Sibling test/upload jobs gate out here: they depend on the build jobs, not on
301+
the selector's outputs, so they carry neither signal.
302+
"""
303+
text = str(job)
304+
if uses_release_label(text):
305+
return True
306+
return any(selector_output_re(name).search(text) for name in selectors)
307+
308+
260309
@dataclass
261310
class WorkflowFile:
262311
doc: Dict[str, Any]
@@ -334,8 +383,16 @@ def collect_release_workflow_paths(files: Dict[str, WorkflowFile]) -> Set[str]:
334383
outputs and so carry no label of their own) -- the release-label signal is
335384
propagated up the ``uses:`` graph rather than matching a hardcoded filename.
336385
From each entry, follow local ``uses:`` references, but only for jobs that
337-
themselves run on a release label, so the build reusable is included while
386+
themselves run on a release runner, so the build reusable is included while
338387
sibling test/upload jobs (which run on other runners) are not.
388+
389+
Both steps use the same notion of "runs on a release runner" -- an inline
390+
label *or* a selector job's outputs. They used to disagree, and the edge test
391+
accepting only inline labels silently dropped
392+
``_build-triton-wheel-linux.yml`` when pytorch/pytorch#193378 split it out of
393+
``build-triton-wheel.yml``: the entry was still discovered via its
394+
``select-runner`` job, but the reusable that actually consumes the runner was
395+
not, so its builds hung unassigned.
339396
"""
340397
label_files = {path for path, wf in files.items() if uses_release_label(wf.raw)}
341398
entry_paths = {
@@ -356,11 +413,12 @@ def collect_release_workflow_paths(files: Dict[str, WorkflowFile]) -> Set[str]:
356413
jobs = wf.doc.get("jobs")
357414
if not isinstance(jobs, dict):
358415
continue
416+
selectors = selector_job_names(wf, label_files)
359417
for job in jobs.values():
360418
local = local_uses(job)
361419
if local is None:
362420
continue
363-
if not uses_release_label(str(job)):
421+
if not runs_on_release_runner(job, selectors):
364422
continue
365423
queue.append(local)
366424
return seen

tools/tests/test_release_manage_runner_groups.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,86 @@ def test_paths_filter_mention_is_not_a_caller(self) -> None:
265265
{".github/workflows/_select-release-runner.yml"},
266266
)
267267

268+
def test_follows_reusable_whose_runner_comes_from_selector_outputs(self) -> None:
269+
# pytorch/pytorch#193378 split the triton build into
270+
# _build-triton-wheel-linux.yml. Its callers pass the runner through from
271+
# the select-runner job's outputs, so the calling job carries no rel-
272+
# label -- the reusable that actually consumes the release runner must
273+
# still be followed. Upload jobs depend on the build jobs rather than the
274+
# selector, so they stay out.
275+
files = {
276+
".github/workflows/build-triton-wheel.yml": m.WorkflowFile(
277+
doc={
278+
"jobs": {
279+
"select-runner": {
280+
"uses": "./.github/workflows/_select-release-runner.yml"
281+
},
282+
"build-wheel-cuda": {
283+
"needs": "select-runner",
284+
"uses": "./.github/workflows/_build-triton-wheel-linux.yml",
285+
"with": {
286+
"runs_on": "${{ needs.select-runner.outputs.x86 }}"
287+
},
288+
},
289+
"build-wheel-win": {"runs-on": "windows.4xlarge"},
290+
"upload-wheel-triton": {
291+
"needs": ["build-wheel-cuda"],
292+
"uses": "./.github/workflows/_upload-triton-wheel.yml",
293+
},
294+
}
295+
},
296+
raw="uses: ./.github/workflows/_select-release-runner.yml\n"
297+
"runs_on: ${{ needs.select-runner.outputs.x86 }}",
298+
),
299+
".github/workflows/_select-release-runner.yml": m.WorkflowFile(
300+
doc={"jobs": {"select": {"runs-on": "ubuntu-24.04"}}},
301+
raw="echo x86=mt-rel-l-x86iavx512-44-340",
302+
),
303+
".github/workflows/_build-triton-wheel-linux.yml": m.WorkflowFile(
304+
doc={"jobs": {"build": {"runs-on": "${{ inputs.runs_on }}"}}},
305+
raw="runs-on: ${{ inputs.runs_on }}",
306+
),
307+
".github/workflows/_upload-triton-wheel.yml": m.WorkflowFile(
308+
doc={"jobs": {"upload": {"runs-on": "ubuntu-24.04"}}},
309+
raw="runs-on: ubuntu-24.04",
310+
),
311+
}
312+
self.assertEqual(
313+
m.collect_release_workflow_paths(files),
314+
{
315+
".github/workflows/build-triton-wheel.yml",
316+
".github/workflows/_select-release-runner.yml",
317+
".github/workflows/_build-triton-wheel-linux.yml",
318+
},
319+
)
320+
321+
def test_selector_outputs_of_a_non_release_job_are_not_a_signal(self) -> None:
322+
# needs.<job>.outputs.* only propagates the release signal when that job
323+
# is a caller of the release-label reusable. A plain matrix-computing job
324+
# must not pull its consumers' reusables in.
325+
files = {
326+
".github/workflows/gen.yml": m.WorkflowFile(
327+
doc={
328+
"jobs": {
329+
"build": {"runs-on": "rel-l-x86iavx512-44-340"},
330+
"matrix": {"runs-on": "ubuntu-24.04"},
331+
"test": {
332+
"needs": "matrix",
333+
"uses": "./.github/workflows/_test.yml",
334+
"with": {"runs_on": "${{ needs.matrix.outputs.runner }}"},
335+
},
336+
}
337+
},
338+
raw="runs-on: rel-l-x86iavx512-44-340",
339+
),
340+
".github/workflows/_test.yml": m.WorkflowFile(
341+
doc={"jobs": {}}, raw="runs-on: ${{ inputs.runs_on }}"
342+
),
343+
}
344+
self.assertEqual(
345+
m.collect_release_workflow_paths(files), {".github/workflows/gen.yml"}
346+
)
347+
268348
def test_ignores_remote_uses(self) -> None:
269349
files = {
270350
".github/workflows/gen.yml": m.WorkflowFile(

0 commit comments

Comments
 (0)