Skip to content

Commit 0b994ee

Browse files
jeanschmidtjeanschmidt
authored
integration-tests: derive resource assertions from runner defs instead of hardcoding (#891)
**Impact:** integration-test workflow generation (CI only) **Risk:** low ## What The generated integration-test workflow now pulls its CPU/memory/GPU assertion values straight from the enabled arc-runners `defs/` at generation time, via `{{VCPU__...}}`/`{{MEMGI__...}}`/`{{TORCHMEM__...}}`/`{{GPU__...}}` placeholders, replacing the hardcoded `EXPECTED=...` literals in `integration-test.yaml.tpl`. ## Why Every time a runner's vcpu/memory/gpu spec changed, the integration-test template's hardcoded expectations drifted and had to be updated by hand (see the recent `-opt` runner right-sizing and the t4-multi memory fixup). Deriving the values from the same runner defs the cluster actually deploys keeps the tests in sync automatically. # Notes - `resource_placeholders()` processes the base `arc-runners` module first, then specialized variants (`-opt`/`-b200`/`-h100`) sorted, so a variant that redefines a shared runner name wins — this is what lets the ue1 `-opt` sizes flow through. - Substitution runs before conditional-block stripping; a dedicated guard after stripping fails generation if a resource placeholder survives inside a live job (unresolved runner def = real bug), while placeholders in removed jobs are cleaned up harmlessly. - Adds an import path to `modules/arc-runners/scripts/python` to reuse `parse_memory_bytes` rather than reimplement memory parsing. Signed-off-by: Jean Schmidt <contato@jschmidt.me>
1 parent 8c40cad commit 0b994ee

2 files changed

Lines changed: 64 additions & 16 deletions

File tree

‎osdc/integration-tests/scripts/python/phases.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616
import yaml
1717

1818
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts" / "python"))
19+
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "modules" / "arc-runners" / "scripts" / "python"))
1920

2021
from conditional_blocks import strip_conditional_block
22+
from generate_runners import parse_memory_bytes
2123
from nodepool_defs import is_excluded_for_region
2224
from run import (
2325
CANARY_REPO,
@@ -279,6 +281,42 @@ def _replace_jobs_with_noop(content: str) -> str:
279281
return "\n".join(out) + suffix
280282

281283

284+
def resource_placeholders(upstream_dir: Path, cluster_modules: list[str]) -> dict[str, str]:
285+
"""Build resource-assertion placeholders from the cluster's arc-runners defs.
286+
287+
For every runner def in each enabled arc-runners* module's defs/ dir, emit
288+
four placeholders keyed by the def name uppercased with hyphens as underscores
289+
(VCPU__/MEMGI__/TORCHMEM__/GPU__<KEY>). These feed the integration-test
290+
template's hardcoded CPU/memory/GPU assertions so they track the deployed
291+
runner spec instead of drifting.
292+
293+
Base module defs are processed first, then specialized variants (-opt/-b200/
294+
-h100), so a variant that redefines a shared label wins.
295+
"""
296+
base = [m for m in cluster_modules if m == "arc-runners"]
297+
specialized = sorted(m for m in cluster_modules if m.startswith("arc-runners") and m != "arc-runners")
298+
placeholders: dict[str, str] = {}
299+
for module in base + specialized:
300+
defs_dir = upstream_dir / "modules" / module / "defs"
301+
if not defs_dir.is_dir():
302+
continue
303+
for def_path in sorted(defs_dir.glob("*.yaml")):
304+
data = yaml.safe_load(def_path.read_text()) or {}
305+
runner = data.get("runner")
306+
if not runner:
307+
continue
308+
name = runner.get("name")
309+
if not name:
310+
continue
311+
key = name.upper().replace("-", "_")
312+
memory_bytes = parse_memory_bytes(runner["memory"])
313+
placeholders[f"{{{{VCPU__{key}}}}}"] = str(runner["vcpu"])
314+
placeholders[f"{{{{MEMGI__{key}}}}}"] = str(memory_bytes // (1024**3))
315+
placeholders[f"{{{{TORCHMEM__{key}}}}}"] = str(memory_bytes)
316+
placeholders[f"{{{{GPU__{key}}}}}"] = str(runner.get("gpu", 0))
317+
return placeholders
318+
319+
282320
def generate_workflow(
283321
upstream_dir: Path,
284322
prefix: str,
@@ -307,6 +345,9 @@ def generate_workflow(
307345
content = content.replace("{{ECR_PULL_RESOLVED_TAG}}", ecr_pull_resolved_tag)
308346
content = content.replace("{{ECR_PULL_SHA}}", ecr_pull_sha)
309347

348+
for placeholder, value in resource_placeholders(upstream_dir, cluster_modules).items():
349+
content = content.replace(placeholder, value)
350+
310351
# Guard: any "{{...}}" left after substitution is a template/code drift bug.
311352
leftover = re.findall(r"\{\{[A-Z_]+\}\}", content)
312353
if leftover:
@@ -328,6 +369,13 @@ def generate_workflow(
328369
for tag in REGION_GATED_BLOCKS:
329370
content = strip_conditional_block(content, tag, keep=tag not in excluded_blocks)
330371

372+
# Resource-assertion guard (after stripping, so placeholders inside a removed
373+
# job — e.g. b200 on a non-b200 cluster — are already gone). A surviving
374+
# resource placeholder means a running job's runner def wasn't found: a real bug.
375+
leftover_res = re.findall(r"\{\{(?:VCPU|MEMGI|TORCHMEM|GPU)__[A-Z0-9_]+\}\}", content)
376+
if leftover_res:
377+
raise RuntimeError(f"Unresolved resource placeholders: {sorted(set(leftover_res))}")
378+
331379
if not _has_any_job(content):
332380
content = _replace_jobs_with_noop(content)
333381

‎osdc/integration-tests/workflows/integration-test.yaml.tpl‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
fi
4040
- name: Verify CPU count
4141
run: |
42-
EXPECTED=8
42+
EXPECTED={{VCPU__L_X86IAMX_8_32}}
4343
ACTUAL=$(nproc)
4444
echo "CPUs: $ACTUAL (expected: $EXPECTED)"
4545
if [ "$ACTUAL" -ne "$EXPECTED" ]; then
@@ -49,7 +49,7 @@ jobs:
4949
echo "PASS: CPU count matches"
5050
- name: Verify memory
5151
run: |
52-
EXPECTED_GI=32
52+
EXPECTED_GI={{MEMGI__L_X86IAMX_8_32}}
5353
if [ -f /sys/fs/cgroup/memory.max ] && [ "$(cat /sys/fs/cgroup/memory.max)" != "max" ]; then
5454
BYTES=$(cat /sys/fs/cgroup/memory.max)
5555
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
@@ -68,7 +68,7 @@ jobs:
6868
- name: Verify TORCH_CI_MAX_MEMORY
6969
run: |
7070
echo "=== TORCH_CI_MAX_MEMORY ==="
71-
EXPECTED=34359738368
71+
EXPECTED={{TORCHMEM__L_X86IAMX_8_32}}
7272
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
7373
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: $EXPECTED)"
7474
if [ -z "$ACTUAL" ]; then
@@ -110,7 +110,7 @@ jobs:
110110
fi
111111
- name: Verify CPU count
112112
run: |
113-
EXPECTED=8
113+
EXPECTED={{VCPU__L_X86IAMX_8_32}}
114114
ACTUAL=$(nproc)
115115
echo "CPUs: $ACTUAL (expected: $EXPECTED)"
116116
if [ "$ACTUAL" -ne "$EXPECTED" ]; then
@@ -120,7 +120,7 @@ jobs:
120120
echo "PASS: CPU count matches"
121121
- name: Verify memory
122122
run: |
123-
EXPECTED_GI=32
123+
EXPECTED_GI={{MEMGI__L_X86IAMX_8_32}}
124124
if [ -f /sys/fs/cgroup/memory.max ] && [ "$(cat /sys/fs/cgroup/memory.max)" != "max" ]; then
125125
BYTES=$(cat /sys/fs/cgroup/memory.max)
126126
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
@@ -139,7 +139,7 @@ jobs:
139139
- name: Verify TORCH_CI_MAX_MEMORY
140140
run: |
141141
echo "=== TORCH_CI_MAX_MEMORY ==="
142-
EXPECTED=34359738368
142+
EXPECTED={{TORCHMEM__L_X86IAMX_8_32}}
143143
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
144144
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: $EXPECTED)"
145145
if [ -z "$ACTUAL" ]; then
@@ -169,7 +169,7 @@ jobs:
169169
echo "PASS: Architecture is $ARCH"
170170
- name: Verify CPU count
171171
run: |
172-
EXPECTED=16
172+
EXPECTED={{VCPU__L_ARM64G3_16_62}}
173173
ACTUAL=$(nproc)
174174
echo "CPUs: $ACTUAL (expected: >= $EXPECTED)"
175175
if [ "$ACTUAL" -lt "$EXPECTED" ]; then
@@ -179,7 +179,7 @@ jobs:
179179
echo "PASS: CPU count within expected range"
180180
- name: Verify memory
181181
run: |
182-
EXPECTED_GI=62
182+
EXPECTED_GI={{MEMGI__L_ARM64G3_16_62}}
183183
if [ -f /sys/fs/cgroup/memory.max ] && [ "$(cat /sys/fs/cgroup/memory.max)" != "max" ]; then
184184
BYTES=$(cat /sys/fs/cgroup/memory.max)
185185
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
@@ -198,7 +198,7 @@ jobs:
198198
- name: Verify TORCH_CI_MAX_MEMORY
199199
run: |
200200
echo "=== TORCH_CI_MAX_MEMORY ==="
201-
EXPECTED=66571993088
201+
EXPECTED={{TORCHMEM__L_ARM64G3_16_62}}
202202
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
203203
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: >= $EXPECTED)"
204204
if [ -z "$ACTUAL" ]; then
@@ -1650,7 +1650,7 @@ jobs:
16501650
nvidia-smi
16511651
echo ""
16521652
GPU_COUNT=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
1653-
if [ "$GPU_COUNT" -lt 1 ]; then
1653+
if [ "$GPU_COUNT" -lt {{GPU__L_X86IAVX512_29_115_T4}} ]; then
16541654
echo "FAIL: Expected at least 1 GPU, found $GPU_COUNT"
16551655
exit 1
16561656
fi
@@ -1673,7 +1673,7 @@ jobs:
16731673
- name: Verify TORCH_CI_MAX_MEMORY
16741674
run: |
16751675
echo "=== TORCH_CI_MAX_MEMORY ==="
1676-
EXPECTED=123480309760
1676+
EXPECTED={{TORCHMEM__L_X86IAVX512_29_115_T4}}
16771677
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
16781678
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: $EXPECTED)"
16791679
if [ -z "$ACTUAL" ]; then
@@ -1697,7 +1697,7 @@ jobs:
16971697
nvidia-smi
16981698
echo ""
16991699
GPU_COUNT=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
1700-
if [ "$GPU_COUNT" -ne 4 ]; then
1700+
if [ "$GPU_COUNT" -ne {{GPU__L_X86IAVX512_45_172_T4_4}} ]; then
17011701
echo "FAIL: Expected 4 GPUs, found $GPU_COUNT"
17021702
exit 1
17031703
fi
@@ -1709,7 +1709,7 @@ jobs:
17091709
- name: Verify TORCH_CI_MAX_MEMORY
17101710
run: |
17111711
echo "=== TORCH_CI_MAX_MEMORY ==="
1712-
EXPECTED=182536110080 # 170Gi = l-x86iavx512-45-172-t4-4 memory; keep in sync with the runner def
1712+
EXPECTED={{TORCHMEM__L_X86IAVX512_45_172_T4_4}}
17131713
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
17141714
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: $EXPECTED)"
17151715
if [ -z "$ACTUAL" ]; then
@@ -1735,15 +1735,15 @@ jobs:
17351735
nvidia-smi
17361736
echo ""
17371737
GPU_COUNT=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
1738-
if [ "$GPU_COUNT" -ne 2 ]; then
1738+
if [ "$GPU_COUNT" -ne {{GPU__L_X86IAMX_44_450_B200_2}} ]; then
17391739
echo "FAIL: Expected 2 GPUs, found $GPU_COUNT"
17401740
exit 1
17411741
fi
17421742
echo "PASS: Found $GPU_COUNT B200 GPUs"
17431743
- name: Verify TORCH_CI_MAX_MEMORY
17441744
run: |
17451745
echo "=== TORCH_CI_MAX_MEMORY ==="
1746-
EXPECTED=483183820800
1746+
EXPECTED={{TORCHMEM__L_X86IAMX_44_450_B200_2}}
17471747
ACTUAL="${TORCH_CI_MAX_MEMORY:-}"
17481748
echo "TORCH_CI_MAX_MEMORY=$ACTUAL (expected: $EXPECTED)"
17491749
if [ -z "$ACTUAL" ]; then
@@ -1920,7 +1920,7 @@ jobs:
19201920
steps:
19211921
- name: Verify CPU count
19221922
run: |
1923-
EXPECTED=16
1923+
EXPECTED={{VCPU__REL_L_ARM64G4_16_62}}
19241924
ACTUAL=$(nproc)
19251925
echo "CPUs: $ACTUAL (expected: $EXPECTED)"
19261926
if [ "$ACTUAL" -ne "$EXPECTED" ]; then

0 commit comments

Comments
 (0)