From ec752ba31d36d18176fb6764b44bdac250afd32c Mon Sep 17 00:00:00 2001 From: Guokai Ma Date: Tue, 1 Sep 2026 09:21:14 +0800 Subject: [PATCH 1/3] Run multi-rank CPU unit tests in CI via LOCAL_SIZE cpu-torch-latest runs on a single-socket runner where CPU_Accelerator.device_count() reports 1 NUMA node, so the per-device gate in tests/unit/common.py skips every test that needs more than one rank. CPU ranks are plain processes over gloo and need no per-rank hardware, so advertise 4 local devices via LOCAL_SIZE, the env var device_count() reads first. The test harness re-sets LOCAL_SIZE per worker, so this value only affects the launch gate. Signed-off-by: Guokai Ma --- .github/workflows/cpu-torch-latest.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml index 541484f05fa1..7fbc2fb2ec13 100644 --- a/.github/workflows/cpu-torch-latest.yml +++ b/.github/workflows/cpu-torch-latest.yml @@ -81,6 +81,12 @@ jobs: runs-on: ubuntu-24.04 env: + # The runner is single-socket, so CPU_Accelerator.device_count() reports 1 + # NUMA node and the per-device gate in tests/unit/common.py skips every + # multi-rank test. CPU ranks are plain processes over gloo, so advertise 4 + # local devices to let world_size<=4 tests run. The test harness re-sets + # LOCAL_SIZE per worker, so this value only affects the launch gate. + LOCAL_SIZE: '4' DEFAULT_TORCH_PRESET: '2.10.0-cpu' DEFAULT_TRANSFORMERS_SOURCE: 'git' # Manual PyPI fallback only; scheduled and default manual runs use Git. From f2b7d0b8d236b6407da7d7641bf334be4b76d183 Mon Sep 17 00:00:00 2001 From: Guokai Ma Date: Wed, 2 Sep 2026 05:21:52 +0800 Subject: [PATCH 2/3] Disable dist-env reuse for the full multi-rank CPU run With LOCAL_SIZE=4 the suite now runs to ~63% and then all xdist workers go silent for hours until the 6h job limit cancels the run - the pool worker cleanup hang that DS_DISABLE_REUSE_DIST_ENV was added for. Fresh pools per test cost some wall time but let the run finish and print the failure summary. Signed-off-by: Guokai Ma --- .github/workflows/cpu-torch-latest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml index 7fbc2fb2ec13..c9f61a3b6b4e 100644 --- a/.github/workflows/cpu-torch-latest.yml +++ b/.github/workflows/cpu-torch-latest.yml @@ -87,6 +87,10 @@ jobs: # local devices to let world_size<=4 tests run. The test harness re-sets # LOCAL_SIZE per worker, so this value only affects the launch gate. LOCAL_SIZE: '4' + # Multi-rank tests churn mp pools for hours; reused pools eventually hang in + # cleanup and stall workers until the 6h job limit. Fresh pools per test are + # slower but let the suite finish (knob documented in tests/unit/common.py). + DS_DISABLE_REUSE_DIST_ENV: '1' DEFAULT_TORCH_PRESET: '2.10.0-cpu' DEFAULT_TRANSFORMERS_SOURCE: 'git' # Manual PyPI fallback only; scheduled and default manual runs use Git. From f2793e70eae367bf56f35d9d3ffb050e27f8c623 Mon Sep 17 00:00:00 2001 From: Guokai Ma Date: Wed, 2 Sep 2026 11:27:21 +0800 Subject: [PATCH 3/3] Split CPU unit tests into halves with per-half timeouts Both full-suite attempts wedge before printing a summary: once the 100th failure trips PYTEST_OPTS' --maxfail, pytest-xdist's interrupt path stalls forever in mp pool teardown (no timeout guards _close_pool), and the 6h job limit cancels the run. Run the suite as two fresh-worker halves, override maxfail so all failures are listed, and cap each half with timeout so the sequential tail always runs. Signed-off-by: Guokai Ma --- .github/workflows/cpu-torch-latest.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml index c9f61a3b6b4e..061c8ee9a573 100644 --- a/.github/workflows/cpu-torch-latest.yml +++ b/.github/workflows/cpu-torch-latest.yml @@ -284,5 +284,12 @@ jobs: run: | unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch cd tests - HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS --forked -n 4 unit/ --torch_ver="$TORCH_TEST_VERSION" - HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS --forked -m 'sequential' unit/ --torch_ver="$TORCH_TEST_VERSION" + # The suite is split so each half gets fresh xdist workers: multi-rank pool + # teardown eventually wedges a worker, and one 3400-test process never + # reaches its summary inside the 6h job limit. maxfail is raised so every + # failure is listed, and timeout caps each half. + overall=0 + timeout 150m pytest $PYTEST_OPTS --maxfail=100000 --forked -n 4 unit/ --ignore=unit/v1 --torch_ver="$TORCH_TEST_VERSION" || overall=$? + timeout 150m pytest $PYTEST_OPTS --maxfail=100000 --forked -n 4 unit/v1 --torch_ver="$TORCH_TEST_VERSION" || overall=$? + HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS --forked -m 'sequential' unit/ --torch_ver="$TORCH_TEST_VERSION" || overall=$? + exit $overall