Skip to content

Commit 1b9a369

Browse files
Alexey-RivkinAlexey-Rivkin
committed
AZP: TEMP - exclude UCXX Python GPU tests from this PR
UCXX-Python pytest wedges the MPS daemon on swx-rdmz-ucx-gpu-01/-02 to the point that UCX CI on the same hosts is also blocked. Disable until a safe recipe is agreed upstream: - ucxx_tests.yml: drop the `test_python` step on GPU slices. GPU `test_cpp` keeps running (it passes); CPU slices unchanged. - ucxx_build.yml: `condition: false` on the three GPU Python test jobs: wheel-tests-ucxx, wheel-tests-distributed-ucxx, conda-python-distributed-tests. CPU build matrix + GPU C++ + docs + devcontainer + checks still run. Restore once a non-wedging recipe lands.
1 parent a44239e commit 1b9a369

3 files changed

Lines changed: 39 additions & 35 deletions

File tree

‎buildlib/pr/main.yml‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -285,19 +285,19 @@ stages:
285285
- template: codestyle.yml
286286

287287
- stage: Basic_compile
288-
condition: false # temp: skip non-UCXX stage; restore before merge
288+
condition: false # temp: skip non-UCXX stage
289289
dependsOn: []
290290
jobs:
291291
- template: basic_compile.yml
292292

293293
- stage: Static_check
294-
condition: false # temp: skip non-UCXX stage; restore before merge
294+
condition: false # temp: skip non-UCXX stage
295295
dependsOn: [Basic_compile]
296296
jobs:
297297
- template: static_checks.yml
298298

299299
- stage: Build
300-
condition: false # temp: skip non-UCXX stage; restore before merge
300+
condition: false # temp: skip non-UCXX stage
301301
dependsOn: [Basic_compile]
302302
jobs:
303303
- template: build_job.yml
@@ -322,7 +322,7 @@ stages:
322322
displayName: Build on aarch64
323323

324324
- stage: ucx_perftest_mad_rte
325-
condition: false # temp: skip non-UCXX stage; restore before merge
325+
condition: false # temp: skip non-UCXX stage
326326
dependsOn: [Basic_compile]
327327
displayName: ucx_perftest over MAD RTE
328328
lockBehavior: sequential
@@ -332,7 +332,7 @@ stages:
332332
- template: mad_tests.yml
333333

334334
- stage: WireCompat
335-
condition: false # temp: skip non-UCXX stage; restore before merge
335+
condition: false # temp: skip non-UCXX stage
336336
dependsOn: [Basic_compile]
337337
jobs:
338338
- template: wire_compat.yml
@@ -364,7 +364,7 @@ stages:
364364
# demands: ucx_iodemo -equals yes
365365

366366
- stage: Coverity
367-
condition: false # temp: skip non-UCXX stage; restore before merge
367+
condition: false # temp: skip non-UCXX stage
368368
dependsOn: [Static_check]
369369
jobs:
370370
- template: coverity.yml
@@ -395,28 +395,28 @@ stages:
395395
- { name: x86_64_cuda13_py311, libucxx_slice: x86_64_cuda13_py314, ucxx_slice: x86_64_cuda13_py311, distributed_slice: x86_64_cuda13_py314,
396396
demands: ucx_gpu -equals yes, rapids_cuda_version: '13.0.2', rapids_py_version: '3.11' }
397397
conda_cpp_slices:
398-
- { name: x86_64_cuda12_py311, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
399-
- { name: x86_64_cuda13_py311, demands: ucx_docker -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
400-
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
401-
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
398+
- { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
399+
- { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
400+
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
401+
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
402402
conda_python_slices:
403-
- { name: x86_64_cuda12_py311, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
404-
- { name: x86_64_cuda13_py311, demands: ucx_docker -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
405-
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
406-
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
403+
- { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
404+
- { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
405+
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
406+
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
407407
wheel_libucxx_slices:
408-
- { name: x86_64_cuda12_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
409-
- { name: x86_64_cuda13_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
410-
- { name: aarch64_cuda12_py314, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
411-
- { name: aarch64_cuda13_py314, demands: ucx_arm64, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
408+
- { name: x86_64_cuda12_py314, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
409+
- { name: x86_64_cuda13_py314, demands: ucx_docker, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
410+
- { name: aarch64_cuda12_py314, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
411+
- { name: aarch64_cuda13_py314, demands: ucx_arm64, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
412412
wheel_ucxx_slices:
413-
- { name: x86_64_cuda12_py311, libucxx_slice: x86_64_cuda12_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
414-
- { name: x86_64_cuda13_py311, libucxx_slice: x86_64_cuda13_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '13.0.2', rapids_py_version: '3.11' }
415-
- { name: aarch64_cuda12_py311, libucxx_slice: aarch64_cuda12_py314, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
416-
- { name: aarch64_cuda13_py311, libucxx_slice: aarch64_cuda13_py314, demands: ucx_arm64, rapids_cuda_version: '13.0.2', rapids_py_version: '3.11' }
413+
- { name: x86_64_cuda12_py311, libucxx_slice: x86_64_cuda12_py314, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
414+
- { name: x86_64_cuda13_py311, libucxx_slice: x86_64_cuda13_py314, demands: ucx_docker, rapids_cuda_version: '13.0.2', rapids_py_version: '3.11' }
415+
- { name: aarch64_cuda12_py311, libucxx_slice: aarch64_cuda12_py314, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
416+
- { name: aarch64_cuda13_py311, libucxx_slice: aarch64_cuda13_py314, demands: ucx_arm64, rapids_cuda_version: '13.0.2', rapids_py_version: '3.11' }
417417
wheel_distributed_ucxx_slices:
418-
- { name: x86_64_cuda12_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
419-
- { name: x86_64_cuda13_py314, demands: ucx_docker -equals yes, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
418+
- { name: x86_64_cuda12_py314, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.14' }
419+
- { name: x86_64_cuda13_py314, demands: ucx_docker, rapids_cuda_version: '13.0.2', rapids_py_version: '3.14' }
420420
docs_slices:
421421
- { name: x86_64_cuda13_py311, cpp_slice: x86_64_cuda13_py311, python_slice: x86_64_cuda13_py311,
422422
demands: ucx_docker -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
@@ -426,7 +426,7 @@ stages:
426426
- { name: x86_64_cuda13_py311, demands: ucx_docker -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
427427

428428
- stage: Tests
429-
condition: false # temp: skip non-UCXX stage; restore before merge
429+
condition: false # temp: skip non-UCXX stage
430430
dependsOn: [Basic_compile]
431431
jobs:
432432
- template: tests.yml
@@ -510,7 +510,7 @@ stages:
510510
demands: ucx_gpu -equals yes
511511

512512
- stage: Build_Static
513-
condition: false # temp: skip non-UCXX stage; restore before merge
513+
condition: false # temp: skip non-UCXX stage
514514
dependsOn: [Basic_compile]
515515
jobs:
516516
- job: build_source
@@ -557,14 +557,14 @@ stages:
557557
558558
559559
- stage: Cuda
560-
condition: false # temp: skip non-UCXX stage; restore before merge
560+
condition: false # temp: skip non-UCXX stage
561561
dependsOn: [Basic_compile]
562562
jobs:
563563
- template: cuda/cuda.yml
564564

565565

566566
- stage: AddressSanitizer
567-
condition: false # temp: skip non-UCXX stage; restore before merge
567+
condition: false # temp: skip non-UCXX stage
568568
dependsOn: [Static_check]
569569
jobs:
570570
- template: tests.yml

‎buildlib/pr/ucxx_build.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ stages:
243243

244244
- ${{ each slice in parameters.wheel_tests_ucxx_slices }}:
245245
- job: ucxx_wheel_tests_ucxx_${{ slice.name }}
246+
condition: false # TEMP: UCXX-Python pytest wedges MPS on ucx_gpu hosts; re-enable once safe recipe is agreed.
246247
workspace:
247248
clean: all
248249
pool:
@@ -289,6 +290,7 @@ stages:
289290

290291
- ${{ each slice in parameters.wheel_tests_distributed_ucxx_slices }}:
291292
- job: ucxx_wheel_tests_dist_ucxx_${{ slice.name }}
293+
condition: false # TEMP: UCXX-Python pytest wedges MPS on ucx_gpu hosts; re-enable once safe recipe is agreed.
292294
workspace:
293295
clean: all
294296
pool:
@@ -344,6 +346,7 @@ stages:
344346

345347
- ${{ each slice in parameters.conda_python_distributed_tests_slices }}:
346348
- job: ucxx_conda_python_dist_tests_${{ slice.name }}
349+
condition: false # TEMP: UCXX-Python pytest wedges MPS on ucx_gpu hosts; re-enable once safe recipe is agreed.
347350
workspace:
348351
clean: all
349352
pool:

‎buildlib/pr/ucxx_tests.yml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ parameters:
66
# TEMP DEBUG: GPU slices listed first so they dispatch ahead of CPU and the
77
# failing leg surfaces sooner each iteration. Restore CPU-first ordering
88
# before merge.
9-
- { name: amd64_cuda1302_py312, gpu: true, demands: ucx_gpu -equals yes, rapids_cuda_version: '13.0.2', rapids_py_version: '3.12' }
10-
- { name: amd64_cuda1322_py313, gpu: true, demands: ucx_gpu -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.13' }
9+
- { name: amd64_cuda1302_py312, gpu: true, demands: ucx_gpu, rapids_cuda_version: '13.0.2', rapids_py_version: '3.12' }
10+
- { name: amd64_cuda1322_py313, gpu: true, demands: ucx_gpu, rapids_cuda_version: '13.2.0', rapids_py_version: '3.13' }
1111
# CPU: builds conda C++/Python pkgs and runs UCXX gtests on host transports.
12-
- { name: x86_64_cuda12_py311, gpu: false, demands: ucx_docker -equals yes, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
13-
- { name: x86_64_cuda13_py311, gpu: false, demands: ucx_docker -equals yes, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
12+
- { name: x86_64_cuda12_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
13+
- { name: x86_64_cuda13_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
1414
- { name: aarch64_cuda12_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '12.9.1', rapids_py_version: '3.11' }
1515
- { name: aarch64_cuda13_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '13.2.0', rapids_py_version: '3.11' }
1616

@@ -50,8 +50,9 @@ stages:
5050
retryCountOnTaskFailure: 5
5151
- bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh build
5252
displayName: Build UCXX
53-
- ${{ if eq(slice.gpu, true) }}:
54-
- bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh test_python
55-
displayName: Run UCXX Python tests
53+
# GPU Python tests excluded: UCXX-Python pytest wedges the MPS daemon
54+
# on swx-rdmz-ucx-gpu-* hosts, blocking UCX CI too. Re-enable once a
55+
# safe recipe is agreed upstream. CPU Python coverage continues
56+
# below via test_cpp on CPU slices and the conda-python-build job.
5657
- bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh test_cpp
5758
displayName: Run UCXX C++ tests

0 commit comments

Comments
 (0)