SDMA HIP device management fixes #797

Workflow file for this run

	name: CCO CI test
	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	IMAGE: rocm/mori:ci-cco
	BASE_IMAGE: rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.12.0
	CONTAINER: mori_cco_ci_${{ github.run_id }}
	CT: podman # MI355X-AINIC runner is podman rootless (no docker daemon)

	jobs:
	cco-unit-test:
	name: CCO unit test (${{ matrix.platform }})
	runs-on: ${{ matrix.runner }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- platform: MI355X_AINIC
	runner: [self-hosted, MI355X-AINIC]
	rdma_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7
	rdma_sl: 3
	rdma_tc: 104
	env:
	MORI_RDMA_DEVICES: ${{ matrix.rdma_devices }}
	MORI_RDMA_SL: ${{ matrix.rdma_sl }}
	MORI_RDMA_TC: ${{ matrix.rdma_tc }}
	timeout-minutes: 30
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true

	- name: Build CI image
	run: \|
	$CT build --network=host --build-arg BASE_IMAGE=$BASE_IMAGE \
	--build-arg GPU_ARCHS="gfx950 gfx1250" --build-arg WITH_FLYDSL=1 \
	-t $IMAGE -f docker/Dockerfile.dev .

	- name: Start container
	run: \|
	$CT rm -f $CONTAINER 2>/dev/null \|\| true
	CONTAINER_RUNTIME=$CT ./docker/ci_run.sh --name $CONTAINER \
	-e MORI_RDMA_DEVICES=$MORI_RDMA_DEVICES \
	-e MORI_RDMA_SL=$MORI_RDMA_SL \
	-e MORI_RDMA_TC=$MORI_RDMA_TC \
	-v $GITHUB_WORKSPACE:$GITHUB_WORKSPACE \
	-w $GITHUB_WORKSPACE \
	$IMAGE sleep infinity
	$CT exec $CONTAINER \
	git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Build mori with tests
	run: \|
	# BUILD_UMBP=OFF: not needed for CCO, and its gtest discovery breaks under cmake 4.4.
	$CT exec $CONTAINER bash -c \
	"cd $GITHUB_WORKSPACE && BUILD_TESTS=ON BUILD_EXAMPLES=ON BUILD_BENCHMARK=ON MORI_WITH_MPI=ON BUILD_UMBP=OFF BUILD_CCO_SDMA=ON pip install ."

	- name: Run CCO unit tests (fork mode, 4 ranks)
	run: \|
	# MORI_CCO_SKIP_GDA_FULL=1: the MI355X-AINIC runner has no intranode
	# cross-rail RDMA, so GDA-FULL tests (gda_flush_async/gda_modes/
	# multiprocess) can't form FULL connections; skip them here.
	$CT exec -e MORI_CCO_SKIP_GDA_FULL=1 $CONTAINER bash $GITHUB_WORKSPACE/tools/run_cco_tests.sh $GITHUB_WORKSPACE/build 4 2>&1

	- name: Run CCO SDMA C++ tests (fork mode, 8 ranks)
	run: \|
	# Every one of these SKIPs and exits 0 when the comm has no SDMA queues,
	# so a build or runner that quietly loses SDMA would turn the whole
	# suite green. Treat a SKIP as a failure.
	$CT exec -e MORI_ENABLE_SDMA=1 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	for t in test_sdma_put test_sdma_get test_sdma_put_mt test_sdma_block test_sdma_edge \
	test_sdma_signal test_sdma_warp_issue; do \
	echo \"=== \$t ===\" && \
	out=\$(timeout 240 ./build/tests/cpp/cco/\$t 8 2>&1) \|\| { echo \"\$out\"; exit 1; }; \
	echo \"\$out\"; \
	case \"\$out\" in SKIP) echo \"\$t SKIPped: SDMA is not active\"; exit 1;; esac; \
	done"

	- name: Run CCO SDMA benchmarks (2 ranks, smoke)
	run: \|
	# Smoke only: a shared runner cannot give stable perf numbers, so this
	# checks that every scope and completion mode still runs and reports.
	# The sizes are picked so a gross regression is visible in the log:
	# latency should sit near 6us up to 8KB, and bandwidth at one op per
	# transfer near 52 GB/s at 1MB and 60 GB/s at 8MB. Whole step is ~35s.
	$CT exec -e MORI_ENABLE_SDMA=1 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && B=./build/benchmark && \
	for s in thread warp block; do \
	for c in quiet signal; do \
	echo \"=== put latency -s \$s -C \$c ===\" && \
	timeout 120 mpirun --allow-run-as-root -np 2 \
	\$B/cco_p2p_put_latency -t sdma -s \$s -C \$c -b 8 -e 64K -n 100 -w 20 \|\| exit 1; \
	done; \
	echo \"=== get latency -s \$s ===\" && \
	timeout 120 mpirun --allow-run-as-root -np 2 \
	\$B/cco_p2p_get_latency -t sdma -s \$s -b 8 -e 64K -n 100 -w 20 \|\| exit 1; \
	done"
	# Bandwidth with the default depth of 1, i.e. one op per transfer.
	# Splitting a transfer costs bandwidth in proportion to how small the
	# pieces get, so -A > 1 here would measure a degenerate config.
	$CT exec -e MORI_ENABLE_SDMA=1 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && B=./build/benchmark && \
	for s in thread warp block; do \
	echo \"=== put bw -s \$s ===\" && \
	timeout 120 mpirun --allow-run-as-root -np 2 \
	\$B/cco_p2p_put_bw -t sdma -s \$s -b 64K -e 8M -n 100 -w 20 \|\| exit 1; \
	echo \"=== get bw -s \$s ===\" && \
	timeout 120 mpirun --allow-run-as-root -np 2 \
	\$B/cco_p2p_get_bw -t sdma -s \$s -b 64K -e 8M -n 100 -w 20 \|\| exit 1; \
	done"
	# Keep the aggregate-doorbell path exercised; it is a different code
	# path (no per-put doorbell, one commit) even though batching does not
	# move bandwidth at these sizes.
	$CT exec -e MORI_ENABLE_SDMA=1 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && B=./build/benchmark && \
	echo \"=== put bw aggregate (-a -A 8) ===\" && \
	timeout 120 mpirun --allow-run-as-root -np 2 \
	\$B/cco_p2p_put_bw -t sdma -s thread -a -A 8 -b 256K -e 1M -n 100 -w 20 \|\| exit 1"

	- name: Run CCO C++ examples (2 ranks)
	run: \|
	$CT exec $CONTAINER bash -c "\
	mpirun --allow-run-as-root -np 2 $GITHUB_WORKSPACE/build/examples/cco_lsa_put"
	# TEMP: cco_gda_put bypassed on the MI355X-AINIC runner (GDA put not
	# working there yet); re-enable once GDA is supported on this runner.
	# mpirun --allow-run-as-root -np 2 $GITHUB_WORKSPACE/build/examples/cco_gda_put

	- name: Run CCO Python examples (2 ranks)
	run: \|
	$CT exec $CONTAINER bash -c "pip install mpi4py && \
	cd $GITHUB_WORKSPACE/examples/cco/python && \
	mpirun --allow-run-as-root -np 2 python 01_barrier/main.py && \
	mpirun --allow-run-as-root -np 2 python 02_lsa_put/main.py && \
	mpirun --allow-run-as-root -np 2 python 04_flydsl_lsa_put/main.py && \
	mpirun --allow-run-as-root -np 2 python 05_flydsl_lsa_allreduce/main.py && \
	mpirun --allow-run-as-root -np 2 python 08_torch_symm_import/main.py"
	# GDA (device-initiated RDMA) examples fail on the runner (cross-rail
	# RDMA); re-enable once that is fixed.
	# mpirun --allow-run-as-root -np 2 python 03_flydsl_put/main.py
	# mpirun --allow-run-as-root -np 2 python 06_flydsl_gda_modes/main.py

	# Lives here rather than in ci.yml because this is the job that shares a runner
	# label with 08_torch_symm_import above, and because MORI_SYMM_FORCE_JIT is what
	# compiles src/allocator/symm_backend.cpp at all: every `pip install .` runs under
	# PEP 517 build isolation, where torch is not importable, so setup.py never
	# prebuilds the extension. Without this step nothing in CI compiles that file.
	- name: MORI torch SymmetricMemory backend
	run: \|
	$CT exec -e PYTHONPATH=$GITHUB_WORKSPACE -e MORI_SYMM_FORCE_JIT=1 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && timeout 600 pytest tests/python/allocator -v"

	- name: Run CCO SDMA Python API tests (spawn, up to 8 GPUs)
	run: \|
	# The build step built with BUILD_CCO_SDMA=ON, which setup.py bakes into
	# the package, so the runtime JIT device wrapper picks up SDMA
	# automatically. (MORI_ENABLE_SDMA is set by the test to build queues.)
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	timeout 300 pytest tests/python/cco/test_sdma_api.py -v"

	- name: Validate CCO ROCm runtime import order
	run: \|
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	timeout 120 pytest -v \
	tests/python/cco/test_rocm_bootstrap.py \
	tests/python/cco/test_rocm_import_order.py"

	- name: Run CCO Triton bindings, LSA, and SDMA tests
	run: \|
	$CT exec \
	-e MORI_ENABLE_SDMA=1 \
	-e BUILD_CCO_SDMA=ON \
	$CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	timeout 300 pytest -v \
	tests/python/cco/test_triton_bindings.py \
	tests/python/cco/test_triton_lsa.py \
	tests/python/cco/test_triton_sdma.py"

	- name: Run CCO Triton benchmark tests
	run: \|
	$CT exec \
	-e MORI_ENABLE_SDMA=1 \
	-e BUILD_CCO_SDMA=ON \
	$CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	timeout 300 pytest -v tests/python/cco/test_triton_benchmark.py"

	- name: Compile CCO Triton GDA variants (AINIC runtime gated)
	run: \|
	# Runtime GDA-FULL tests and examples are already gated on this runner:
	# its intranode AINIC path faults even when both ranks use one HCA.
	# Still create the resources and compile/link every Triton GDA variant.
	FIRST_RDMA_DEVICE="${MORI_RDMA_DEVICES%%,*}"
	$CT exec \
	-e MORI_CCO_TRITON_GDA_DEVICE=$FIRST_RDMA_DEVICE \
	-e MORI_CCO_TRITON_GDA_COMPILE_ONLY=1 \
	-e MORI_RDMA_DEVICES=$FIRST_RDMA_DEVICE \
	-e MORI_DISABLE_TOPO=1 \
	-e MORI_DEVICE_NIC=ionic \
	-e BUILD_CCO_SDMA=ON \
	$CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	timeout 300 pytest -v tests/python/cco/test_triton_gda.py"
	- name: Run JIT v2 C++ tests + host/device split guard
	run: \|
	# Registered by tests/cpp/jit/CMakeLists.txt but selected by no job before
	# this one. Neither needs a GPU peer: jit_core exercises render/cache/schema,
	# the guard only checks the Cfg headers still compile without hipcc.
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/build && \
	timeout 600 ctest --output-on-failure -R '^jit_'"

	- name: Run JIT v2 Python binding tests (1 rank, outside the repo)
	run: \|
	# The ctypes/C-ABI seam: schema-vs-sizeof, request coercion, unknown-field
	# rejection. Single rank -- EpDispatchPlan(world_size=1) needs no communicator.
	#
	# Run from /tmp rather than the checkout, and that is the point: a plan needs
	# libmori_jit.so, libmori_ops_v2.so and the v2 kernel sources, all of which
	# resolve out of build/ and src/ from a repo cwd even when the install ships
	# none of them. Outside the repo only the packaged copies can answer.
	$CT exec $CONTAINER bash -c "\
	cd /tmp && \
	timeout 600 python -m pytest \
	$GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2/test_jit_binding.py -v"

	- name: Run ops-v2 dispatch/combine test (EP8, 8 ranks)
	run: \|
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE && \
	python -m pytest tests/python/ops/dispatch_combine_v2/test_dispatch_combine_v2_intranode.py -v"

	- name: Run ops-v2 HIP backend (EP8, 8 ranks)
	run: \|
	# The step above drives the FlyDSL backend; this one drives the HIP kernels
	# behind the same op. test_op.py reports per-case PASS/FAIL and does not exit
	# non-zero on a failed CHECK, so grep the output too -- and treat "no PASS at
	# all" as failure, or a run that died before the first case reads as green.
	$CT exec -e MORI_V2_KERNEL_BACKEND=hip $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2 && \
	out=\$(timeout 900 torchrun --standalone --nproc_per_node=8 test_op.py 2>&1) \|\| \
	{ echo \"\$out\"; exit 1; }; \
	echo \"\$out\"; \
	case \"\$out\" in FAIL) exit 1;; esac; \
	case \"\$out\" in PASS) ;; *) echo 'no PASS line: nothing ran'; exit 1;; esac"

	- name: Run ops-v2 HIP backend with per-token scales (EP8, 8 ranks)
	run: \|
	# SCALE_DIM>0 turns on the forwarded per-token scale row and makes
	# test_op.py verify it through the routing reverse map (OP-SCALES). It is
	# a separate step rather than an env added to the one above so the
	# no-scale path keeps its own coverage.
	$CT exec -e MORI_V2_KERNEL_BACKEND=hip -e SCALE_DIM=32 $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2 && \
	out=\$(timeout 900 torchrun --standalone --nproc_per_node=8 test_op.py 2>&1) \|\| \
	{ echo \"\$out\"; exit 1; }; \
	echo \"\$out\"; \
	case \"\$out\" in FAIL) exit 1;; esac; \
	case \"\$out\" in OP-SCALES) ;; *) echo 'no OP-SCALES line: scales never ran'; exit 1;; esac"

	- name: Run ops-v2 scales on a quantizing wire (fp8 and fp4 dispatch, EP8)
	run: \|
	# The step above can only test scales on a BF16 wire: test_op.py gives
	# dispatch and combine one dtype, and the HIP backend refuses an fp8
	# combine (it sums). fp8/fp4 dispatch with a bf16 combine is what a real
	# quantizing wire is, and it is the only configuration where the scale
	# row is not decoration -- so it gets its own step.
	#
	# SCALE_DIM=224 on purpose: hidden/32 for hidden 7168, and NOT a multiple
	# of 128, so it exercises the padding mori adds to the destination stride
	# (EpScaleStride). A row that is already aligned would not.
	#
	# This runner has no gfx1250, so what actually runs here is the portable
	# body -- the one that cannot be exercised by hand on the TDM boxes.
	for disp in fp8 fp4; do
	$CT exec -e MORI_V2_KERNEL_BACKEND=hip -e SCALE_DIM=224 -e HIDDEN=7168 \
	-e DISP=$disp $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2 && \
	out=\$(timeout 900 torchrun --standalone --nproc_per_node=8 test_asym_dtype.py 2>&1) \|\| \
	{ echo \"\$out\"; exit 1; }; \
	echo \"\$out\"; \
	case \"\$out\" in FAIL) exit 1;; esac; \
	case \"\$out\" in ASYM-SCALES) ;; *) echo 'no ASYM-SCALES line: scales never ran'; exit 1;; esac" \|\| exit 1
	done

	- name: Run ops-v2 backend parity (FlyDSL vs HIP, EP8)
	run: \|
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2 && \
	timeout 900 torchrun --standalone --nproc_per_node=8 test_ep_backend_parity.py"

	- name: Run ops-v2 dispatch/combine bench (EP8 smoke, both backends)
	run: \|
	# bench_ep.py gates every point on an identity-expert check and exits
	# non-zero if one fails, so this is a smoke test as well as a bench.
	$CT exec $CONTAINER bash -c "\
	cd $GITHUB_WORKSPACE/tests/python/ops/dispatch_combine_v2 && \
	BACKENDS=flydsl,hip MODES=graph SWEEP=128,512,4096,8192 \
	ITERS=20 WARMUP=10 \
	torchrun --standalone --nproc_per_node=8 bench_ep.py"


	- name: Cleanup
	if: always()
	run: \|
	# Restore workspace ownership (build ran as root in the container).
	$CT exec $CONTAINER chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE 2>/dev/null \|\| true
	# Graceful stop so GPU ranks release HIP/KFD contexts (not SIGKILLed).
	CONTAINER_RUNTIME=$CT bash docker/ci_stop.sh $CONTAINER \|\| true

Navigation Menu

Uh oh!

SDMA HIP device management fixes #797

Workflow file

SDMA HIP device management fixes #797

Uh oh!

Workflow file for this run