[CRL] Add unified MR registry with P2P engine integration #86
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Unit Tests: device_api" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| unittest-device-api: | |
| runs-on: [self-hosted, cx-build] | |
| container: | |
| image: localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538 | |
| options: --gpus all --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| # FORCE_DEFAULT_PATH only affects kernel object compilation (comm traits); | |
| # host-side library and vendor API dispatch are unchanged. | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| export MPI_HOME=/usr/local/mpi | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Build device_api correctness tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/unittest/device_api | |
| export MPI_HOME=/usr/local/mpi | |
| make -j$(nproc) USE_NVIDIA=1 FORCE_DEFAULT_PATH=1 | |
| - name: Run intra-node tests (8 GPUs as single node) | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/unittest/device_api | |
| export MPI_HOME=/usr/local/mpi | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH" | |
| FLAGS="-b 1M -e 4M -f 2 -R 2" | |
| mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_api ${FLAGS} | |
| mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_ir_intra ${FLAGS} | |
| - name: Run inter-node tests (simulate 2 nodes, 4 GPUs each) | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/unittest/device_api | |
| export MPI_HOME=/usr/local/mpi | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH" | |
| FLAGS="-b 1M -e 4M -f 2 -R 2" | |
| NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3" | |
| NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7" | |
| mpirun --allow-run-as-root \ | |
| -np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_api ${FLAGS} \ | |
| : -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_api ${FLAGS} | |
| mpirun --allow-run-as-root \ | |
| -np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_ir_inter ${FLAGS} \ | |
| : -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_ir_inter ${FLAGS} |