Adding Support for SGLang CI Tests

.azure-pipelines/integration-test.yml

@@ -19,11 +19,11 @@ pr:
   drafts: false
   paths:
     exclude:
-      - .devcontainer/**
-      - .github/**
-      - docker/**
-      - docs/**
-      - '**/*.md'
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
 
 jobs:
 - job: IntegrationTestA100
@@ -43,9 +43,9 @@ jobs:
   steps:
   - template: templates/integration-test.yml
     parameters:
-      subscription:     mscclpp-ci
-      vmssName:         mscclpp-ci
-      gpuArch:          '80'
+      subscription: mscclpp-ci
+      vmssName: mscclpp-ci
+      gpuArch: '80'
 
 - job: IntegrationTestH100
   displayName: Integration test H100
@@ -62,7 +62,7 @@ jobs:
   steps:
   - template: templates/integration-test.yml
     parameters:
-      subscription:     mscclpp-ci-h100
-      vmssName:         mscclpp-h100-ci
+      subscription: mscclpp-ci-h100
+      vmssName: mscclpp-h100-ci
       perfBaselineFile: test/deploy/perf_ndmv5.jsonl
-      gpuArch:          '90'
+      gpuArch: '90'

.azure-pipelines/multi-nodes-test.yml

@@ -14,7 +14,6 @@ trigger:
 # Do not run multi-nodes-test for PR, we can trigger it manually
 pr: none
 
-
 parameters:
 - name: vmssName
   type: string
@@ -79,10 +78,10 @@ jobs:
 
   - template: templates/deploy.yml
     parameters:
-      subscription:  mscclpp-ci-h100
-      vmssName:      ${{ parameters.vmssName }}
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
       resourceGroup: mscclpp
-      gpuArch:       '90'
+      gpuArch: '90'
 
   - template: templates/run-remote-task.yml
     parameters:
@@ -119,6 +118,6 @@ jobs:
 
   - template: templates/stop.yml
     parameters:
-      subscription:  mscclpp-ci-h100
-      vmssName:      ${{ parameters.vmssName }}
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
       resourceGroup: mscclpp

.azure-pipelines/sglang-multi-node-test.yml

@@ -0,0 +1,136 @@
+# =============================================================================
+# Multi-node SGLang integration test pipeline.
+#
+# This pipeline runs MSCCL++ SGLang tests across two H100 VMSS GPU nodes.
+# High-level flow:
+#   1. The pipeline agent runs inside a container on the `mscclpp-multi-node`
+#      pool. The agent itself has no GPUs.
+#   2. SSH/host configuration is generated so the agent can reach the two
+#      pre-provisioned VMSS GPU nodes.
+#   3. `templates/deploy.yml` builds and ships MSCCL++ to the GPU nodes.
+#   4. `templates/sglang-multi-test.yml` runs the SGLang multi-node tests.
+#   5. `templates/stop.yml` tears down / stops the VMSS nodes.
+#
+# Docs / non-code changes are excluded from triggering this pipeline.
+# =============================================================================
+
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+  drafts: false
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+parameters:
+# Name of the pre-provisioned Azure VMSS that hosts the GPU test nodes.
+# Node hostnames are derived as "${vmssName}000000" and "${vmssName}000001".
+- name: vmssName
+  type: string
+  default: mscclpp-h100-multinode-ci
+# Static /etc/hosts entries mapping VMSS node hostnames to their private IPs.
+# These IPs are tied to the specific VMSS above; update both together if the
+# VMSS is reprovisioned or renamed.
+- name: hostEntries
+  type: string
+  default: |
+    10.0.0.5 mscclpp-h100-multinode-ci000000
+    10.0.0.4 mscclpp-h100-multinode-ci000001
+
+jobs:
+- job: SGlangTestMultiNode
+  displayName: SGLANG Test Multi Node
+  strategy:
+    matrix:
+      cuda12:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
+  pool:
+    name: mscclpp-multi-node
+  container:
+    image: $(containerImage)
+
+  steps:
+  # Ensure the VMSS node hostnames resolve from the pipeline agent container.
+  # Idempotent: only appends lines that are not already present in /etc/hosts.
+  - task: Bash@3
+    displayName: Add HostEntry
+    inputs:
+      targetType: 'inline'
+      script: |
+        while IFS= read -r line; do
+          [ -z "$line" ] && continue
+          if ! grep -qxF "$line" /etc/hosts; then
+            echo "Adding to /etc/hosts: $line"
+            echo "$line" | sudo tee -a /etc/hosts
+          else
+            echo "Entry already exists: $line"
+          fi
+        done <<< "${{ parameters.hostEntries }}"
+
+  # Generate the SSH config and hostfile consumed by the deploy / test
+  # templates below:
+  #   - config   : SSH client config (custom port + key) for each node
+  #   - hostfile : user@host list used by deploy / test scripts (parallel-ssh)
+  - task: Bash@3
+    displayName: Generate deploy files
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -e
+        VMSS="${{ parameters.vmssName }}"
+        DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
+        NODE0="${VMSS}000000"
+        NODE1="${VMSS}000001"
+
+        echo "Host ${NODE0}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no
+        Host ${NODE1}
+          Port 22345
+          IdentityFile /root/mscclpp/sshkey
+          StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
+
+        printf '%s\n%s\n' "azureuser@${NODE0}" "azureuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
+
+  # Build MSCCL++ and deploy it onto the VMSS GPU nodes.
+  - template: templates/deploy.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+      resourceGroup: mscclpp
+      gpuArch: '90'
+      deployArgs: 'multi-node-test true cuda'
+      containerName: 'sglang-mscclpp-test'
+
+  # Run the SGLang multi-node tests across the two GPU nodes.
+  - template: templates/sglang-multi-test.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+
+  # Stop/deallocate the VMSS GPU nodes to release resources.
+  - template: templates/stop.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: ${{ parameters.vmssName }}
+      resourceGroup: mscclpp

.azure-pipelines/sglang-test.yml

@@ -0,0 +1,56 @@
+# =============================================================================
+# Single-node SGLang integration test pipeline.
+#
+# Runs MSCCL++ SGLang tests on a single H100 GPU node from the `msccl-ci-h100`
+# pool. All deploy / run / teardown logic is delegated to
+# `templates/sglang-test.yml`.
+#
+# Docs / non-code changes are excluded from triggering this pipeline.
+# =============================================================================
+
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+  drafts: false
+  paths:
+    exclude:
+    - .devcontainer/**
+    - .github/**
+    - docker/**
+    - docs/**
+    - '**/*.md'
+
+jobs:
+- job: SGlangTest
+  displayName: SGLANG Test
+  strategy:
+    matrix:
+      cuda12:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9
+  pool:
+    name: msccl-ci-h100
+  container:
+    image: $(containerImage)
+
+  steps:
+  # Deploy MSCCL++ to the GPU node and run the SGLang single-node tests.
+  - template: templates/sglang-test.yml
+    parameters:
+      subscription: mscclpp-ci-h100
+      vmssName: mscclpp-h100-ci
+      gpuArch: '90'

.azure-pipelines/templates/deploy.yml

@@ -32,6 +32,9 @@ parameters:
 - name: deployArgs
   type: string
   default: ''
+- name: containerName
+  type: string
+  default: 'mscclpp-test'
 
 steps:
 # 0. Ensure Azure CLI exists before running AzureCLI@2 tasks.
@@ -56,6 +59,8 @@ steps:
     targetType: 'inline'
     script: |
       set -e
+      sudo apt-get update -y
+      sudo apt-get install cmake -y
       rm -rf build
       mkdir -p build && cd build
       BUILD_TESTS_ARG=""
@@ -147,5 +152,5 @@ steps:
   inputs:
     targetType: filePath
     filePath: test/deploy/deploy.sh
-    arguments: ${{ parameters.deployArgs }}
+    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }}
     workingDirectory: '$(System.DefaultWorkingDirectory)'

.azure-pipelines/templates/sglang-multi-test.yml

@@ -0,0 +1,104 @@
+# =============================================================================
+# SGLang multi-node test template.
+#
+# Runs on the pipeline agent and dispatches remote steps to the two VMSS GPU
+# nodes (via run-remote-task.yml + the SSH config / hostfile produced by the
+# caller pipeline). Steps:
+#   1. Build and install MSCCL++ on each node.
+#   2. Install a (currently forked) SGLang on each node, replacing any
+#      pre-baked copy from the base image.
+#   3. Run a 2-node sglang.bench_one_batch smoke test with MSCCL++ enabled.
+#   4. Run the MSCCL++ all-reduce micro-benchmark via torchrun across both
+#      nodes.
+# =============================================================================
+
+parameters:
+- name: subscription
+  type: string
+- name: vmssName
+  type: string
+- name: containerName
+  type: string
+  default: 'sglang-mscclpp-test'
+
+steps:
+# TODO: Switch to the official upstream sglang repo once Caio's PR is merged.
+# Tracking: the fork below (`caiomcbr/sglang` @ caiorocha/mscclpp) is a personal
+# branch and should not remain a long-term CI dependency.
+- template: run-remote-task.yml
+  parameters:
+    name: InstallSGLang
+    displayName: Install SGLang
+    runRemoteArgs: '--container ${{ parameters.containerName }} --hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --user azureuser'
+    remoteScript: |
+      # Remove any pre-baked sglang from the container image so all nodes
+      # use the freshly cloned fork (otherwise rank 0 imports
+      # /sgl-workspace/sglang while rank 1 imports our fork, causing
+      # version mismatch and NCCL/CUDA errors).
+      pip uninstall -y sglang sglang-router || true
+      rm -rf /sgl-workspace/sglang || true
+      rm -rf sglang
+      git clone -b caiorocha/mscclpp https://github.com/caiomcbr/sglang.git
+      cd sglang
+      pip install -e "python"
+      # Sanity check: confirm sglang resolves to our fork on every node.
+      python -c "import sglang, os; p=os.path.dirname(sglang.__file__); print('sglang from:', p); assert '/sgl-workspace' not in p, 'stock sglang still active'"
+
+# Smoke test: 2-node tensor-parallel benchmark of Qwen3-8B with MSCCL++.
+# Port 20003 is the SGLang distributed-init rendezvous port (arbitrary, must
+# match across ranks and be free on node 0).
+- template: run-remote-task.yml
+  parameters:
+    name: RunSGLangMultiBenchOneBatch
+    displayName: Run SGLang Multi-Node Bench One Batch
+    runRemoteArgs: '--container ${{ parameters.containerName }} --hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --user azureuser'
+    remoteScript: |
+      export FLASHINFER_DISABLE_VERSION_CHECK=1
+      VMSS="${{ parameters.vmssName }}"
+      HOSTNAME=$(hostname)
+      # Explicit 2-node mapping: hostname suffix -> SGLang node rank.
+      if [ "$HOSTNAME" = "${VMSS}000000" ]; then
+        NODE_RANK=0
+      elif [ "$HOSTNAME" = "${VMSS}000001" ]; then
+        NODE_RANK=1
+      else
+        echo "Unknown hostname: $HOSTNAME"
+        exit 1
+      fi
+      python -m sglang.bench_one_batch --model-path Qwen/Qwen3-8B --batch 1 2 4 8 16 32 64 128 256 512 --input-len 256 --output-len 256 --tp-size 16 --dist-init-addr ${VMSS}000000:20003 --nnodes 2 --node-rank $NODE_RANK --enable-mscclpp
+
+# Depends on the `sglang/` source tree cloned by the InstallSGLang step above
+# (steps on the same remote share a working directory).
+- template: run-remote-task.yml
+  parameters:
+    name: RunSGLangMultiTestAllReduce
+    displayName: Run SGLang Multi-Node Test All Reduce
+    runRemoteArgs: '--container ${{ parameters.containerName }} --hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --user azureuser'
+    remoteScript: |
+      export FLASHINFER_DISABLE_VERSION_CHECK=1
+      VMSS="${{ parameters.vmssName }}"
+      HOSTNAME=$(hostname)
+      # Explicit 2-node mapping: hostname suffix -> torchrun node rank.
+      if [ "$HOSTNAME" = "${VMSS}000000" ]; then
+        NODE_RANK=0
+      elif [ "$HOSTNAME" = "${VMSS}000001" ]; then
+        NODE_RANK=1
+      else
+        echo "Unknown hostname: $HOSTNAME"
+        exit 1
+      fi
+
+      export NODE_SIZE=2
+      export WORLD_SIZE=8
+
+      cd sglang
+
+      # Port 20004 is the torchrun rendezvous port (arbitrary, must match
+      # across ranks and be free on node 0). Distinct from 20003 used by
+      # sglang.bench_one_batch above.
+      torchrun --nproc_per_node $WORLD_SIZE \
+        --nnodes $NODE_SIZE \
+        --node_rank $NODE_RANK \
+        --master_addr ${VMSS}000000 \
+        --master_port 20004 \
+        benchmark/kernels/all_reduce/benchmark_mscclpp.py