Merge pull request #13 from swiss-ai/nccl_fix

NCCL Libfabric fix. Added new model configurations

Author: davmels

serving/README.md

@@ -126,6 +126,71 @@ python serving/submit_job.py \
 
 </details>
 
+#### `Mistral-Small-24B-Instruct-2501`
+
+<details>
+
+<summary>SGLang, vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 1 \
+  --serving-framework sglang \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/sglang.toml \
+    --framework-args "--model-path mistralai/Mistral-Small-24B-Instruct-2501 \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --served-model-name mistralai/Mistral-Small-24B-Instruct-2501-$(whoami) \
+    --dp-size 4"
+```
+
+</details>
+
+#### `Mistral-Large-3-675B-Instruct-2512`
+
+<details>
+
+<summary>vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 4 \
+  --serving-framework vllm \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/vllm.toml \
+  --disable-ocf \
+  --framework-args "--model mistralai/Mistral-Large-3-675B-Instruct-2512 \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --served-model-name mistralai/Mistral-Large-3-675B-Instruct-2512-$(whoami) \
+    --tensor-parallel-size 16"
+```
+
+</details>
+
+#### `Mixtral-8x22B-Instruct-v0.1`
+
+<details>
+
+<summary>SGLang, vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 2 \
+  --serving-framework sglang \
+  --disable-ocf \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/sglang.toml \
+  --framework-args "--model mistralai/Mixtral-8x22B-Instruct-v0.1 \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --tp-size 8 \
+    --served-model-name mistralai/Mixtral-8x22B-Instruct-v0.1-$(whoami)"
+```
+
+</details>
+
 ### Snowflake
 
 #### `snowflake-arctic-embed-l-v2.0`
@@ -149,6 +214,48 @@ python serving/submit_job.py \
 
 ### Qwen
 
+#### `Qwen3-8B`
+
+<details>
+
+<summary>SGLang, vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 1 \
+  --serving-framework sglang \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/sglang.toml \
+  --framework-args "--model-path Qwen/Qwen3-8B \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --served-model-name Qwen/Qwen3-8B-$(whoami) \
+    --dp-size 4"
+```
+
+</details>
+
+#### `Qwen3-32B`
+
+<details>
+
+<summary>SGLang, vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 1 \
+  --serving-framework sglang \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/sglang.toml \
+  --framework-args "--model-path Qwen/Qwen3-32B \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --served-model-name Qwen/Qwen3-32B-$(whoami) \
+    --dp-size 4"
+```
+
+</details>
+
 #### `Qwen3-Next-80B-A3B-Instruct`
 
 <details>
@@ -168,6 +275,50 @@ python serving/submit_job.py \
 
 </details>
 
+#### `Qwen3-235B-A22B-Instruct-2507`
+
+<details>
+
+<summary>SGLang, vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 2 \
+  --serving-framework sglang \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/sglang.toml \
+  --disable-ocf \
+  --framework-args "--model-path Qwen/Qwen3-235B-A22B-Instruct-2507 \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --served-model-name Qwen/Qwen3-235B-A22B-Instruct-2507-$(whoami) \
+    --tp-size 8"
+```
+
+</details>
+
+#### `Qwen3.5-397B-A17B`
+
+<details>
+
+<summary>vLLM (tested ✅)</summary>
+
+```bash
+python serving/submit_job.py \
+  --slurm-nodes 4 \
+  --serving-framework vllm \
+  --disable-ocf \
+  --worker-port 8080 \
+  --slurm-environment $(pwd)/serving/envs/vllm_qwen35.toml \
+  --framework-args "--model Qwen/Qwen3.5-397B-A17B \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --tensor-parallel-size 16 \
+    --served-model-name Qwen/Qwen3.5-397B-A17B-$(whoami)"
+```
+
+</details>
+
 ### DeepSeek
 
 #### `DeepSeek-V3.1`

serving/envs/sglang.toml

@@ -16,7 +16,6 @@ workdir = "/opt"
 [env]
 # NCCL_DEBUG = "INFO"  # uncomment for debugging
 # NCCL_DEBUG_SUBSYS = "INIT,NET"  # uncomment for debugging
-LD_LIBRARY_PATH = "/opt/cscs/aws-ofi-ccl-plugin/cuda12:/opt/cray/libfabric/lib64:/usr/lib:${LD_LIBRARY_PATH:-}"
 NCCL_NET_PLUGIN = "/opt/cscs/aws-ofi-ccl-plugin/cuda12/libnccl-net.so"
 NCCL_NET = "AWS Libfabric"
 NCCL_CROSS_NIC = "1"

serving/envs/sglang_glm.toml

@@ -1,4 +1,4 @@
-image = "/iopsstor/scratch/cscs/ahadinia/sglang_glm.sqsh"
+image = "/capstor/store/cscs/swissai/infra01/container-images/sglang_glm5_nightly.sqsh"
 
 # "src_path:trg_path" mounts the src_path on the host inside the container at the trg_path.
 mounts = [

serving/envs/sglang_kimi.toml

@@ -8,14 +8,15 @@ mounts = [
   "/usr/lib64/libhwloc.so.15:/usr/lib/libhwloc.so.15",
   "/usr/lib64/libpciaccess.so.0:/usr/lib/libpciaccess.so.0",
   "/usr/lib64/libxml2.so.2:/usr/lib/libxml2.so.2",
+  "/opt/cray/libfabric/1.22.0/lib64:/opt/cray/libfabric/lib64",
+  "/opt/cscs/aws-ofi-ccl-plugin/cuda12:/opt/cscs/aws-ofi-ccl-plugin/cuda12",
 ]
 
 workdir = "/opt" 
 
 [env]
-# NCCL_DEBUG = "info"  # uncomment for debugging
-NCCL_NET_PLUGIN = "ofi"
-NCCL_NET = "Socket"
+NCCL_NET_PLUGIN = "/opt/cscs/aws-ofi-ccl-plugin/cuda12/libnccl-net.so"
+NCCL_NET = "AWS Libfabric"
 NCCL_CROSS_NIC = "1"
 NCCL_NET_GDR_LEVEL = "PHB"
 NCCL_SOCKET_IFNAME = "hsn"
@@ -28,3 +29,7 @@ FI_CXI_DEFAULT_TX_SIZE = "32768"
 FI_CXI_DISABLE_HOST_REGISTER = "1"
 OFI_NCCL_DISABLE_DMABUF = "1"
 SGL_ENABLE_JIT_DEEPGEMM = "0"
+
+[annotations]
+com.hooks.aws_ofi_nccl.enabled = "true"
+com.hooks.aws_ofi_nccl.variant = "cuda12"

serving/envs/vllm_qwen35.toml

@@ -0,0 +1,37 @@
+image = "vllm/vllm-openai:nightly"
+
+mounts = [
+  "/iopsstor/store/cscs/swissai/a09/xyao/bin:/ocfbin",
+  "/capstor",
+  "/iopsstor",
+  "/usr/lib64/libhwloc.so.15:/usr/lib/libhwloc.so.15",
+  "/usr/lib64/libpciaccess.so.0:/usr/lib/libpciaccess.so.0",
+  "/usr/lib64/libxml2.so.2:/usr/lib/libxml2.so.2",
+  "/opt/cray/libfabric/1.22.0/lib64:/opt/cray/libfabric/lib64",
+  "/opt/cscs/aws-ofi-ccl-plugin/cuda12:/opt/cscs/aws-ofi-ccl-plugin/cuda12",
+]
+
+workdir = "/opt" 
+
+[env]
+LD_LIBRARY_PATH = "/opt/cscs/aws-ofi-ccl-plugin/cuda12:/opt/cray/libfabric/lib64:/usr/lib:${LD_LIBRARY_PATH:-}"
+NCCL_NET_PLUGIN = "/opt/cscs/aws-ofi-ccl-plugin/cuda12/libnccl-net.so"
+NCCL_NET = "AWS Libfabric"
+NCCL_CROSS_NIC = "1"
+NCCL_NET_GDR_LEVEL = "PHB"
+NCCL_SOCKET_IFNAME = "hsn"
+NCCL_PROTO = "^LL128"
+NCCL_DEBUG = "INFO"
+NCCL_DEBUG_SUBSYS = "INIT,NET"
+FI_CXI_COMPAT = "0"
+FI_MR_CACHE_MONITOR = "userfaultfd"
+FI_CXI_RX_MATCH_MODE = "software"
+FI_CXI_DEFAULT_CQ_SIZE = "131072"
+FI_CXI_DEFAULT_TX_SIZE = "32768"
+FI_CXI_DISABLE_HOST_REGISTER = "1"
+OFI_NCCL_DISABLE_DMABUF = "1"
+VLLM_ALLREDUCE_USE_SYMM_MEM = "0"
+
+[annotations]
+com.hooks.aws_ofi_nccl.enabled = "true"
+com.hooks.aws_ofi_nccl.variant = "cuda12"

serving/template.jinja

@@ -59,7 +59,7 @@ case "$FRAMEWORK" in
         FRAMEWORK_LAUNCH="python3 -m sglang.launch_server"
         ;;
     vllm)
-        FRAMEWORK_ENV_SETUP="export no_proxy=\"0.0.0.0,\$no_proxy\"; export NO_PROXY=\"0.0.0.0,\$NO_PROXY\""
+        FRAMEWORK_ENV_SETUP="export RAY_CGRAPH_get_timeout=1800; export no_proxy=\"0.0.0.0,\$no_proxy\"; export NO_PROXY=\"0.0.0.0,\$NO_PROXY\""
         FRAMEWORK_LAUNCH="python3 -m vllm.entrypoints.openai.api_server"
         ;;
 esac