vllm-omni exmaple

Author: robmsmt

examples/qwen/Qwen3-Omni-30B-A3B-Captioner-vllm.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+sml advanced \
+   --slurm-nodes 1 \
+   --slurm-time 6:00:00 \
+   --serving-framework vllm \
+   --slurm-environment src/swiss_ai_model_launch/assets/envs/vllm_qwen3_omni.toml \
+   --framework-args "--model /capstor/store/cscs/swissai/infra01/hf_models/models/swiss-ai/Qwen/Qwen3-Omni-30B-A3B-Captioner \
+    --served-model-name Qwen/Qwen3-Omni-30B-A3B-Captioner-$(whoami) \
+    --tensor-parallel-size 4 \
+    --host 0.0.0.0 \
+    --port 8080 \
+    --dtype bfloat16 --max-model-len 32768 --trust-remote-code"

src/swiss_ai_model_launch/assets/envs/vllm_qwen3_omni.toml

@@ -0,0 +1,35 @@
+image = "/capstor/store/cscs/swissai/infra01/container-images/vllm-qwen-omni-13.0.sqsh"
+
+mounts = [
+  "/capstor/store/cscs/swissai/infra01/ocf-share:/ocfbin",
+  "/capstor",
+  "/iopsstor",
+  "/usr/lib64/libhwloc.so.15:/usr/lib/libhwloc.so.15",
+  "/usr/lib64/libpciaccess.so.0:/usr/lib/libpciaccess.so.0",
+  "/usr/lib64/libxml2.so.2:/usr/lib/libxml2.so.2",
+  "/usr/lib64/libnuma.so.1:/usr/lib/libnuma.so.1",
+]
+
+workdir = "/opt" 
+
+[env]
+# NCCL_DEBUG = "INFO"  # uncomment for debugging
+# NCCL_DEBUG_SUBSYS = "INIT,NET"  # uncomment for debugging
+NCCL_NET = "AWS Libfabric"
+NCCL_CROSS_NIC = "1"
+NCCL_NET_GDR_LEVEL = "PHB"
+NCCL_SOCKET_IFNAME = "hsn"
+NCCL_PROTO = "^LL128"
+FI_CXI_COMPAT = "0"
+FI_MR_CACHE_MONITOR = "userfaultfd"
+FI_CXI_RX_MATCH_MODE = "software"
+FI_CXI_DEFAULT_CQ_SIZE = "131072"
+FI_CXI_DEFAULT_TX_SIZE = "32768"
+FI_CXI_DISABLE_HOST_REGISTER = "1"
+OFI_NCCL_DISABLE_DMABUF = "1"
+VLLM_ALLREDUCE_USE_SYMM_MEM = "0"
+
+[annotations]
+com.hooks.aws_ofi_nccl.enabled = "true"
+com.hooks.aws_ofi_nccl.variant = "cuda13"
+com.hooks.cxi.enabled = "true"