Sync ishandhanani/srt-slurm history into NVIDIA/srt-slurm (#14)

Author: csahithi

configs/rebuild-deepep.sh

@@ -15,7 +15,7 @@ fi
 cd "$DEEPEP_SRC"
 
 # Find NVSHMEM
-NVSHMEM_DIR=$(find /usr/local -name "nvshmem" -type d 2>/dev/null | head -1)
+NVSHMEM_DIR=$(find /usr/local -name "nvshmem" -type d -not -path "*/flashinfer*" 2>/dev/null | head -1)
 if [ -z "${NVSHMEM_DIR:-}" ]; then
     echo "ERROR: NVSHMEM installation not found under /usr/local" >&2
     exit 1

recipes/gb200-fp8/8k1k/low-latency.yaml

@@ -1,117 +1,128 @@
-name: "gb200-fp8-8k1k-low-latency"
+base:
+  name: "gb200-fp8-8k1k-low-latency"
 
-dynamo:
-  version: 0.8.1
+  dynamo:
+    version: 0.8.1
 
-frontend:
-  type: dynamo
-  enable_multiple_frontends: true
-  num_additional_frontends: 2
-  nginx_container: nginx
+  frontend:
+    type: dynamo
+    enable_multiple_frontends: true
+    num_additional_frontends: 2
+    nginx_container: nginx
 
-model:
-  path: "dsr1-fp8"
-  container: "dynamo-sglang"
-  precision: "fp8"
+  model:
+    path: "dsr1-fp8"
+    container: "dynamo-sglang"
+    precision: "fp8"
 
-resources:
-  gpu_type: "gb200"
-  prefill_nodes: 2
-  decode_nodes: 2
-  prefill_workers: 1
-  decode_workers: 1
-  gpus_per_node: 4
+  resources:
+    gpu_type: "gb200"
+    gpus_per_node: 4
+    prefill_nodes: 2
+    prefill_workers: 1
 
-backend:
-  prefill_environment:
-    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
-    PYTHONUNBUFFERED: "1"
-    DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
-    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
-    SGLANG_ENABLE_JIT_DEEPGEMM: "false"
-    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
-    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
-    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
-    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
-    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
-    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
-    MC_TE_METRIC: "true"
-    MC_FORCE_MNNVL: "1"
-    NCCL_MNNVL_ENABLE: "1"
-    NCCL_CUMEM_ENABLE: "1"
 
-  decode_environment:
-    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
-    PYTHONUNBUFFERED: "1"
-    DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
-    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
-    SGLANG_ENABLE_JIT_DEEPGEMM: "false"
-    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
-    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
-    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
-    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
-    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
-    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
-    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
-    MC_TE_METRIC: "true"
-    MC_FORCE_MNNVL: "1"
-    NCCL_MNNVL_ENABLE: "1"
-    NCCL_CUMEM_ENABLE: "1"
+  backend:
+    prefill_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+      PYTHONUNBUFFERED: "1"
+      DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
+      MC_TE_METRIC: "true"
+      MC_FORCE_MNNVL: "1"
+      NCCL_MNNVL_ENABLE: "1"
+      NCCL_CUMEM_ENABLE: "1"
 
-  sglang_config:
-    prefill:
-      served-model-name: "deepseek-ai/DeepSeek-R1"
-      trust-remote-code: true
-      kv-cache-dtype: "fp8_e4m3"
-      attention-backend: "trtllm_mla"
-      quantization: "fp8"
-      moe-runner-backend: "flashinfer_trtllm"
-      disable-radix-cache: true
-      watchdog-timeout: 1000000
-      context-length: 9600 
-      disaggregation-mode: "prefill"
-      mem-fraction-static: 0.8
-      max-total-tokens: 32768 
-      chunked-prefill-size: 24576 
-      cuda-graph-max-bs: 512 
-      max-running-requests: 512
-      load-balance-method: "round_robin"
-      scheduler-recv-interval: 10
-      tensor-parallel-size: 8
-      data-parallel-size: 1
-      expert-parallel-size: 1
-      fp8-gemm-backend: "flashinfer_trtllm"
-      disaggregation-bootstrap-port: 30001
-      disaggregation-transfer-backend: nixl
+    decode_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+      PYTHONUNBUFFERED: "1"
+      DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+      SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
+      MC_TE_METRIC: "true"
+      MC_FORCE_MNNVL: "1"
+      NCCL_MNNVL_ENABLE: "1"
+      NCCL_CUMEM_ENABLE: "1"
 
-    decode:
-      served-model-name: "deepseek-ai/DeepSeek-R1"
-      trust-remote-code: true
-      kv-cache-dtype: "fp8_e4m3"
-      attention-backend: "trtllm_mla"
-      quantization: "fp8"
-      moe-runner-backend: "flashinfer_trtllm"
-      disable-radix-cache: true
-      watchdog-timeout: 1000000
-      context-length: 9600 
-      disaggregation-mode: "decode"
-      mem-fraction-static: 0.8
-      chunked-prefill-size: 8192
-      cuda-graph-max-bs: 512 
-      max-running-requests: 512
-      scheduler-recv-interval: 10
-      enable-symm-mem: true
-      prefill-round-robin-balance: true
-      tensor-parallel-size: 8
-      data-parallel-size: 1
-      expert-parallel-size: 1
-      fp8-gemm-backend: "flashinfer_trtllm"
-      disaggregation-bootstrap-port: 30001
-      disaggregation-transfer-backend: nixl
+    sglang_config:
+      prefill:
+        served-model-name: "deepseek-ai/DeepSeek-R1"
+        trust-remote-code: true
+        kv-cache-dtype: "fp8_e4m3"
+        attention-backend: "trtllm_mla"
+        quantization: "fp8"
+        moe-runner-backend: "flashinfer_trtllm"
+        disable-radix-cache: true
+        watchdog-timeout: 1000000
+        context-length: 9600
+        disaggregation-mode: "prefill"
+        mem-fraction-static: 0.6
+        max-running-requests: 32
+        cuda-graph-max-bs: 32
+        load-balance-method: "round_robin"
+        scheduler-recv-interval: 10
+        tensor-parallel-size: 8
+        data-parallel-size: 1
+        expert-parallel-size: 1
+        fp8-gemm-backend: "flashinfer_trtllm"
+        disaggregation-bootstrap-port: 30001
+        disaggregation-transfer-backend: nixl
 
-benchmark:
-  type: "sa-bench"
-  isl: 8192 
-  osl: 1024
-  concurrencies: "4x8x16"
-  req_rate: "inf"
+      decode:
+        served-model-name: "deepseek-ai/DeepSeek-R1"
+        trust-remote-code: true
+        kv-cache-dtype: "fp8_e4m3"
+        attention-backend: "trtllm_mla"
+        quantization: "fp8"
+        moe-runner-backend: "flashinfer_trtllm"
+        disable-radix-cache: true
+        watchdog-timeout: 1000000
+        context-length: 9600
+        disaggregation-mode: "decode"
+        mem-fraction-static: 0.7
+        cuda-graph-max-bs: 32
+        max-running-requests: 32
+        scheduler-recv-interval: 10
+        enable-symm-mem: true
+        prefill-round-robin-balance: true
+        tensor-parallel-size: 8
+        data-parallel-size: 1
+        expert-parallel-size: 1
+        fp8-gemm-backend: "flashinfer_trtllm"
+        disaggregation-bootstrap-port: 30001
+        disaggregation-transfer-backend: nixl
+
+  benchmark:
+    type: "sa-bench"
+    isl: 8192
+    osl: 1024
+    concurrencies: "4x8x16"
+    req_rate: "inf"
+
+zip_override_lowlat:
+  resources:
+    decode_nodes: [14, 14, 6]
+    decode_workers: [7, 7, 3]
+  name:
+    - "gb200-fp8-8k1k-low-latency-c8"
+    - "gb200-fp8-8k1k-low-latency-c16-32"
+    - "gb200-fp8-8k1k-low-latency-c64"
+  backend:
+    sglang_config:
+      prefill:
+        max-prefill-tokens: [8192, 16384, 32768]
+        chunked-prefill-size: [8192, 16384, 32768]
+      decode:
+        max-running-requests: [1, 4, 16]
+  benchmark:
+    concurrencies: ["8", "16x32", "64"]