Qwen3.5-122B.yaml

x-logging-conf: &logging-conf
  driver: "json-file"
  options:
    max-size: "100m"
    max-file: "10"
    labels: "com.datadoghq.ad.logs"

x-nvidia: &nvidia
  runtime: nvidia
  ipc: host
  ulimits:
    memlock: -1
    nofile:
      soft: 65535
      hard: 65535

x-vllm-proxy-common: &vllm-proxy-common
  image: nearaidev/vllm-proxy-rs@sha256:59e42dd68faa15eb0c23521029a2fc3d80d86a4143f9f766542357918be33a8c
  user: root
  privileged: true
  <<: *nvidia
  extra_hosts:
    - "compose-manager:host-gateway"
  volumes:
    - /var/run/dstack.sock:/var/run/dstack.sock
    - certs:/etc/letsencrypt:ro
  restart: unless-stopped
  logging: *logging-conf

x-sg-qwen35-122b-common: &sg-qwen35-122b-common
  <<: *nvidia
  init: true
  # SGLang v0.5.12 (cu129) — migrated off vLLM 2026-05-22.
  # Context dropped from 1.01M (yarn rope override) to native 262144.
  # EAGLE spec decoding enabled (Spec V2 default since 0.5.11).
  image: lmsysorg/sglang:v0.5.12-cu129@sha256:9e02c8e1fe2790a1c445bd5f6814305fe43639a4adb01c8ad1e8e21e750bf581
  command: >
      sglang serve
      --model-path Qwen/Qwen3.5-122B-A10B
      --revision dc4d348443bc740c68e2d77492492c11606384d5
      --tp 4
      --reasoning-parser qwen3
      --tool-call-parser qwen3_coder
      --speculative-algorithm EAGLE
      --speculative-num-steps 3
      --speculative-eagle-topk 1
      --speculative-num-draft-tokens 4
      --mamba-scheduler-strategy extra_buffer
      --kv-cache-dtype fp8_e4m3
      --mem-fraction-static 0.88
      --context-length 262144
      --num-continuous-decode-steps 5
      --model-loader-extra-config '{"enable_multithread_load": "true", "num_threads": 64}'
      --enable-mixed-chunk
      --chunked-prefill-size 16384
      --port 8000
      --host 0.0.0.0
      --enable-cache-report
      --enable-metrics
      --trust-remote-code
      --log-requests-level 0
      --served-model-name Qwen/Qwen3.5-122B-A10B
  volumes:
    - huggingface_cache:/root/.cache/huggingface
    - kernel_cache:/root/.cache/deep_gemm
  environment:
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    - OPENBLAS_L2_SIZE=2097152
    - NCCL_DEBUG=WARN
    - SGLANG_ENABLE_SPEC_V2=1
  restart: unless-stopped
  stop_grace_period: 5m
  logging: *logging-conf

services:
  model-downloader:
    image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim@sha256:4f5d923c9dcea037f57bda425dd209f3ec643da2f0b74227f68d09dab0b3bb36
    container_name: model-downloader
    restart: "no"
    entrypoint: ["sh", "-c"]
    command:
      - |
        set -e
        echo "Downloading Qwen/Qwen3.5-122B-A10B..."
        uvx --from 'huggingface_hub[hf_xet]' hf download Qwen/Qwen3.5-122B-A10B --revision dc4d348443bc740c68e2d77492492c11606384d5
        echo "Download complete."
    volumes:
      - huggingface_cache:/root/.cache/huggingface
    environment:
      - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    logging: *logging-conf

  proxy-nginx:
    image: nginx@sha256:1d13701a5f9f3fb01aaa88cef2344d65b6b5bf6b7d9fa4cf0dca557a8d7702ba
    container_name: proxy-nginx
    command: /bin/sh -c 'while :; do sleep 6h; nginx -s reload; done & nginx -g "daemon off;"'
    ports:
      - "8000:80"
      - "8444:443"
    volumes:
      - certs:/etc/letsencrypt:ro
    configs:
      - source: nginx_conf
        target: /etc/nginx/conf.d/default.conf
        mode: 0644
    restart: unless-stopped
    logging: *logging-conf

  model-proxy-registrar:
    image: curlimages/curl@sha256:d94d07ba9e7d6de898b6d96c1a072f6f8266c687af78a74f380087a0addf5d17
    container_name: model-proxy-registrar
    entrypoint: ["sh", "/register.sh"]
    restart: unless-stopped
    environment:
      - HOST_IP=${HOST_IP}
      - HTTP_PORT=${HTTP_PORT:-8000}
      - TLS_PORT=${TLS_PORT:-8444}
      - MODEL_PROXY_TOKEN=${MODEL_PROXY_TOKEN}
    configs:
      - source: registrar_script
        target: /register.sh
        mode: 0755
    logging: *logging-conf

  # --- Single proxy for both backends ---

  proxy-qwen35:
    <<: *vllm-proxy-common
    container_name: proxy-qwen35
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3.5-122B-A10B
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BACKEND_URLS=http://model-sg-qwen35-122b-1:8000,http://model-sg-qwen35-122b-2:8000
      - VLLM_PROXY_MAX_REQUEST_SIZE=104857600
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3.5-122B-A10B","ip:${HOST_IP}","port:8000"]}]'

  # --- Qwen3.5-122B-A10B instance 1 (GPUs 0-3) ---

  model-sg-qwen35-122b-1:
    <<: *sg-qwen35-122b-common
    container_name: model-sg-qwen35-122b-1
    depends_on:
      model-downloader:
        condition: service_completed_successfully
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0","1","2","3"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["openmetrics"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "sglang", "service": "sglang", "tags":["model:Qwen/Qwen3.5-122B-A10B","ip:${HOST_IP}","port:8000","instance:1"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-qwen35-122b-1:8000/metrics", "metrics":["sglang:*"], "histogram_buckets_as_distributions": true, "service": "qwen35-1", "tags":["model:Qwen/Qwen3.5-122B-A10B","ip:${HOST_IP}","port:8000"]}]'

  # --- Qwen3.5-122B-A10B instance 2 (GPUs 4-7) ---

  model-sg-qwen35-122b-2:
    <<: *sg-qwen35-122b-common
    container_name: model-sg-qwen35-122b-2
    depends_on:
      model-downloader:
        condition: service_completed_successfully
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["4","5","6","7"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["openmetrics"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "sglang", "service": "sglang", "tags":["model:Qwen/Qwen3.5-122B-A10B","ip:${HOST_IP}","port:8001","instance:2"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-qwen35-122b-2:8000/metrics", "metrics":["sglang:*"], "histogram_buckets_as_distributions": true, "service": "qwen35-2", "tags":["model:Qwen/Qwen3.5-122B-A10B","ip:${HOST_IP}","port:8001"]}]'

  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-distroless
    container_name: dcgm-exporter
    runtime: nvidia
    cap_add:
      - SYS_ADMIN
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
    ports:
      - "9400:9400"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    logging: *logging-conf
    labels:
      com.datadoghq.ad.check_names: '["dcgm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint": "http://%%host%%:9400/metrics", "tags":["model:Qwen/Qwen3.5-122B-A10B","deployment:Qwen3.5-122B","ip:${HOST_IP}"]}]'
      com.datadoghq.ad.logs: '[{"source": "dcgm-exporter", "service": "dcgm-exporter", "tags":["model:Qwen/Qwen3.5-122B-A10B","deployment:Qwen3.5-122B","ip:${HOST_IP}"]}]'

networks:
  default:
    external: true
    name: dstack_default

volumes:
  huggingface_cache:
  kernel_cache:
  certs:
    external: true
    name: certs

configs:
  registrar_script:
    content: |
      #!/bin/sh
      PROXY_URL="https://completions.near.ai"
      TOKEN="$${MODEL_PROXY_TOKEN}"
      ENDPOINT="$${HOST_IP}:$${HTTP_PORT}"
      REGISTERED=false
      FAILURE_COUNT=0
      MAX_RETRIES=3

      register_endpoint() {
        echo "Registering endpoint $$1 with routing port $$2"
        curl -sS --max-time 10 -X POST "$$PROXY_URL/register/endpoint" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"endpoint\":\"$$1\",\"routing_port\":$$2}"
      }

      unregister_endpoint() {
        echo "Unregistering endpoint $$1"
        curl -sS --max-time 10 -X POST "$$PROXY_URL/unregister/endpoint" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"endpoint\":\"$$1\"}"
      }

      register_model() {
        curl -sS --max-time 10 -X POST "$$PROXY_URL/register/model" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"model\":\"$$1\",\"domain\":\"$$2\"}"
      }

      cleanup() {
        echo "SIGTERM received, unregistering $$ENDPOINT"
        unregister_endpoint "$$ENDPOINT"
        REGISTERED=false
        exit 0
      }
      trap cleanup TERM INT

      # Health check directly on backend (no auth needed on raw sglang container)
      check_inference() {
        echo "Performing health check on backend..."
        curl -sSf --max-time 45 -X POST "http://model-sg-qwen35-122b-1:8000/v1/chat/completions" \
          -H "Content-Type: application/json" \
          -d '{"model":"Qwen/Qwen3.5-122B-A10B","messages":[{"role":"user","content":"hi"}],"max_tokens":1}'
      }

      echo "Waiting for model to be ready..."
      until curl -sf http://proxy-nginx:80/v1/models > /dev/null 2>&1; do sleep 30; done
      echo "Model ready, starting registration loop"

      while true; do
        if check_inference; then
          FAILURE_COUNT=0
          register_endpoint "$$ENDPOINT" "$${TLS_PORT}"
          register_model "Qwen/Qwen3.5-122B-A10B" "qwen35-122b.completions.near.ai"
          if [ "$$REGISTERED" = false ]; then
            echo "[Qwen3.5-122B] Registered Qwen/Qwen3.5-122B-A10B at $$ENDPOINT"
          fi
          REGISTERED=true
        else
          FAILURE_COUNT=$$((FAILURE_COUNT + 1))
          echo "[Qwen3.5-122B] Health check failed ($$FAILURE_COUNT/$$MAX_RETRIES)"
          if [ "$$REGISTERED" = true ] && [ "$$FAILURE_COUNT" -ge "$$MAX_RETRIES" ]; then
            echo "[Qwen3.5-122B] Health check failed and Retry limit reached, unregistering $$ENDPOINT"
            unregister_endpoint "$$ENDPOINT"
            REGISTERED=false
            echo "[Qwen3.5-122B] Exiting to refresh DNS (container will auto-restart)"
            exit 1
          fi
        fi
        sleep 60
      done
  nginx_conf:
    content: |
      log_format json_combined escape=json
        '{'  '"time":"$$time_iso8601"'
        ',"request_id":"$$http_x_request_id"'
        ',"org_id":"$$http_x_org_id"'
        ',"workspace_id":"$$http_x_workspace_id"'
        ',"host":"$$host"'
        ',"method":"$$request_method"'
        ',"uri":"$$uri"'
        ',"status":$$status'
        ',"request_length":$$request_length'
        ',"bytes_sent":$$bytes_sent'
        ',"request_time":$$request_time'
        ',"upstream_addr":"$$upstream_addr"'
        '}';

      access_log /var/log/nginx/access.log json_combined;

      proxy_http_version 1.1;
      proxy_set_header Host $$host;
      proxy_set_header X-Real-IP $$remote_addr;
      proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
      proxy_set_header X-Forwarded-Proto $$scheme;
      proxy_set_header Connection '';
      proxy_buffering off;
      proxy_cache off;
      proxy_read_timeout 3600s;
      client_max_body_size 100m;
      client_body_buffer_size 1m;

      # :80 — single proxy handles both backends
      server {
        listen 80 default_server;
        location / { proxy_pass http://proxy-qwen35:8000; }
      }

      ssl_certificate /etc/letsencrypt/live/completions.near.ai/fullchain.pem;
      ssl_certificate_key /etc/letsencrypt/live/completions.near.ai/privkey.pem;
      ssl_protocols TLSv1.2 TLSv1.3;

      server {
        listen 443 ssl http2;
        server_name qwen35-122b.completions.near.ai;
        # Keep H2 connections from cloud-api alive across long idle gaps so the
        # bucket-pinned TCP connection survives between chats. Without this,
        # nginx defaults (75s idle, 1000 req/conn) close the connection and the
        # next request opens a new TCP via model-proxy's L4 LB → may land on a
        # different backend → signature 404. Pairs with cloud-api H2 keepalive
        # PINGs (http2_keep_alive_while_idle).
        keepalive_timeout 1h;
        keepalive_requests 1000000;
        location / { proxy_pass http://proxy-qwen35:8000; }
      }