small-models.yaml

x-logging-conf: &logging-conf
  driver: "json-file"
  options:
    max-size: "100m"
    max-file: "10"
    labels: "com.datadoghq.ad.logs"

# Non-chat models (FLUX, embedding, reranker, whisper) use /v1/models

x-nvidia: &nvidia
  runtime: nvidia
  ipc: host
  ulimits:
    memlock: -1
    nofile:
      soft: 65535
      hard: 65535

x-vllm-common: &vllm-common
  <<: *nvidia
  init: true
  volumes:
    - hugginface_cache:/root/.cache/huggingface
    - vllm_cache:/root/.cache/vllm
  restart: unless-stopped
  stop_grace_period: 5m
  logging: *logging-conf

x-vllm-proxy-common: &vllm-proxy-common
  image: nearaidev/vllm-proxy-rs@sha256:59e42dd68faa15eb0c23521029a2fc3d80d86a4143f9f766542357918be33a8c
  user: root
  privileged: true
  <<: *nvidia
  extra_hosts:
    - "compose-manager:host-gateway"
  volumes:
    - /var/run/dstack.sock:/var/run/dstack.sock
    - certs:/etc/letsencrypt:ro
  restart: unless-stopped
  logging: *logging-conf

x-vllm-env:
  environment: &vllm-env
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - VLLM_LOGGING_LEVEL=INFO
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    - OPENBLAS_L2_SIZE=2097152
    - NCCL_DEBUG=INFO
    - VLLM_CACHE_ROOT=/root/.cache/vllm

x-sglang-env:
  environment: &sglang-env
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - VLLM_LOGGING_LEVEL=INFO
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    - OPENBLAS_L2_SIZE=2097152
    - NCCL_DEBUG=INFO
    - VLLM_CACHE_ROOT=/root/.cache/vllm
    - SGLANG_CACHE_DIT_ENABLED=true

x-gpt-oss-common: &gpt-oss-common
  <<: *vllm-common
  image: vllm/vllm-openai@sha256:6766ce0c459e24b76f3e9ba14ffc0442131ef4248c904efdcbf0d89e38be01fe
  # Cache subpath is configurable so a corrupted torch.compile artifact in the
  # shared vllm_cache volume can be sidestepped without a new tag: pass
  # env={"GPT_OSS_CACHE_SUBPATH":"v3"} to /compose/up to force a fresh cache.
  # Default is "v2" because v1 (the original /root/.cache/vllm root) holds the
  # corrupted 1603e894a6/... artifact from a prior interrupted compile.
  command: >
      openai/gpt-oss-120b
      --revision b5c939de8f754692c1647ca79fbf85e8c1e70f8a
      --tensor-parallel-size 1
      --gpu-memory-utilization 0.90
      --enable-prefix-caching
      --async-scheduling
      --max-num-seqs 64
      --tool-call-parser openai
      --enable-auto-tool-choice
      --max-model-len 128K
      --max-num-batched-tokens 8192
      --load-format runai_streamer
      --model-loader-extra-config '{"distributed":true, "concurrency":48}'
      --enable-prompt-tokens-details
  volumes:
    - hugginface_cache:/root/.cache/huggingface
    - vllm_cache:/root/.cache/vllm
  environment:
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - VLLM_LOGGING_LEVEL=INFO
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    - VLLM_MXFP4_USE_MARLIN=1
    - VLLM_CACHE_ROOT=/root/.cache/vllm/${GPT_OSS_CACHE_SUBPATH:-v2}

x-qwen3-6-common: &qwen3-6-common
  <<: *nvidia
  init: true
  # SGLang v0.5.12 (cu129), matched to GLM-5.1's pin. Spec V2 default since 0.5.11
  # (SGLANG_ENABLE_SPEC_V2=1 below kept for explicitness).
  # https://lmsysorg.mintlify.app/cookbook/autoregressive/Qwen/Qwen3.6
  image: lmsysorg/sglang:v0.5.12-cu129@sha256:9e02c8e1fe2790a1c445bd5f6814305fe43639a4adb01c8ad1e8e21e750bf581
  command: >
      sglang serve
      --model-path Qwen/Qwen3.6-35B-A3B-FP8
      --revision 95a723d08a9490559dae23d0cff1d9466213d989
      --tp 1
      --reasoning-parser qwen3
      --tool-call-parser qwen3_coder
      --speculative-algorithm EAGLE
      --speculative-num-steps 3
      --speculative-eagle-topk 1
      --speculative-num-draft-tokens 4
      --mamba-scheduler-strategy extra_buffer
      --mem-fraction-static 0.85
      --max-running-requests 128
      --context-length 262144
      --num-continuous-decode-steps 5
      --model-loader-extra-config '{"enable_multithread_load": "true", "num_threads": 64}'
      --enable-mixed-chunk
      --chunked-prefill-size 16384
      --port 8000
      --host 0.0.0.0
      --enable-cache-report
      --enable-metrics
      --trust-remote-code
      --log-requests-level 0
      --served-model-name Qwen/Qwen3.6-35B-A3B-FP8
  volumes:
    - hugginface_cache:/root/.cache/huggingface
    - kernel_cache:/root/.cache/deep_gemm
  environment:
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    - OPENBLAS_L2_SIZE=2097152
    - NCCL_DEBUG=WARN
    - SGLANG_ENABLE_SPEC_V2=1
  restart: unless-stopped
  stop_grace_period: 5m
  logging: *logging-conf

x-flux-common: &flux-common
  <<: *vllm-common
  image: sglang-diffusion
  build:
    context: .
    dockerfile_inline: |
      # Pinned pre-v0.5.11 because the cu129-tagged v0.5.12 image actually
      # ships PyTorch built against CUDA 13.0, which mismatches the cu12.9
      # torchvision pulled in by `python[diffusion]` and breaks sglang's
      # `from torchvision.io import decode_jpeg` at import time.
      FROM lmsysorg/sglang@sha256:8ece90ad52faa8b56149f0117227d9009db34513213e35990da468aeb6fe0b75
      RUN python3 -m pip install -e "python[diffusion]"
      RUN python3 -m pip install accelerate einops
  command: >
      sglang serve
      --model-path black-forest-labs/FLUX.2-klein-4B
      --tensor-parallel-size 1
      --port 8000
      --host 0.0.0.0
      --trust-remote-code
      --log-requests-level 0
      --enable-torch-compile
      --mem-fraction-static 0.15
      --enable-metrics
  volumes:
    - hugginface_cache:/root/.cache/huggingface
    - vllm_cache:/root/.cache/vllm
  environment: *sglang-env
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            device_ids: ["7"]
            capabilities: [gpu]

x-privacy-filter-common: &privacy-filter-common
  <<: *nvidia
  init: true
  image: privacy-filter-hf
  build:
    context: .
    dockerfile_inline: |
      # syntax=docker/dockerfile:1.4
      FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
      RUN pip install --no-cache-dir \
          "transformers>=4.46" \
          accelerate \
          fastapi \
          "uvicorn[standard]"
      WORKDIR /app
      COPY <<'PYEOF' /app/server.py
      import os
      import sys
      import threading
      import time
      import torch
      from fastapi import FastAPI, HTTPException
      from pydantic import BaseModel, Field
      from transformers import AutoTokenizer, pipeline

      MODEL_ID = "openai/privacy-filter"
      MODEL_REVISION = "7ffa9a043d54d1be65afb281eddf0ffbe629385b"


      def _env(name, default, cast):
          # Tolerate absent/blank/garbage env so a typo can't crash-loop the boot.
          try:
              return cast(os.environ[name])
          except (KeyError, ValueError):
              return default


      # GPU 7 is shared with Qwen3-VL / FLUX / embeddings / reranker / whisper.
      # Left unbounded, the HF pipeline's CUDA caching allocator ratchets its
      # reserved memory up under traffic and never releases it, slowly hoarding the
      # card and starving the co-located models until they OOM (Qwen3-VL crash-loops).
      # Defence in depth (all tunable via env, no rebuild):
      #   * PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (set in compose) is the
      #     root-cause fix — reserved segments can shrink again instead of ratcheting.
      #   * A watchdog returns idle cached blocks to the driver on an interval (never
      #     per request: empty_cache() is a synchronizing cudaFree that would stall
      #     the shared GPU) and hard-restarts the container if this process's VRAM
      #     ever climbs toward starving its neighbours — a real restart (clean reclaim)
      #     rather than silently serving 500s behind a still-healthy /v1/models probe.
      #   * An acute CUDA-OOM in a request also exits, for the same clean recycle.
      # Inputs are deliberately NOT truncated — a privacy filter must see the whole
      # text or it would miss PII; per-request peak is bounded by PRIVACY_BATCH_SIZE.
      BATCH_SIZE = max(1, _env("PRIVACY_BATCH_SIZE", 32, int))
      GPU_MEM_LIMIT_GB = max(1.0, _env("GPU_MEM_LIMIT_GB", 32.0, float))
      WATCHDOG_INTERVAL_S = max(1.0, _env("WATCHDOG_INTERVAL_S", 30.0, float))

      USE_CUDA = torch.cuda.is_available()

      clf = pipeline(
          "token-classification",
          model=MODEL_ID,
          revision=MODEL_REVISION,
          aggregation_strategy="simple",
          device=0 if USE_CUDA else -1,
          torch_dtype=torch.bfloat16,
          trust_remote_code=True,
      )
      tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, revision=MODEL_REVISION, trust_remote_code=True)


      def _gpu_watchdog():
          while True:
              time.sleep(WATCHDOG_INTERVAL_S)
              torch.cuda.empty_cache()
              reserved_gb = torch.cuda.memory_reserved(0) / 1024 ** 3
              if reserved_gb > GPU_MEM_LIMIT_GB:
                  sys.stderr.write(
                      "privacy-filter: reserved %.1f GB > %.1f GB limit; "
                      "exiting for a clean restart\n" % (reserved_gb, GPU_MEM_LIMIT_GB)
                  )
                  sys.stderr.flush()
                  os._exit(1)


      if USE_CUDA:
          threading.Thread(target=_gpu_watchdog, daemon=True).start()

      app = FastAPI()


      class ClassifyRequest(BaseModel):
          input: str | list[str]
          threshold: float = Field(default=0.0, ge=0.0, le=1.0)


      @app.get("/v1/models")
      def models():
          return {"object": "list", "data": [{"id": MODEL_ID, "object": "model"}]}


      @app.post("/v1/privacy/classify")
      def classify(req: ClassifyRequest):
          texts = [req.input] if isinstance(req.input, str) else req.input
          if not texts or any(not isinstance(t, str) for t in texts):
              raise HTTPException(400, "input must be a non-empty string or list of strings")

          # The pipeline runs the forward under torch.no_grad() internally.
          try:
              raw = clf(texts, batch_size=BATCH_SIZE)
          except torch.cuda.OutOfMemoryError:
              # Don't 500-storm behind a healthy /v1/models probe: recycle the
              # container so restart:unless-stopped reclaims the VRAM cleanly.
              torch.cuda.empty_cache()
              os._exit(1)

          # Single batched tokenize for usage counts instead of N sequential calls.
          tok_lens = [len(ids) for ids in tokenizer(texts).input_ids]

          data = []
          for i, spans in enumerate(raw):
              kept = [
                  {
                      "category": s["entity_group"],
                      "score": float(s["score"]),
                      "text": s["word"],
                      "start": int(s["start"]),
                      "end": int(s["end"]),
                  }
                  for s in spans
                  if float(s["score"]) >= req.threshold
              ]
              data.append({"index": i, "spans": kept, "usage": {"input_tokens": tok_lens[i]}})

          return {"model": MODEL_ID, "data": data}
      PYEOF
      EXPOSE 8000
      CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]
  volumes:
    - hugginface_cache:/root/.cache/huggingface
  environment:
    - HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
    - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    # Root-cause fix for the VRAM leak: let the CUDA caching allocator shrink
    # reserved segments instead of ratcheting them up and hoarding the shared GPU.
    - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
    # Watchdog/batch knobs consumed by server.py — tunable per host, no rebuild.
    # GPU_MEM_LIMIT_GB caps this process's blast radius on the 140 GB H200 (~10x
    # the classifier's footprint), leaving the rest for Qwen3-VL/FLUX/etc.
    - GPU_MEM_LIMIT_GB=32
    - PRIVACY_BATCH_SIZE=32
  restart: unless-stopped
  stop_grace_period: 5m
  logging: *logging-conf
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            device_ids: ["7"]
            capabilities: [gpu]

services:
  proxy-nginx:
    image: nginx@sha256:1d13701a5f9f3fb01aaa88cef2344d65b6b5bf6b7d9fa4cf0dca557a8d7702ba
    container_name: proxy-nginx
    command: /bin/sh -c 'while :; do sleep 6h; nginx -s reload; done & nginx -g "daemon off;"'
    ports:
      - "80:80"
      - "8000:8000"
      - "8001:8001"
      - "8002:8002"
      - "8003:8003"
      - "8004:8004"
      - "8005:8005"
      - "8006:8006"
      - "8007:8007"
      - "8008:8008"
      - "8009:8009"
      - "8444:443"
    volumes:
      - certs:/etc/letsencrypt:ro
    configs:
      - source: nginx_conf
        target: /etc/nginx/conf.d/default.conf
        mode: 0644
    restart: unless-stopped
    logging: *logging-conf

  model-proxy-registrar:
    image: curlimages/curl@sha256:d94d07ba9e7d6de898b6d96c1a072f6f8266c687af78a74f380087a0addf5d17
    container_name: model-proxy-registrar
    entrypoint: ["sh", "/register.sh"]
    restart: unless-stopped
    environment:
      - HOST_IP=${HOST_IP}
      - MODEL_PROXY_TOKEN=${MODEL_PROXY_TOKEN}
    configs:
      - source: registrar_script
        target: /register.sh
        mode: 0755
    logging: *logging-conf

  # --- Qwen3-30B (GPUs 0-1) — single instance, single proxy ---

  proxy-qwen3-30b:
    <<: *vllm-proxy-common
    container_name: proxy-qwen3-30b
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3-30B-A3B-Instruct-2507
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-vllm-qwen3-30b:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3-30B-A3B-Instruct-2507", "ip:${HOST_IP}", "port:8000"]}]'

  model-vllm-qwen3-30b:
    <<: *vllm-common
    image: vllm/vllm-openai@sha256:ccd6a6dbf4aba4e94c6f7052d1835d6e742082b6a5095276552e9b7a5a47c2e5
    container_name: model-vllm-qwen3-30b
    command: >
        Qwen/Qwen3-30B-A3B-Instruct-2507
        --revision 0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe
        --tensor-parallel-size 1
        --enable-prefix-caching
        --gpu-memory-utilization 0.95
        --tool-call-parser hermes
        --enable-auto-tool-choice
        --max-model-len 256K
        --max-num-batched-tokens 16K
        --stream-interval 10
        --load-format runai_streamer
        --dtype float16
        --model-loader-extra-config '{"concurrency":48}'
        --enable-prompt-tokens-details
        --speculative-config '{"method":"eagle3","model":"lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex","num_speculative_tokens":3,"draft_tensor_parallel_size":1}'
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - vllm_cache:/root/.cache/vllm
    environment: *vllm-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:Qwen/Qwen3-30B-A3B-Instruct-2507","ip:${HOST_IP}", "port:8000"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-qwen3-30b:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-qwen3-30b", "tags":["model:Qwen/Qwen3-30B-A3B-Instruct-2507","ip:${HOST_IP}", "port:8000"]}]'

  # --- GPT-OSS x2 (GPUs 2, 3) — single proxy, multi-backend ---

  proxy-gpt-oss:
    <<: *vllm-proxy-common
    container_name: proxy-gpt-oss
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=openai/gpt-oss-120b
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BACKEND_URLS=http://model-vllm-gpt-oss-1:8000,http://model-vllm-gpt-oss-2:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:openai/gpt-oss-120b", "ip:${HOST_IP}", "port:8001"]}]'

  model-vllm-gpt-oss-1:
    <<: *gpt-oss-common
    container_name: model-vllm-gpt-oss-1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["2"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:openai/gpt-oss-120b","ip:${HOST_IP}", "port:8001","instance:1"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-gpt-oss-1:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-gpt-oss-1", "tags":["model:openai/gpt-oss-120b","ip:${HOST_IP}", "port:8001"]}]'

  model-vllm-gpt-oss-2:
    <<: *gpt-oss-common
    container_name: model-vllm-gpt-oss-2
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["3"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:openai/gpt-oss-120b","ip:${HOST_IP}", "port:8001","instance:2"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-gpt-oss-2:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-gpt-oss-2", "tags":["model:openai/gpt-oss-120b","ip:${HOST_IP}", "port:8001"]}]'

  # --- Qwen3.6-35B-A3B-FP8 x2 (GPUs 1, 4) — SGLang, TP=1, EAGLE-v2 spec, single proxy multi-backend ---

  proxy-qwen3-6-35b:
    <<: *vllm-proxy-common
    container_name: proxy-qwen3-6-35b
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3.6-35B-A3B-FP8
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BACKEND_URLS=http://model-sg-qwen3-6-35b-1:8000,http://model-sg-qwen3-6-35b-2:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3.6-35B-A3B-FP8", "ip:${HOST_IP}", "port:8008"]}]'

  model-sg-qwen3-6-35b-1:
    <<: *qwen3-6-common
    container_name: model-sg-qwen3-6-35b-1
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["1"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["openmetrics"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "sglang", "service": "sglang", "tags":["model:Qwen/Qwen3.6-35B-A3B-FP8","ip:${HOST_IP}", "port:8008","instance:1"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-qwen3-6-35b-1:8000/metrics", "histogram_buckets_as_distributions": true, "metrics":["sglang:*"], "service": "model-sg-qwen3-6-35b-1", "tags":["model:Qwen/Qwen3.6-35B-A3B-FP8","ip:${HOST_IP}", "port:8008"]}]'

  model-sg-qwen3-6-35b-2:
    <<: *qwen3-6-common
    container_name: model-sg-qwen3-6-35b-2
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["4"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["openmetrics"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "sglang", "service": "sglang", "tags":["model:Qwen/Qwen3.6-35B-A3B-FP8","ip:${HOST_IP}", "port:8008","instance:2"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-qwen3-6-35b-2:8000/metrics", "histogram_buckets_as_distributions": true, "metrics":["sglang:*"], "service": "model-sg-qwen3-6-35b-2", "tags":["model:Qwen/Qwen3.6-35B-A3B-FP8","ip:${HOST_IP}", "port:8008"]}]'

  # --- Gemma-4-31B-it (GPUs 5-6 TP=2, BF16) — single instance, single proxy ---

  proxy-gemma-4-31b:
    <<: *vllm-proxy-common
    container_name: proxy-gemma-4-31b
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=google/gemma-4-31B-it
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-sg-gemma-4-31b:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:google/gemma-4-31B-it", "ip:${HOST_IP}", "port:8009"]}]'

  model-sg-gemma-4-31b:
    <<: *nvidia
    init: true
    # SGLang gemma4 build (CUDA 12.9). Switched off vLLM because vLLM was
    # returning 500 for media-fetch failures (broken FB/YouTube URLs) and
    # cloud-api was retrying them as transient errors. SGLang's gemma4 tag also
    # auto-selects the Triton attention backend required for bidirectional
    # image-token attention during prefill.
    # https://lmsysorg.mintlify.app/cookbook/autoregressive/Google/Gemma4
    image: lmsysorg/sglang:gemma4@sha256:87cecd3c9f4d17632c44b2d7cd1a20c50377c42b461d9ca39b153b4bb2b6e6ae
    container_name: model-sg-gemma-4-31b
    command: >
        sglang serve
        --model-path google/gemma-4-31B-it
        --revision ba74f5b6c647c0911554e50278d6f6f4477f9010
        --tp 2
        --reasoning-parser gemma4
        --tool-call-parser gemma4
        --mem-fraction-static 0.85
        --max-running-requests 64
        --chunked-prefill-size 8192
        --num-continuous-decode-steps 5
        --enable-mixed-chunk
        --disable-fast-image-processor
        --model-loader-extra-config '{"enable_multithread_load": "true", "num_threads": 64}'
        --port 8000
        --host 0.0.0.0
        --enable-cache-report
        --enable-metrics
        --trust-remote-code
        --log-requests-level 0
        --served-model-name google/gemma-4-31B-it
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - kernel_cache:/root/.cache/deep_gemm
    environment: *sglang-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["5","6"]
              capabilities: [gpu]
    restart: unless-stopped
    stop_grace_period: 5m
    logging: *logging-conf
    labels:
      com.datadoghq.ad.check_names: '["openmetrics"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "sglang", "service": "sglang", "tags":["model:google/gemma-4-31B-it","ip:${HOST_IP}", "port:8009"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-gemma-4-31b:8000/metrics", "histogram_buckets_as_distributions": true, "metrics":["sglang:*"], "service": "model-sg-gemma-4-31b", "tags":["model:google/gemma-4-31B-it","ip:${HOST_IP}", "port:8009"]}]'

  # --- FLUX x1 (GPU 7, shared with small models) ---

  proxy-flux:
    <<: *vllm-proxy-common
    container_name: proxy-flux
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=black-forest-labs/FLUX.2-klein-4B
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-sg-flux-2-klein-4b-1:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:black-forest-labs/FLUX.2-klein-4B", "ip:${HOST_IP}", "port:8002"]}]'

  model-sg-flux-2-klein-4b-1:
    <<: *flux-common
    container_name: model-sg-flux-2-klein-4b-1
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:black-forest-labs/FLUX.2-klein-4B","ip:${HOST_IP}", "port:8002","instance:1"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-sg-flux-2-klein-4b-1:8000/metrics", "histogram_buckets_as_distributions": true, "metrics":["sglang:*"], "service": "model-sg-flux-2-klein-4b-1", "tags":["model:black-forest-labs/FLUX.2-klein-4B","ip:${HOST_IP}", "port:8002"]}]'

  # --- Qwen3-VL FP8 (GPU 7, shared with small models) ---

  proxy-qwen3-vl:
    <<: *vllm-proxy-common
    container_name: proxy-qwen3-vl
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3-VL-30B-A3B-Instruct
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-vllm-qwen3-vl:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3-VL-30B-A3B-Instruct", "ip:${HOST_IP}", "port:8003"]}]'

  model-vllm-qwen3-vl:
    <<: *vllm-common
    image: vllm/vllm-openai@sha256:ccd6a6dbf4aba4e94c6f7052d1835d6e742082b6a5095276552e9b7a5a47c2e5
    container_name: model-vllm-qwen3-vl
    command: >
        Qwen/Qwen3-VL-30B-A3B-Instruct-FP8
        --revision d9748a51ae66354c4dad665aab2c71f26cf2c8cd
        --served-model-name Qwen/Qwen3-VL-30B-A3B-Instruct
        --enable-prefix-caching
        --tensor-parallel-size 1
        --gpu-memory-utilization 0.35
        --max-model-len 16384
        --max-num-seqs 16
        --max-num-batched-tokens 8K
        --async-scheduling
        --enable-prompt-tokens-details
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - vllm_cache:/root/.cache/vllm
    environment: *vllm-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["7"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:Qwen/Qwen3-VL-30B-A3B-Instruct","ip:${HOST_IP}", "port:8003"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-qwen3-vl:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-qwen3-vl", "tags":["model:Qwen/Qwen3-VL-30B-A3B-Instruct","ip:${HOST_IP}", "port:8003"]}]'

  # --- Embeddings, Reranker, Whisper (GPU 7) — single instance each ---

  proxy-qwen3-embeddings:
    <<: *vllm-proxy-common
    container_name: proxy-qwen3-embeddings
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3-Embedding-0.6B
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-vllm-qwen3-embeddings:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3-Embedding-0.6B", "ip:${HOST_IP}", "port:8004"]}]'

  model-vllm-qwen3-embeddings:
    <<: *vllm-common
    image: vllm/vllm-openai@sha256:ccd6a6dbf4aba4e94c6f7052d1835d6e742082b6a5095276552e9b7a5a47c2e5
    container_name: model-vllm-qwen3-embeddings
    command: >
        Qwen/Qwen3-Embedding-0.6B
        --revision 97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3
        --runner pooling
        --gpu-memory-utilization 0.06
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - vllm_cache:/root/.cache/vllm
    environment: *vllm-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["7"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:Qwen/Qwen3-Embedding-0.6B","ip:${HOST_IP}", "port:8004"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-qwen3-embeddings:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-qwen3-embeddings", "tags":["model:Qwen/Qwen3-Embedding-0.6B","ip:${HOST_IP}", "port:8004"]}]'

  proxy-qwen3-reranker:
    <<: *vllm-proxy-common
    container_name: proxy-qwen3-reranker
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=Qwen/Qwen3-Reranker-0.6B
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-vllm-qwen3-reranker:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:Qwen/Qwen3-Reranker-0.6B", "ip:${HOST_IP}", "port:8005"]}]'

  model-vllm-qwen3-reranker:
    <<: *vllm-common
    image: vllm/vllm-openai@sha256:ccd6a6dbf4aba4e94c6f7052d1835d6e742082b6a5095276552e9b7a5a47c2e5
    container_name: model-vllm-qwen3-reranker
    command: >
        Qwen/Qwen3-Reranker-0.6B
        --revision e61197ed45024b0ed8a2d74b80b4d909f1255473
        --runner pooling
        --gpu-memory-utilization 0.06
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - vllm_cache:/root/.cache/vllm
    environment: *vllm-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["7"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:Qwen/Qwen3-Reranker-0.6B","ip:${HOST_IP}", "port:8005"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-qwen3-reranker:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-qwen3-reranker", "tags":["model:Qwen/Qwen3-Reranker-0.6B","ip:${HOST_IP}", "port:8005"]}]'

  proxy-whisper3-large:
    <<: *vllm-proxy-common
    container_name: proxy-whisper3-large
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=openai/whisper-large-v3
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-vllm-whisper3-large:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:openai/whisper-large-v3", "ip:${HOST_IP}", "port:8005"]}]'

  model-vllm-whisper3-large:
    <<: *vllm-common
    image: vllm-with-audio
    build:
      context: .
      dockerfile_inline: |
        FROM vllm/vllm-openai@sha256:ccd6a6dbf4aba4e94c6f7052d1835d6e742082b6a5095276552e9b7a5a47c2e5
        RUN pip install openai-whisper torchaudio librosa vllm[audio]
    container_name: model-vllm-whisper3-large
    command: >
        openai/whisper-large-v3
        --revision 06f233fe06e710322aca913c1bc4249a0d71fce1
        --gpu-memory-utilization 0.06
    volumes:
      - hugginface_cache:/root/.cache/huggingface
      - vllm_cache:/root/.cache/vllm
    environment: *vllm-env
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["7"]
              capabilities: [gpu]
    labels:
      com.datadoghq.ad.check_names: '["vllm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.logs: '[{"source": "vllm", "service": "vllm", "tags":["model:openai/whisper-large-v3","ip:${HOST_IP}", "port:8005"]}]'
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint":"http://model-vllm-whisper3-large:8000/metrics", "histogram_buckets_as_distributions": true, "service": "model-vllm-whisper3-large", "tags":["model:openai/whisper-large-v3","ip:${HOST_IP}", "port:8005"]}]'

  # --- Privacy filter (GPU 7) — HF transformers token-classification ---

  proxy-privacy-filter:
    <<: *vllm-proxy-common
    container_name: proxy-privacy-filter
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CLOUD_API_URL=https://cloud-api.near.ai
      - CLOUD_API_USAGE_TOKEN=${CLOUD_API_USAGE_TOKEN}
      - COMPOSE_MANAGER_URL=http://compose-manager:8080
      - LOG_FORMAT=json
      - MODEL_NAME=openai/privacy-filter
      - OHTTP_ENABLED=true
      - TOKEN=${PROXY_TOKEN}
      - VLLM_BASE_URL=http://model-privacy-filter:8000
      - TLS_CERT_PATH=/etc/letsencrypt/live/completions.near.ai/fullchain.pem
      - USE_NV_ATTESTATION_SDK=true
    labels:
      com.datadoghq.ad.logs: '[{"source": "vllm-proxy", "service": "vllm-proxy", "tags": ["model:openai/privacy-filter", "ip:${HOST_IP}", "port:8007"]}]'

  model-privacy-filter:
    <<: *privacy-filter-common
    container_name: model-privacy-filter
    labels:
      com.datadoghq.ad.logs: '[{"source": "privacy-filter", "service": "privacy-filter", "tags":["model:openai/privacy-filter","ip:${HOST_IP}", "port:8007"]}]'

  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.2-4.8.1-distroless
    container_name: dcgm-exporter
    runtime: nvidia
    cap_add:
      - SYS_ADMIN
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
    ports:
      - "9400:9400"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    logging: *logging-conf
    labels:
      com.datadoghq.ad.check_names: '["dcgm"]'
      com.datadoghq.ad.init_configs: "[{}]"
      com.datadoghq.ad.instances: '[{"openmetrics_endpoint": "http://%%host%%:9400/metrics", "tags":["deployment:small-models","ip:${HOST_IP}"]}]'
      com.datadoghq.ad.logs: '[{"source": "dcgm-exporter", "service": "dcgm-exporter", "tags":["deployment:small-models","ip:${HOST_IP}"]}]'

networks:
  default:
    external: true
    name: dstack_default

volumes:
  hugginface_cache:
  vllm_cache:
  kernel_cache:
  certs:
    external: true
    name: certs

configs:
  registrar_script:
    content: |
      #!/bin/sh
      PROXY_URL="https://completions.near.ai"
      TOKEN="$${MODEL_PROXY_TOKEN}"
      TLS_PORT=8444

      # Track registration state per port
      for port in 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009; do
        eval "REG_$$port=false"
        eval "FAIL_COUNT_$$port=0"
      done

      register_endpoint() {
        curl -sS --max-time 10 -X POST "$$PROXY_URL/register/endpoint" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"endpoint\":\"$$1\",\"routing_port\":$$TLS_PORT}" || true
      }

      unregister_endpoint() {
        curl -sS --max-time 10 -X POST "$$PROXY_URL/unregister/endpoint" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"endpoint\":\"$$1\"}" || true
      }

      register_model() {
        curl -sS --max-time 10 -X POST "$$PROXY_URL/register/model" \
          -H "Authorization: Bearer $$TOKEN" \
          -H "Content-Type: application/json" \
          -d "{\"model\":\"$$1\",\"domain\":\"$$2\"}" || true
      }

      cleanup() {
        echo "SIGTERM received, unregistering all endpoints"
        for port in 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009; do
          unregister_endpoint "$${HOST_IP}:$$port"
        done
        exit 0
      }
      trap cleanup TERM INT

      # Health check directly on backends (no auth needed on raw vLLM/SGLang containers)
      check_chat() {
        curl -sf --max-time 45 -X POST "http://$$1:8000/v1/chat/completions" \
          -H "Content-Type: application/json" \
          -d "{\"model\":\"$$2\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}],\"max_tokens\":1}" \
          > /dev/null 2>&1
      }

      check_port() {
        case $$1 in
          8000) check_chat model-vllm-qwen3-30b "Qwen/Qwen3-30B-A3B-Instruct-2507" ;;
          8001) check_chat model-vllm-gpt-oss-1 "openai/gpt-oss-120b" ;;
          8002) curl -sf --max-time 10 "http://model-sg-flux-2-klein-4b-1:8000/v1/models" > /dev/null 2>&1 ;;
          8003) check_chat model-vllm-qwen3-vl "Qwen/Qwen3-VL-30B-A3B-Instruct" ;;
          8004) curl -sf --max-time 10 "http://model-vllm-qwen3-embeddings:8000/v1/models" > /dev/null 2>&1 ;;
          8005) curl -sf --max-time 10 "http://model-vllm-qwen3-reranker:8000/v1/models" > /dev/null 2>&1 ;;
          8006) curl -sf --max-time 10 "http://model-vllm-whisper3-large:8000/v1/models" > /dev/null 2>&1 ;;
          8007) curl -sf --max-time 10 "http://model-privacy-filter:8000/v1/models" > /dev/null 2>&1 ;;
          8008) check_chat model-sg-qwen3-6-35b-1 "Qwen/Qwen3.6-35B-A3B-FP8" ;;
          8009) check_chat model-sg-gemma-4-31b "google/gemma-4-31B-it" ;;
        esac
      }

      echo "Waiting for first model to be ready..."
      until curl -sf http://proxy-nginx:8000/v1/models > /dev/null 2>&1; do sleep 30; done
      echo "First model ready, starting registration loop"

      while true; do
        for port in 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009; do
          eval "was_reg=\$$REG_$$port"
          eval "fails=\$$FAIL_COUNT_$$port"
          if check_port $$port; then
            eval "FAIL_COUNT_$$port=0"
            register_endpoint "$${HOST_IP}:$$port" "$$TLS_PORT"
            if [ "$$was_reg" = false ]; then
              echo "Registered $${HOST_IP}:$$port"
            fi
            eval "REG_$$port=true"
          else
            fails=$$((fails + 1))
            eval "FAIL_COUNT_$$port=$$fails"
            echo "Health check failed for $${HOST_IP}:$$port (fail count: $$fails)"
            eval "REG_$$port=false"
          fi
        done

        # Model-to-domain mappings
        register_model "Qwen/Qwen3-30B-A3B-Instruct-2507" "qwen3-30b.completions.near.ai"
        register_model "openai/gpt-oss-120b" "gpt-oss-120b.completions.near.ai"
        register_model "black-forest-labs/FLUX.2-klein-4B" "flux2-klein.completions.near.ai"
        register_model "Qwen/Qwen3-VL-30B-A3B-Instruct" "qwen3-vl-30b.completions.near.ai"
        register_model "Qwen/Qwen3-Embedding-0.6B" "qwen3-embedding.completions.near.ai"
        register_model "Qwen/Qwen3-Reranker-0.6B" "qwen3-reranker.completions.near.ai"
        register_model "openai/whisper-large-v3" "whisper-large-v3.completions.near.ai"
        register_model "openai/privacy-filter" "privacy-filter.completions.near.ai"
        register_model "Qwen/Qwen3.6-35B-A3B-FP8" "qwen3-6-35b.completions.near.ai"
        register_model "google/gemma-4-31B-it" "gemma-4-31b.completions.near.ai"
        sleep 60
      done
  nginx_conf:
    content: |
      log_format json_combined escape=json
        '{'  '"time":"$$time_iso8601"'
        ',"request_id":"$$http_x_request_id"'
        ',"org_id":"$$http_x_org_id"'
        ',"workspace_id":"$$http_x_workspace_id"'
        ',"host":"$$host"'
        ',"method":"$$request_method"'
        ',"uri":"$$uri"'
        ',"status":$$status'
        ',"request_length":$$request_length'
        ',"bytes_sent":$$bytes_sent'
        ',"request_time":$$request_time'
        ',"upstream_addr":"$$upstream_addr"'
        '}';

      access_log /var/log/nginx/access.log json_combined;

      # Common proxy settings
      proxy_http_version 1.1;
      proxy_set_header Host $$host;
      proxy_set_header X-Real-IP $$remote_addr;
      proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
      proxy_set_header X-Forwarded-Proto $$scheme;
      proxy_set_header Connection '';
      proxy_buffering off;
      proxy_cache off;
      proxy_read_timeout 3600s;
      client_max_body_size 100m;
      client_body_buffer_size 1m;

      # :8000 - Qwen3-30B
      server {
          listen 8000;
          location / { proxy_pass http://proxy-qwen3-30b:8000; }
      }

      # :8001 - GPT-OSS (single proxy, 2 backends)
      server {
          listen 8001;
          location / { proxy_pass http://proxy-gpt-oss:8000; }
      }

      # :8002 - FLUX (single proxy, 4 backends)
      server {
          listen 8002;
          location / { proxy_pass http://proxy-flux:8000; }
      }

      # :8003 - Qwen3-VL
      server {
          listen 8003;
          location / { proxy_pass http://proxy-qwen3-vl:8000; }
      }

      # :8004 - Embeddings
      server {
          listen 8004;
          location / { proxy_pass http://proxy-qwen3-embeddings:8000; }
      }

      # :8005 - Reranker
      server {
          listen 8005;
          location / { proxy_pass http://proxy-qwen3-reranker:8000; }
      }

      # :8006 - Whisper
      server {
          listen 8006;
          location / { proxy_pass http://proxy-whisper3-large:8000; }
      }

      # :8007 - Privacy-filter
      server {
          listen 8007;
          location / { proxy_pass http://proxy-privacy-filter:8000; }
      }

      # :8008 - Qwen3.6-35B-A3B-FP8
      server {
          listen 8008;
          location / { proxy_pass http://proxy-qwen3-6-35b:8000; }
      }

      # :8009 - Gemma-4-31B-it
      server {
          listen 8009;
          location / { proxy_pass http://proxy-gemma-4-31b:8000; }
      }

      # ── TLS server blocks (port 443 = host 8444) ──

      ssl_certificate /etc/letsencrypt/live/completions.near.ai/fullchain.pem;
      ssl_certificate_key /etc/letsencrypt/live/completions.near.ai/privkey.pem;
      ssl_protocols TLSv1.2 TLSv1.3;

      # Each model accepts both its canonical SNI and the rotation-SNI form
      # `<canonical>-i<N>.completions.near.ai` used by cloud-api's discovery
      # (model-proxy's parse_indexed_sni). Without the regex match, rotation
      # SNIs fall through to the default server (lexically first model) and
      # return the wrong model's signing pubkey, breaking E2EE — see
      # nearai/infra#143 and the L4-SNI-rewrite analysis in
      # nearai/model-proxy#30.

      server {
          listen 443 ssl http2;
          server_name qwen3-30b.completions.near.ai
                      ~^qwen3-30b-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-qwen3-30b:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name gpt-oss-120b.completions.near.ai
                      ~^gpt-oss-120b-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-gpt-oss:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name flux2-klein.completions.near.ai
                      ~^flux2-klein-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-flux:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name qwen3-vl-30b.completions.near.ai
                      ~^qwen3-vl-30b-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-qwen3-vl:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name qwen3-embedding.completions.near.ai
                      ~^qwen3-embedding-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-qwen3-embeddings:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name qwen3-reranker.completions.near.ai
                      ~^qwen3-reranker-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-qwen3-reranker:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name whisper-large-v3.completions.near.ai
                      ~^whisper-large-v3-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-whisper3-large:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name privacy-filter.completions.near.ai
                      ~^privacy-filter-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-privacy-filter:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name qwen3-6-35b.completions.near.ai
                      ~^qwen3-6-35b-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-qwen3-6-35b:8000; }
      }

      server {
          listen 443 ssl http2;
          server_name gemma-4-31b.completions.near.ai
                      ~^gemma-4-31b-i\d+\.completions\.near\.ai$$;
          location / { proxy_pass http://proxy-gemma-4-31b:8000; }
      }