deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: open-llm-proxy
  labels:
    app: open-llm-proxy
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      # Never drop below the desired ready count during a rollout: bring a new
      # pod up (surge) and only retire an old one once the new one is Ready.
      # (maxUnavailable:1 on a single replica previously caused a full outage.)
      maxUnavailable: 0
      maxSurge: 1
  selector:
    matchLabels:
      app: open-llm-proxy
  template:
    metadata:
      labels:
        app: open-llm-proxy
    spec:
      # DNS resilience for intermittent upstream "Temporary failure in name
      # resolution" (EAI_AGAIN) → 502s. See issue #28.
      #
      # Resolution goes through the per-pod `dns-cache` CoreDNS sidecar on
      # 127.0.0.1, which forwards to cluster CoreDNS and CACHES with serve_stale
      # — so when cluster CoreDNS blips, the sidecar keeps serving the last-good
      # answer instead of failing. Cluster CoreDNS (10.96.0.10) is kept as a
      # FALLBACK nameserver: if the sidecar is unreachable, resolution degrades
      # to today's direct-to-CoreDNS behavior (never worse than before).
      #
      # dnsPolicy: None is required to put 127.0.0.1 first (ClusterFirst would
      # prepend 10.96.0.10). We therefore specify searches explicitly; the
      # node-specific search domain is intentionally dropped (app only resolves
      # in-cluster services + external FQDNs). ndots:2 keeps external FQDNs to a
      # single query while short names like rook-ceph-rgw-nautiluss3.rook (S3
      # flush) still resolve via the search list.
      dnsPolicy: None
      dnsConfig:
        nameservers:
          - 127.0.0.1
          - 10.96.0.10
        searches:
          - biodiversity.svc.cluster.local
          - svc.cluster.local
          - cluster.local
        options:
          - name: ndots
            value: "2"
          - name: attempts
            value: "3"
          - name: timeout
            value: "2"
      affinity:
        # Spread replicas across nodes so one stuck/unhealthy node can't take
        # the whole service down.
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app: open-llm-proxy
                topologyKey: kubernetes.io/hostname
        nodeAffinity:
          # TEMPORARY: these nodes are stuck/failing image pulls (2026-06-17),
          # which the scheduler otherwise keeps selecting. Remove entries as the
          # nodes are healed/cordoned.
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/hostname
                    operator: NotIn
                    values:
                      - gp-argo.usd.edu
                      - cph-blade01.humboldt.edu
                      - cph-blade03.humboldt.edu
                      - cph-blade05.humboldt.edu
      initContainers:
      - name: git-sync
        image: alpine/git:latest
        command:
        - sh
        - -c
        - |
          git clone https://github.com/boettiger-lab/open-llm-proxy.git /tmp/repo
          cp -r /tmp/repo/. /app/
        volumeMounts:
        - name: app-code
          mountPath: /app
      containers:
      # Per-pod caching DNS resolver (see dns-cache-configmap.yaml + #28). The
      # app points at 127.0.0.1; this forwards to cluster CoreDNS and serves
      # stale entries through upstream blips. Falls back are handled by the
      # second nameserver in dnsConfig if this container is unavailable.
      - name: dns-cache
        image: registry.k8s.io/coredns/coredns:v1.11.3
        args: ["-conf", "/etc/coredns/Corefile"]
        securityContext:
          capabilities:
            add: ["NET_BIND_SERVICE"]   # bind :53
        livenessProbe:
          httpGet: { path: /health, port: 8080 }
          initialDelaySeconds: 5
          periodSeconds: 10
        readinessProbe:
          httpGet: { path: /ready, port: 8181 }
          periodSeconds: 5
        resources:
          requests: { cpu: "10m", memory: "32Mi" }
          limits:   { cpu: "200m", memory: "128Mi" }
        volumeMounts:
        - name: dns-config
          mountPath: /etc/coredns
      - name: open-llm-proxy
        image: astral/uv:python3.12-bookworm-slim
        command: ["uvx", "--from", "uvicorn", "--with", "fastapi", "--with", "httpx", "--with", "pydantic", "--with", "boto3", "uvicorn", "llm_proxy:app", "--host", "0.0.0.0", "--port", "8002", "--workers", "4"]
        workingDir: /app
        ports:
        - containerPort: 8002
        env:
        - name: NRP_API_KEY
          valueFrom:
            secretKeyRef:
              name: open-llm-proxy-secrets
              key: nrp-api-key
        - name: PROXY_KEY
          valueFrom:
            secretKeyRef:
              name: open-llm-proxy-secrets
              key: proxy-key
        # Optional additional revocable client keys (comma-separated), e.g.
        # per-user eval keys. Accepted exactly like PROXY_KEY; revoke by removing
        # from the secret value and restarting. optional=true so the pod starts
        # fine when the secret key is absent (backward compatible).
        - name: PROXY_KEYS_EXTRA
          valueFrom:
            secretKeyRef:
              name: open-llm-proxy-secrets
              key: proxy-keys-extra
              optional: true
        - name: OPENROUTER_KEY
          valueFrom:
            secretKeyRef:
              name: openrouter-key
              key: OPENROUTER_KEY
        - name: NIMBUS_API_KEY
          valueFrom:
            secretKeyRef:
              name: nimbus-api-key
              key: NIMBUS_API_KEY
        - name: ANTHROPIC_API_KEY
          valueFrom:
            secretKeyRef:
              name: anthropic-api-key
              key: ANTHROPIC_API_KEY
        - name: CACHE_SALT
          valueFrom:
            secretKeyRef:
              name: open-llm-proxy-secrets
              key: cache-salt
        # Logging fidelity (see LOGGING.md). Credentials are always scrubbed.
        # "summary" (default): full response content + generously-capped inputs.
        # "full": additionally log the entire scrubbed `messages` array per turn
        #         (training-grade), with the system prompt de-duplicated by hash.
        - name: LOG_CAPTURE_MODE
          value: "summary"
        # Per-field caps (chars; 0 = uncapped). tool_calls args are always full.
        # Defaults keep full final answer + full tool calls, bounded reasoning.
        - name: LOG_REASONING_MAX
          value: "4000"
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef: { name: aws, key: AWS_ACCESS_KEY_ID }
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef: { name: aws, key: AWS_SECRET_ACCESS_KEY }
        resources:
          requests:
            cpu: "1000m"
            memory: "1Gi"
          limits:
            cpu: "2000m"
            memory: "2Gi"
        volumeMounts:
        - name: app-code
          mountPath: /app
        livenessProbe:
          httpGet:
            path: /health
            port: 8002
          initialDelaySeconds: 10
          periodSeconds: 30
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /health
            port: 8002
          initialDelaySeconds: 5
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        startupProbe:
          httpGet:
            path: /health
            port: 8002
          initialDelaySeconds: 0
          periodSeconds: 5
          timeoutSeconds: 5
          failureThreshold: 30
      tolerations:
      - key: "nautilus.io/issue"
        operator: "Equal"
        value: "1476"
        effect: "NoSchedule"
      volumes:
      - name: app-code
        emptyDir: {}
      - name: dns-config
        configMap:
          name: open-llm-proxy-dns-cache