open-llm-proxy/ingress.yaml at main · boettiger-lab/open-llm-proxy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: open-llm-proxy-ingress
  annotations:
    # CORS IS ENFORCED HERE, not in the app. The FastAPI CORSMiddleware in
    # llm_proxy.py is effectively dead config — this haproxy ingress answers the
    # browser preflight and wins. `cors-allow-headers` is an EXPLICIT list (it
    # does not reflect requested headers), so any custom request header a browser
    # client sends MUST be added below or the preflight fails and the ENTIRE
    # request is blocked (not just that header). e.g. to let geo-agent send an
    # X-Client version header, append `,X-Client` to cors-allow-headers and
    # deploy this BEFORE the client starts sending it. Verify with:
    #   curl -i -X OPTIONS https://open-llm-proxy.nrp-nautilus.io/v1/chat/completions \
    #     -H "Origin: https://tpl.nrp-nautilus.io" \
    #     -H "Access-Control-Request-Method: POST" \
    #     -H "Access-Control-Request-Headers: authorization,content-type,x-client,x-session-id"
    haproxy-ingress.github.io/cors-enable: "true"
    haproxy-ingress.github.io/cors-allow-origin: "*"
    haproxy-ingress.github.io/cors-allow-methods: "GET, POST, OPTIONS"
    haproxy-ingress.github.io/cors-allow-headers: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Accept,Accept-Encoding,Accept-Language,Connection,Host,Origin,Referer,Sec-Fetch-Dest,Sec-Fetch-Mode,Sec-Fetch-Site,Priority,Authorization,X-Client,X-Session-Id"
    haproxy-ingress.github.io/cors-allow-credentials: "false"
    haproxy-ingress.github.io/cors-max-age: "600"
    # Keep timeout-client aligned with timeout-server (hygiene). Because the proxy
    # calls upstream non-streaming, a long completion leaves BOTH sides idle — the
    # backend silent (timeout-server) and the client waiting (timeout-client) — so
    # the two should match. NOTE: this does NOT resolve #82. A ~300s ceiling still
    # cuts long single generations (client sees `fetch failed` at ~300s); testing
    # showed NRP's shared haproxy enforces that client timeout and this per-Ingress
    # annotation does not override it. The real fix is streaming end-to-end (proxy
    # + geo-agent) or an NRP-side timeout bump — tracked in #82.
    haproxy-ingress.github.io/timeout-server: "600s"
    haproxy-ingress.github.io/timeout-client: "600s"
    haproxy-ingress.github.io/timeout-tunnel: "3600s"
spec:
  ingressClassName: haproxy
  tls:
    - hosts:
        - open-llm-proxy.nrp-nautilus.io
  rules:
    - host: open-llm-proxy.nrp-nautilus.io
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: open-llm-proxy
                port:
                  number: 8002