fix(llm-request-router): require explicit remote Watch URIs

deploy/helm/llm-request-router/README.md

@@ -40,6 +40,10 @@ exposes a dashed-IP SRV alias. A multi-replica StatefulSet can instead run
 without the backend router and retain direct headless Service SRV discovery.
 `llmRequestRouter.discovery.watchHeartbeatMs` controls the maximum interval
 between unchanged Watch snapshots from both Stargate and the backend router.
+`llmRequestRouter.discovery.remoteWatchUrls` accepts only explicit `https://`
+Watch URIs. Development plaintext endpoints require an explicit `http://` URI
+and `allowInsecureRemoteWatchHttp=true`; scheme-less and unsupported values are
+rejected instead of defaulting to plaintext.
 
 `llmRequestRouter.kubernetes.advertisedHostnameTemplate` supports the Stargate
 placeholders `{pod_name}` and `{namespace}`. Stargate resolves both placeholders
@@ -133,7 +137,7 @@ dial addresses to the external endpoints that workers can resolve:
 llmRequestRouter:
   backendRouter:
     enabled: true
-    pylonGrpcDialAddress: llm-router.example.com:443
+    pylonGrpcDialAddress: https://llm-router.example.com:443
     pylonReverseTunnelDialAddress: llm-router.example.com:8080
 ```
 

deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl

@@ -80,6 +80,31 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- dig "workload" "kind" "Deployment" .Values.llmRequestRouter | toString -}}
 {{- end -}}
 
+{{- define "llm-request-router.isExplicitHttpUri" -}}
+{{- $uri := . | toString | trim -}}
+{{- $authorityValid := regexMatch "^https?://([A-Za-z0-9._~-]+|\\[[0-9A-Fa-f:.]+\\])(:[0-9]+)?/?$" $uri -}}
+{{- $portMatch := regexFind ":[0-9]+/?$" $uri -}}
+{{- $portText := $portMatch | trimPrefix ":" | trimSuffix "/" -}}
+{{- $portValid := or
+      (not $portMatch)
+      (and (le (len $portText) 5) (le ($portText | int) 65535)) -}}
+{{- if and $authorityValid $portValid -}}true{{- end -}}
+{{- end -}}
+
+{{- define "llm-request-router.validateRemoteWatchUrls" -}}
+{{- $discovery := .Values.llmRequestRouter.discovery | default dict -}}
+{{- $allowHttp := dig "allowInsecureRemoteWatchHttp" false $discovery -}}
+{{- range $remoteWatchUrl := dig "remoteWatchUrls" (list) $discovery -}}
+{{- $remoteWatchUrl = $remoteWatchUrl | toString | trim -}}
+{{- if ne (include "llm-request-router.isExplicitHttpUri" $remoteWatchUrl) "true" -}}
+{{- fail "llmRequestRouter.discovery.remoteWatchUrls entries must be explicit http:// or https:// URIs" -}}
+{{- end -}}
+{{- if and (hasPrefix "http://" $remoteWatchUrl) (not $allowHttp) -}}
+{{- fail "llmRequestRouter.discovery.remoteWatchUrls requires https://; set allowInsecureRemoteWatchHttp=true only for development plaintext endpoints" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 An unset backendRouter.enabled follows the workload contract: a multi-replica
 Deployment needs the EndpointSlice router, while StatefulSet and single-replica
@@ -297,9 +322,12 @@ externally reachable address.
 {{- $backendRouter := .Values.llmRequestRouter.backendRouter | default dict -}}
 {{- $configured := dig "pylonGrpcDialAddress" "" $backendRouter | toString | trim -}}
 {{- if $configured -}}
+{{- if ne (include "llm-request-router.isExplicitHttpUri" $configured) "true" -}}
+{{- fail "llmRequestRouter.backendRouter.pylonGrpcDialAddress must be an explicit http:// or https:// URI" -}}
+{{- end -}}
 {{- $configured -}}
 {{- else -}}
-{{- printf "%s.%s.svc.cluster.local:%v" (include "llm-request-router.backendRouterName" .) (include "llm-request-router.namespace" .) (dig "service" "grpcPort" 50071 $backendRouter) -}}
+{{- printf "http://%s.%s.svc.cluster.local:%v" (include "llm-request-router.backendRouterName" .) (include "llm-request-router.namespace" .) (dig "service" "grpcPort" 50071 $backendRouter) -}}
 {{- end -}}
 {{- end -}}
 

deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml

@@ -84,6 +84,12 @@ spec:
             - --advertised-grpc-port={{ .Values.llmRequestRouter.service.grpcPort }}
             - --grpc-pylon-dial-addr={{ include "llm-request-router.backendRouterGrpcDialAddress" . }}
             - --watch-heartbeat-ms={{ .Values.llmRequestRouter.discovery.watchHeartbeatMs }}
+            {{- range .Values.llmRequestRouter.discovery.remoteWatchUrls }}
+            - --remote-stargate-url={{ . }}
+            {{- end }}
+            {{- if .Values.llmRequestRouter.discovery.allowInsecureRemoteWatchHttp }}
+            - --allow-insecure-remote-watch-http
+            {{- end }}
             - --grpc-port-name=grpc
             - --quic-port-name=quic
             - --tunnel-protocol=raw-quic

deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml

@@ -17,6 +17,7 @@
 {{- if not (has $workloadKind (list "Deployment" "StatefulSet")) -}}
 {{- fail (printf "llmRequestRouter.workload.kind must be Deployment or StatefulSet, got %q" $workloadKind) -}}
 {{- end }}
+{{- include "llm-request-router.validateRemoteWatchUrls" . }}
 apiVersion: apps/v1
 kind: {{ $workloadKind }}
 metadata:
@@ -124,6 +125,12 @@ spec:
             {{- with dig "discovery" "watchHeartbeatMs" "" .Values.llmRequestRouter }}
             - --watch-heartbeat-ms={{ . }}
             {{- end }}
+            {{- range .Values.llmRequestRouter.discovery.remoteWatchUrls }}
+            - --remote-stargate-url={{ . }}
+            {{- end }}
+            {{- if .Values.llmRequestRouter.discovery.allowInsecureRemoteWatchHttp }}
+            - --allow-insecure-remote-watch-http
+            {{- end }}
             - --shutdown-drain-timeout-ms={{ .Values.llmRequestRouter.shutdown.drainTimeoutMs }}
             - --quic-connect-timeout-ms={{ .Values.llmRequestRouter.transport.quicConnectTimeoutMs }}
             - --quic-request-timeout-ms={{ .Values.llmRequestRouter.transport.quicRequestTimeoutMs }}

deploy/helm/llm-request-router/llm-request-router/values.yaml

@@ -119,10 +119,10 @@ llmRequestRouter:
       repository: ""
       tag: ""
       pullPolicy: ""
-    # Addresses workers dial to reach the router. Both default to the
-    # backend-router Service inside the cluster, which is correct when workers
-    # run alongside the control plane. Override both with externally reachable
-    # addresses when workers run in a separate cluster or region.
+    # Addresses workers dial to reach the router. The gRPC address is an
+    # explicit HTTP(S) URI; the reverse-tunnel address remains host:port. Both
+    # default to the backend-router Service inside the cluster. Override both
+    # with externally reachable addresses for a separate cluster or region.
     pylonGrpcDialAddress: ""
     pylonReverseTunnelDialAddress: ""
     serviceAccount:
@@ -238,6 +238,10 @@ llmRequestRouter:
     # Maximum interval between unchanged WatchStargates snapshots. This is
     # shared by Stargate and the EndpointSlice backend router.
     watchHeartbeatMs: 5000
+    # Recursive remote Watch endpoints must include their HTTP transport
+    # scheme. HTTPS is required unless the development-only opt-in is enabled.
+    remoteWatchUrls: []
+    allowInsecureRemoteWatchHttp: false
 
   transport:
     quicConnectTimeoutMs: 2000

deploy/helm/llm-request-router/scripts/check-backend-router-render.sh

@@ -20,7 +20,7 @@ helm template llm-request-router "$chart_dir" \
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.certificate.enabled=true \
   --set llmRequestRouter.certificate.issuerRef.name=test-issuer \
@@ -166,13 +166,13 @@ assert_zero_config_contains() {
   fi
 }
 
-assert_zero_config_contains "--grpc-pylon-dial-addr=llm-request-router-backend-router.nvcf.svc.cluster.local:50071" \
+assert_zero_config_contains "--grpc-pylon-dial-addr=http://llm-request-router-backend-router.nvcf.svc.cluster.local:50071" \
   "gRPC dial address must default to the in-cluster backend-router Service"
 assert_zero_config_contains "--reverse-tunnel-pylon-dial-addr=llm-request-router-backend-router.nvcf.svc.cluster.local:50072" \
   "reverse-tunnel dial address must default to the in-cluster backend-router Service"
 
 # An explicitly configured address must still win over the default.
-assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \
+assert_contains "--grpc-pylon-dial-addr=https://llm-router.example.invalid:443" \
   "configured gRPC dial address must override the in-cluster default"
 
 # Each replica terminates QUIC itself and cannot resume another replica's
@@ -199,15 +199,15 @@ assert_contains "--advertised-hostname-template={pod_name}.llm-request-router-he
   "backend router authority and SNI template must match Stargate"
 assert_contains "--advertised-grpc-port=50071" \
   "backend router Watch snapshots must advertise the Stargate gRPC port"
-assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \
+assert_contains "--grpc-pylon-dial-addr=https://llm-router.example.invalid:443" \
   "backend router Watch snapshots must preserve the Pylon dial endpoint"
 assert_contains "- '*.llm-request-router-headless.nvcf.svc.cluster.local'" \
   "request-router certificate must cover pod-specific backend routing hostnames"
 assert_contains "image: registry.example.invalid/nvcf/stargate:next" \
   "backend router must use its explicitly pinned Stargate image"
 assert_contains "app.kubernetes.io/version: \"next\"" \
   "backend router labels must identify the explicitly pinned image version"
-assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \
+assert_contains "--grpc-pylon-dial-addr=https://llm-router.example.invalid:443" \
   "Stargate must advertise the external gRPC endpoint to pylon"
 assert_contains "--reverse-tunnel-pylon-dial-addr=llm-router.example.invalid:8080" \
   "Stargate must advertise the external reverse-tunnel endpoint to pylon"
@@ -238,7 +238,7 @@ assert_render_fails "llmRequestRouter.kubernetes.advertisedHostnameTemplate must
   --set-string 'llmRequestRouter.kubernetes.advertisedHostnameTemplate=\{pod_name\}\{pod_name\}' \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080
 
 assert_render_fails "llmRequestRouter.transport.reverseTunnelListenAddr port 50073 must match llmRequestRouter.service.reverseTunnelPort 50072 when backend routing is enabled" \
@@ -271,7 +271,7 @@ assert_render_fails "llmRequestRouter.backendRouter.serviceAccount.name is requi
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.backendRouter.serviceAccount.create=false
 
@@ -280,7 +280,7 @@ assert_render_fails "llmRequestRouter.backendRouter.serviceAccount.name is requi
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.backendRouter.serviceAccount.create=false \
   --set llmRequestRouter.rbac.create=false
@@ -291,7 +291,7 @@ helm template llm-request-router "$chart_dir" \
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.backendRouter.serviceAccount.create=false \
   --set llmRequestRouter.backendRouter.serviceAccount.name=external-backend-router \
@@ -314,7 +314,7 @@ helm template llm-request-router "$chart_dir" \
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.certificate.enabled=true \
   --set llmRequestRouter.certificate.issuerRef.name=test-issuer \
@@ -342,15 +342,15 @@ assert_render_fails "llmRequestRouter.kubernetes.advertisedHostnameTemplate must
   --set llmRequestRouter.kubernetes.advertisedHostnameTemplate=llm-request-router.nvcf.svc.cluster.local \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080
 
 assert_render_fails "llmRequestRouter backend routing requires a TLS Secret and cert/key paths when tls.quicInsecure is false" \
   --set llmRequestRouter.image.registry=registry.example.invalid \
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.tls.quicInsecure=false
 
@@ -359,7 +359,7 @@ assert_render_fails "llmRequestRouter backend routing requires tls.secretName (o
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt
 
@@ -368,7 +368,7 @@ assert_render_fails "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyP
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.tls.secretName=stargate-quic-tls \
   --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \
@@ -388,7 +388,7 @@ assert_render_fails "llmRequestRouter.tls.mountPath must match the directory con
   --set llmRequestRouter.image.repository=nvcf/stargate \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \
   --set llmRequestRouter.tls.secretName=stargate-quic-tls \
   --set llmRequestRouter.tls.mountPath=/var/run/stargate \
@@ -402,7 +402,7 @@ single_replica="$(helm template llm-request-router "$chart_dir" \
   --set llmRequestRouter.replicaCount=1 \
   --set llmRequestRouter.backendRouter.enabled=true \
   --set llmRequestRouter.backendRouter.image.tag=next \
-  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \
+  --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=https://llm-router.example.invalid:443 \
   --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080)"
 if ! grep -Fq -- "--advertised-hostname-template={pod_name}.llm-request-router-headless.nvcf.svc.cluster.local" <<<"$single_replica"; then
   echo "FAIL: backend routing must retain per-pod authority and SNI for one replica" >&2