OSMO-6648 - Move service auth to Kubernetes Secret

Store the stable service authentication identity in an externally managed Kubernetes Secret, migrate existing PostgreSQL-backed identities safely, and wire the unified OSMO chart as the sole deployment interface.

Author: vvnpn-nv

bzl/tests/BUILD

@@ -86,6 +86,19 @@ py_test(
     main = "python_image_runfiles_dedup_test.py",
 )
 
+py_test(
+    name = "service_auth_bootstrap_python_runfiles_dedup_test",
+    srcs = ["python_image_runfiles_dedup_test.py"],
+    args = [
+        "$(rootpath //src/service/core:service_image_x86_64)",
+        "src/service/core/service_binary.runfiles",
+        "usr/bin/service-auth-bootstrap",
+        "src/service/core/service_auth_bootstrap.py",
+    ],
+    data = ["//src/service/core:service_image_x86_64"],
+    main = "python_image_runfiles_dedup_test.py",
+)
+
 py_test(
     name = "mcp_python_runfiles_dedup_test",
     srcs = ["python_image_runfiles_dedup_test.py"],

deployments/charts/osmo/README.md

@@ -30,7 +30,8 @@ It installs:
 - the compute backend listener and worker;
 - persistent CloudNativePG, Valkey, and RustFS instances;
 - generated development credentials, object-storage buckets, configuration,
-  and a CPU-only default pool.
+  and a CPU-only default pool. Service auth is generated explicitly before
+  installation so every process uses the same identity.
 
 ### Prerequisites
 
@@ -62,9 +63,22 @@ helm --kube-context kind-osmo upgrade --install cnpg cnpg/cloudnative-pg \
 
 ### Install OSMO
 
-Install the unified chart with the single quick-start values file:
+Generate the shared development service-auth identity, create its Secret, then
+install the unified chart with the single quick-start values file:
 
 ```bash
+OSMO_SERVICE_AUTH_DIRECTORY="$(mktemp -d)"
+docker run --rm --user "$(id -u):$(id -g)" \
+  --entrypoint service-auth-bootstrap \
+  --volume "${OSMO_SERVICE_AUTH_DIRECTORY}:/output" \
+  nvcr.io/nvidia/osmo/service:latest \
+  generate --output /output/authentication-config.json
+kubectl --context kind-osmo create namespace osmo \
+  --dry-run=client --output=yaml | kubectl --context kind-osmo apply -f -
+kubectl --context kind-osmo --namespace osmo create secret generic \
+  osmo-service-auth \
+  --from-file="authentication-config.json=${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+
 helm dependency build deployments/charts/osmo
 helm --kube-context kind-osmo upgrade --install osmo deployments/charts/osmo \
   --namespace osmo \
@@ -73,6 +87,9 @@ helm --kube-context kind-osmo upgrade --install osmo deployments/charts/osmo \
   --set-string compute.backendName=default \
   --wait \
   --timeout 20m
+
+rm "${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+rmdir "${OSMO_SERVICE_AUTH_DIRECTORY}"
 ```
 
 Inspect the release without reading generated Secret values:
@@ -197,9 +214,22 @@ kubectl --context kind-osmo --namespace cnpg-system rollout status \
   deployment/cnpg-cloudnative-pg --timeout=10m
 ```
 
-Install OSMO with one values file:
+Generate and create the shared service-auth Secret, then install OSMO with one
+values file:
 
 ```bash
+OSMO_SERVICE_AUTH_DIRECTORY="$(mktemp -d)"
+docker run --rm --user "$(id -u):$(id -g)" \
+  --entrypoint service-auth-bootstrap \
+  --volume "${OSMO_SERVICE_AUTH_DIRECTORY}:/output" \
+  nvcr.io/nvidia/osmo/service:latest \
+  generate --output /output/authentication-config.json
+kubectl --context kind-osmo create namespace osmo \
+  --dry-run=client --output=yaml | kubectl --context kind-osmo apply -f -
+kubectl --context kind-osmo --namespace osmo create secret generic \
+  osmo-service-auth \
+  --from-file="authentication-config.json=${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+
 helm dependency build deployments/charts/osmo
 helm --kube-context kind-osmo upgrade --install osmo deployments/charts/osmo \
   --namespace osmo \
@@ -208,6 +238,9 @@ helm --kube-context kind-osmo upgrade --install osmo deployments/charts/osmo \
   --set-string compute.backendName=default \
   --wait \
   --timeout 20m
+
+rm "${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+rmdir "${OSMO_SERVICE_AUTH_DIRECTORY}"
 ```
 
 Check the database, workloads, storage, and Services without reading Secret
@@ -219,7 +252,7 @@ kubectl --context kind-osmo --namespace osmo wait \
 kubectl --context kind-osmo --namespace osmo get pods,pvc,services,jobs
 kubectl --context kind-osmo --namespace osmo get secret \
   osmo-backend-token osmo-master-encryption-key osmo-valkey-credentials \
-  osmo-rustfs-credentials
+  osmo-rustfs-credentials osmo-service-auth
 ```
 
 In one terminal, forward the gateway:
@@ -591,6 +624,7 @@ may reference a separate Secret. The defaults expect these keys:
 | `secrets.objectStorage` | `object-storage.yaml` | Workflow data, logs, and apps |
 | `secrets.masterEncryptionKey` | `mek.yaml` | OSMO encryption-key configuration |
 | `secrets.backendApiTokens.credentials[]` | `token`, optional `previous-token` | Backend authentication |
+| `secrets.serviceAuth` | `authentication-config.json` | Stable JWT signing identity |
 
 Generated backend-token and MEK Secrets are intentionally retained because
 replacing either can disconnect the compute plane or make encrypted database
@@ -620,6 +654,109 @@ postgresql:
         name: osmo-postgresql-credentials
 ```
 
+### Service auth identity
+
+The OSMO JWT signing identity is installation-scoped secret material. Every
+control-plane installation requires an externally persisted Kubernetes Secret
+containing canonical `AuthenticationConfig` JSON under
+`authentication-config.json`. The chart references and mounts this Secret but
+never renders its private key into Helm values or release state. Runtime
+services do not read or write `service_auth` through PostgreSQL.
+
+For a fresh installation, generate the identity offline with the service image,
+then create the Secret before installing the chart:
+
+```bash
+OSMO_SERVICE_AUTH_DIRECTORY="$(mktemp -d)"
+docker run --rm --user "$(id -u):$(id -g)" \
+  --entrypoint service-auth-bootstrap \
+  --volume "${OSMO_SERVICE_AUTH_DIRECTORY}:/output" \
+  <service-image> \
+  generate --output /output/authentication-config.json
+kubectl create secret generic osmo-service-auth --namespace <namespace> \
+  --from-file="authentication-config.json=${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+rm "${OSMO_SERVICE_AUTH_DIRECTORY}/authentication-config.json"
+rmdir "${OSMO_SERVICE_AUTH_DIRECTORY}"
+```
+
+The generator creates the output with mode `0600`, refuses to overwrite an
+existing file, validates the keypair, and never writes private material to
+stdout. Store and back up the Secret through the installation's normal secret
+management system.
+
+For an existing PostgreSQL-backed installation, first establish a maintenance
+window that prevents the old configuration API from changing `service_auth`.
+Delete its HPA, scale the old API deployment to zero, and verify that no old API
+pod remains before starting the upgrade. Replace the example release and
+namespace if needed.
+
+```bash
+OSMO_RELEASE_NAME=osmo
+OSMO_NAMESPACE=osmo
+OSMO_API_SELECTOR="app.kubernetes.io/instance=${OSMO_RELEASE_NAME},app.kubernetes.io/component=api"
+OSMO_API_DEPLOYMENT="$(kubectl --namespace "${OSMO_NAMESPACE}" get deployment \
+  --selector "${OSMO_API_SELECTOR}" \
+  --output=jsonpath='{.items[0].metadata.name}')"
+test -n "${OSMO_API_DEPLOYMENT}"
+kubectl --namespace "${OSMO_NAMESPACE}" delete hpa \
+  --selector "${OSMO_API_SELECTOR}" --ignore-not-found
+kubectl --namespace "${OSMO_NAMESPACE}" scale deployment \
+  "${OSMO_API_DEPLOYMENT}" --replicas=0
+kubectl --namespace "${OSMO_NAMESPACE}" rollout status deployment \
+  "${OSMO_API_DEPLOYMENT}" --timeout=5m
+if kubectl --namespace "${OSMO_NAMESPACE}" get pods \
+  --selector "${OSMO_API_SELECTOR}" \
+  --output=name | grep -q .; then
+  echo "old API pods still exist; do not continue" >&2
+  exit 1
+fi
+```
+
+With writers stopped, pre-provision an empty Secret and authorize it for the
+exact Helm release:
+
+```bash
+kubectl create secret generic osmo-service-auth \
+  --namespace "${OSMO_NAMESPACE}"
+kubectl annotate secret osmo-service-auth \
+  --namespace "${OSMO_NAMESPACE}" \
+  "osmo.nvidia.com/service-auth-bootstrap-placeholder=${OSMO_RELEASE_NAME}"
+```
+
+Upgrade with `secrets.serviceAuth.existingSecret.name=osmo-service-auth` and
+`secrets.serviceAuth.migration.enabled=true`. A pre-upgrade Job reads and
+decrypts the legacy DB identity, validates every public/private keypair, and
+writes canonical plaintext JSON into the authorized placeholder. It then reads
+the DB identity again and aborts if the stable authority changed during the
+migration. An already populated Secret is preserved only when its complete
+stable identity matches. Temporary hook RBAC grants only `get` and `update` on
+that named Secret and is removed after the hook completes.
+
+The 6.4 workloads start only after the hook succeeds and read the copied
+identity exclusively from the mounted Secret, so existing tokens remain valid.
+Wait for the Secret-backed API deployment to become ready and confirm that Helm
+has recreated its HPA when autoscaling is enabled. After the successful upgrade,
+disable `migration.enabled`. Retain the legacy DB row and its MEK through the
+rollback window so an older binary can still use the same identity; 6.4 runtime
+services ignore that row.
+
+```bash
+kubectl --namespace "${OSMO_NAMESPACE}" rollout status deployment \
+  "${OSMO_API_DEPLOYMENT}" --timeout=10m
+kubectl --namespace "${OSMO_NAMESPACE}" get hpa \
+  --selector "${OSMO_API_SELECTOR}"
+```
+
+The migration uses `secrets.masterEncryptionKey` to decrypt legacy MEK/JWE
+values. Missing, malformed, changing, or mismatched identity data fails closed
+without generating a replacement key. `login_info` remains deployment-derived
+and is overlaid only in memory.
+
+Intentional key rotation requires a staged keyset rollout: add the new key,
+roll all consumers, switch `active_key`, retain the old verification key until
+all tokens it signed have expired, and remove it in a later rollout. Change
+`rolloutNonce` on each Secret update.
+
 The MEK is mounted through the typed
 `secrets.masterEncryptionKey.existingSecret.{name,key}` reference. Use
 `managementMode: external` for an operator-owned read-only Secret. Use

deployments/charts/osmo/profiles/README.md

@@ -11,8 +11,8 @@ values take precedence.
 
 | File | Directly installable | Required environment input |
 | --- | --- | --- |
-| `quickstart.yaml` | Yes, on a development cluster | KAI Scheduler, the CloudNativePG operator, and a default dynamic StorageClass installed separately; `compute.backendName` set explicitly at install time |
-| `kind-self-contained.yaml` | Yes, on kind | KAI Scheduler and the CloudNativePG operator installed separately; `compute.backendName` set explicitly at install time |
+| `quickstart.yaml` | Yes, on a development cluster | KAI Scheduler, the CloudNativePG operator, and a default dynamic StorageClass installed separately; `osmo-service-auth` generated and created as documented; `compute.backendName` set explicitly at install time |
+| `kind-self-contained.yaml` | Yes, on kind | KAI Scheduler and the CloudNativePG operator installed separately; `osmo-service-auth` generated and created as documented; `compute.backendName` set explicitly at install time |
 | `split-plane-control.yaml` | Base overlay | PostgreSQL, Valkey, and object-storage endpoints; Kubernetes Secrets; and `externalUrl` |
 | `split-plane-compute.yaml` | Base overlay | A control-plane `externalUrl`, a compute authentication Secret, and `compute.backendName` set explicitly at install time |
 
@@ -21,7 +21,8 @@ for browser, CLI, and CPU hello-world verification. It exposes the UI and API
 through gateway NodePort `30080` while omitting other optional services. The kind
 profile retains a broader local-development surface. Both profiles are
 development-only and intentionally use `latest` OSMO images, one replica per
-component, generated credentials, and embedded stateful dependencies. The split
+component, explicitly generated service auth, other generated credentials, and
+embedded stateful dependencies. The split
 profiles contain example names and endpoints; copy them into an environment
 values file before installation.
 

deployments/charts/osmo/profiles/kind-self-contained.yaml

@@ -50,6 +50,10 @@ secrets:
       key: mek.yaml
     bootstrap:
       enabled: true
+  serviceAuth:
+    existingSecret:
+      name: osmo-service-auth
+      key: authentication-config.json
 
 services:
   mcp:

deployments/charts/osmo/profiles/quickstart.yaml

@@ -41,8 +41,16 @@ secrets:
       managedSecret:
         name: osmo-backend-token
   masterEncryptionKey:
-    generate: true
-    existingSecret: ''
+    managementMode: osmo
+    existingSecret:
+      name: osmo-master-encryption-key
+      key: mek.yaml
+    bootstrap:
+      enabled: true
+  serviceAuth:
+    existingSecret:
+      name: osmo-service-auth
+      key: authentication-config.json
 
 services:
   ui:

deployments/charts/osmo/templates/_helpers.tpl

@@ -412,6 +412,33 @@ osmo.nvidia.com/mek-rollout: {{ . | quote }}
 {{- end }}
 {{- end -}}
 
+{{- define "osmo.secrets.serviceAuthRolloutAnnotation" -}}
+osmo.nvidia.com/service-auth-rollout: {{ .Values.secrets.serviceAuth.rolloutNonce | quote }}
+{{- end -}}
+
+{{- define "osmo.secrets.serviceAuthArgs" -}}
+- --service_auth_file
+- /etc/osmo/service-auth/authentication-config.json
+{{- end -}}
+
+{{- define "osmo.secrets.serviceAuthVolumeMount" -}}
+- name: service-auth
+  mountPath: /etc/osmo/service-auth/authentication-config.json
+  subPath: authentication-config.json
+  readOnly: true
+{{- end -}}
+
+{{- define "osmo.secrets.serviceAuthVolume" -}}
+{{- $serviceAuth := .Values.secrets.serviceAuth -}}
+- name: service-auth
+  secret:
+    defaultMode: 292
+    secretName: {{ required "secrets.serviceAuth.existingSecret.name is required for the control plane" $serviceAuth.existingSecret.name | quote }}
+    items:
+    - key: {{ required "secrets.serviceAuth.existingSecret.key is required" $serviceAuth.existingSecret.key | quote }}
+      path: authentication-config.json
+{{- end -}}
+
 {{- define "osmo.valkey.fullname" -}}
 {{- $name := default "valkey" (dig "nameOverride" "" .Values.valkey) -}}
 {{- $fullnameOverride := dig "fullnameOverride" "" .Values.valkey -}}

deployments/charts/osmo/templates/agent-service.yaml

@@ -47,6 +47,7 @@ spec:
       annotations:
         {{- include "osmo.pod.annotations" (dict "root" . "component" .Values.services.agent) | nindent 8 }}
         {{- include "osmo.secrets.mekRolloutAnnotation" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthRolloutAnnotation" . | nindent 8 }}
     spec:
       automountServiceAccountToken: {{ .Values.services.agent.serviceAccount.automountServiceAccountToken }}
       securityContext:
@@ -125,6 +126,7 @@ spec:
         - {{ .Values.logging.logFormat | default "text" }}
         {{- end }}
         {{- include "osmo.configuration.args" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthArgs" . | nindent 8 }}
         {{- range $arg := .Values.services.agent.extraArgs }}
         - {{ $arg | quote }}
         {{- end }}
@@ -144,6 +146,7 @@ spec:
           name: mek-volume
           readOnly: true
         {{- end }}
+        {{- include "osmo.secrets.serviceAuthVolumeMount" . | nindent 8 }}
         {{- include "osmo.configuration.volumeMounts" . | nindent 8 }}
         {{- include "osmo.externalDependencies.caVolumeMounts" . | nindent 8 }}
         {{- include "osmo.component.extraVolumeMounts" .Values.services.agent | nindent 8 }}
@@ -183,6 +186,7 @@ spec:
         {{- include "osmo.pod.extraVolumes" .Values.services.agent | nindent 8 }}
         {{- include "osmo.gateway.upstreamTlsVolume" (dict "context" . "secretName" .Values.gateway.tls.upstreamCerts.agent) | nindent 8 }}
         {{- include "osmo.secrets.mekVolume" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthVolume" . | nindent 8 }}
         {{- include "osmo.configuration.volumes" . | nindent 8 }}
         {{- include "osmo.externalDependencies.caVolumes" . | nindent 8 }}
 

deployments/charts/osmo/templates/api-service.yaml

@@ -48,6 +48,7 @@ spec:
       annotations:
         {{- include "osmo.pod.annotations" (dict "root" . "component" .Values.services.api "protectedAnnotations" $protectedPodAnnotations) | nindent 8 }}
         {{- include "osmo.secrets.mekRolloutAnnotation" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthRolloutAnnotation" . | nindent 8 }}
     spec:
       automountServiceAccountToken: {{ .Values.services.api.serviceAccount.automountServiceAccountToken }}
       securityContext:
@@ -155,6 +156,7 @@ spec:
         - /etc/osmo/backend-tokens
         {{- end }}
         {{- include "osmo.configuration.args" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthArgs" . | nindent 8 }}
         {{- range $arg := .Values.services.api.extraArgs }}
         - {{ $arg | quote }}
         {{- end }}
@@ -186,6 +188,7 @@ spec:
           name: mek-volume
           readOnly: true
         {{- end }}
+        {{- include "osmo.secrets.serviceAuthVolumeMount" . | nindent 8 }}
         {{- if .Values.secrets.backendApiTokens.enabled }}
         {{- range .Values.secrets.backendApiTokens.credentials }}
         - name: backend-token-{{ .name }}
@@ -220,6 +223,7 @@ spec:
         {{- include "osmo.pod.extraVolumes" .Values.services.api | nindent 8 }}
         {{- include "osmo.gateway.upstreamTlsVolume" (dict "context" . "secretName" .Values.gateway.tls.upstreamCerts.api) | nindent 8 }}
         {{- include "osmo.secrets.mekVolume" . | nindent 8 }}
+        {{- include "osmo.secrets.serviceAuthVolume" . | nindent 8 }}
         {{- include "osmo.configuration.volumes" . | nindent 8 }}
         {{- include "osmo.externalDependencies.caVolumes" . | nindent 8 }}
         {{- if .Values.secrets.backendApiTokens.enabled }}