fix(nats): extra accounts (#13)

match existing secret format

add local e2e test coverage

Signed-off-by: Frank Spitulski <fspitulski@nvidia.com>

Author: FrankSpitulski

deploy/README.md

@@ -151,8 +151,11 @@ CSC-side secrets to authorize incoming CPC leaf connections.
 
 | Secret | Keys | Purpose |
 |--------|------|---------|
-| `nats-leaf-csc` | seed, pubkey | CPC to CSC leaf (CPC only) |
-| `nats-leaf-cpc-{id}` | seed, pubkey | CPC leaf key pair (CSC only, pubkey via auth-callout.extraEnvs) |
+| `nats-leaf-csc` | seed | CPC to CSC leaf (CPC only) |
+| `nats-leaf-cpc-{id}` | pubkey | CPC leaf user (CSC only, via auth-callout.extraEnvs) |
+| `nats-leaf-{account}-csc` | seed | Extra-account CPC to CSC leaf (CPC only) |
+| `nats-leaf-{account}-cpc-{id}` | pubkey | Extra-account CPC leaf user (CSC only, via auth-callout.extraEnvs) |
+| `nats-{account}-client` | seed, pubkey | Generated extra-account NKey client credential; only active when added to permissions |
 
 ### Generating Secrets
 
@@ -167,17 +170,26 @@ An example script is provided to generate all required secrets to local files:
 
 # Options:
 #   -o, --output DIR         Output root directory (default: deploy/secrets)
+#       --extra-account NAME Generate client and CPC-to-CSC leaf keys for an extra account
 #   -h, --help               Show help message
 
 # Examples:
 ./scripts/generate-nkeys.sh                    # Generate CSC only
 ./scripts/generate-nkeys.sh 1 2 3              # Generate CSC, CPC-1, CPC-2, CPC-3
+./scripts/generate-nkeys.sh --extra-account LaunchLayer 1 2
 ./scripts/generate-nkeys.sh -o ./my-secrets 4  # Generate CSC and CPC-4 under ./my-secrets
 ```
 
-The script is additive. Existing cluster outputs are left unchanged. For each
-requested CPC, the same CPC-to-CSC leaf key pair is stored in CSC as
-`nats-leaf-cpc-{id}` and in that CPC as `nats-leaf-csc`.
+The script requires `nsc` and `nk` on `PATH`.
+
+The script is additive. Existing cluster outputs are left unchanged except for
+unused leaf credential keys from older layouts. For each requested CPC, the
+CPC-to-CSC leaf seed is stored in that CPC as `nats-leaf-csc`; CSC stores only
+the matching public key as `nats-leaf-cpc-{id}`. For each extra account, the
+same split is used with `nats-leaf-{account}-csc` on the CPC and
+`nats-leaf-{account}-cpc-{id}` on CSC. Each cluster also gets a
+`nats-{account}-client` NKey pair for explicit client permissions and local
+functional tests; leaf credentials should not be reused as client credentials.
 
 Generated secret files are written with mode `0600`, and generated secret
 directories are written with mode `0700`. Treat the full output directory as
@@ -197,7 +209,9 @@ deploy/secrets/
 │       ├── nats-mtls-sys-leaf/{seed,pubkey}
 │       ├── nats-surveyor/{seed,pubkey}
 │       ├── auth-callout-keys/{nkey-seed,issuer-seed,xkey-seed}
-│       └── nats-leaf-cpc-{id}/{seed,pubkey}
+│       ├── nats-{account}-client/{seed,pubkey}
+│       ├── nats-leaf-cpc-{id}/pubkey
+│       └── nats-leaf-{account}-cpc-{id}/pubkey
 └── cpc-{id}/
     └── nkeys/
         ├── nats-auth-signing/{seed,pubkey}
@@ -209,7 +223,9 @@ deploy/secrets/
         ├── nats-mtls-sys-leaf/{seed,pubkey}
         ├── nats-surveyor/{seed,pubkey}
         ├── auth-callout-keys/{nkey-seed,issuer-seed,xkey-seed}
-        └── nats-leaf-csc/{seed,pubkey}
+        ├── nats-{account}-client/{seed,pubkey}
+        ├── nats-leaf-csc/seed
+        └── nats-leaf-{account}-csc/seed
 ```
 
 ## Chart Dependencies
@@ -351,7 +367,22 @@ eventBus:
     Kiwi: {}  # Minimal account with defaults
 ```
 
-All properties are passed through to the NATS account configuration. On CPCs, `jetstream` is always forced to `false` and a JetStream domain mapping to CSC is automatically added.
+Extra account names must use letters and numbers only, start with a letter, and
+must not use built-in account names (`SYS`, `AUTH`, `AUTHX`, `CSC`, `CPC`) or
+start with `cpc` in any case.
+All properties are passed through to the NATS account configuration except
+`enabled`. On CPCs, `jetstream` is always forced to `false` and a JetStream
+domain mapping to CSC is automatically added. Every enabled extra account gets a
+CPC-to-CSC leaf connection by default. Provide the CPC seed env
+`LEAF_{ACCOUNT}_USER_SEED` from `nats-leaf-{account}-csc` and the CSC pubkey env
+`NKEY_LEAF_{ACCOUNT}_CPC_{id}_PUBKEY` from `nats-leaf-{account}-cpc-{id}`.
+The chart fails rendering if those explicit secret refs are missing, point at a
+different secret/key, or are marked optional.
+
+The chart generates `eventBus.auth.permissions.nkey` entries named
+`leaf-cpc-{id}` and `leaf-{account}-cpc-{id}` for leaf authorization. Do not
+define manual NKey permission entries with those names; use distinct names for
+operator-managed clients.
 
 ### mTLS MQTT Endpoint
 

deploy/nats-event-bus/templates/_helpers.tpl

@@ -14,6 +14,52 @@ CPC
 {{- end -}}
 {{- end -}}
 
+{{/*
+extraAccountEnvName: Converts an extra account name into a stable env-var token.
+*/}}
+{{- define "nats-event-bus.extraAccountEnvName" -}}
+{{- regexReplaceAll "[^A-Z0-9]" (upper .) "_" -}}
+{{- end -}}
+
+{{/*
+extraAccountSecretName: Converts an extra account name into a stable Kubernetes
+secret-name token.
+*/}}
+{{- define "nats-event-bus.extraAccountSecretName" -}}
+{{- trimAll "-" (regexReplaceAll "[^a-z0-9-]" (lower .) "-") -}}
+{{- end -}}
+
+{{/*
+validateSecretKeyRef: Fails rendering unless a value is a required
+valueFrom.secretKeyRef pointing at the expected secret key.
+*/}}
+{{- define "nats-event-bus.validateSecretKeyRef" -}}
+{{- $value := .value -}}
+{{- $path := .path -}}
+{{- $secretName := .secretName -}}
+{{- $key := .key -}}
+{{- if not (kindIs "map" $value) -}}
+{{- fail (printf "%s must be a valueFrom.secretKeyRef for secret %s key %s." $path $secretName $key) -}}
+{{- end -}}
+{{- $valueFrom := get $value "valueFrom" | default dict -}}
+{{- if not (kindIs "map" $valueFrom) -}}
+{{- fail (printf "%s.valueFrom must be set to secretKeyRef for secret %s key %s." $path $secretName $key) -}}
+{{- end -}}
+{{- $secretKeyRef := get $valueFrom "secretKeyRef" | default dict -}}
+{{- if not (kindIs "map" $secretKeyRef) -}}
+{{- fail (printf "%s.valueFrom.secretKeyRef must reference secret %s key %s." $path $secretName $key) -}}
+{{- end -}}
+{{- if ne (get $secretKeyRef "name") $secretName -}}
+{{- fail (printf "%s.valueFrom.secretKeyRef.name must be %s." $path $secretName) -}}
+{{- end -}}
+{{- if ne (get $secretKeyRef "key") $key -}}
+{{- fail (printf "%s.valueFrom.secretKeyRef.key must be %s." $path $key) -}}
+{{- end -}}
+{{- if eq (toString (get $secretKeyRef "optional")) "true" -}}
+{{- fail (printf "%s.valueFrom.secretKeyRef.optional must not be true; missing leaf secrets must fail fast." $path) -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 natsConfFields: Renders a NATS configuration block body from a flat map.
 Strings are quoted; booleans and numbers are unquoted.

deploy/nats-event-bus/templates/auth-callout-permissions.yaml

@@ -20,6 +20,22 @@ Pubkeys use ${ENV_VAR} syntax for runtime expansion from secrets.
 {{- range .Values.eventBus.cpcIds }}
 {{- $autoNkeys = set $autoNkeys (printf "leaf-cpc-%s" .) (dict "public_key" (printf "${NKEY_LEAF_CPC_%s_PUBKEY}" .) "account" "CSC") }}
 {{- end }}
+{{- range $accountName, $config := .Values.eventBus.extraAccounts }}
+{{- $enabled := true -}}
+{{- if hasKey $config "enabled" -}}
+{{- $enabled = $config.enabled -}}
+{{- end -}}
+{{- if $enabled }}
+{{- $accountEnvName := include "nats-event-bus.extraAccountEnvName" $accountName -}}
+{{- $accountSecretName := include "nats-event-bus.extraAccountSecretName" $accountName -}}
+{{- $pubkeyEnvPrefix := printf "NKEY_LEAF_%s_CPC" $accountEnvName -}}
+{{- range $.Values.eventBus.cpcIds }}
+{{- $cpcId := toString . -}}
+{{- $pubkeyEnv := printf "%s_%s_PUBKEY" $pubkeyEnvPrefix $cpcId -}}
+{{- $autoNkeys = set $autoNkeys (printf "leaf-%s-cpc-%s" $accountSecretName $cpcId) (dict "public_key" (printf "${%s}" $pubkeyEnv) "account" $accountName) }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{- end }}
 {{- $userNkeys := .Values.eventBus.auth.permissions.nkey | default dict -}}
 {{- $mergedNkeys := merge $userNkeys $autoNkeys -}}

deploy/nats-event-bus/templates/nats-accounts-config.yaml

@@ -22,11 +22,19 @@ data:
       jetstream: true
     }
   {{- range $name, $config := $.Values.eventBus.extraAccounts }}
+    {{- $enabled := true -}}
+    {{- if hasKey $config "enabled" -}}
+    {{- $enabled = $config.enabled -}}
+    {{- end -}}
+    {{- if $enabled }}
     {{ $name }} {
       {{- range $key, $value := $config }}
+      {{- if and (ne $key "enabled") (ne $key "leaf") }}
       {{ $key }}: {{ $value }}
       {{- end }}
+      {{- end }}
     }
+    {{- end }}
   {{- end }}
 {{ else if eq .Values.eventBus.clusterType "cpc" }}
 {{- $clusterId := .Values.eventBus.clusterId -}}
@@ -79,16 +87,22 @@ data:
       ]
     }
   {{- range $name, $config := $.Values.eventBus.extraAccounts }}
+    {{- $enabled := true -}}
+    {{- if hasKey $config "enabled" -}}
+    {{- $enabled = $config.enabled -}}
+    {{- end -}}
+    {{- if $enabled }}
     {{ $name }} {
       jetstream: false
       mappings: {
         "$JS.API.>": "$JS.CSC.API.>"
       }
       {{- range $key, $value := $config }}
-      {{- if ne $key "jetstream" }}
+      {{- if and (ne $key "jetstream") (ne $key "enabled") (ne $key "leaf") }}
       {{ $key }}: {{ $value }}
       {{- end }}
       {{- end }}
     }
+    {{- end }}
   {{- end }}
 {{- end }}

deploy/nats-event-bus/templates/nats-leafnodes-config.yaml

@@ -30,5 +30,23 @@ data:
         {{- if not (empty $remoteTls) }}
 {{ include "nats-event-bus.natsConfBlock" (dict "name" "tls" "fields" $remoteTls) | nindent 8 }}{{- end }}
       }
+{{- range $name, $config := $.Values.eventBus.extraAccounts }}
+{{- $enabled := true -}}
+{{- if hasKey $config "enabled" -}}
+{{- $enabled = $config.enabled -}}
+{{- end -}}
+{{- if $enabled }}
+{{- $accountEnvName := include "nats-event-bus.extraAccountEnvName" $name -}}
+{{- $seedEnv := printf "LEAF_%s_USER_SEED" $accountEnvName -}}
+{{ printf "\n" }}
+      {
+        urls: [{{ $.Values.eventBus.cscEndpoint | quote }}]
+        nkey: ${{ $seedEnv }}
+        account: {{ $name | quote }}
+        {{- if not (empty $remoteTls) }}
+{{ include "nats-event-bus.natsConfBlock" (dict "name" "tls" "fields" $remoteTls) | nindent 8 }}{{- end }}
+      }
+{{- end }}
+{{- end }}
     ]
 {{- end }}

deploy/nats-event-bus/templates/validation.yaml

@@ -7,12 +7,100 @@ This template emits no resources; it fails rendering on unsafe combinations.
 */}}
 {{- $authCallout := index .Values "auth-callout" | default dict -}}
 {{- $extraEnvs := get $authCallout "extraEnvs" | default dict -}}
+{{- $nats := .Values.nats | default dict -}}
+{{- $natsContainer := get $nats "container" | default dict -}}
+{{- $natsEnvs := get $natsContainer "env" | default dict -}}
+{{- $extraAccountEnvNames := dict -}}
+{{- $extraAccountSecretNames := dict -}}
+{{- $reservedAccountNames := dict "SYS" true "AUTH" true "AUTHX" true "CSC" true "CPC" true -}}
+{{- $cpcIds := dict -}}
+{{- range .Values.eventBus.cpcIds }}
+{{- $cpcId := toString . -}}
+{{- if not (regexMatch "^[a-z0-9]+$" $cpcId) -}}
+{{- fail (printf "eventBus.cpcIds includes %q, but CPC IDs must use lower-case letters and numbers only so generated secret names and env vars are valid." $cpcId) -}}
+{{- end -}}
+{{- if hasKey $cpcIds $cpcId -}}
+{{- fail (printf "eventBus.cpcIds includes duplicate ID %q." $cpcId) -}}
+{{- end -}}
+{{- $_ := set $cpcIds $cpcId true -}}
+{{- end -}}
+{{- range $accountName, $config := .Values.eventBus.extraAccounts }}
+{{- $enabled := true -}}
+{{- if hasKey $config "enabled" -}}
+{{- $enabled = $config.enabled -}}
+{{- end -}}
+{{- if $enabled -}}
+{{- if not (regexMatch "^[A-Za-z][A-Za-z0-9]*$" $accountName) -}}
+{{- fail (printf "eventBus.extraAccounts.%s is not a valid NATS account name. Use letters and numbers only, starting with a letter." $accountName) -}}
+{{- end -}}
+{{- if hasKey $reservedAccountNames (upper $accountName) -}}
+{{- fail (printf "eventBus.extraAccounts.%s conflicts with a built-in NATS account name." $accountName) -}}
+{{- end -}}
+{{- if regexMatch "^CPC" (upper $accountName) -}}
+{{- fail (printf "eventBus.extraAccounts.%s is not allowed because extra account names must not start with cpc." $accountName) -}}
+{{- end -}}
+{{- $envToken := include "nats-event-bus.extraAccountEnvName" $accountName -}}
+{{- $secretToken := include "nats-event-bus.extraAccountSecretName" $accountName -}}
+{{- if eq $envToken "" -}}
+{{- fail (printf "eventBus.extraAccounts includes %q, but its env token is empty after normalization." $accountName) -}}
+{{- end -}}
+{{- if eq $secretToken "" -}}
+{{- fail (printf "eventBus.extraAccounts includes %q, but its secret-name token is empty after normalization." $accountName) -}}
+{{- end -}}
+{{- if hasKey $extraAccountEnvNames $envToken -}}
+{{- fail (printf "eventBus.extraAccounts.%s normalizes to env token %s, which is already used by eventBus.extraAccounts.%s." $accountName $envToken (get $extraAccountEnvNames $envToken)) -}}
+{{- end -}}
+{{- if hasKey $extraAccountSecretNames $secretToken -}}
+{{- fail (printf "eventBus.extraAccounts.%s normalizes to secret token %s, which is already used by eventBus.extraAccounts.%s." $accountName $secretToken (get $extraAccountSecretNames $secretToken)) -}}
+{{- end -}}
+{{- $_ := set $extraAccountEnvNames $envToken $accountName -}}
+{{- $_ := set $extraAccountSecretNames $secretToken $accountName -}}
+{{- end -}}
+{{- end -}}
 {{- if and (eq .Values.eventBus.clusterType "csc") (gt (len .Values.eventBus.cpcIds) 0) -}}
 {{- range .Values.eventBus.cpcIds }}
 {{- $cpcId := toString . -}}
 {{- $envName := printf "NKEY_LEAF_CPC_%s_PUBKEY" $cpcId -}}
 {{- if not (hasKey $extraEnvs $envName) -}}
 {{- fail (printf "eventBus.cpcIds includes %q, but auth-callout.extraEnvs.%s is missing. Add a secretKeyRef to nats-leaf-cpc-%s key pubkey so CSC can authorize the CPC leaf connection." $cpcId $envName $cpcId) -}}
 {{- end -}}
+{{- include "nats-event-bus.validateSecretKeyRef" (dict "value" (get $extraEnvs $envName) "path" (printf "auth-callout.extraEnvs.%s" $envName) "secretName" (printf "nats-leaf-cpc-%s" $cpcId) "key" "pubkey") -}}
+{{- end -}}
+{{- end -}}
+{{- if eq .Values.eventBus.clusterType "cpc" -}}
+{{- range $accountName, $config := .Values.eventBus.extraAccounts }}
+{{- $enabled := true -}}
+{{- if hasKey $config "enabled" -}}
+{{- $enabled = $config.enabled -}}
+{{- end -}}
+{{- if $enabled -}}
+{{- $accountEnvName := include "nats-event-bus.extraAccountEnvName" $accountName -}}
+{{- $accountSecretName := include "nats-event-bus.extraAccountSecretName" $accountName -}}
+{{- $seedEnvName := printf "LEAF_%s_USER_SEED" $accountEnvName -}}
+{{- if not (hasKey $natsEnvs $seedEnvName) -}}
+{{- fail (printf "eventBus.extraAccounts.%s is enabled on a CPC cluster, but nats.container.env.%s is missing. Add a secretKeyRef to nats-leaf-%s-csc key seed so the CPC can connect this account leaf to CSC." $accountName $seedEnvName $accountSecretName) -}}
+{{- end -}}
+{{- include "nats-event-bus.validateSecretKeyRef" (dict "value" (get $natsEnvs $seedEnvName) "path" (printf "nats.container.env.%s" $seedEnvName) "secretName" (printf "nats-leaf-%s-csc" $accountSecretName) "key" "seed") -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- if and (eq .Values.eventBus.clusterType "csc") (gt (len .Values.eventBus.cpcIds) 0) -}}
+{{- range $accountName, $config := .Values.eventBus.extraAccounts }}
+{{- $enabled := true -}}
+{{- if hasKey $config "enabled" -}}
+{{- $enabled = $config.enabled -}}
+{{- end -}}
+{{- if $enabled -}}
+{{- $accountEnvName := include "nats-event-bus.extraAccountEnvName" $accountName -}}
+{{- $accountSecretName := include "nats-event-bus.extraAccountSecretName" $accountName -}}
+{{- range $.Values.eventBus.cpcIds }}
+{{- $cpcId := toString . -}}
+{{- $envName := printf "NKEY_LEAF_%s_CPC_%s_PUBKEY" $accountEnvName $cpcId -}}
+{{- if not (hasKey $extraEnvs $envName) -}}
+{{- fail (printf "eventBus.extraAccounts.%s is enabled on CSC, but auth-callout.extraEnvs.%s is missing. Add a secretKeyRef to nats-leaf-%s-cpc-%s key pubkey so CSC can authorize this extra-account CPC leaf." $accountName $envName $accountSecretName $cpcId) -}}
+{{- end -}}
+{{- include "nats-event-bus.validateSecretKeyRef" (dict "value" (get $extraEnvs $envName) "path" (printf "auth-callout.extraEnvs.%s" $envName) "secretName" (printf "nats-leaf-%s-cpc-%s" $accountSecretName $cpcId) "key" "pubkey") -}}
+{{- end -}}
+{{- end -}}
 {{- end -}}
 {{- end -}}