feat: persist provided registry tokens as hdb_secret refs; resolve on every install

Completes the reference-not-embed story: a provided registry token is no longer a
transient, this-node-only credential — it is ingested into the encrypted, replicated
hdb_secret store and referenced everywhere, so package deploys survive rollback,
reboot, and new peers without the operator re-supplying the token.

- ingestRegistryAuth() (secretOperations): a literal `{ registry, token }` is sealed
  into hdb_secret (via set_secret, custody required) under a derived name
  (deploy.<component>.<registry>) granted to the component, and returned as a
  `{ registry, secret }` reference. Idempotent on rotation. Already-reference entries
  and (no-custody) literal tokens pass through untouched.
- deployComponent: ingests up front, then records references in TWO durable places —
  the component config (applicationConfig.registryAuth, read on every cold install) and
  the hdb_deployment row (registry_auth, the rollback source). req.registryAuth now
  replicates as references, never tokens; peers resolve from their own replicated
  hdb_secret copy (resolveRegistryAuth gains a bounded waitMs to cover the row arriving
  just behind the deploy op). No-custody core stays on the transient #1158 fallback.
- installApplications(): resolves applicationConfig.registryAuth at cold install so a
  fresh/wiped node or new peer authenticates from the store (best-effort — logs and
  installs without auth if custody isn't up yet, rather than blocking boot).
- assertApplicationConfig + deployment row + config type: registryAuth is references
  only; a literal token on disk is rejected.

Fabric NPM_CONFIG_USERCONFIG injection (harper-pro) becomes redundant for auth once
peers resolve from the store — a coordinated harper-pro follow-up removes the token
injection; core keeps userconfig *inheritance* for non-auth npm config (proxy/cafile).

Tests: ingest seal/round-trip/passthrough/no-custody/idempotent, derived-name
sanitization, resolve bounded-wait (times-out-404 + replicates-in-mid-wait),
assertApplicationConfig references-only.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Kris Zyp

components/Application.ts

@@ -38,6 +38,10 @@ interface ApplicationConfig {
 		timeout?: number;
 		allowInstallScripts?: boolean;
 	};
+	// Private-registry auth in reference form only — each entry names an hdb_secret row, never a
+	// token. Recorded by deploy_component so every (cold) install — reboot, new peer, rollback —
+	// re-resolves the credential from the store rather than needing it re-supplied.
+	registryAuth?: { registry: string; secret: string; scope?: string }[];
 	// an application config can have other arbitrary properties
 	[key: string]: unknown;
 }
@@ -111,6 +115,28 @@ export function assertApplicationConfig(
 			);
 		}
 	}
+	if ('registryAuth' in applicationConfig && applicationConfig.registryAuth !== undefined) {
+		const entries = applicationConfig.registryAuth;
+		if (!Array.isArray(entries)) {
+			throw new (class InvalidRegistryAuthError extends TypeError {})(
+				`Invalid 'registryAuth' property for application ${applicationName}: expected array, got ${typeof entries}`
+			);
+		}
+		for (const entry of entries) {
+			// Config carries references only — a literal `token` here would mean a plaintext credential
+			// was persisted to disk, which the deploy path is designed to prevent.
+			if (
+				typeof entry !== 'object' ||
+				entry === null ||
+				typeof (entry as any).registry !== 'string' ||
+				typeof (entry as any).secret !== 'string'
+			) {
+				throw new (class InvalidRegistryAuthError extends TypeError {})(
+					`Invalid 'registryAuth' entry for application ${applicationName}: expected { registry, secret, scope? } reference`
+				);
+			}
+		}
+	}
 }
 
 /**
@@ -699,10 +725,30 @@ export async function installApplications() {
 			// This will throw if the config is invalid
 			assertApplicationConfig(name, applicationConfig);
 
+			// Resolve any private-registry auth references from the store so a cold install (fresh
+			// node, wiped components dir, new peer that never installed) can authenticate without the
+			// token being re-supplied. Best-effort: if custody isn't available yet or a referenced
+			// secret is missing, log and install without it (a truly private package then fails in
+			// npm with its own error) rather than blocking boot.
+			let resolvedRegistryAuth: RegistryAuthEntry[] | undefined;
+			if (applicationConfig.registryAuth?.length) {
+				try {
+					const { resolveRegistryAuth } = await import('./secretOperations.ts');
+					resolvedRegistryAuth = (await resolveRegistryAuth(applicationConfig.registryAuth, name)) as
+						| RegistryAuthEntry[]
+						| undefined;
+				} catch (error) {
+					logger.warn?.(
+						`Could not resolve registryAuth for application ${name} at install time: ${(error as Error).message}`
+					);
+				}
+			}
+
 			const application = new Application({
 				name,
 				packageIdentifier: applicationConfig.package,
 				install: applicationConfig.install,
+				registryAuth: resolvedRegistryAuth,
 			});
 
 			// Lock check: only install if not already installed with matching configuration

components/deploymentRecorder.ts

@@ -15,6 +15,7 @@ import { Readable, Transform, pipeline } from 'node:stream';
 import { databases } from '../resources/databases.ts';
 import { createBlob, isSaving, deleteBlob } from '../resources/blob.ts';
 import * as terms from '../utility/hdbTerms.ts';
+import type { RegistryAuthReference } from './secretOperations.ts';
 import { ClientError } from '../utility/errors/hdbError.ts';
 import { logger } from '../utility/logging/logger.ts';
 import { hostname } from 'node:os';
@@ -61,6 +62,10 @@ interface CreateOptions {
 	user?: string;
 	restart_mode?: 'immediate' | 'rolling' | null;
 	rollback_of?: string | null;
+	// Registry-auth in reference form (`{ registry, secret, scope? }`) — never a literal token. Kept
+	// so a rollback can re-resolve the private-registry credential from hdb_secret without the
+	// operator re-supplying it. Null when the deploy used no auth or a no-custody transient token.
+	registry_auth?: RegistryAuthReference[] | null;
 	emitter?: ProgressEmitter;
 }
 
@@ -98,6 +103,7 @@ export class DeploymentRecorder {
 			completed_at: null,
 			user: options.user ?? null,
 			rollback_of: options.rollback_of ?? null,
+			registry_auth: options.registry_auth ?? null,
 			error: null,
 		};
 		const recorder = new DeploymentRecorder(deploymentId, record);
@@ -446,7 +452,7 @@ export class DeploymentRecorder {
 // take well over the original 30s to arrive on a peer (harper-pro#402). 120s matches the
 // blob-stream receive default and gives a loaded cluster room to converge. Override per-deploy
 // via the `deployment_timeout` operation parameter.
-const DEFAULT_AWAIT_ROW_TIMEOUT_MS = 120_000;
+export const DEFAULT_AWAIT_ROW_TIMEOUT_MS = 120_000;
 
 /**
  * Peer-side helper — wait for the hdb_deployment row to arrive via table replication,

components/operations.js

@@ -26,7 +26,11 @@ const { packageDirectory } = require('../components/packageComponent.ts');
 const { Resources } = require('../resources/Resources.ts');
 const { Application, prepareApplication, ASIDE_STAGING_DIR } = require('./Application.ts');
 const { server } = require('../server/Server.ts');
-const { DeploymentRecorder, awaitDeploymentRow } = require('./deploymentRecorder.ts');
+const {
+	DeploymentRecorder,
+	awaitDeploymentRow,
+	DEFAULT_AWAIT_ROW_TIMEOUT_MS,
+} = require('./deploymentRecorder.ts');
 const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts');
 
 /**
@@ -370,6 +374,16 @@ async function deployComponent(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
+	// Ingest any provided registry token into the secrets store so the credential lives as
+	// replicated ciphertext (reference, not embed); already-reference entries pass through, and with
+	// no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers
+	// re-running a replicated deploy already carry references and never re-ingest.
+	const { ingestRegistryAuth, resolveRegistryAuth } = require('./secretOperations.ts');
+	req.registryAuth = await ingestRegistryAuth(req, req.registryAuth, req.project);
+	// References are safe to persist (config + deployment row) and replicate; a no-custody literal
+	// token is not — it is used only for this node's install below, then stripped before replication.
+	const registryAuthReferences = (req.registryAuth ?? []).filter((entry) => entry && entry.secret !== undefined);
+
 	// Write to root config if the request contains a package identifier
 	if (req.package) {
 		// Check if trying to overwrite a core component (requires force)
@@ -393,6 +407,9 @@ async function deployComponent(req) {
 			};
 		}
 		if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath;
+		// Persist registry-auth references (never tokens) so every cold install of this component —
+		// reboot, new peer, rollback — re-resolves the credential from the store.
+		if (registryAuthReferences.length) applicationConfig.registryAuth = registryAuthReferences;
 		await configUtils.addConfig(req.project, applicationConfig);
 	}
 
@@ -417,6 +434,8 @@ async function deployComponent(req) {
 				package_identifier: req.package ?? null,
 				user: req.hdb_user?.username,
 				restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null,
+				// Reference form only — the rollback source for re-resolving registry auth.
+				registry_auth: registryAuthReferences.length ? registryAuthReferences : null,
 				emitter,
 			});
 	if (recorder) req._deploymentId = recorder.deploymentId;
@@ -454,13 +473,18 @@ async function deployComponent(req) {
 			extractionPayload = row.payload_blob.stream();
 		}
 
-		// Resolve any registryAuth entries that reference an hdb_secret row into concrete tokens
-		// (literal `token` entries pass through unchanged). Runs on the main thread, where deploys
-		// dispatch and secrets custody is registered; a bad reference (missing/ungranted/undecryptable)
-		// fails the deploy with a precise error. The resolved tokens stay in-memory for this node's
-		// npm pack/install and are stripped from req below — same transient handling as a literal token.
-		const { resolveRegistryAuth } = require('./secretOperations.ts');
-		const resolvedRegistryAuth = await resolveRegistryAuth(req.registryAuth, req.project);
+		// Resolve registryAuth references into concrete tokens for this node's npm pack/install
+		// (a no-custody literal-token fallback passes through unchanged). On a peer running a
+		// replicated deploy, the referenced hdb_secret row may arrive just behind the deploy op, so
+		// allow a bounded grace period (same budget as the payload-row wait) for it to replicate in.
+		let registryAuthWaitMs = 0;
+		if (isReplicatedExecution) {
+			const requested = Number(req.deployment_timeout);
+			registryAuthWaitMs = Number.isFinite(requested) && requested >= 0 ? requested : DEFAULT_AWAIT_ROW_TIMEOUT_MS;
+		}
+		const resolvedRegistryAuth = await resolveRegistryAuth(req.registryAuth, req.project, {
+			waitMs: registryAuthWaitMs,
+		});
 
 		const application = new Application({
 			name: req.project,
@@ -478,17 +502,16 @@ async function deployComponent(req) {
 				installCapture.push(manager, stream, line);
 				if (emitter) emit('install', { manager, stream, line });
 			},
-			// Private-registry auth (already resolved above), used here for this node's npm
-			// pack/install. The Application ctor captures it into application.registryAuth; we strip
-			// registryAuth from req immediately (below) so neither a literal token nor a secret
-			// reference is persisted or sent to peers — peers authenticate via their own
-			// fabric-injected NPM_CONFIG_USERCONFIG.
+			// Private-registry auth (already resolved above), used here for this node's npm pack/install.
 			registryAuth: resolvedRegistryAuth,
 		});
-		// Strip registryAuth from req immediately after the ctor captures the resolved tokens, so it
-		// can't survive into an error/log path if prepareApplication or loadComponent throws below
-		// (the previous strip point after loadComponent only ran on the success path, leaking on failure).
-		delete req.registryAuth;
+		// Reduce req.registryAuth to references only (never a token) before it can reach an error/log
+		// path or replication: references are what peers resolve from their own replicated hdb_secret
+		// copy; a no-custody literal token is dropped entirely (peers fall back to their fabric-injected
+		// NPM_CONFIG_USERCONFIG, as before). This also fixes the prior success-only strip that leaked a
+		// literal token on a prepare/load failure.
+		if (registryAuthReferences.length) req.registryAuth = registryAuthReferences;
+		else delete req.registryAuth;
 
 		emit('phase', { phase: 'prepare', status: 'start' });
 		await prepareApplication(application);

components/secretOperations.ts

@@ -332,36 +332,120 @@ export interface ResolvedRegistryAuthEntry {
 	scope?: string;
 }
 
+/** A registry-auth entry after ingestion: a reference into hdb_secret, never a token. */
+export interface RegistryAuthReference {
+	registry: string;
+	secret: string;
+	scope?: string;
+}
+
+/**
+ * Deterministic name for the auto-minted secret backing a literal registry token: keyed by the
+ * deploying component and the registry, so re-supplying (or rotating) the token on a later deploy
+ * overwrites the same row rather than accumulating one per deploy. Sanitized to the set_secret name
+ * grammar (`\w.-`), since a registry can carry a scheme, port, or path.
+ */
+export function deriveRegistrySecretName(component: string, registry: string): string {
+	const registryKey = registry
+		.trim()
+		.replace(/^https?:\/\//i, '')
+		.replace(/^\/\//, '')
+		.replace(/\/+$/, '')
+		.toLowerCase()
+		.replace(/[^\w.-]+/g, '_');
+	const componentKey = String(component).replace(/[^\w.-]+/g, '_');
+	return `deploy.${componentKey}.${registryKey}`;
+}
+
+/**
+ * Ingest deploy_component `registryAuth` into the secrets store so a provided registry token lives
+ * as ciphertext in the replicated, audited `hdb_secret` store (reference, not embed) rather than
+ * travelling in the operation body. Returns the entries in reference form (`{ registry, secret }`).
+ *
+ * - A literal `{ registry, token }` entry is encrypted (via `set_secret`, custody required) under a
+ *   derived name granted to `component`, and returned as a reference. Overwrites an existing
+ *   derived row so token rotation is idempotent.
+ * - An already-reference `{ registry, secret }` entry passes through unchanged (peers re-running a
+ *   replicated deploy already carry references — they never re-ingest).
+ * - With no custody on this node (OSS core, or key not held here), literal tokens CANNOT be sealed;
+ *   they pass through untouched so the caller can fall back to the transient, this-node-only path.
+ *   The reference-form result therefore contains only entries the caller may persist and replicate.
+ *
+ * Runs on the deploying main thread where custody is registered.
+ */
+export async function ingestRegistryAuth(req: any, registryAuth: any[] | undefined, component: string): Promise<any[]> {
+	if (!Array.isArray(registryAuth) || registryAuth.length === 0) return registryAuth ?? [];
+	const custody = getSecretCustody();
+	const out: any[] = [];
+	for (const entry of registryAuth) {
+		// Already a reference (or nothing to seal without custody): leave as-is.
+		if (entry.secret !== undefined || !custody) {
+			out.push(entry);
+			continue;
+		}
+		const name = deriveRegistrySecretName(component, entry.registry);
+		// Reuse set_secret's seal-and-store path (encrypt with custody, grant to the component, audit
+		// the mutation). The deploy request is already super_user, which set_secret requires.
+		await setSecret({
+			operation: terms.OPERATIONS_ENUM.SET_SECRET,
+			hdb_user: req?.hdb_user,
+			name,
+			value: entry.token,
+			grants: [component],
+			processEnv: false,
+		});
+		out.push(entry.scope === undefined ? { registry: entry.registry, secret: name } : { registry: entry.registry, secret: name, scope: entry.scope });
+	}
+	return out;
+}
+
+/** Wait (bounded) for a secret row to appear — covers the replicated-deploy race where a peer runs
+ * the deploy before the origin's hdb_secret row has replicated in. */
+async function waitForSecretRow(table: any, name: string, waitMs: number): Promise<any> {
+	const deadline = Date.now() + waitMs;
+	let row = await table.get(name);
+	while (!row && Date.now() < deadline) {
+		await new Promise((resolve) => setTimeout(resolve, Math.min(200, Math.max(1, waitMs))));
+		row = await table.get(name);
+	}
+	return row;
+}
+
 /**
  * Resolve deploy_component `registryAuth` entries that reference a stored secret
  * (`{ registry, secret }`) into concrete token entries (`{ registry, token }`) by decrypting the
  * named hdb_secret row on this thread. Entries that already carry a literal `token` pass through
- * unchanged, so this is a no-op for the token-only form.
+ * unchanged, so this is a no-op for the token-only fallback form.
  *
  * This is intentionally NOT a `get_secret` operation — the store never returns plaintext across the
- * API boundary. The resolved token is handed back to the deploy handler in-memory, fed to the
- * transient .npmrc, and stripped from the request before replication; it is never written back to
- * the request body, replicated, or logged.
+ * API boundary. The resolved token is handed back to the install path in-memory, fed to the
+ * transient .npmrc, and never written back to the request body, replicated, or logged.
  *
  * Authority mirrors the accessor model (#1550): the referenced secret must be usable by the
  * component being deployed — either a `processEnv` (global) secret or a scoped secret granted to
  * `component`. Without that, the deploy path would let a super_user pull an arbitrary scoped secret
  * into an .npmrc, sidestepping the grant that is the store's authority. A missing row, missing
  * grant, absent custody, or decrypt failure throws a ClientError naming the precise reason.
  *
+ * `options.waitMs` gives a bounded grace period for a referenced row to replicate in (used on the
+ * peer side of a replicated deploy, where the deploy op can arrive just ahead of the hdb_secret
+ * row); it defaults to 0 (the origin wrote the row before it resolves, so no wait is needed).
+ *
  * Runs on the main thread, where the operations API dispatches deploys and the Pro secrets
  * component registers custody — the same place set_secret decrypts. On a node without custody
  * (OSS core, or a custody key not held here) a referenced secret cannot be resolved and the deploy
  * fails loudly rather than silently installing without auth.
  */
 export async function resolveRegistryAuth(
 	registryAuth: any[] | undefined,
-	component: string
+	component: string,
+	options: { waitMs?: number } = {}
 ): Promise<ResolvedRegistryAuthEntry[] | undefined> {
 	if (!Array.isArray(registryAuth) || registryAuth.length === 0) return registryAuth;
 	// Token-only requests must not require custody or a provisioned store — keep the fast path pure.
 	if (!registryAuth.some((entry) => entry && entry.secret !== undefined)) return registryAuth;
 
+	const waitMs = options.waitMs ?? 0;
 	const table = secretTable();
 	const resolved: ResolvedRegistryAuthEntry[] = [];
 	for (const entry of registryAuth) {
@@ -370,7 +454,7 @@ export async function resolveRegistryAuth(
 			continue;
 		}
 		const name: string = entry.secret;
-		const row = await table.get(name);
+		const row = waitMs > 0 ? await waitForSecretRow(table, name, waitMs) : await table.get(name);
 		if (!row) {
 			throw new ClientError(
 				`registryAuth references secret '${name}', which does not exist`,

unitTests/components/Application.test.js

@@ -5,7 +5,7 @@ const assert = require('node:assert');
 const testUtils = require('../testUtils.js');
 testUtils.preTestPrep();
 
-const { isSSHAuthFailure } = require('#src/components/Application');
+const { isSSHAuthFailure, assertApplicationConfig } = require('#src/components/Application');
 
 describe('isSSHAuthFailure', () => {
 	it('returns true for "Could not read from remote repository"', () => {
@@ -65,3 +65,39 @@ npm error 404 '@scope/nonexistent-pkg@latest' is not in this registry.
 		assert.strictEqual(isSSHAuthFailure(''), false);
 	});
 });
+
+describe('assertApplicationConfig registryAuth', () => {
+	it('accepts a config with no registryAuth', () => {
+		assert.doesNotThrow(() => assertApplicationConfig('app', { package: 'npm:@org/app@1.0.0' }));
+	});
+
+	it('accepts registryAuth reference entries', () => {
+		assert.doesNotThrow(() =>
+			assertApplicationConfig('app', {
+				package: 'npm:@org/app@1.0.0',
+				registryAuth: [{ registry: 'https://npm.pkg.github.com', secret: 'deploy.app.gh', scope: '@org' }],
+			})
+		);
+	});
+
+	it('rejects registryAuth that is not an array', () => {
+		assert.throws(
+			() => assertApplicationConfig('app', { package: 'p', registryAuth: { registry: 'r', secret: 's' } }),
+			/expected array/
+		);
+	});
+
+	it('rejects a registryAuth entry carrying a literal token (references only on disk)', () => {
+		assert.throws(
+			() => assertApplicationConfig('app', { package: 'p', registryAuth: [{ registry: 'r', token: 'tok' }] }),
+			/expected \{ registry, secret, scope\? \} reference/
+		);
+	});
+
+	it('rejects a registryAuth entry missing registry/secret', () => {
+		assert.throws(
+			() => assertApplicationConfig('app', { package: 'p', registryAuth: [{ secret: 's' }] }),
+			/reference/
+		);
+	});
+});