Skip to content

Commit 136a10e

Browse files
kriszypclaude
kriszyp
and
claude
committed
feat(deploy): hdb_deployment system table + audit record on every deploy
Slice A of #641. Every deploy_component call now writes a row to a new system.hdb_deployment table capturing the project, package identifier, sha256 of the payload tarball, payload size, status (pending → success or failed), error info, and the upload payload itself as a Blob attribute. The deployment_id is returned in the deploy response and is the join key Studio/CLI will use to subscribe to live progress in Slice B. Includes: - json/systemSchema.json: hdb_deployment table definition (deployment_id hash, with attributes mirroring the lifecycle) - utility/hdbTerms.ts: SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME + LIST_DEPLOYMENTS / GET_DEPLOYMENT / GET_DEPLOYMENT_PAYLOAD / DELETE_DEPLOYMENT_PAYLOAD operation enums - upgrade/directives/5-2-0.ts: provisions the table on existing installs (fresh installs get it via mount_hdb's systemSchema iteration) - components/deploymentRecorder.ts: lifecycle wrapper used by deployComponent — creates the row up front, ingests the payload into a Blob attribute with sha256 + size, then commits success or failure - components/deploymentOperations.ts: handlers for list_deployments (with project/status/since/until/limit/offset filters) and get_deployment; payload bytes are stripped from these responses - components/operations.js: deployComponent now wraps prepareApplication in a try/catch driven by the recorder; payload is re-sourced from the persisted blob so extraction reads exactly what was recorded - server/serverHelpers/serverUtilities.ts: registers the two new ops - integrationTests/deploy/deploy-tracking.test.ts: end-to-end coverage for the happy path, list filtering, and failure recording Updates the brittle deepStrictEqual deploy-response assertions in 4 existing tests to allow the new deployment_id field. Slice A scope is deliberately single-node; Slice B will replace the in-memory buffer in ingestPayload with a streaming variant and add peer-side reads from the replicated blob. Refs #641 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 74d0a0c commit 136a10e

14 files changed

Lines changed: 651 additions & 58 deletions

‎components/deploymentOperations.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
'use strict';
2+
3+
// Read-side operations against system.hdb_deployment. Slice A of issue #641.
4+
// Write-side lives in deploymentRecorder.ts; this module only reads.
5+
6+
import { databases } from '../resources/databases.ts';
7+
import * as terms from '../utility/hdbTerms.ts';
8+
import { ClientError } from '../utility/errors/hdbError.ts';
9+
10+
const DEPLOYMENT_TABLE = terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME;
11+
12+
interface ListRequest {
13+
project?: string;
14+
status?: string;
15+
since?: number;
16+
until?: number;
17+
limit?: number;
18+
offset?: number;
19+
}
20+
21+
interface GetRequest {
22+
deployment_id: string;
23+
}
24+
25+
function deploymentTable() {
26+
const table = (databases as any).system?.[DEPLOYMENT_TABLE];
27+
if (!table) {
28+
throw new ClientError(
29+
`Deployment tracking is not initialized on this node (system.${DEPLOYMENT_TABLE} missing). ` +
30+
`Run upgrade or restart the server to provision the table.`
31+
);
32+
}
33+
return table;
34+
}
35+
36+
// Strip the blob attribute from a row; the bytes never travel over the operations API.
37+
// Callers wanting bytes use get_deployment_payload (added in Slice B).
38+
function stripBlob(row: any): any {
39+
if (!row || typeof row !== 'object') return row;
40+
const { payload_blob, ...rest } = row;
41+
rest.payload_blob_present = payload_blob != null;
42+
return rest;
43+
}
44+
45+
export async function handleListDeployments(req: ListRequest = {}): Promise<{ deployments: any[]; total: number }> {
46+
const table = deploymentTable();
47+
const conditions: any[] = [];
48+
if (req.project) conditions.push({ attribute: 'project', value: req.project });
49+
if (req.status) conditions.push({ attribute: 'status', value: req.status });
50+
if (req.since != null) conditions.push({ attribute: 'started_at', value: req.since, comparator: 'greater_than_equal' });
51+
if (req.until != null) conditions.push({ attribute: 'started_at', value: req.until, comparator: 'less_than_equal' });
52+
53+
const collected: any[] = [];
54+
for await (const row of table.search(conditions)) {
55+
collected.push(stripBlob(row));
56+
}
57+
// Newest first by started_at; ties broken by deployment_id for stability.
58+
collected.sort((a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id));
59+
60+
const total = collected.length;
61+
const offset = Math.max(0, req.offset ?? 0);
62+
const limit = req.limit != null ? Math.max(0, req.limit) : collected.length;
63+
return { deployments: collected.slice(offset, offset + limit), total };
64+
}
65+
66+
export async function handleGetDeployment(req: GetRequest): Promise<any> {
67+
if (!req || !req.deployment_id) {
68+
throw new ClientError(`'deployment_id' is required`);
69+
}
70+
const table = deploymentTable();
71+
const row = await table.get(req.deployment_id);
72+
if (!row) {
73+
throw new ClientError(`No deployment found with id '${req.deployment_id}'`);
74+
}
75+
return stripBlob(row);
76+
}

‎components/deploymentRecorder.ts‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
'use strict';
2+
3+
// DeploymentRecorder — Slice A scope.
4+
//
5+
// Owns the lifecycle of one row in system.hdb_deployment: creates the pending row at deploy
6+
// start, streams the upload payload into the row's payload_blob (computing sha256 + size
7+
// alongside), and writes the terminal status at the end. Slice B will extend this with
8+
// ProgressEmitter subscription and event_log writes; Slice C will add rollback sourcing
9+
// from the blob.
10+
11+
import { randomUUID } from 'node:crypto';
12+
import { createHash, Hash } from 'node:crypto';
13+
import { Transform } from 'node:stream';
14+
import type { Readable } from 'node:stream';
15+
import { databases } from '../resources/databases.ts';
16+
import { createBlob } from '../resources/blob.ts';
17+
import * as terms from '../utility/hdbTerms.ts';
18+
import { hostname } from 'node:os';
19+
20+
type DeploymentStatus = 'pending' | 'extracting' | 'installing' | 'loading' | 'replicating' | 'restarting' | 'success' | 'failed' | 'rolled_back';
21+
22+
interface CreateOptions {
23+
project?: string;
24+
package_identifier?: string;
25+
user?: string;
26+
restart_mode?: 'immediate' | 'rolling' | null;
27+
rollback_of?: string | null;
28+
}
29+
30+
export class DeploymentRecorder {
31+
readonly deploymentId: string;
32+
private readonly record: Record<string, any>;
33+
private hash: Hash | null = null;
34+
private byteCount = 0;
35+
private finished = false;
36+
37+
private constructor(deploymentId: string, initial: Record<string, any>) {
38+
this.deploymentId = deploymentId;
39+
this.record = initial;
40+
}
41+
42+
static async create(options: CreateOptions): Promise<DeploymentRecorder> {
43+
const deploymentId = randomUUID();
44+
const startedAt = Date.now();
45+
const record: Record<string, any> = {
46+
deployment_id: deploymentId,
47+
project: options.project ?? null,
48+
package_identifier: options.package_identifier ?? null,
49+
payload_hash: null,
50+
payload_size: null,
51+
payload_blob: null,
52+
status: 'pending' as DeploymentStatus,
53+
phase: 'pending',
54+
event_log: [],
55+
peer_results: [],
56+
origin_node: hostname(),
57+
restart_mode: options.restart_mode ?? null,
58+
started_at: startedAt,
59+
completed_at: null,
60+
user: options.user ?? null,
61+
rollback_of: options.rollback_of ?? null,
62+
error: null,
63+
};
64+
const recorder = new DeploymentRecorder(deploymentId, record);
65+
await recorder.put();
66+
return recorder;
67+
}
68+
69+
/**
70+
* Drain a payload source (Buffer or Readable) into the row's payload_blob attribute,
71+
* computing sha256 and byte count alongside. After this resolves the row has been
72+
* committed once with the final hash and size, and `this.row.payload_blob.stream()`
73+
* yields a fresh Readable that callers can pass to extraction.
74+
*
75+
* Slice A buffers the payload in memory so the hash/size are known synchronously before
76+
* we commit and so the blob's `saveBlob` lifecycle doesn't race with our digest() call.
77+
* Slice B will swap this for a true streaming path once we also gain the ProgressEmitter
78+
* subscriber that benefits from chunk-level progress events.
79+
*/
80+
async ingestPayload(source: Readable | Buffer | string): Promise<void> {
81+
const hash = createHash('sha256');
82+
let byteCount = 0;
83+
let buffer: Buffer;
84+
if (Buffer.isBuffer(source)) {
85+
buffer = source;
86+
} else if (typeof source === 'string') {
87+
// Legacy CBOR/JSON path: payload arrives as a base64-encoded string.
88+
buffer = Buffer.from(source, 'base64');
89+
} else {
90+
const chunks: Buffer[] = [];
91+
for await (const chunk of source as AsyncIterable<Buffer | string>) {
92+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as any));
93+
}
94+
buffer = Buffer.concat(chunks);
95+
}
96+
hash.update(buffer);
97+
byteCount = buffer.length;
98+
this.record.payload_blob = createBlob(buffer, { type: 'application/gzip' });
99+
this.record.payload_hash = hash.digest('hex');
100+
this.record.payload_size = byteCount;
101+
// Touch the unused private fields so the type system stays happy in Slice B when we
102+
// reintroduce the streaming variant that uses them.
103+
this.hash = hash;
104+
this.byteCount = byteCount;
105+
await this.put();
106+
}
107+
108+
async transitionPhase(phase: string, status?: DeploymentStatus): Promise<void> {
109+
this.record.phase = phase;
110+
if (status) this.record.status = status;
111+
await this.put();
112+
}
113+
114+
async finish(status: 'success' | 'failed' | 'rolled_back', error?: unknown): Promise<void> {
115+
if (this.finished) return;
116+
this.finished = true;
117+
this.record.status = status;
118+
this.record.completed_at = Date.now();
119+
if (error) {
120+
const e = error as { message?: string; code?: string | number; stack?: string };
121+
this.record.error = {
122+
message: e?.message ?? String(error),
123+
code: e?.code,
124+
phase: this.record.phase,
125+
};
126+
}
127+
await this.put();
128+
}
129+
130+
get row(): Record<string, any> {
131+
return this.record;
132+
}
133+
134+
private async put(): Promise<void> {
135+
const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME];
136+
if (!table) {
137+
// Table missing means the upgrade directive hasn't run yet (or the table got dropped).
138+
// We tolerate this — tracking is observability; the deploy itself must still succeed.
139+
return;
140+
}
141+
await table.put(this.record);
142+
}
143+
}

‎components/operations.js‎

Lines changed: 78 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ const { packageDirectory } = require('../components/packageComponent.ts');
1818
const { Resources } = require('../resources/Resources.ts');
1919
const { Application, prepareApplication } = require('./Application.ts');
2020
const { server } = require('../server/Server.ts');
21+
const { DeploymentRecorder } = require('./deploymentRecorder.ts');
2122

2223
/**
2324
* Read the settings.js file and return the
@@ -361,7 +362,6 @@ async function deployComponent(req) {
361362
}
362363

363364
// Write to root config if the request contains a package identifier
364-
// TODO: how can we keep record of the `payload`? Its often too large to stuff into a config file; especially the root config. Maybe we can write it to a file and reference that way?
365365
if (req.package) {
366366
// Check if trying to overwrite a core component (requires force)
367367
// Lazy-load to avoid circular dependency with componentLoader
@@ -386,60 +386,89 @@ async function deployComponent(req) {
386386
await configUtils.addConfig(req.project, applicationConfig);
387387
}
388388

389-
const application = new Application({
390-
name: req.project,
391-
payload: req.payload,
392-
packageIdentifier: req.package,
393-
install: {
394-
command: req.install_command,
395-
timeout: req.install_timeout,
396-
allowInstallScripts: req.install_allow_scripts,
397-
},
389+
// Slice A of issue #641: create a hdb_deployment row up front so the deploy is
390+
// observable and auditable even if the CLI disconnects. The row also holds the payload
391+
// in a Blob attribute — Slice B will use that for peer delivery; for now it's the
392+
// audit record and the rollback source.
393+
const recorder = await DeploymentRecorder.create({
394+
project: req.project,
395+
package_identifier: req.package ?? null,
396+
user: req.hdb_user?.username,
397+
restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null,
398398
});
399+
req._deploymentId = recorder.deploymentId;
399400

400-
await prepareApplication(application);
401-
402-
// now we attempt to actually load the component in case there is
403-
// an error we can immediately detect and report, but app code should not run on the main thread
404-
if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
405-
const pseudoResources = new Resources();
406-
pseudoResources.isWorker = true;
407-
408-
const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts');
409-
let lastError;
410-
componentLoader.setErrorReporter((error) => (lastError = error));
411-
await componentLoader.loadComponent(
412-
application.dirPath,
413-
pseudoResources,
414-
undefined,
415-
false,
416-
undefined,
417-
false,
418-
req.project
419-
);
401+
let extractionPayload = req.payload;
402+
try {
403+
// If a tarball came in (Buffer or Readable from the multipart parser), tee it through
404+
// a hash-and-size tap into the row's payload_blob, then re-source extraction from the
405+
// persisted blob. This means we read the upload exactly once into local storage; the
406+
// blob is the staging area and (in Slice B) the channel peers will replicate from.
407+
if (req.payload != null) {
408+
await recorder.ingestPayload(req.payload);
409+
extractionPayload = recorder.row.payload_blob.stream();
410+
}
420411

421-
if (lastError) throw lastError;
422-
}
423-
const rollingRestart = req.restart === 'rolling';
424-
// if doing a rolling restart set restart to false so that other nodes don't also restart.
425-
req.restart = rollingRestart ? false : req.restart;
426-
let response = await server.replication.replicateOperation(req);
427-
if (req.restart === true) {
428-
manageThreads.restartWorkers('http');
429-
response.message = `Successfully deployed: ${application.name}, restarting Harper`;
430-
} else if (rollingRestart) {
431-
const serverUtilities = require('../server/serverHelpers/serverUtilities.ts');
432-
const jobResponse = await serverUtilities.executeJob({
433-
operation: 'restart_service',
434-
service: 'http',
435-
replicated: true,
412+
const application = new Application({
413+
name: req.project,
414+
payload: extractionPayload,
415+
packageIdentifier: req.package,
416+
install: {
417+
command: req.install_command,
418+
timeout: req.install_timeout,
419+
allowInstallScripts: req.install_allow_scripts,
420+
},
436421
});
437422

438-
response.restartJobId = jobResponse.job_id;
439-
response.message = `Successfully deployed: ${application.name}, restarting Harper`;
440-
} else response.message = `Successfully deployed: ${application.name}`;
423+
await prepareApplication(application);
441424

442-
return response;
425+
// now we attempt to actually load the component in case there is
426+
// an error we can immediately detect and report, but app code should not run on the main thread
427+
if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
428+
const pseudoResources = new Resources();
429+
pseudoResources.isWorker = true;
430+
431+
const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts');
432+
let lastError;
433+
componentLoader.setErrorReporter((error) => (lastError = error));
434+
await componentLoader.loadComponent(
435+
application.dirPath,
436+
pseudoResources,
437+
undefined,
438+
false,
439+
undefined,
440+
false,
441+
req.project
442+
);
443+
444+
if (lastError) throw lastError;
445+
}
446+
const rollingRestart = req.restart === 'rolling';
447+
// if doing a rolling restart set restart to false so that other nodes don't also restart.
448+
req.restart = rollingRestart ? false : req.restart;
449+
let response = await server.replication.replicateOperation(req);
450+
if (req.restart === true) {
451+
manageThreads.restartWorkers('http');
452+
response.message = `Successfully deployed: ${application.name}, restarting Harper`;
453+
} else if (rollingRestart) {
454+
const serverUtilities = require('../server/serverHelpers/serverUtilities.ts');
455+
const jobResponse = await serverUtilities.executeJob({
456+
operation: 'restart_service',
457+
service: 'http',
458+
replicated: true,
459+
});
460+
461+
response.restartJobId = jobResponse.job_id;
462+
response.message = `Successfully deployed: ${application.name}, restarting Harper`;
463+
} else response.message = `Successfully deployed: ${application.name}`;
464+
465+
response.deployment_id = recorder.deploymentId;
466+
await recorder.finish('success');
467+
return response;
468+
} catch (err) {
469+
await recorder.finish('failed', err);
470+
throw err;
471+
}
443472
}
444473

445474
/**

‎integrationTests/components/early-hints.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
* conversion, empty hints handling, and response length limits.
77
*/
88
import { suite, test, before, after } from 'node:test';
9-
import { strictEqual, ok, deepStrictEqual, match } from 'node:assert/strict';
9+
import { strictEqual, ok, match } from 'node:assert/strict';
1010
import { join, dirname } from 'node:path';
1111
import { fileURLToPath } from 'node:url';
1212

@@ -26,7 +26,8 @@ suite('Component: early-hints', (ctx: ContextWithHarper) => {
2626
package: join(__dirname, '../fixtures/template-early-hints-2.0.0.tgz'),
2727
restart: true,
2828
});
29-
deepStrictEqual(deployBody, { message: 'Successfully deployed: early-hints, restarting Harper' });
29+
strictEqual(deployBody.message, 'Successfully deployed: early-hints, restarting Harper');
30+
ok(typeof deployBody.deployment_id === 'string', `expected deployment_id, got ${deployBody.deployment_id}`);
3031

3132
// poll until /hints endpoint is registered and seed data is loaded
3233
const seedDeadline = Date.now() + 60_000;

0 commit comments

Comments
 (0)