Merge remote-tracking branch 'origin/main' into feat-mcp-metadata-1095

# Conflicts:
#	resources/graphql.ts

Author: kylebernhardy

.github/workflows/integration-tests.yml

@@ -74,194 +74,6 @@ jobs:
             package.json
           retention-days: 1
 
-  run-integration-apiTests:
-    name: Integration API Tests (Node.js v${{ matrix.node-version }})
-    needs: [generate-node-version-matrix, build]
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      matrix:
-        node-version: ${{ fromJSON(needs.generate-node-version-matrix.outputs.node-versions) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: ${{ matrix.node-version }}
-          package-manager-cache: false
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-node-${{ matrix.node-version }}
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: '/tmp/hdb'
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          rm -rf /tmp/hdb
-          mkdir -p /tmp/hdb/log
-          node ./dist/bin/harper.js install > /tmp/hdb/log/install-stdout.log 2> /tmp/hdb/log/install-stderr.log
-          node ./dist/bin/harper.js start > /tmp/hdb/log/start-stdout.log 2> /tmp/hdb/log/start-stderr.log &
-          echo "Waiting for Harper to be ready on port 9925..."
-          timeout 120 bash -c 'until nc -z localhost 9925 2>/dev/null; do sleep 0.5; done'
-          echo "Harper is ready."
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-node-${{ matrix.node-version }}
-          path: /tmp/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
-  run-integration-apiTests-bun:
-    name: Integration API Tests (Bun)
-    if: false
-    needs: [build]
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: 24
-          package-manager-cache: false
-
-      - name: Setup Bun
-        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-node-24
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: '/tmp/hdb'
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          rm -rf /tmp/hdb
-          mkdir -p /tmp/hdb/log
-          node ./dist/bin/harper.js install > /tmp/hdb/log/install-stdout.log 2> /tmp/hdb/log/install-stderr.log
-          sleep 10
-          bun ./dist/bin/harper.js start > /tmp/hdb/log/start-stdout.log 2> /tmp/hdb/log/start-stderr.log &
-          sleep 10
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-bun
-          path: /tmp/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
-  run-integration-apiTests-windows:
-    name: Integration API Tests (Windows, Node.js v24)
-    needs: [build-windows]
-    runs-on: windows-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js 24
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: 24
-          package-manager-cache: false
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-windows
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: ${{ runner.temp }}/hdb
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          New-Item -ItemType Directory -Force -Path ${{ runner.temp }}/hdb/log
-          node ./dist/bin/harper.js install > ${{ runner.temp }}/hdb/log/install-stdout.log 2> ${{ runner.temp }}/hdb/log/install-stderr.log
-          Start-Process node -ArgumentList "./dist/bin/harper.js start" -RedirectStandardOutput "${{ runner.temp }}/hdb/log/start-stdout.log" -RedirectStandardError "${{ runner.temp }}/hdb/log/start-stderr.log" -PassThru
-          Write-Host "Waiting for Harper to be ready on port 9925..."
-          $deadline = (Get-Date).AddSeconds(120)
-          do {
-            Start-Sleep -Milliseconds 500
-            $client = New-Object System.Net.Sockets.TcpClient
-            $connected = $client.ConnectAsync('localhost', 9925).Wait(2000)
-            if ($connected) { $client.Close(); $ready = $true }
-            else { $client.Close() }
-          } until ($ready -or (Get-Date) -gt $deadline)
-          if (-not $ready) { Write-Error "Harper did not start within 120s"; exit 1 }
-          Write-Host "Harper is ready."
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-windows
-          path: ${{ runner.temp }}/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
   run-integration-tests:
     name: Integration Tests ${{matrix.shard}}/4 (Node.js v${{ matrix.node-version }})
     runs-on: ubuntu-latest

.gitignore

@@ -50,6 +50,7 @@ harper-*.tgz
 # AI
 .antigravitycli/
 .claude/
+cache/
 
 # YCSB benchmark results (generated)
 benchmarks/ycsb/results/

AGENTS.md

@@ -43,6 +43,8 @@ npx mocha unitTests/resources/mytest.js
 
 TypeScript is stripped at runtime via `--conditions=typestrip` (Node.js native type stripping) — no compilation required for development. Use `npm run test:unit:typestrip` to run tests with this mode.
 
+**Test timing:** prefer condition-waits over fixed `delay(N)` sleeps. `await delay(N); assert(sideEffectHappened)` races against loaded runners and is the root cause of a class of flakiness (#1138). Use the shared `waitFor(condition, timeout?, interval?)` helper in `unitTests/waitFor.js` to poll until the actual condition holds. Reserve fixed sleeps for genuinely modeling elapsed time (TTL/expiry windows) or asserting a non-event (that something has _not_ happened yet).
+
 ---
 
 ## Architecture

components/componentLoader.ts

@@ -77,6 +77,7 @@ export function loadComponentDirectories(loadedPluginModules?: Map<any, any>, lo
 	}
 	const hdbAppFolder = process.env.RUN_HDB_APP;
 	if (hdbAppFolder) {
+		if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder);
 		cfsLoaded.push(
 			loadComponent(hdbAppFolder, resources, hdbAppFolder, {
 				isRoot: false,

components/deploymentRecorder.ts

@@ -72,6 +72,7 @@ export class DeploymentRecorder {
 	private unsubscribe: (() => void) | null = null;
 	private pendingPut: Promise<void> | null = null;
 	private dirty = false;
+	private sealed = false;
 
 	private constructor(deploymentId: string, initial: Record<string, any>) {
 		this.deploymentId = deploymentId;
@@ -151,6 +152,12 @@ export class DeploymentRecorder {
 	// the record dirty; the chained continuation issues a follow-up put once the prior one
 	// settles. This keeps event_log writes O(1) puts per burst rather than O(N) per event.
 	private scheduleFlush(): void {
+		if (this.sealed) {
+			// Sealed: accumulate state in memory but don't write. finish() does the single
+			// terminal write. See seal() for why. The emitter still emits live SSE events.
+			this.dirty = true;
+			return;
+		}
 		if (this.pendingPut) {
 			this.dirty = true;
 			return;
@@ -264,6 +271,24 @@ export class DeploymentRecorder {
 		for (const result of results) this.recordPeer(result);
 	}
 
+	/**
+	 * Stop persisting intermediate row updates; accumulate them in memory so finish() writes
+	 * the terminal state in a single put. Called before the replicate phase, where the row
+	 * otherwise receives a tight burst of puts (replicate phase + per-peer + finish) within
+	 * a few ms. That burst can commit out of order on a loaded peer, where an older full
+	 * update reverts the terminal `success` write — the row stays stuck at `replicating` and
+	 * never converges (harperdb/harper#1170). Collapsing to one terminal write isolates it
+	 * from any concurrent same-key write so the receiver converges.
+	 *
+	 * Tradeoff: the origin's get_deployment *polling* view skips the transient `replicating`
+	 * status and incremental peer_results during the final phase; live SSE tailing is
+	 * unaffected (the emitter still emits in real time). Once #1170 lands this seal can be
+	 * removed to restore incremental peer_results persistence.
+	 */
+	seal(): void {
+		this.sealed = true;
+	}
+
 	async finish(status: 'success' | 'failed' | 'rolled_back', error?: unknown): Promise<void> {
 		if (this.finished) return;
 		// Send a terminal sentinel through the emitter (if any) BEFORE we unsubscribe and

components/operations.js

@@ -384,6 +384,7 @@ async function deployComponent(req) {
 				allowInstallScripts: req.install_allow_scripts,
 			};
 		}
+		if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath;
 		await configUtils.addConfig(req.project, applicationConfig);
 	}
 
@@ -507,6 +508,11 @@ async function deployComponent(req) {
 					emit('peer', result);
 				}
 			: undefined;
+		// Seal the recorder before the replicate phase so the row's terminal write (finish())
+		// isn't part of the tight put burst that can commit out of order on a peer and revert
+		// it (harperdb/harper#1170). onPeerResult/peer_results accumulate in memory and land in
+		// finish()'s single write; live SSE 'peer' events still fire below.
+		recorder?.seal();
 		emit('phase', { phase: 'replicate', status: 'start' });
 		let response = await server.replication.replicateOperation(req, { onPeerResult });
 		emit('phase', { phase: 'replicate', status: 'done' });

components/operationsValidation.js

@@ -242,7 +242,15 @@ function deployComponentValidator(req) {
 		install_timeout: Joi.number().optional(),
 		install_allow_scripts: Joi.boolean().optional(),
 		force: Joi.boolean().optional(),
-	});
+		urlPath: Joi.string()
+			.min(1)
+			.custom((value, helpers) => {
+				if (value.includes('..')) return helpers.error('any.invalid');
+				return value;
+			})
+			.optional()
+			.messages({ 'any.invalid': 'urlPath must not contain ".."' }),
+	}).with('urlPath', 'package');
 
 	return validator.validateBySchema(req, deployProjSchema);
 }

config-root.schema.json

@@ -460,6 +460,10 @@
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
+				"randomAccessFields": {
+					"type": "boolean",
+					"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas should leave this off. Applies to each primary table on open, so changing it switches encoding for tables that have not pinned the choice via the @table(randomAccessFields:) directive (safe at runtime: existing records still decode, only new writes change). Default: false"
+				},
 				"prefetchWrites": { "type": "boolean", "description": "Load data prior to write transactions. Default: true" },
 				"path": { "type": "string", "description": "Directory for all database files. Default: <rootPath>/database" },
 				"blobPaths": {