Merge remote-tracking branch 'origin/main' into fix/schemadefined-on-replicated-tables-rebase

# Conflicts:
#	resources/databases.ts
#	unitTests/resources/databases.test.js

Author: ldt1996

.github/workflows/claude-issue-to-pr.yml

@@ -22,11 +22,11 @@ concurrency:
 
 jobs:
   work:
-    uses: HarperFast/ai-review-prompts/.github/workflows/_claude-issue-to-pr.yml@6463b3da6326f0ca4646fb6b5139c2ecf92130f6 # main 2026-05-06 (post #11/#12/#14)
+    uses: HarperFast/ai-review-prompts/.github/workflows/_claude-issue-to-pr.yml@441d2a310cb360adf419d3546dd27566164f1ec2 # main 2026-06-09 (post #58: Opus for :bug/:test, efficiency guidance)
     with:
       # Same SHA as the `uses:` ref above. See the comment in
       # claude-mention.yml for why the duplication is unavoidable.
-      ai-review-prompts-ref: 6463b3da6326f0ca4646fb6b5139c2ecf92130f6
+      ai-review-prompts-ref: 441d2a310cb360adf419d3546dd27566164f1ec2
       repo-specific-conventions: |
         ## Harper core notes
 

.github/workflows/claude-review.yml

@@ -33,7 +33,7 @@ jobs:
     # Note: the `claude-review` label name is matched there too —
     # `_claude-review.yml`'s authorize `if:`, not in this caller.
     if: ${{ vars.CLAUDE_ALWAYS_ON == 'true' || github.event.action == 'labeled' }}
-    uses: HarperFast/ai-review-prompts/.github/workflows/_claude-review.yml@2be0f702b114e39b90eed90962922ef64df58482 # main 2026-06-03 (post #52 — week-of-05-25 calibration: Harper-aware best-practices wiring, skills-ref v1.7.0, surface-findings rule, access-control blocker guidance, public-skill dedup)
+    uses: HarperFast/ai-review-prompts/.github/workflows/_claude-review.yml@3f2300a39910a0983eb74967dedeebd8c3c98adc # main 2026-06-08 (post #57 — week-of-06-01 calibration: severity-discipline non-blocker bullets, async-commit-ordering check, non-critical-work-on-critical-path guard)
     with:
       # Same SHA as the `uses:` ref above. The reusable uses this to
       # check out HarperFast/ai-review-prompts (layer files + bash
@@ -44,7 +44,7 @@ jobs:
       # introspect their own ref (`github.workflow_ref` resolves to the
       # CALLER's ref in `workflow_call` context), and `uses: …@<ref>`
       # is parsed literally so we can't interpolate a variable.
-      ai-review-prompts-ref: 2be0f702b114e39b90eed90962922ef64df58482
+      ai-review-prompts-ref: 3f2300a39910a0983eb74967dedeebd8c3c98adc
       review-layers: |
         universal
         harper/common

.github/workflows/gemini-review.yml

@@ -52,7 +52,7 @@ jobs:
     # Note: the `gemini-review` label name is matched there too —
     # `_gemini-review.yml`'s authorize `if:`, not in this caller.
     if: ${{ vars.GEMINI_ALWAYS_ON == 'true' || github.event.action == 'labeled' }}
-    uses: HarperFast/ai-review-prompts/.github/workflows/_gemini-review.yml@2be0f702b114e39b90eed90962922ef64df58482 # main 2026-06-03 (post #52 — calibration: Harper-aware wiring for the Gemini leg, skills-ref v1.7.0, surface-findings rule, access-control blocker guidance, public-skill dedup)
+    uses: HarperFast/ai-review-prompts/.github/workflows/_gemini-review.yml@3f2300a39910a0983eb74967dedeebd8c3c98adc # main 2026-06-08 (post #57 — week-of-06-01 calibration: severity-discipline non-blocker bullets, async-commit-ordering check, non-critical-work-on-critical-path guard)
     # Caller-side permissions at the calling-job level (NOT workflow-
     # level — that placement caps the reusable's per-job grants below
     # what they need and breaks the workflow at startup; see
@@ -70,7 +70,7 @@ jobs:
       # duplication is unavoidable: reusable workflows can't introspect
       # their own ref (`github.workflow_ref` resolves to the CALLER's
       # ref in workflow_call context), and `uses: …@<ref>` is literal.
-      ai-review-prompts-ref: 2be0f702b114e39b90eed90962922ef64df58482
+      ai-review-prompts-ref: 3f2300a39910a0983eb74967dedeebd8c3c98adc
       review-layers: |
         universal
         harper/common

.github/workflows/integration-tests.yml

@@ -74,194 +74,6 @@ jobs:
             package.json
           retention-days: 1
 
-  run-integration-apiTests:
-    name: Integration API Tests (Node.js v${{ matrix.node-version }})
-    needs: [generate-node-version-matrix, build]
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      matrix:
-        node-version: ${{ fromJSON(needs.generate-node-version-matrix.outputs.node-versions) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: ${{ matrix.node-version }}
-          package-manager-cache: false
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-node-${{ matrix.node-version }}
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: '/tmp/hdb'
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          rm -rf /tmp/hdb
-          mkdir -p /tmp/hdb/log
-          node ./dist/bin/harper.js install > /tmp/hdb/log/install-stdout.log 2> /tmp/hdb/log/install-stderr.log
-          node ./dist/bin/harper.js start > /tmp/hdb/log/start-stdout.log 2> /tmp/hdb/log/start-stderr.log &
-          echo "Waiting for Harper to be ready on port 9925..."
-          timeout 120 bash -c 'until nc -z localhost 9925 2>/dev/null; do sleep 0.5; done'
-          echo "Harper is ready."
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-node-${{ matrix.node-version }}
-          path: /tmp/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
-  run-integration-apiTests-bun:
-    name: Integration API Tests (Bun)
-    if: false
-    needs: [build]
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: 24
-          package-manager-cache: false
-
-      - name: Setup Bun
-        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-node-24
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: '/tmp/hdb'
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          rm -rf /tmp/hdb
-          mkdir -p /tmp/hdb/log
-          node ./dist/bin/harper.js install > /tmp/hdb/log/install-stdout.log 2> /tmp/hdb/log/install-stderr.log
-          sleep 10
-          bun ./dist/bin/harper.js start > /tmp/hdb/log/start-stdout.log 2> /tmp/hdb/log/start-stderr.log &
-          sleep 10
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-bun
-          path: /tmp/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
-  run-integration-apiTests-windows:
-    name: Integration API Tests (Windows, Node.js v24)
-    needs: [build-windows]
-    runs-on: windows-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Setup Node.js 24
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: 24
-          package-manager-cache: false
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: harper-build-artifacts-windows
-
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: ${{ runner.temp }}/hdb
-          OPERATIONSAPI_NETWORK_PORT: 9925
-          LOGGING_LEVEL: 'debug'
-          LOGGING_STDSTREAMS: true
-          THREADS_COUNT: 1
-          THREADS_DEBUG: false
-          NODE_HOSTNAME: 'localhost'
-        run: |
-          New-Item -ItemType Directory -Force -Path ${{ runner.temp }}/hdb/log
-          node ./dist/bin/harper.js install > ${{ runner.temp }}/hdb/log/install-stdout.log 2> ${{ runner.temp }}/hdb/log/install-stderr.log
-          Start-Process node -ArgumentList "./dist/bin/harper.js start" -RedirectStandardOutput "${{ runner.temp }}/hdb/log/start-stdout.log" -RedirectStandardError "${{ runner.temp }}/hdb/log/start-stderr.log" -PassThru
-          Write-Host "Waiting for Harper to be ready on port 9925..."
-          $deadline = (Get-Date).AddSeconds(120)
-          do {
-            Start-Sleep -Milliseconds 500
-            $client = New-Object System.Net.Sockets.TcpClient
-            $connected = $client.ConnectAsync('localhost', 9925).Wait(2000)
-            if ($connected) { $client.Close(); $ready = $true }
-            else { $client.Close() }
-          } until ($ready -or (Get-Date) -gt $deadline)
-          if (-not $ready) { Write-Error "Harper did not start within 120s"; exit 1 }
-          Write-Host "Harper is ready."
-
-      - name: Run API Tests
-        id: run-api-tests
-        env:
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-        run: npm run test:integration:api-tests
-
-      - name: Upload Harper logs
-        if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: harper-integration-api-test-logs-windows
-          path: ${{ runner.temp }}/hdb/log/
-          retention-days: 3
-          if-no-files-found: ignore
-
   run-integration-tests:
     name: Integration Tests ${{matrix.shard}}/4 (Node.js v${{ matrix.node-version }})
     runs-on: ubuntu-latest

.gitignore

@@ -50,6 +50,7 @@ harper-*.tgz
 # AI
 .antigravitycli/
 .claude/
+cache/
 
 # YCSB benchmark results (generated)
 benchmarks/ycsb/results/

AGENTS.md

@@ -43,6 +43,8 @@ npx mocha unitTests/resources/mytest.js
 
 TypeScript is stripped at runtime via `--conditions=typestrip` (Node.js native type stripping) — no compilation required for development. Use `npm run test:unit:typestrip` to run tests with this mode.
 
+**Test timing:** prefer condition-waits over fixed `delay(N)` sleeps. `await delay(N); assert(sideEffectHappened)` races against loaded runners and is the root cause of a class of flakiness (#1138). Use the shared `waitFor(condition, timeout?, interval?)` helper in `unitTests/waitFor.js` to poll until the actual condition holds. Reserve fixed sleeps for genuinely modeling elapsed time (TTL/expiry windows) or asserting a non-event (that something has _not_ happened yet).
+
 ---
 
 ## Architecture

DESIGN.md

@@ -13,6 +13,16 @@ Records stored in tables are plain objects given a `RecordObject` prototype, whi
 - To give a plain JS object the RecordObject prototype without copying it (preserving mutability), use `Object.setPrototypeOf(obj, primaryStore.encoder.structPrototype)` then `entryMap.set(obj, entry)`.
 - Do **not** copy the object (e.g. via `Object.assign` into a new instance) if any code still holds a reference to the original and expects to mutate it — see below.
 
+## Struct mode is gated to primary DBIs (downgrade compatibility)
+
+`RecordEncoder` extends a structon encoder, whose random-access "struct" encoding uses header bytes in `0x20–0x3f`. That range overlaps msgpack positive-fixints, so a reader without struct support (msgpackr v1, i.e. harperdb v4) decodes a struct header as an integer. harperdb v4 only enabled struct mode (its `randomAccessStructure` option) on **primary** DBIs, so non-primary DBIs — notably the `__dbis__` metadata store (table/attribute defs) — were plain records mode. If v5 writes `__dbis__` in struct mode, a v5→v4 downgrade can't decode the metadata, silently treats the instance as a fresh pre-3.0 install, and refuses to boot.
+
+To match v4: `OpenDBIObject` sets `randomAccessStructure = isPrimary`, and `RecordEncoder`, when `randomAccessStructure` is false, makes the struct **write** hook bail (`this._writeStruct = () => 0`) so objects are written in records mode. Two subtleties:
+
+- We **bail** the hook (return 0) rather than clear it (`undefined`). Clearing it shifts msgpackr's positive-fixint boundary from `0x20` to `0x40`, so top-level integers `0x20–0x3f` (e.g. a `NEXT_TABLE_ID` ≥ 32 stored in `__dbis__`) would be written as bare fixints — which the still-installed struct **read** hook misreads as struct headers. Bailing keeps the boundary at `0x20`, so those integers are written as `uint8`.
+- The struct **read** hook is left intact, so records a prior v5 already wrote in struct mode still decode (read-compat); only new writes switch to records mode.
+- Companion change in `structon`: `prepareStructures` saves the shared structures in the legacy plain-array form when there are no typed structs (instead of the `{named, typed}` Map), so the `Symbol.for('structures')` buffer for a records-mode `__dbis__` is also v4-decodable.
+
 ## getFromSource() timing: promise resolves before commit runs
 
 In `getFromSource()` (`Table.ts`), the promise that callers await resolves with the entry **before** the `dbTxn.addWrite` commit callback runs. The commit callback mutates `updatedRecord` in-place to set fields like `createdAt` and `updatedAt`. Since the resolved entry's `.value` is the same reference as `updatedRecord`, those mutations are visible to the caller after resolution.

components/Scope.ts

@@ -271,10 +271,14 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 			if (key[0] === 'files' || key[0] === 'urlPath') {
 				// TODO: validate options
 
-				// If not entry handler exists then likely the config did not have `files` initially
-				// Now, it does, so create a default entry handler.
+				// If no entry handler exists yet, the plugin's handleApplication has not called
+				// handleEntry() yet — or it hasn't been called at all for this scope. Either way,
+				// when handleEntry() is eventually called it will read the current config via
+				// getFilesOption() and create the handler with the correct, up-to-date values.
+				// Eagerly creating an entry handler here would start chokidar's initial scan
+				// before the plugin's callback is attached, causing the initial `add` events to
+				// be missed (RE-8).
 				if (!scope.#entryHandler) {
-					scope.#entryHandler = scope.#createEntryHandler(config as FileAndURLPathConfig);
 					return;
 				}
 

components/componentLoader.ts

@@ -77,6 +77,7 @@ export function loadComponentDirectories(loadedPluginModules?: Map<any, any>, lo
 	}
 	const hdbAppFolder = process.env.RUN_HDB_APP;
 	if (hdbAppFolder) {
+		if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder);
 		cfsLoaded.push(
 			loadComponent(hdbAppFolder, resources, hdbAppFolder, {
 				isRoot: false,

components/deploymentRecorder.ts

@@ -72,6 +72,7 @@ export class DeploymentRecorder {
 	private unsubscribe: (() => void) | null = null;
 	private pendingPut: Promise<void> | null = null;
 	private dirty = false;
+	private sealed = false;
 
 	private constructor(deploymentId: string, initial: Record<string, any>) {
 		this.deploymentId = deploymentId;
@@ -151,6 +152,12 @@ export class DeploymentRecorder {
 	// the record dirty; the chained continuation issues a follow-up put once the prior one
 	// settles. This keeps event_log writes O(1) puts per burst rather than O(N) per event.
 	private scheduleFlush(): void {
+		if (this.sealed) {
+			// Sealed: accumulate state in memory but don't write. finish() does the single
+			// terminal write. See seal() for why. The emitter still emits live SSE events.
+			this.dirty = true;
+			return;
+		}
 		if (this.pendingPut) {
 			this.dirty = true;
 			return;
@@ -264,6 +271,24 @@ export class DeploymentRecorder {
 		for (const result of results) this.recordPeer(result);
 	}
 
+	/**
+	 * Stop persisting intermediate row updates; accumulate them in memory so finish() writes
+	 * the terminal state in a single put. Called before the replicate phase, where the row
+	 * otherwise receives a tight burst of puts (replicate phase + per-peer + finish) within
+	 * a few ms. That burst can commit out of order on a loaded peer, where an older full
+	 * update reverts the terminal `success` write — the row stays stuck at `replicating` and
+	 * never converges (harperdb/harper#1170). Collapsing to one terminal write isolates it
+	 * from any concurrent same-key write so the receiver converges.
+	 *
+	 * Tradeoff: the origin's get_deployment *polling* view skips the transient `replicating`
+	 * status and incremental peer_results during the final phase; live SSE tailing is
+	 * unaffected (the emitter still emits in real time). Once #1170 lands this seal can be
+	 * removed to restore incremental peer_results persistence.
+	 */
+	seal(): void {
+		this.sealed = true;
+	}
+
 	async finish(status: 'success' | 'failed' | 'rolled_back', error?: unknown): Promise<void> {
 		if (this.finished) return;
 		// Send a terminal sentinel through the emitter (if any) BEFORE we unsubscribe and