fix: cap maxOwnStructures at 32 to avoid two-byte record corruption

RecordEncoder set maxOwnStructures=256 (the OOM cap). With msgpackr's default
maxSharedStructures of 32, that pushes maxOwn+maxShared past 64, flipping on
msgpackr's two-byte record-id encoding. That path mis-serializes over-cap "own"
structures when a shared-structures store is present (every primary store has
one): it writes an out-of-range record-id reference instead of inlining the
structure, so once a table exceeds the shared cap, new records become
undecodable ("Record id is not defined for N") — on BOTH the typed default and
the randomAccessFields=false opt-out. Confirmed via a high-cardinality repro
(2900+/3000 records undecodable at maxOwn=256; 0 at maxOwn<=32).

Lowering to 32 keeps msgpackr on the one-byte path, which inlines over-cap
shapes correctly and bounds memory at least as tightly. Adds a regression test
that writes 500 distinct shapes through a shared store and asserts every record
decodes, on both the typed and readOnlyStructures paths. The deeper msgpackr
two-byte fix (to make >32 shared structures correct) is tracked separately.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: kriszyp

resources/RecordEncoder.ts

@@ -88,10 +88,18 @@ export class RecordEncoder extends Encoder {
 	name: string;
 	constructor(options) {
 		options.useBigIntExtension = true;
-		// Bound the per-encoder typed-structure dictionary. It is append-only and pinned on the
-		// long-lived primary store, so a wide/sparse schema (whose records vary by per-field value
-		// width) can grow it unbounded and exhaust memory. Caller-overridable; default caps it.
-		options.maxOwnStructures ??= 256;
+		// Bound the per-encoder structure dictionary. It is append-only and pinned on the long-lived
+		// primary store, so a wide/sparse schema (whose records vary by per-field value width) can grow
+		// it unbounded and exhaust memory. Caller-overridable; default caps it.
+		//
+		// Must stay <= 32: with msgpackr's default maxSharedStructures (32), maxOwnStructures + 32 > 64
+		// flips on the two-byte record-id encoding, which mis-serializes over-cap "own" structures when a
+		// shared-structures store is present (the primary store always has one) — it writes an out-of-range
+		// record-id reference instead of inlining the structure, so records become undecodable
+		// ("Record id is not defined for N") once a table exceeds the shared cap. The one-byte path
+		// (<= 32) inlines over-cap shapes correctly and bounds memory at least as tightly. (A prior 256
+		// triggered this on both the typed default and the randomAccessFields=false opt-out.)
+		options.maxOwnStructures ??= 32;
 		// When random-access fields are disabled (storage.randomAccessFields=false), write records as
 		// classic shared structures instead of typed random-access structures. randomAccessStructure stays
 		// on so reads still decode either form — existing typed-struct data remains readable; only new

unitTests/resources/recordEncoder.test.js

@@ -177,3 +177,45 @@ describe('RecordEncoder random-access fields opt-out (readOnlyStructures)', () =
 		});
 	});
 });
+
+describe('RecordEncoder structure dictionary cap (no two-byte over-cap corruption)', () => {
+	// Regression: harper's default maxOwnStructures must keep msgpackr on the one-byte record path
+	// (maxOwnStructures + the default maxSharedStructures of 32 must stay <= 64). A larger value (the
+	// prior 256) flips on the two-byte path, which mis-serializes over-cap "own" structures when a
+	// shared-structures store is present — records written after a table exceeds the shared cap become
+	// undecodable ("Record id is not defined for N"), on both the typed default and the
+	// randomAccessFields=false opt-out. These write far more distinct shapes than the shared cap through
+	// a shared store and assert every record still decodes.
+	function diverseRecords(count) {
+		const records = [];
+		for (let i = 0; i < count; i++) {
+			const rec = { id: i };
+			const attrCount = 1 + (i % 6);
+			for (let a = 0; a < attrCount; a++) rec['attr_' + ((i * 7 + a) % 200)] = i % 2 ? 's' + i : i;
+			records.push(rec);
+		}
+		return records;
+	}
+
+	function assertAllDecode(extra) {
+		const store = sharedStore();
+		const writer = makeEncoder(store, extra);
+		const reader = makeEncoder(store, extra);
+		const records = diverseRecords(500); // far exceeds the 32 shared-structure cap
+		const buffers = records.map((r) => Buffer.from(writer.encode(r)));
+		let failures = 0;
+		for (let i = 0; i < buffers.length; i++) {
+			const decoded = reader.decode(buffers[i]);
+			if (!decoded || decoded.id !== records[i].id) failures++;
+		}
+		assert.strictEqual(failures, 0, `all ${records.length} records should decode after exceeding the shared cap`);
+	}
+
+	it('decodes every record past the shared cap on the typed path (default)', () => {
+		assertAllDecode(undefined);
+	});
+
+	it('decodes every record past the shared cap with randomAccessFields off (readOnlyStructures)', () => {
+		assertAllDecode({ readOnlyStructures: true });
+	});
+});