Make typed random-access structures opt-in (default off) in 5.1

config-root.schema.json

@@ -460,6 +460,10 @@
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
+				"randomAccessFields": {
+					"type": "boolean",
+					"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas should leave this off. Applies to each primary table on open, so changing it switches encoding for tables that have not pinned the choice via the @table(randomAccessFields:) directive (safe at runtime: existing records still decode, only new writes change). Default: false"
+				},
 				"prefetchWrites": { "type": "boolean", "description": "Load data prior to write transactions. Default: true" },
 				"path": { "type": "string", "description": "Directory for all database files. Default: <rootPath>/database" },
 				"blobPaths": {

resources/RecordEncoder.ts

@@ -262,7 +262,14 @@ export class RecordEncoder extends StructonEncoder {
 		let nextByte = buffer[start];
 		let metadataFlags = 0;
 		try {
-			if ((this.isRocksDB && nextByte === 66) || (nextByte < 32 && end > 2)) {
+			// The metadata/timestamp prefix is detected heuristically by the first byte. For rocksdb a
+			// local-timestamp prefix starts with 66 — but 66 (0x42) is also classic shared-structure
+			// record-id #2, so a timestamp-less classic record beginning with that id is misread as a
+			// timestamped record (8 bytes stripped → corrupt). Callers that pass a value known to have no
+			// prefix (e.g. the audit store's getValue) set options.noMetadata to skip the heuristic. Typed
+			// structs start at 0x20-0x3f and never hit this, which is why it only surfaces with classic
+			// structures (typed structures off).
+			if (!options?.noMetadata && ((this.isRocksDB && nextByte === 66) || (nextByte < 32 && end > 2))) {
 				// record with metadata
 				// this means that the record starts with a local timestamp (that was assigned by lmdb-js).
 				// we copy it so we can decode it as float-64; we need to do it first because if structural data

resources/Table.ts

@@ -108,6 +108,19 @@ const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404,
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
+// A frozen record we may need to copy-on-mutate before stamping it (records are immutable — decoded
+// records are frozen and 5.2 record caching relies on it). Only plain/record objects qualify: never
+// a Buffer/typed-array (spreading would corrupt the binary into a {0:.., 1:..} object) or a primitive
+// (which reports as frozen and would spread into character/index keys).
+function isFrozenRecordObject(value: any): boolean {
+	return (
+		value !== null &&
+		typeof value === 'object' &&
+		!ArrayBuffer.isView(value) &&
+		!(value instanceof ArrayBuffer) &&
+		Object.isFrozen(value)
+	);
+}
 export const INVALIDATED = 1;
 export const EVICTED = 8; // note that 2 is reserved for timestamps
 const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
@@ -1666,6 +1679,12 @@ export function makeTable(options) {
 					if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
 						if (!(context as any)?.source) {
 							transaction.checkOverloaded();
+							// Records are intentionally immutable: decoded records are frozen (and 5.2 record
+							// caching relies on it), so mutating in place would corrupt cached/shared state.
+							// validate() coerces values and we stamp created/updated times + the primary key
+							// below, so copy-on-mutate when recordUpdate is frozen (e.g. a record decoded during
+							// log replay) instead of writing through the frozen object.
+							if (isFrozenRecordObject(recordUpdate)) recordUpdate = { ...recordUpdate };
 							this.validate(recordUpdate, !fullUpdate);
 							if (updatedTimeProperty) {
 								recordUpdate[updatedTimeProperty.name] =
@@ -4382,6 +4401,10 @@ export function makeTable(options) {
 								}
 							}
 							if (typeof updatedRecord.toJSON === 'function') updatedRecord = updatedRecord.toJSON();
+							// updatedRecord may still be a frozen record (e.g. a reused existingRecord); copy-on-mutate
+							// before stamping the primary key and created/updated times below (records are immutable —
+							// 5.2 record caching relies on it — so we must not write through the frozen object).
+							if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
 							if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
 						}
 						resolved = true;

resources/auditStore.ts

@@ -563,7 +563,10 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 				if (action & HAS_RECORD || (action & HAS_PARTIAL_RECORD && !fullRecord)) {
 					if (!value) {
 						value = decodeFromDatabase(
-							() => store.decoder.decode(buffer.subarray(decoder.position, end)),
+							// the audit value has no on-disk timestamp/metadata prefix (the audit entry carries
+							// its own time), so skip the prefix heuristic — otherwise a classic record whose
+							// structure-id byte is 66 (0x42) is misread as a rocksdb timestamp. See RecordEncoder.decode.
+							() => store.decoder.decode(buffer.subarray(decoder.position, end), { noMetadata: true }),
 							store.rootStore
 						);
 					}

resources/databases.ts

@@ -586,6 +586,9 @@ function initStores(
 					envGet(CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
 				dbiInit.compression.threshold = compressionThreshold;
 			}
+			// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+			if (typeof primaryAttribute.randomAccessFields === 'boolean')
+				dbiInit.randomAccessStructure = primaryAttribute.randomAccessFields;
 			if (rootStore instanceof RocksDatabase) {
 				primaryStore = handleLocalTimeForGets(
 					openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key } as any),
@@ -710,6 +713,7 @@ interface TableDefinition {
 	sealed?: boolean;
 	splitSegments?: boolean;
 	replicate?: boolean;
+	randomAccessFields?: boolean;
 	trackDeletes?: boolean;
 	attributes: any[];
 	schemaDefined?: boolean;
@@ -878,6 +882,14 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 	const objectStorage =
 		attribute.isPrimaryKey || (attribute.indexed.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
 	const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
+	// Custom-index object stores (e.g. HNSW vector graphs) hold fixed-shape internal nodes —
+	// numeric-keyed per-level connection arrays and quantized bins — that rely on random-access
+	// struct encoding. Keep them in struct mode regardless of the table's storage.randomAccessFields
+	// setting: their node shapes are controlled, so the wide/variably-typed OOM + divergence risks
+	// that motivate the table-level default-off don't apply, and disabling structs corrupts the graph.
+	if (attribute.indexed?.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore) {
+		dbiInit.randomAccessStructure = true;
+	}
 	let dbi:
 		| LMDBDatabase
 		| (RocksDatabase & {
@@ -929,6 +941,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		sealed,
 		splitSegments,
 		replicate,
+		randomAccessFields,
 		trackDeletes,
 		schemaDefined,
 		origin,
@@ -982,13 +995,23 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		primaryKeyAttribute.splitSegments = splitSegments; // always default to not splitting segments going forward
 		if (typeof sealed === 'boolean') primaryKeyAttribute.sealed = sealed;
 		if (typeof replicate === 'boolean') primaryKeyAttribute.replicate = replicate;
+		// An explicit directive PINS this table's encoding: we persist the boolean, so later changes
+		// to the global storage.randomAccessFields default never affect this table. Tables WITHOUT the
+		// directive are intentionally not persisted here — they follow the current global default on
+		// each open (a runtime lever to flip encoding fleet-wide). Switching either way is safe: the
+		// struct READ hook always stays on and struct (0x20-0x3f) vs classic-record (0x40-0x7f) bytes
+		// are disjoint, so already-written records still decode; only the encoding of NEW writes changes.
+		if (typeof randomAccessFields === 'boolean') primaryKeyAttribute.randomAccessFields = randomAccessFields;
 		if (origin) {
 			if (!primaryKeyAttribute.origins) primaryKeyAttribute.origins = [origin];
 			else if (!primaryKeyAttribute.origins.includes(origin)) primaryKeyAttribute.origins.push(origin);
 		}
 		logger.trace(`${tableName} table loading, opening primary store`);
 		const dbiInit = createOpenDBIObject(false, true);
 		dbiInit.compression = primaryKeyAttribute.compression;
+		// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+		if (typeof primaryKeyAttribute.randomAccessFields === 'boolean')
+			dbiInit.randomAccessStructure = primaryKeyAttribute.randomAccessFields;
 		const dbiName = tableName + '/';
 
 		if (rootStore instanceof RocksDatabase) {

resources/graphql.ts

@@ -77,6 +77,9 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 						if (typeDef.schema) typeDef.database = typeDef.schema;
 						if (!typeDef.table) typeDef.table = typeName;
 						if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
+						// Boolean directive args arrive as actual booleans; tolerate string forms too.
+						if (typeDef.randomAccessFields !== undefined)
+							typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true';
 						typeDef.attributes = typeDef.properties;
 						tables.push(typeDef);
 					}

schema.graphql

@@ -113,6 +113,13 @@ directive @table(
 	table definition.
 	"""
 	replicate: Boolean
+	"""
+	Encode this table's records as typed random-access structures, optimizing for fast field
+	access and smaller records. Best for tables with stable, homogeneous field types; leave off
+	for wide or variably-typed schemas. Pins this table's encoding, overriding the global
+	storage.randomAccessFields config (which otherwise applies to each table dynamically on open).
+	"""
+	randomAccessFields: Boolean
 ) on OBJECT
 
 """

unitTests/resources/databases.test.js

@@ -19,3 +19,33 @@ describe('flushDatabases', () => {
 		await assert.doesNotReject(() => flushDatabases());
 	});
 });
+
+describe('table() randomAccessFields directive', () => {
+	before(function () {
+		setupTestDBPath();
+		setMainIsWorker(true);
+	});
+
+	it('defaults to classic structures (struct writes disabled) when the directive is absent', function () {
+		const DefaultTable = table({
+			table: 'RafDefault',
+			database: 'test',
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+		const encoder = DefaultTable.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.strictEqual(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+
+	it('enables typed random-access structures when @table(randomAccessFields: true)', function () {
+		const RafTable = table({
+			table: 'RafEnabled',
+			database: 'test',
+			randomAccessFields: true,
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+		const encoder = RafTable.primaryStore.encoder;
+		assert.strictEqual(encoder.randomAccessStructure, true);
+		assert.ok(encoder._writeStruct.length > 0, 'expected the real struct-write hook');
+	});
+});

unitTests/resources/models/randomAccessFieldsDirective.test.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const { setupTestDBPath } = require('../../testUtils');
+const { loadGQLSchema } = require('#src/resources/graphql');
+const { tables } = require('#src/resources/databases');
+
+// Parse-level behavior of the @table(randomAccessFields:) directive: the boolean is coerced from the
+// GraphQL string value and flows through to the primary store's encoder, which keeps or stubs its
+// struct-write hook. storage.randomAccessFields defaults off, so an absent directive leaves writes off.
+describe('@table(randomAccessFields:) directive parsing', () => {
+	before(() => setupTestDBPath());
+
+	it('enables typed random-access structures when randomAccessFields: true', async () => {
+		await loadGQLSchema(`type RafOn @table(randomAccessFields: true) {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafOn.primaryStore.encoder;
+		assert.equal(encoder.randomAccessStructure, true);
+		assert.ok(encoder._writeStruct.length > 0, 'expected the real struct-write hook');
+	});
+
+	it('keeps writes disabled when randomAccessFields: false', async () => {
+		await loadGQLSchema(`type RafOff @table(randomAccessFields: false) {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafOff.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.equal(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+
+	it('defaults to disabled when the directive is absent', async () => {
+		await loadGQLSchema(`type RafAbsent @table {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafAbsent.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.equal(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+});

unitTests/utility/lmdb/OpenDBIObject.test.js

@@ -0,0 +1,62 @@
+require('../../testUtils');
+const assert = require('assert');
+const { OpenDBIObject } = require('#src/utility/lmdb/OpenDBIObject');
+const envMngr = require('#src/utility/environment/environmentManager');
+const { CONFIG_PARAMS } = require('#src/utility/hdbTerms');
+
+// Covers the global storage.randomAccessFields path in the OpenDBIObject constructor. The directive
+// tests (databases.test.js / randomAccessFieldsDirective.test.js) stamp dbiInit.randomAccessStructure
+// in databases.ts before the store opens, bypassing this constructor branch — so a wrong config key,
+// a non-boolean value, or a hdbTerms/YAML name mismatch would go uncaught.
+//
+// The constructor reads the value via envMngr.get(CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS). We
+// temporarily override that single getter via defineProperty (restored in a finally) rather than
+// using sinon or envMngr.setProperty: another test in the full unit suite leaves envMngr.get wrapped
+// by sinon, which (a) makes setProperty's value invisible behind that stub and (b) makes a second
+// sinon.stub throw "already wrapped". Saving/replacing/restoring the property descriptor is immune to
+// that — the replacement delegates to whatever get currently is (real or another test's stub) for
+// every other key, and restores the exact prior descriptor afterward.
+function withRandomAccessFields(value, fn) {
+	const previousDescriptor = Object.getOwnPropertyDescriptor(envMngr, 'get');
+	const currentGet = envMngr.get;
+	Object.defineProperty(envMngr, 'get', {
+		configurable: true,
+		writable: true,
+		value: (key) => (key === CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS ? value : currentGet(key)),
+	});
+	try {
+		fn();
+	} finally {
+		Object.defineProperty(envMngr, 'get', previousDescriptor);
+	}
+}
+
+describe('OpenDBIObject storage.randomAccessFields global config', () => {
+	it('enables randomAccessStructure on a primary DBI when the global config is true', () => {
+		withRandomAccessFields(true, () => {
+			assert.strictEqual(new OpenDBIObject(false, true).randomAccessStructure, true);
+		});
+	});
+
+	it('leaves randomAccessStructure off on a primary DBI when the global config is false (default)', () => {
+		withRandomAccessFields(false, () => {
+			assert.strictEqual(new OpenDBIObject(false, true).randomAccessStructure, false);
+		});
+	});
+
+	it('treats a non-boolean truthy config value as off (strict === true)', () => {
+		// envMngr should hand back a real boolean; the strict === true guards against a stray truthy
+		// (e.g. the string "true") silently flipping encoding on.
+		withRandomAccessFields('true', () => {
+			assert.strictEqual(new OpenDBIObject(false, true).randomAccessStructure, false);
+		});
+	});
+
+	it('keeps randomAccessStructure off on non-primary DBIs even when the global config is true', () => {
+		// Non-primary stores (e.g. the __dbis__ metadata DBI) must stay in records mode for
+		// v4-downgrade decodability, regardless of the global setting.
+		withRandomAccessFields(true, () => {
+			assert.strictEqual(new OpenDBIObject(false, false).randomAccessStructure, false);
+		});
+	});
+});

utility/hdbTerms.ts

@@ -604,6 +604,7 @@ export const CONFIG_PARAMS = {
 	STORAGE_CACHING: 'storage_caching',
 	STORAGE_COMPRESSION: 'storage_compression',
 	STORAGE_NOREADAHEAD: 'storage_noReadAhead',
+	STORAGE_RANDOMACCESSFIELDS: 'storage_randomAccessFields',
 	STORAGE_PREFETCHWRITES: 'storage_prefetchWrites',
 	STORAGE_ENCRYPTION: 'storage_encryption',
 	STORAGE_MAXTRANSACTIONQUEUETIME: 'storage_maxTransactionQueueTime',

utility/lmdb/OpenDBIObject.ts

@@ -42,7 +42,13 @@ export class OpenDBIObject {
 		// non-primary stores (e.g. the __dbis__ metadata DBI) stay in records mode and remain
 		// decodable after a downgrade. RecordEncoder still reads struct data so existing v5
 		// struct entries decode.
-		this.randomAccessStructure = isPrimary;
+		// As of 5.1 the primary default is itself opt-in via storage.randomAccessFields (default
+		// off; overridable per-table via databases.ts before the store opens). Typed structures key
+		// on per-field value WIDTH, so wide/variably-typed schemas can mint an unbounded dictionary
+		// (OOM) and diverge across replicas. Read the config here at construction — not at module
+		// import — so an env/CLI override applied during startup is honored (DBIs open after config
+		// is finalized).
+		this.randomAccessStructure = isPrimary && envMngr.get(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS) === true;
 		if (isPrimary) {
 			this.cache = LMDB_CACHING && { validated: true };
 			this.freezeData = true;