feat: make typed random-access structures configurable (config + @table directive)

main already gates struct-mode writes to primary DBIs (a2d0faf, for v4-downgrade
compat). This makes the primary-DBI default itself controllable:

- storage.randomAccessFields config (default off) — global default for primary stores
- @table(randomAccessFields: true) directive — per-table override, persisted at creation
  (like sealed/compression)
- OpenDBIObject: primary randomAccessStructure now follows the config
  (isPrimary && RANDOM_ACCESS_FIELDS); non-primary stays off (v4-downgrade compat unchanged)

Typed structures key on per-field value WIDTH, so wide/variably-typed schemas mint an
unbounded per-encoder dictionary (OOM) and diverge across replicas (decode failures);
defaulting them off for primary stores is the conservative choice, with opt-in where safe.

Also keep custom-index object stores (e.g. HNSW vector graphs) in struct mode regardless
of the config — their internal node shapes (numeric-keyed per-level connection arrays and
quantized bins) are fixed and rely on struct encoding, so the table-level default-off would
corrupt the graph.

Adds unit coverage for the config + directive wiring (databases.test.js,
randomAccessFieldsDirective.test.js).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: kriszyp

config-root.schema.json

@@ -460,6 +460,10 @@
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
+				"randomAccessFields": {
+					"type": "boolean",
+					"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas should leave this off. Can be overridden per-table with the @table(randomAccessFields:) schema directive. Default: false"
+				},
 				"prefetchWrites": { "type": "boolean", "description": "Load data prior to write transactions. Default: true" },
 				"path": { "type": "string", "description": "Directory for all database files. Default: <rootPath>/database" },
 				"blobPaths": {

resources/databases.ts

@@ -586,6 +586,9 @@ function initStores(
 					envGet(CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
 				dbiInit.compression.threshold = compressionThreshold;
 			}
+			// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+			if (typeof primaryAttribute.randomAccessFields === 'boolean')
+				dbiInit.randomAccessStructure = primaryAttribute.randomAccessFields;
 			if (rootStore instanceof RocksDatabase) {
 				primaryStore = handleLocalTimeForGets(
 					openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key } as any),
@@ -710,6 +713,7 @@ interface TableDefinition {
 	sealed?: boolean;
 	splitSegments?: boolean;
 	replicate?: boolean;
+	randomAccessFields?: boolean;
 	trackDeletes?: boolean;
 	attributes: any[];
 	schemaDefined?: boolean;
@@ -878,6 +882,14 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
 	const objectStorage =
 		attribute.isPrimaryKey || (attribute.indexed.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
 	const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
+	// Custom-index object stores (e.g. HNSW vector graphs) hold fixed-shape internal nodes —
+	// numeric-keyed per-level connection arrays and quantized bins — that rely on random-access
+	// struct encoding. Keep them in struct mode regardless of the table's storage.randomAccessFields
+	// setting: their node shapes are controlled, so the wide/variably-typed OOM + divergence risks
+	// that motivate the table-level default-off don't apply, and disabling structs corrupts the graph.
+	if (attribute.indexed?.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore) {
+		dbiInit.randomAccessStructure = true;
+	}
 	let dbi:
 		| LMDBDatabase
 		| (RocksDatabase & {
@@ -929,6 +941,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		sealed,
 		splitSegments,
 		replicate,
+		randomAccessFields,
 		trackDeletes,
 		schemaDefined,
 		origin,
@@ -982,13 +995,19 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		primaryKeyAttribute.splitSegments = splitSegments; // always default to not splitting segments going forward
 		if (typeof sealed === 'boolean') primaryKeyAttribute.sealed = sealed;
 		if (typeof replicate === 'boolean') primaryKeyAttribute.replicate = replicate;
+		// Like compression, the record encoding is fixed at creation: existing typed-struct data
+		// still decodes either way, but we don't rewrite already-persisted records.
+		if (typeof randomAccessFields === 'boolean') primaryKeyAttribute.randomAccessFields = randomAccessFields;
 		if (origin) {
 			if (!primaryKeyAttribute.origins) primaryKeyAttribute.origins = [origin];
 			else if (!primaryKeyAttribute.origins.includes(origin)) primaryKeyAttribute.origins.push(origin);
 		}
 		logger.trace(`${tableName} table loading, opening primary store`);
 		const dbiInit = createOpenDBIObject(false, true);
 		dbiInit.compression = primaryKeyAttribute.compression;
+		// per-table override of the storage.randomAccessFields default (see OpenDBIObject)
+		if (typeof primaryKeyAttribute.randomAccessFields === 'boolean')
+			dbiInit.randomAccessStructure = primaryKeyAttribute.randomAccessFields;
 		const dbiName = tableName + '/';
 
 		if (rootStore instanceof RocksDatabase) {

resources/graphql.ts

@@ -77,6 +77,9 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 						if (typeDef.schema) typeDef.database = typeDef.schema;
 						if (!typeDef.table) typeDef.table = typeName;
 						if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
+						// Boolean directive args arrive as actual booleans; tolerate string forms too.
+						if (typeDef.randomAccessFields !== undefined)
+							typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true';
 						typeDef.attributes = typeDef.properties;
 						tables.push(typeDef);
 					}

schema.graphql

@@ -113,6 +113,13 @@ directive @table(
 	table definition.
 	"""
 	replicate: Boolean
+	"""
+	Encode this table's records as typed random-access structures, optimizing for fast field
+	access and smaller records. Best for tables with stable, homogeneous field types; leave off
+	for wide or variably-typed schemas. Overrides the storage.randomAccessFields config for this
+	table. Applied when the table is created.
+	"""
+	randomAccessFields: Boolean
 ) on OBJECT
 
 """

unitTests/resources/databases.test.js

@@ -19,3 +19,33 @@ describe('flushDatabases', () => {
 		await assert.doesNotReject(() => flushDatabases());
 	});
 });
+
+describe('table() randomAccessFields directive', () => {
+	before(function () {
+		setupTestDBPath();
+		setMainIsWorker(true);
+	});
+
+	it('defaults to classic structures (struct writes disabled) when the directive is absent', function () {
+		const DefaultTable = table({
+			table: 'RafDefault',
+			database: 'test',
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+		const encoder = DefaultTable.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.strictEqual(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+
+	it('enables typed random-access structures when @table(randomAccessFields: true)', function () {
+		const RafTable = table({
+			table: 'RafEnabled',
+			database: 'test',
+			randomAccessFields: true,
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+		const encoder = RafTable.primaryStore.encoder;
+		assert.strictEqual(encoder.randomAccessStructure, true);
+		assert.ok(encoder._writeStruct.length > 0, 'expected the real struct-write hook');
+	});
+});

unitTests/resources/models/randomAccessFieldsDirective.test.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const { setupTestDBPath } = require('../../testUtils');
+const { loadGQLSchema } = require('#src/resources/graphql');
+const { tables } = require('#src/resources/databases');
+
+// Parse-level behavior of the @table(randomAccessFields:) directive: the boolean is coerced from the
+// GraphQL string value and flows through to the primary store's encoder, which keeps or stubs its
+// struct-write hook. storage.randomAccessFields defaults off, so an absent directive leaves writes off.
+describe('@table(randomAccessFields:) directive parsing', () => {
+	before(() => setupTestDBPath());
+
+	it('enables typed random-access structures when randomAccessFields: true', async () => {
+		await loadGQLSchema(`type RafOn @table(randomAccessFields: true) {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafOn.primaryStore.encoder;
+		assert.equal(encoder.randomAccessStructure, true);
+		assert.ok(encoder._writeStruct.length > 0, 'expected the real struct-write hook');
+	});
+
+	it('keeps writes disabled when randomAccessFields: false', async () => {
+		await loadGQLSchema(`type RafOff @table(randomAccessFields: false) {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafOff.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.equal(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+
+	it('defaults to disabled when the directive is absent', async () => {
+		await loadGQLSchema(`type RafAbsent @table {
+			id: ID @primaryKey
+			name: String
+		}`);
+		const encoder = tables.RafAbsent.primaryStore.encoder;
+		assert.ok(!encoder.randomAccessStructure);
+		assert.equal(encoder._writeStruct.length, 0, 'expected the no-op write stub');
+	});
+});

utility/hdbTerms.ts

@@ -604,6 +604,7 @@ export const CONFIG_PARAMS = {
 	STORAGE_CACHING: 'storage_caching',
 	STORAGE_COMPRESSION: 'storage_compression',
 	STORAGE_NOREADAHEAD: 'storage_noReadAhead',
+	STORAGE_RANDOMACCESSFIELDS: 'storage_randomAccessFields',
 	STORAGE_PREFETCHWRITES: 'storage_prefetchWrites',
 	STORAGE_ENCRYPTION: 'storage_encryption',
 	STORAGE_MAXTRANSACTIONQUEUETIME: 'storage_maxTransactionQueueTime',

utility/lmdb/OpenDBIObject.ts

@@ -5,6 +5,11 @@ import { RecordEncoder } from '../../resources/RecordEncoder.ts';
 envMngr.initSync();
 
 const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false;
+// Typed random-access structures are opt-in (default off). They optimize for fast field access and
+// smaller records, but key structures on per-field value WIDTH, so a wide/variably-typed schema can
+// mint an unbounded dictionary. Enable globally with storage.randomAccessFields, or per-table with
+// the @table(randomAccessFields:) directive (which overrides this default).
+const RANDOM_ACCESS_FIELDS = envMngr.get(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS) === true;
 
 /**
  * Defines how a DBI will be created/opened
@@ -42,7 +47,10 @@ export class OpenDBIObject {
 		// non-primary stores (e.g. the __dbis__ metadata DBI) stay in records mode and remain
 		// decodable after a downgrade. RecordEncoder still reads struct data so existing v5
 		// struct entries decode.
-		this.randomAccessStructure = isPrimary;
+		// Non-primary DBIs stay off (v4-downgrade compat — their metadata must be readable by a
+		// struct-unaware decoder). Primary DBIs follow the storage.randomAccessFields default (off),
+		// overridable per-table via databases.ts before the store opens.
+		this.randomAccessStructure = isPrimary && RANDOM_ACCESS_FIELDS;
 		if (isPrimary) {
 			this.cache = LMDB_CACHING && { validated: true };
 			this.freezeData = true;