Skip to content

Commit 68aa9b8

Browse files
authored
Merge pull request #1169 from HarperFast/kris/v5.0-randomaccessfields
Add storage.randomAccessFields config to disable typed structures (v5.0)
2 parents 0f38c49 + df07f09 commit 68aa9b8

10 files changed

Lines changed: 259 additions & 8 deletions

File tree

config-root.schema.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,10 @@
412412
"description": "Disable fsync for faster writes (risk of data loss on crash). Default: false"
413413
},
414414
"caching": { "type": "boolean", "description": "Enable in-memory caching of records. Default: true" },
415+
"randomAccessFields": {
416+
"type": "boolean",
417+
"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas can set this to false to fall back to classic shared-structure encoding. Applies to each primary table on open: existing records still decode either way, only new writes change. Default: true"
418+
},
415419
"compression": {
416420
"oneOf": [
417421
{ "type": "boolean" },

package-lock.json

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@
205205
"minimist": "1.2.8",
206206
"moment": "2.30.1",
207207
"mqtt-packet": "~9.0.1",
208-
"msgpackr": "1.11.14",
208+
"msgpackr": "1.12.0",
209209
"needle": "3.5.0",
210210
"node-forge": "^1.3.1",
211211
"node-stream-zip": "1.15.0",

resources/RecordEncoder.ts

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
*/
77

88
import { Encoder } from 'msgpackr';
9+
import { get as envGet } from '../utility/environment/environmentManager.js';
10+
import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
911
import {
1012
HAS_PREVIOUS_RESIDENCY_ID,
1113
HAS_CURRENT_RESIDENCY_ID,
@@ -90,6 +92,16 @@ export class RecordEncoder extends Encoder {
9092
// long-lived primary store, so a wide/sparse schema (whose records vary by per-field value
9193
// width) can grow it unbounded and exhaust memory. Caller-overridable; default caps it.
9294
options.maxOwnStructures ??= 256;
95+
// When random-access fields are disabled (storage.randomAccessFields=false), write records as
96+
// classic shared structures instead of typed random-access structures. randomAccessStructure stays
97+
// on so reads still decode either form — existing typed-struct data remains readable; only new
98+
// writes change. lmdb-js does not forward non-whitelisted encoder options, so the flag is derived
99+
// from the global config here rather than passed through the store options. Read at construction
100+
// (DBI open, a cold path) so env/CLI config overrides are applied; an explicit option still wins
101+
// (e.g. rocksdb-js's option spread, or tests).
102+
if (options.readOnlyStructures === undefined && envGet(CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS) === false) {
103+
options.readOnlyStructures = true;
104+
}
93105
/**
94106
* The base class for records that provides the read-only methods for accessing
95107
* metadata and will be assigned computed property getters. On its own, these instances
@@ -265,7 +277,14 @@ export class RecordEncoder extends Encoder {
265277
let nextByte = buffer[start];
266278
let metadataFlags = 0;
267279
try {
268-
if ((this.isRocksDB && nextByte === 66) || (nextByte < 32 && end > 2)) {
280+
// The metadata/timestamp prefix is detected heuristically by the first byte. For rocksdb a
281+
// local-timestamp prefix starts with 66 — but 66 (0x42) is also classic shared-structure
282+
// record-id #2, so a timestamp-less classic record beginning with that id is misread as a
283+
// timestamped record (8 bytes stripped → corrupt). Callers that pass a value known to have no
284+
// prefix (e.g. the audit store's getValue) set options.noMetadata to skip the heuristic. Typed
285+
// structs start at 0x20-0x3f and never hit this, which is why it only surfaces with classic
286+
// structures (random-access fields off).
287+
if (!options?.noMetadata && ((this.isRocksDB && nextByte === 66) || (nextByte < 32 && end > 2))) {
269288
// record with metadata
270289
// this means that the record starts with a local timestamp (that was assigned by lmdb-js).
271290
// we copy it so we can decode it as float-64; we need to do it first because if structural data
@@ -362,6 +381,21 @@ export class RecordEncoder extends Encoder {
362381
}
363382
}
364383
}
384+
385+
/**
386+
* Encoder for custom-index object stores (e.g. HNSW vector graphs). These hold fixed-shape internal
387+
* nodes — numeric-keyed per-level connection arrays and quantized bins — that depend on random-access
388+
* struct encoding and are mutated in place during graph maintenance. Keep them writing typed structs
389+
* even when storage.randomAccessFields disables structs for user tables: their node shapes are
390+
* controlled, so the wide/heterogeneous OOM risk that motivates opt-out doesn't apply, and classic
391+
* (frozen) decoding would break the in-place graph mutation.
392+
*/
393+
export class IndexRecordEncoder extends RecordEncoder {
394+
constructor(options) {
395+
options.readOnlyStructures = false;
396+
super(options);
397+
}
398+
}
365399
function getTimestamp() {
366400
TIMESTAMP_HOLDER[0] = TIMESTAMP_HOLDER[0] ^ 0x40; // restore the first byte, we xor to differentiate the first byte from structures
367401
return TIMESTAMP_VIEW.getFloat64(0);

resources/Table.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,19 @@ const RECORD_PRUNING_INTERVAL = 60000; // one minute
9494
envMngr.initSync();
9595
const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
9696
const LOCK_TIMEOUT = 10000;
97+
98+
// True only for frozen plain/record objects (the immutable decoded records). Excludes Buffers,
99+
// TypedArrays, ArrayBuffers and primitives/null, which must not be shallow-copied via spread (that
100+
// would corrupt binary values). Used to decide when to copy-on-mutate before stamping a record.
101+
function isFrozenRecordObject(value: any): boolean {
102+
return (
103+
value !== null &&
104+
typeof value === 'object' &&
105+
!ArrayBuffer.isView(value) &&
106+
!(value instanceof ArrayBuffer) &&
107+
Object.isFrozen(value)
108+
);
109+
}
97110
export const INVALIDATED = 1;
98111
export const EVICTED = 8; // note that 2 is reserved for timestamps
99112
const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
@@ -1639,6 +1652,12 @@ export function makeTable(options) {
16391652
if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
16401653
if (!context?.source) {
16411654
transaction.checkOverloaded();
1655+
// Records are intentionally immutable: decoded records are frozen (and 5.2 record
1656+
// caching relies on it), so mutating in place would corrupt cached/shared state.
1657+
// validate() coerces values and we stamp created/updated times + the primary key
1658+
// below, so copy-on-mutate when recordUpdate is frozen (e.g. a record decoded during
1659+
// log replay) instead of writing through the frozen object.
1660+
if (isFrozenRecordObject(recordUpdate)) recordUpdate = { ...recordUpdate };
16421661
this.validate(recordUpdate, !fullUpdate);
16431662
if (updatedTimeProperty) {
16441663
recordUpdate[updatedTimeProperty.name] =
@@ -4255,6 +4274,10 @@ export function makeTable(options) {
42554274
}
42564275
}
42574276
if (typeof updatedRecord.toJSON === 'function') updatedRecord = updatedRecord.toJSON();
4277+
// updatedRecord may still be a frozen record (e.g. a reused existingRecord); copy-on-mutate
4278+
// before stamping the primary key below (records are immutable — 5.2 record caching relies
4279+
// on it — so we must not write through the frozen object).
4280+
if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
42584281
if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
42594282
}
42604283
resolved = true;

resources/auditStore.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,10 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
552552
if (action & HAS_RECORD || (action & HAS_PARTIAL_RECORD && !fullRecord)) {
553553
if (!value) {
554554
value = decodeFromDatabase(
555-
() => store.decoder.decode(buffer.subarray(decoder.position, end)),
555+
// the audit value has no on-disk timestamp/metadata prefix (the audit entry carries
556+
// its own time), so skip the prefix heuristic — otherwise a classic record whose
557+
// structure-id byte is 66 (0x42) is misread as a rocksdb timestamp. See RecordEncoder.decode.
558+
() => store.decoder.decode(buffer.subarray(decoder.position, end), { noMetadata: true }),
556559
store.rootStore
557560
);
558561
}

resources/databases.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import harperLogger from '../utility/logging/harper_logger.js';
2222
const { forComponent } = harperLogger;
2323
import * as manageThreads from '../server/threads/manageThreads.js';
2424
import { openAuditStore, readAuditEntry, createAuditEntry, type AuditRecord } from './auditStore.ts';
25-
import { handleLocalTimeForGets } from './RecordEncoder.ts';
25+
import { handleLocalTimeForGets, IndexRecordEncoder } from './RecordEncoder.ts';
2626
import { deleteRootBlobPathsForDB } from './blob.ts';
2727
import { CUSTOM_INDEXES } from './indexes/customIndexes.ts';
2828
import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
@@ -818,6 +818,12 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any)
818818
const objectStorage =
819819
attribute.isPrimaryKey || (attribute.indexed.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
820820
const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
821+
// Custom-index object stores (e.g. HNSW vector graphs) must keep writing typed structs regardless
822+
// of the storage.randomAccessFields opt-out — their internal nodes are mutated in place and depend
823+
// on random-access struct encoding (see IndexRecordEncoder).
824+
if (attribute.indexed?.type && CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore) {
825+
dbiInit.encoder = { Encoder: IndexRecordEncoder };
826+
}
821827
let dbi:
822828
| LMDBDatabase
823829
| (RocksDatabase & {

static/defaultConfig.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ rootPath: null
6969
storage:
7070
writeAsync: false
7171
caching: true
72+
randomAccessFields: true
7273
compression: true
7374
noReadAhead: false
7475
path: null
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
require('../testUtils');
2+
const assert = require('assert');
3+
const { RecordEncoder, IndexRecordEncoder } = require('#src/resources/RecordEncoder');
4+
const env = require('#js/utility/environment/environmentManager');
5+
const terms = require('#src/utility/hdbTerms');
6+
7+
// In-memory shared structures (mirrors how a DBI shares the structures array under
8+
// Symbol.for('structures')) so struct/record structures cross between encoder instances. We keep the
9+
// live array reference rather than round-tripping through encode/decode, since msgpackr attaches
10+
// bookkeeping to the structures array that a re-decode would strip.
11+
function sharedStore() {
12+
let structures = [];
13+
return {
14+
save(s) {
15+
structures = s;
16+
return true;
17+
},
18+
get() {
19+
return structures;
20+
},
21+
};
22+
}
23+
24+
function makeEncoder(store, extra) {
25+
return new RecordEncoder({
26+
// randomAccessStructure stays on regardless of the opt-out — reads must keep decoding typed
27+
// structs so existing data is still readable; only writes change.
28+
randomAccessStructure: true,
29+
getStructures: store.get,
30+
saveStructures: store.save,
31+
...extra,
32+
});
33+
}
34+
35+
const record = { name: 'price', type: 'Float', indexed: true };
36+
37+
describe('RecordEncoder random-access fields opt-out (readOnlyStructures)', () => {
38+
it('writes typed random-access structs by default (random-access fields on)', () => {
39+
const store = sharedStore();
40+
const enc = makeEncoder(store);
41+
const bytes = enc.encode(record);
42+
assert.ok(bytes[0] >= 0x20 && bytes[0] < 0x40, `expected typed-struct header byte, got 0x${bytes[0].toString(16)}`);
43+
});
44+
45+
it('writes classic shared structures when readOnlyStructures is set (opt-out)', () => {
46+
const store = sharedStore();
47+
const enc = makeEncoder(store, { readOnlyStructures: true });
48+
const bytes = enc.encode(record);
49+
assert.ok(
50+
bytes[0] >= 0x40 && bytes[0] < 0x80,
51+
`expected classic shared-structure byte, got 0x${bytes[0].toString(16)}`
52+
);
53+
});
54+
55+
it('opt-out encoder still reads typed-struct data written before opting out', () => {
56+
// Existing data was written as typed structs; after the user opts out the same store must keep
57+
// decoding it (randomAccessStructure stays on), only new writes switch to classic.
58+
const store = sharedStore();
59+
const typedWriter = makeEncoder(store);
60+
const typedBytes = typedWriter.encode(record);
61+
assert.ok(typedBytes[0] >= 0x20 && typedBytes[0] < 0x40, 'precondition: typed-struct bytes');
62+
63+
const optedOutReader = makeEncoder(store, { readOnlyStructures: true });
64+
const decoded = optedOutReader.decode(Buffer.from(typedBytes));
65+
assert.ok(decoded, 'typed-struct data should still decode (not swallowed to null)');
66+
assert.strictEqual(decoded.name, record.name);
67+
assert.strictEqual(decoded.type, record.type);
68+
assert.strictEqual(decoded.indexed, record.indexed);
69+
});
70+
71+
it('classic records and typed structs round-trip together on the same store', () => {
72+
const store = sharedStore();
73+
const typedWriter = makeEncoder(store);
74+
const classicWriter = makeEncoder(store, { readOnlyStructures: true });
75+
const reader = makeEncoder(store);
76+
77+
const a = { id: 1, kind: 'typed' };
78+
const b = { id: 2, kind: 'classic', extra: true };
79+
const typedBytes = typedWriter.encode(a);
80+
const classicBytes = classicWriter.encode(b);
81+
assert.ok(typedBytes[0] >= 0x20 && typedBytes[0] < 0x40, 'precondition: typed bytes');
82+
assert.ok(classicBytes[0] >= 0x40 && classicBytes[0] < 0x80, 'precondition: classic bytes');
83+
84+
// Typed structs decode with the RecordObject prototype, classic records as plain objects; spread
85+
// to compare field values regardless of prototype.
86+
assert.deepStrictEqual({ ...reader.decode(Buffer.from(typedBytes)) }, a);
87+
assert.deepStrictEqual({ ...reader.decode(Buffer.from(classicBytes)) }, b);
88+
});
89+
90+
it('IndexRecordEncoder keeps writing typed structs even with readOnlyStructures requested', () => {
91+
// Object-store indexes (HNSW) must stay in struct mode regardless of the table opt-out.
92+
const store = sharedStore();
93+
const enc = new IndexRecordEncoder({
94+
randomAccessStructure: true,
95+
readOnlyStructures: true,
96+
getStructures: store.get,
97+
saveStructures: store.save,
98+
});
99+
const bytes = enc.encode(record);
100+
assert.ok(
101+
bytes[0] >= 0x20 && bytes[0] < 0x40,
102+
`index encoder should ignore readOnlyStructures and write typed structs, got 0x${bytes[0].toString(16)}`
103+
);
104+
});
105+
106+
it('decodes a classic record whose structure-id byte is 66 (0x42) when noMetadata is set', () => {
107+
// Regression: 66 (0x42) is classic shared-structure record-id #2 and also a rocksdb local-timestamp
108+
// marker. On a rocksdb store the prefix heuristic strips 8 bytes and corrupts the record, decoding
109+
// to null (the MQTT "publish non-JSON" failure). The audit store passes { noMetadata: true } to skip
110+
// the heuristic for values that have no on-disk timestamp prefix.
111+
const store = sharedStore();
112+
const writer = makeEncoder(store, { readOnlyStructures: true });
113+
// The 3rd distinct classic structure starts with byte 0x42; encode three shapes to reach it.
114+
writer.encode({ a: 1 });
115+
writer.encode({ b: 2, c: 3 });
116+
const target = { d: 4, e: 5, f: 6 };
117+
const bytes = Buffer.from(writer.encode(target));
118+
assert.strictEqual(bytes[0], 66, 'precondition: target record begins with structure-id byte 66 (0x42)');
119+
120+
const reader = makeEncoder(store);
121+
// Warm up the structure cache with a normal (non-rocksdb) decode so getStructures isn't needed
122+
// once we flip isRocksDB below (the rocksdb getStructures path needs a rootStore we don't mock).
123+
assert.deepStrictEqual(reader.decode(bytes), target, 'baseline classic decode works');
124+
125+
// Simulate the rocksdb decode path, where byte 66 collides with the local-timestamp marker.
126+
reader.isRocksDB = true;
127+
assert.deepStrictEqual(
128+
reader.decode(bytes, { noMetadata: true }),
129+
target,
130+
'with noMetadata the classic record still decodes correctly'
131+
);
132+
// Without noMetadata the rocksdb timestamp heuristic misreads byte 66 and corrupts the decode.
133+
assert.notDeepStrictEqual(
134+
reader.decode(bytes),
135+
target,
136+
'without noMetadata the 0x42 collision corrupts the decode (demonstrating why the flag is needed)'
137+
);
138+
});
139+
140+
describe('storage.randomAccessFields config drives the default', () => {
141+
let previous;
142+
beforeEach(() => {
143+
previous = env.get(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS);
144+
});
145+
afterEach(() => {
146+
env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, previous);
147+
});
148+
149+
it('writes classic structures when storage.randomAccessFields is false (no explicit option)', () => {
150+
env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, false);
151+
const enc = makeEncoder(sharedStore()); // no explicit readOnlyStructures
152+
const bytes = enc.encode(record);
153+
assert.ok(
154+
bytes[0] >= 0x40 && bytes[0] < 0x80,
155+
`config off should write classic structures, got 0x${bytes[0].toString(16)}`
156+
);
157+
});
158+
159+
it('writes typed structs when storage.randomAccessFields is true', () => {
160+
env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, true);
161+
const enc = makeEncoder(sharedStore());
162+
const bytes = enc.encode(record);
163+
assert.ok(
164+
bytes[0] >= 0x20 && bytes[0] < 0x40,
165+
`config on should write typed structs, got 0x${bytes[0].toString(16)}`
166+
);
167+
});
168+
169+
it('an explicit readOnlyStructures option overrides the config', () => {
170+
env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, false);
171+
const enc = makeEncoder(sharedStore(), { readOnlyStructures: false });
172+
const bytes = enc.encode(record);
173+
assert.ok(
174+
bytes[0] >= 0x20 && bytes[0] < 0x40,
175+
`explicit readOnlyStructures:false should keep typed structs despite config, got 0x${bytes[0].toString(16)}`
176+
);
177+
});
178+
});
179+
});

utility/hdbTerms.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ export const CONFIG_PARAMS = {
554554
STORAGE_WRITEASYNC: 'storage_writeAsync',
555555
STORAGE_OVERLAPPINGSYNC: 'storage_overlappingSync',
556556
STORAGE_CACHING: 'storage_caching',
557+
STORAGE_RANDOMACCESSFIELDS: 'storage_randomAccessFields',
557558
STORAGE_COMPRESSION: 'storage_compression',
558559
STORAGE_NOREADAHEAD: 'storage_noReadAhead',
559560
STORAGE_PREFETCHWRITES: 'storage_prefetchWrites',

0 commit comments

Comments
 (0)