|
| 1 | +require('../testUtils'); |
| 2 | +const assert = require('assert'); |
| 3 | +const { RecordEncoder, IndexRecordEncoder } = require('#src/resources/RecordEncoder'); |
| 4 | +const env = require('#js/utility/environment/environmentManager'); |
| 5 | +const terms = require('#src/utility/hdbTerms'); |
| 6 | + |
| 7 | +// In-memory shared structures (mirrors how a DBI shares the structures array under |
| 8 | +// Symbol.for('structures')) so struct/record structures cross between encoder instances. We keep the |
| 9 | +// live array reference rather than round-tripping through encode/decode, since msgpackr attaches |
| 10 | +// bookkeeping to the structures array that a re-decode would strip. |
| 11 | +function sharedStore() { |
| 12 | + let structures = []; |
| 13 | + return { |
| 14 | + save(s) { |
| 15 | + structures = s; |
| 16 | + return true; |
| 17 | + }, |
| 18 | + get() { |
| 19 | + return structures; |
| 20 | + }, |
| 21 | + }; |
| 22 | +} |
| 23 | + |
| 24 | +function makeEncoder(store, extra) { |
| 25 | + return new RecordEncoder({ |
| 26 | + // randomAccessStructure stays on regardless of the opt-out — reads must keep decoding typed |
| 27 | + // structs so existing data is still readable; only writes change. |
| 28 | + randomAccessStructure: true, |
| 29 | + getStructures: store.get, |
| 30 | + saveStructures: store.save, |
| 31 | + ...extra, |
| 32 | + }); |
| 33 | +} |
| 34 | + |
| 35 | +const record = { name: 'price', type: 'Float', indexed: true }; |
| 36 | + |
| 37 | +describe('RecordEncoder random-access fields opt-out (readOnlyStructures)', () => { |
| 38 | + it('writes typed random-access structs by default (random-access fields on)', () => { |
| 39 | + const store = sharedStore(); |
| 40 | + const enc = makeEncoder(store); |
| 41 | + const bytes = enc.encode(record); |
| 42 | + assert.ok(bytes[0] >= 0x20 && bytes[0] < 0x40, `expected typed-struct header byte, got 0x${bytes[0].toString(16)}`); |
| 43 | + }); |
| 44 | + |
| 45 | + it('writes classic shared structures when readOnlyStructures is set (opt-out)', () => { |
| 46 | + const store = sharedStore(); |
| 47 | + const enc = makeEncoder(store, { readOnlyStructures: true }); |
| 48 | + const bytes = enc.encode(record); |
| 49 | + assert.ok( |
| 50 | + bytes[0] >= 0x40 && bytes[0] < 0x80, |
| 51 | + `expected classic shared-structure byte, got 0x${bytes[0].toString(16)}` |
| 52 | + ); |
| 53 | + }); |
| 54 | + |
| 55 | + it('opt-out encoder still reads typed-struct data written before opting out', () => { |
| 56 | + // Existing data was written as typed structs; after the user opts out the same store must keep |
| 57 | + // decoding it (randomAccessStructure stays on), only new writes switch to classic. |
| 58 | + const store = sharedStore(); |
| 59 | + const typedWriter = makeEncoder(store); |
| 60 | + const typedBytes = typedWriter.encode(record); |
| 61 | + assert.ok(typedBytes[0] >= 0x20 && typedBytes[0] < 0x40, 'precondition: typed-struct bytes'); |
| 62 | + |
| 63 | + const optedOutReader = makeEncoder(store, { readOnlyStructures: true }); |
| 64 | + const decoded = optedOutReader.decode(Buffer.from(typedBytes)); |
| 65 | + assert.ok(decoded, 'typed-struct data should still decode (not swallowed to null)'); |
| 66 | + assert.strictEqual(decoded.name, record.name); |
| 67 | + assert.strictEqual(decoded.type, record.type); |
| 68 | + assert.strictEqual(decoded.indexed, record.indexed); |
| 69 | + }); |
| 70 | + |
| 71 | + it('classic records and typed structs round-trip together on the same store', () => { |
| 72 | + const store = sharedStore(); |
| 73 | + const typedWriter = makeEncoder(store); |
| 74 | + const classicWriter = makeEncoder(store, { readOnlyStructures: true }); |
| 75 | + const reader = makeEncoder(store); |
| 76 | + |
| 77 | + const a = { id: 1, kind: 'typed' }; |
| 78 | + const b = { id: 2, kind: 'classic', extra: true }; |
| 79 | + const typedBytes = typedWriter.encode(a); |
| 80 | + const classicBytes = classicWriter.encode(b); |
| 81 | + assert.ok(typedBytes[0] >= 0x20 && typedBytes[0] < 0x40, 'precondition: typed bytes'); |
| 82 | + assert.ok(classicBytes[0] >= 0x40 && classicBytes[0] < 0x80, 'precondition: classic bytes'); |
| 83 | + |
| 84 | + // Typed structs decode with the RecordObject prototype, classic records as plain objects; spread |
| 85 | + // to compare field values regardless of prototype. |
| 86 | + assert.deepStrictEqual({ ...reader.decode(Buffer.from(typedBytes)) }, a); |
| 87 | + assert.deepStrictEqual({ ...reader.decode(Buffer.from(classicBytes)) }, b); |
| 88 | + }); |
| 89 | + |
| 90 | + it('IndexRecordEncoder keeps writing typed structs even with readOnlyStructures requested', () => { |
| 91 | + // Object-store indexes (HNSW) must stay in struct mode regardless of the table opt-out. |
| 92 | + const store = sharedStore(); |
| 93 | + const enc = new IndexRecordEncoder({ |
| 94 | + randomAccessStructure: true, |
| 95 | + readOnlyStructures: true, |
| 96 | + getStructures: store.get, |
| 97 | + saveStructures: store.save, |
| 98 | + }); |
| 99 | + const bytes = enc.encode(record); |
| 100 | + assert.ok( |
| 101 | + bytes[0] >= 0x20 && bytes[0] < 0x40, |
| 102 | + `index encoder should ignore readOnlyStructures and write typed structs, got 0x${bytes[0].toString(16)}` |
| 103 | + ); |
| 104 | + }); |
| 105 | + |
| 106 | + it('decodes a classic record whose structure-id byte is 66 (0x42) when noMetadata is set', () => { |
| 107 | + // Regression: 66 (0x42) is classic shared-structure record-id #2 and also a rocksdb local-timestamp |
| 108 | + // marker. On a rocksdb store the prefix heuristic strips 8 bytes and corrupts the record, decoding |
| 109 | + // to null (the MQTT "publish non-JSON" failure). The audit store passes { noMetadata: true } to skip |
| 110 | + // the heuristic for values that have no on-disk timestamp prefix. |
| 111 | + const store = sharedStore(); |
| 112 | + const writer = makeEncoder(store, { readOnlyStructures: true }); |
| 113 | + // The 3rd distinct classic structure starts with byte 0x42; encode three shapes to reach it. |
| 114 | + writer.encode({ a: 1 }); |
| 115 | + writer.encode({ b: 2, c: 3 }); |
| 116 | + const target = { d: 4, e: 5, f: 6 }; |
| 117 | + const bytes = Buffer.from(writer.encode(target)); |
| 118 | + assert.strictEqual(bytes[0], 66, 'precondition: target record begins with structure-id byte 66 (0x42)'); |
| 119 | + |
| 120 | + const reader = makeEncoder(store); |
| 121 | + // Warm up the structure cache with a normal (non-rocksdb) decode so getStructures isn't needed |
| 122 | + // once we flip isRocksDB below (the rocksdb getStructures path needs a rootStore we don't mock). |
| 123 | + assert.deepStrictEqual(reader.decode(bytes), target, 'baseline classic decode works'); |
| 124 | + |
| 125 | + // Simulate the rocksdb decode path, where byte 66 collides with the local-timestamp marker. |
| 126 | + reader.isRocksDB = true; |
| 127 | + assert.deepStrictEqual( |
| 128 | + reader.decode(bytes, { noMetadata: true }), |
| 129 | + target, |
| 130 | + 'with noMetadata the classic record still decodes correctly' |
| 131 | + ); |
| 132 | + // Without noMetadata the rocksdb timestamp heuristic misreads byte 66 and corrupts the decode. |
| 133 | + assert.notDeepStrictEqual( |
| 134 | + reader.decode(bytes), |
| 135 | + target, |
| 136 | + 'without noMetadata the 0x42 collision corrupts the decode (demonstrating why the flag is needed)' |
| 137 | + ); |
| 138 | + }); |
| 139 | + |
| 140 | + describe('storage.randomAccessFields config drives the default', () => { |
| 141 | + let previous; |
| 142 | + beforeEach(() => { |
| 143 | + previous = env.get(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS); |
| 144 | + }); |
| 145 | + afterEach(() => { |
| 146 | + env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, previous); |
| 147 | + }); |
| 148 | + |
| 149 | + it('writes classic structures when storage.randomAccessFields is false (no explicit option)', () => { |
| 150 | + env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, false); |
| 151 | + const enc = makeEncoder(sharedStore()); // no explicit readOnlyStructures |
| 152 | + const bytes = enc.encode(record); |
| 153 | + assert.ok( |
| 154 | + bytes[0] >= 0x40 && bytes[0] < 0x80, |
| 155 | + `config off should write classic structures, got 0x${bytes[0].toString(16)}` |
| 156 | + ); |
| 157 | + }); |
| 158 | + |
| 159 | + it('writes typed structs when storage.randomAccessFields is true', () => { |
| 160 | + env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, true); |
| 161 | + const enc = makeEncoder(sharedStore()); |
| 162 | + const bytes = enc.encode(record); |
| 163 | + assert.ok( |
| 164 | + bytes[0] >= 0x20 && bytes[0] < 0x40, |
| 165 | + `config on should write typed structs, got 0x${bytes[0].toString(16)}` |
| 166 | + ); |
| 167 | + }); |
| 168 | + |
| 169 | + it('an explicit readOnlyStructures option overrides the config', () => { |
| 170 | + env.setProperty(terms.CONFIG_PARAMS.STORAGE_RANDOMACCESSFIELDS, false); |
| 171 | + const enc = makeEncoder(sharedStore(), { readOnlyStructures: false }); |
| 172 | + const bytes = enc.encode(record); |
| 173 | + assert.ok( |
| 174 | + bytes[0] >= 0x20 && bytes[0] < 0x40, |
| 175 | + `explicit readOnlyStructures:false should keep typed structs despite config, got 0x${bytes[0].toString(16)}` |
| 176 | + ); |
| 177 | + }); |
| 178 | + }); |
| 179 | +}); |
0 commit comments