diff --git a/DESIGN.md b/DESIGN.md index fce648d95..01ce6b2a0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -77,6 +77,8 @@ When `migrateOnStart` opens a source LMDB primary store to read records out for Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match. +The persisted `compression` value itself is LMDB-era and loosely shaped: `getDefaultCompression()` historically stored whatever falsy value the config resolved to (`''`, `false`, `null`) when `storage.compression` was disabled, and `{ startingOffset, threshold, dictionary? }` when enabled. lmdb-js interprets falsy as "no compression", but rocksdb-js >= 2.6 validates the option strictly (`''`/booleans throw `Unsupported compression algorithm`) and treats UNSET as "use the build default (lz4)" — the inverse default of lmdb. Every RocksDB open must therefore route through `toRocksCompression()` in `resources/databases.ts` (applied inside `openRocksDatabase`, the single chokepoint), which maps defined-falsy → `'none'` and `true` → unset. Don't pass persisted attribute compression to a RocksDB open directly. + The same source-dbi open has a second non-obvious requirement: assign `sourceDbi.encoder.rootStore = sourceRootStore` for the primary store. The primary dbi decodes through a `RecordEncoder`, and decoding a record that holds a file-backed blob reference resolves that reference against `rootStore` (it locates the blob file). With `rootStore` unset, the `Blob` msgpackr extension throws `No store specified, cannot load blob from storage`; `RecordEncoder.decode` swallows the error and yields `null`, and `copyDbToRocks` then skips the `null` value — so every record with a file-backed blob is silently dropped from the migration. The runtime path gets `rootStore` from `handleLocalTimeForGets`; the migration path opens the source dbi raw and must set it explicitly (issue #857). ## Schema migration and `runIndexing` internals (`databases.ts`) diff --git a/resources/databases.ts b/resources/databases.ts index 027e6d14f..4cd676e82 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -176,8 +176,23 @@ export const databaseEventsEmitter = new EventEmitter() export const tables: Tables = Object.create(null); export const databases: Databases = Object.create(null); +/** + * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata + * carries values where a defined falsy value (false, '') means compression was explicitly + * disabled and true/{ threshold, ... } mean enabled with defaults. rocksdb-js >= 2.6 + * validates `compression` strictly and treats UNSET as "use the build default (lz4)", so + * disabled must be mapped to 'none' explicitly, and true to unset. + */ +export function toRocksCompression(compression: unknown): unknown { + if (compression === true) return undefined; + if (compression !== undefined && !compression) return 'none'; + return compression; +} + function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { options.disableWAL ??= true; + const legacyOptions = options as { compression?: unknown }; + legacyOptions.compression = toRocksCompression(legacyOptions.compression); // Apply read-only mode if enabled if (isReadOnlyMode()) { options.readOnly = true; @@ -2265,7 +2280,9 @@ export function getDefaultCompression() { if (STORAGE_COMPRESSION_DICTIONARY) LMDB_COMPRESSION_OPTS['dictionary'] = readFileSync(STORAGE_COMPRESSION_DICTIONARY); if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD; - return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS; + // normalize disabled to false so a falsy config value ('' or null) is never persisted + // into table metadata as-is (openRocksDatabase maps defined-falsy to 'none') + return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false; } /** diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index 0c653cdc7..0b958c013 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -28,7 +28,9 @@ describe('ComponentLoader Status Integration', function () { if (key === 'MAX_HEADER_SIZE') return 8192; if (key === 'HTTP_PORT') return 9925; if (key === 'CUSTOM_FUNCTIONS') return false; - return ''; + // Unknown keys must read as unset — '' is a real (and sometimes invalid) + // config value, e.g. storage.compression rejects '' as an algorithm name + return undefined; }); // Get both the lifecycle and internal objects diff --git a/unitTests/components/startOnMainThreadOnce.test.js b/unitTests/components/startOnMainThreadOnce.test.js index 4b63bbc1c..6c68cf071 100644 --- a/unitTests/components/startOnMainThreadOnce.test.js +++ b/unitTests/components/startOnMainThreadOnce.test.js @@ -32,7 +32,9 @@ describe('startOnMainThread once-per-component (#460)', function () { if (key === 'MAX_HEADER_SIZE') return 8192; if (key === 'HTTP_PORT') return 9925; if (key === 'CUSTOM_FUNCTIONS') return false; - return ''; + // Unknown keys must read as unset — '' is a real (and sometimes invalid) + // config value, e.g. storage.compression rejects '' as an algorithm name + return undefined; }); const configUtils = require('#src/config/configUtils'); diff --git a/unitTests/resources/rocksCompression.test.js b/unitTests/resources/rocksCompression.test.js new file mode 100644 index 000000000..9dd0de51d --- /dev/null +++ b/unitTests/resources/rocksCompression.test.js @@ -0,0 +1,34 @@ +// Regression guard for the rocksdb-js 2.6 upgrade (HarperFast/harper#2036): persisted table +// metadata carries LMDB-era compression values (false/'' = explicitly disabled, true/object = +// enabled with defaults). rocksdb-js >= 2.6 validates the `compression` option strictly — a +// bare '' or boolean throws ("Unsupported compression algorithm") — and treats UNSET as "use +// the build default (lz4)", so disabled values must be mapped to 'none' before they reach a +// RocksDB open. +const assert = require('node:assert'); +const { toRocksCompression } = require('#src/resources/databases'); + +describe('toRocksCompression', function () { + it('maps explicitly disabled values to none', function () { + assert.strictEqual(toRocksCompression(false), 'none'); + assert.strictEqual(toRocksCompression(''), 'none'); + assert.strictEqual(toRocksCompression(null), 'none'); + assert.strictEqual(toRocksCompression(0), 'none'); + }); + + it('maps true to unset so the build default applies', function () { + assert.strictEqual(toRocksCompression(true), undefined); + }); + + it('leaves unset unset (internal DBIs keep the build default)', function () { + assert.strictEqual(toRocksCompression(undefined), undefined); + }); + + it('passes through LMDB-era option objects (rocksdb-js treats a missing algorithm as unset)', function () { + const legacyOptions = { startingOffset: 32, threshold: 4036 }; + assert.strictEqual(toRocksCompression(legacyOptions), legacyOptions); + }); + + it('passes through explicit algorithm names', function () { + assert.strictEqual(toRocksCompression('zstd'), 'zstd'); + }); +});