From 1e7ed0139ec3268ad3080abfc0e0f4c6f48ac7f9 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Fri, 31 Jul 2026 18:41:32 -0500 Subject: [PATCH 1/2] Map legacy compression metadata to valid rocksdb-js compression options rocksdb-js >= 2.6 validates the compression option strictly: '' and booleans throw ("Unsupported compression algorithm"), and unset means the build default (lz4). Harper's persisted table metadata carries LMDB-era values where defined falsy (false, '') means explicitly disabled, so map those to 'none' and true to unset at the single chokepoint every RocksDB open flows through. Also normalize getDefaultCompression's disabled return to false so raw config garbage ('') is never persisted, and fix two component test stubs that returned '' for every unknown config key - which is how CI on the rocksdb-js 2.6.1 bump first hit this. Co-Authored-By: Claude Fable 5 --- resources/databases.ts | 19 ++++++++++- unitTests/components/componentLoader.test.js | 4 ++- .../components/startOnMainThreadOnce.test.js | 4 ++- unitTests/resources/rocksCompression.test.js | 34 +++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 unitTests/resources/rocksCompression.test.js diff --git a/resources/databases.ts b/resources/databases.ts index 027e6d14f2..4cd676e82e 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -176,8 +176,23 @@ export const databaseEventsEmitter = new EventEmitter() export const tables: Tables = Object.create(null); export const databases: Databases = Object.create(null); +/** + * Map a persisted (LMDB-era) compression value to what rocksdb-js accepts. Table metadata + * carries values where a defined falsy value (false, '') means compression was explicitly + * disabled and true/{ threshold, ... } mean enabled with defaults. rocksdb-js >= 2.6 + * validates `compression` strictly and treats UNSET as "use the build default (lz4)", so + * disabled must be mapped to 'none' explicitly, and true to unset. + */ +export function toRocksCompression(compression: unknown): unknown { + if (compression === true) return undefined; + if (compression !== undefined && !compression) return 'none'; + return compression; +} + function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { options.disableWAL ??= true; + const legacyOptions = options as { compression?: unknown }; + legacyOptions.compression = toRocksCompression(legacyOptions.compression); // Apply read-only mode if enabled if (isReadOnlyMode()) { options.readOnly = true; @@ -2265,7 +2280,9 @@ export function getDefaultCompression() { if (STORAGE_COMPRESSION_DICTIONARY) LMDB_COMPRESSION_OPTS['dictionary'] = readFileSync(STORAGE_COMPRESSION_DICTIONARY); if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD; - return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS; + // normalize disabled to false so a falsy config value ('' or null) is never persisted + // into table metadata as-is (openRocksDatabase maps defined-falsy to 'none') + return LMDB_COMPRESSION ? LMDB_COMPRESSION_OPTS : false; } /** diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index 0c653cdc7c..0b958c0130 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -28,7 +28,9 @@ describe('ComponentLoader Status Integration', function () { if (key === 'MAX_HEADER_SIZE') return 8192; if (key === 'HTTP_PORT') return 9925; if (key === 'CUSTOM_FUNCTIONS') return false; - return ''; + // Unknown keys must read as unset — '' is a real (and sometimes invalid) + // config value, e.g. storage.compression rejects '' as an algorithm name + return undefined; }); // Get both the lifecycle and internal objects diff --git a/unitTests/components/startOnMainThreadOnce.test.js b/unitTests/components/startOnMainThreadOnce.test.js index 4b63bbc1ce..6c68cf071f 100644 --- a/unitTests/components/startOnMainThreadOnce.test.js +++ b/unitTests/components/startOnMainThreadOnce.test.js @@ -32,7 +32,9 @@ describe('startOnMainThread once-per-component (#460)', function () { if (key === 'MAX_HEADER_SIZE') return 8192; if (key === 'HTTP_PORT') return 9925; if (key === 'CUSTOM_FUNCTIONS') return false; - return ''; + // Unknown keys must read as unset — '' is a real (and sometimes invalid) + // config value, e.g. storage.compression rejects '' as an algorithm name + return undefined; }); const configUtils = require('#src/config/configUtils'); diff --git a/unitTests/resources/rocksCompression.test.js b/unitTests/resources/rocksCompression.test.js new file mode 100644 index 0000000000..9dd0de51d7 --- /dev/null +++ b/unitTests/resources/rocksCompression.test.js @@ -0,0 +1,34 @@ +// Regression guard for the rocksdb-js 2.6 upgrade (HarperFast/harper#2036): persisted table +// metadata carries LMDB-era compression values (false/'' = explicitly disabled, true/object = +// enabled with defaults). rocksdb-js >= 2.6 validates the `compression` option strictly — a +// bare '' or boolean throws ("Unsupported compression algorithm") — and treats UNSET as "use +// the build default (lz4)", so disabled values must be mapped to 'none' before they reach a +// RocksDB open. +const assert = require('node:assert'); +const { toRocksCompression } = require('#src/resources/databases'); + +describe('toRocksCompression', function () { + it('maps explicitly disabled values to none', function () { + assert.strictEqual(toRocksCompression(false), 'none'); + assert.strictEqual(toRocksCompression(''), 'none'); + assert.strictEqual(toRocksCompression(null), 'none'); + assert.strictEqual(toRocksCompression(0), 'none'); + }); + + it('maps true to unset so the build default applies', function () { + assert.strictEqual(toRocksCompression(true), undefined); + }); + + it('leaves unset unset (internal DBIs keep the build default)', function () { + assert.strictEqual(toRocksCompression(undefined), undefined); + }); + + it('passes through LMDB-era option objects (rocksdb-js treats a missing algorithm as unset)', function () { + const legacyOptions = { startingOffset: 32, threshold: 4036 }; + assert.strictEqual(toRocksCompression(legacyOptions), legacyOptions); + }); + + it('passes through explicit algorithm names', function () { + assert.strictEqual(toRocksCompression('zstd'), 'zstd'); + }); +}); From 784a746ec044376a7e1d39db16c85f6ea2d154ac Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Fri, 31 Jul 2026 18:42:40 -0500 Subject: [PATCH 2/2] Document legacy compression metadata semantics in DESIGN.md Co-Authored-By: Claude Fable 5 --- DESIGN.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index fce648d95c..01ce6b2a09 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -77,6 +77,8 @@ When `migrateOnStart` opens a source LMDB primary store to read records out for Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match. +The persisted `compression` value itself is LMDB-era and loosely shaped: `getDefaultCompression()` historically stored whatever falsy value the config resolved to (`''`, `false`, `null`) when `storage.compression` was disabled, and `{ startingOffset, threshold, dictionary? }` when enabled. lmdb-js interprets falsy as "no compression", but rocksdb-js >= 2.6 validates the option strictly (`''`/booleans throw `Unsupported compression algorithm`) and treats UNSET as "use the build default (lz4)" — the inverse default of lmdb. Every RocksDB open must therefore route through `toRocksCompression()` in `resources/databases.ts` (applied inside `openRocksDatabase`, the single chokepoint), which maps defined-falsy → `'none'` and `true` → unset. Don't pass persisted attribute compression to a RocksDB open directly. + The same source-dbi open has a second non-obvious requirement: assign `sourceDbi.encoder.rootStore = sourceRootStore` for the primary store. The primary dbi decodes through a `RecordEncoder`, and decoding a record that holds a file-backed blob reference resolves that reference against `rootStore` (it locates the blob file). With `rootStore` unset, the `Blob` msgpackr extension throws `No store specified, cannot load blob from storage`; `RecordEncoder.decode` swallows the error and yields `null`, and `copyDbToRocks` then skips the `null` value — so every record with a file-backed blob is silently dropped from the migration. The runtime path gets `rootStore` from `handleLocalTimeForGets`; the migration path opens the source dbi raw and must set it explicitly (issue #857). ## Schema migration and `runIndexing` internals (`databases.ts`)