fix(storage): translate the LMDB compression descriptor for RocksDB stores

Harper's `storage.compression` - and the `compression` descriptor persisted on a
table's primary attribute - describe LMDB's per-value compression:
`{ startingOffset, threshold, dictionary }`. That value was passed straight
through to every RocksDB store open. rocksdb-js ignored it until 2.6.0, which
gives `compression` a real meaning (block/blob algorithm) and validates it. 2.6.1
accepts the descriptor object as "unset", but an unset `storage.compression`
resolves to `''` and throws `Unsupported compression algorithm ""`, and the
booleans throw as a malformed option. Every table open throws, taking component
loading and the status table down with it.

Translate at openRocksDatabase, the single point every RocksDB store opens, so
re-opens of existing tables - whose persisted descriptor is LMDB-shaped - are
covered along with new ones. `''`, `true` and the LMDB descriptor leave the
option unset and take rocksdb-js's own default of lz4 (naming lz4 explicitly
would throw on a build without it); `storage.compression: false` maps to 'none';
an algorithm name or { algorithm, level } passes through.

The 2.6 binaries link lz4/zstd/bzip2 for the first time - 2.5.0 exported no
`supportedCompression` and its binary carries no LZ4_compress/ZSTD_compress - so
RocksDB stores were uncompressed until now. `false` therefore preserves today's
behavior and the default (lz4) is the change worth a release note.

main pinned `^2.5.0`, which already resolved 2.6.1 for the Docker image, which
installs from a tarball and so resolves fresh from package.json rather than the
published shrinkwrap. Pin `~2.6.1` so the dependency and the translation move
together.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: kriszyp

package-lock.json

@@ -177,7 +177,7 @@
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^2.5.0",
+		"@harperfast/rocksdb-js": "~2.6.1",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",

package.json

@@ -176,8 +176,35 @@ export const databaseEventsEmitter = new EventEmitter<DatabaseWatcherEventMap>()
 export const tables: Tables = Object.create(null);
 export const databases: Databases = Object.create(null);
 
+/**
+ * Harper's `storage.compression` config - and the `compression` descriptor persisted on a table's
+ * primary attribute - describe LMDB's per-value compression: `{ startingOffset, threshold,
+ * dictionary }`. RocksDB compresses whole SST blocks and blob files itself, and as of rocksdb-js
+ * 2.6 the `compression` store option means that: an algorithm name, or `{ algorithm, level }`.
+ * 2.6.1 accepts an object with no `algorithm` as "unset", but `''` (an unset `storage.compression`)
+ * and the booleans still throw, so the config has to be translated before it reaches the store -
+ * for re-opens of existing tables (whose persisted descriptor is LMDB-shaped) as much as new ones.
+ *
+ * Exported for unit test: every branch here is a startup path, and the ones that throw without it
+ * are exactly the ones a table-level test cannot reach.
+ */
+export function resolveRocksCompression(compression: any) {
+	if (compression == null) return undefined;
+	// a blank string is how an unset `storage.compression` reaches here (it survives the `&&` in
+	// getDefaultCompression), and rocksdb-js rejects it as an unknown algorithm rather than ignoring it
+	if (typeof compression === 'string' && compression.trim() === '') return undefined;
+	// already RocksDB-shaped ('lz4', { algorithm: 'zstd', level: 3 }, ...): pass through
+	if (typeof compression === 'string') return compression;
+	if (typeof compression === 'object' && compression.algorithm != null) return compression;
+	// LMDB descriptor: compression is on, so take the rocksdb-js default (lz4 where the build
+	// supports it) by leaving the option unset - naming lz4 explicitly would instead throw on a
+	// build without it. `storage.compression: false` reaches here as false and disables it.
+	return compression ? undefined : 'none';
+}
+
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
+	(options as any).compression = resolveRocksCompression((options as any).compression);
 	// Apply read-only mode if enabled
 	if (isReadOnlyMode()) {
 		options.readOnly = true;
@@ -745,7 +772,9 @@ function initStores(
 			}
 			const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
 			dbiInit.compression = primaryAttribute.compression;
-			if (dbiInit.compression) {
+			// threshold only applies to the LMDB descriptor shape; a RocksDB algorithm name has no
+			// threshold, and assigning a property to a string primitive throws under strict mode
+			if (dbiInit.compression && typeof dbiInit.compression === 'object') {
 				const compressionThreshold =
 					envGet(CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
 				dbiInit.compression.threshold = compressionThreshold;

resources/databases.ts

@@ -3,7 +3,14 @@ const assert = require('assert');
 const { setupTestDBPath } = require('../testUtils');
 const { existsSync, mkdirSync, writeFileSync } = require('node:fs');
 const { dirname, join } = require('node:path');
-const { table, flushDatabases, dropDatabase, getDatabases, resetDatabases } = require('#src/resources/databases');
+const {
+	table,
+	flushDatabases,
+	dropDatabase,
+	getDatabases,
+	resetDatabases,
+	resolveRocksCompression,
+} = require('#src/resources/databases');
 const { setMainIsWorker } = require('#js/server/threads/manageThreads');
 const { RocksDatabase } = require('@harperfast/rocksdb-js');
 const { beginRestore, completeRestore, RESTORE_META_DIR } = require('#src/dataLayer/restoreMarker');
@@ -54,6 +61,61 @@ describe('table() randomAccessFields directive', () => {
 	});
 });
 
+describe('table() compression', () => {
+	before(function () {
+		setupTestDBPath();
+		setMainIsWorker(true);
+	});
+
+	it('opens the primary store with lz4 under the default config', function () {
+		const CompressionTable = table({
+			table: 'CompressionDefault',
+			database: 'test',
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+		if (!(CompressionTable.primaryStore instanceof RocksDatabase)) return this.skip();
+		assert.strictEqual(CompressionTable.primaryStore.compression?.algorithm, 'lz4');
+	});
+});
+
+// The default config yields the LMDB descriptor object, which rocksdb-js 2.6.1 already accepts as
+// "unset" - so the table-level test above passes with or without the translation. The inputs that
+// actually throw are `''` (an unset storage.compression, rejected as an unknown algorithm) and the
+// booleans (rejected as a malformed option), and only a direct test reaches them.
+describe('resolveRocksCompression', () => {
+	it("treats an unset storage.compression ('') as unset rather than an algorithm name", () => {
+		assert.strictEqual(resolveRocksCompression(''), undefined);
+		assert.strictEqual(resolveRocksCompression('   '), undefined);
+	});
+
+	it('maps the LMDB descriptor and `true` to unset, taking the rocksdb-js default', () => {
+		assert.strictEqual(resolveRocksCompression({ startingOffset: 32, threshold: 4036 }), undefined);
+		assert.strictEqual(resolveRocksCompression(true), undefined);
+	});
+
+	it('maps `storage.compression: false` to no compression', () => {
+		assert.strictEqual(resolveRocksCompression(false), 'none');
+	});
+
+	it('passes an already-RocksDB-shaped option through untouched', () => {
+		assert.strictEqual(resolveRocksCompression('zstd'), 'zstd');
+		const descriptor = { algorithm: 'zstd', level: 3 };
+		assert.strictEqual(resolveRocksCompression(descriptor), descriptor);
+	});
+
+	it('leaves an omitted option unset', () => {
+		assert.strictEqual(resolveRocksCompression(undefined), undefined);
+		assert.strictEqual(resolveRocksCompression(null), undefined);
+	});
+
+	// the resolver runs on every store open, so a throw here fails startup rather than one table
+	it('never throws, whatever shape the persisted attribute holds', () => {
+		for (const input of [0, 1, [], [6], NaN, Symbol('x'), () => {}]) {
+			assert.doesNotThrow(() => resolveRocksCompression(input));
+		}
+	});
+});
+
 describe('schemaDefined backfill on replicas missing the flag', () => {
 	const TABLE = 'SchemaDefinedBackfillTest';
 	const DB = 'test';