From f769c8de746cd7717831e3bbc57fc4718ab4ee8e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 25 May 2026 22:34:28 -0600 Subject: [PATCH 1/3] Expose RocksDB WriteBufferManager config params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three new config params for the process-wide RocksDB WriteBufferManager exposed by @harperfast/rocksdb-js#584: - storage.rocks.writeBufferManagerSize — bytes the WBM caps across all DBs in the process (0 disables). The structural cap on total memtable + maintain-window memory. - storage.rocks.writeBufferManagerCostToCache — when true, memtable charges show up in block-cache-usage as pinned entries (single observability metric for read cache + memtable footprint). - storage.rocks.writeBufferManagerAllowStall — when true, writes stall once the cap is reached instead of allowing memtables to briefly exceed it. Values are read inside openRocksDatabase() and passed via spread, so unset values don't surface unknown options to older bindings of rocksdb-js where the WBM isn't yet wired up. This makes the change forward-compatible: it's a no-op until the env var is set AND the underlying binding supports it. Co-Authored-By: Claude Sonnet 4.7 (cherry picked from commit f24bae042e63d1c82d2943b7bc3c5ebf71ec687a) --- resources/databases.ts | 25 ++++++++++++++++++++++--- utility/hdbTerms.ts | 4 ++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index accec15b7..08546c9bb 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -110,11 +110,30 @@ export const databaseEventsEmitter = new EventEmitter() export const tables: Tables = Object.create(null); export const databases: Databases = Object.create(null); -const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; // 25% of available memory - function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { options.disableWAL ??= true; - RocksDatabase.config({ blockCacheSize: MEMORY_FOR_ROCKS_DB }); + // Read RocksDB memory config lazily so env/CLI overrides applied after module load are + // respected. The block cache falls back to 25% of constrained (cgroup) memory when not + // configured; the WriteBufferManager is opt-in (0 disables). + // + // Note: writeBufferManagerCostToCache and writeBufferManagerAllowStall are fixed at WBM + // creation time inside rocksdb-js (the underlying RocksDB API doesn't support changing + // costToCache on a live manager, and allowStall is only re-applied when explicitly changed). + // In practice that's fine — these come from process-level config that doesn't change. + const configuredBlockCacheSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE); + const blockCacheSize = + configuredBlockCacheSize > 0 + ? configuredBlockCacheSize + : Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; + const writeBufferManagerSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE); + const writeBufferManagerCostToCache = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE); + const writeBufferManagerAllowStall = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL); + RocksDatabase.config({ + blockCacheSize, + ...(writeBufferManagerSize > 0 ? { writeBufferManagerSize } : {}), + ...(writeBufferManagerCostToCache != null ? { writeBufferManagerCostToCache } : {}), + ...(writeBufferManagerAllowStall != null ? { writeBufferManagerAllowStall } : {}), + }); if (!existsSync(path)) { mkdirSync(path, { recursive: true }); } diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index a163578ce..c35946450 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -577,6 +577,10 @@ export const CONFIG_PARAMS = { STORAGE_RECLAMATION_INTERVAL: 'storage_reclamation_interval', STORAGE_RECLAMATION_EVICTIONFACTOR: 'storage_reclamation_evictionFactor', STORAGE_ENGINE: 'storage_engine', + STORAGE_ROCKS_BLOCKCACHESIZE: 'storage_rocks_blockCacheSize', + STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE: 'storage_rocks_writeBufferManagerSize', + STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE: 'storage_rocks_writeBufferManagerCostToCache', + STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL: 'storage_rocks_writeBufferManagerAllowStall', DATABASES: 'databases', IGNORE_SCRIPTS: 'ignoreScripts', MQTT_NETWORK_PORT: 'mqtt_network_port', From ea65fc1b05b4054de114db49eef598f3b9cc53d8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 26 May 2026 05:48:15 -0600 Subject: [PATCH 2/3] Coerce envGet values to native types before passing to RocksDB envGet may return raw strings from process.env (configUtils doesn't cast every code path), so RocksDatabase.config previously could receive "12345" instead of 12345 for the size params and "false" (truthy in JS) instead of false for the booleans. - Numeric params (blockCacheSize, writeBufferManagerSize): wrap with Number(). Falsy results (NaN, 0) fall through the > 0 guard. - Boolean params (writeBufferManagerCostToCache, allowStall): inline toBool() that recognizes booleans, 'true'/'false' strings, and generic truthy/falsy fallback. Returns undefined when unset so the spread skips the property. Per PR review from @cb1kenobi (PR #780, line 169). Co-Authored-By: Claude Sonnet 4.7 (cherry picked from commit bc725ea49ead22a59fe7a69f8231a92d528770d9) --- resources/databases.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 08546c9bb..b7667f850 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -116,23 +116,33 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo // respected. The block cache falls back to 25% of constrained (cgroup) memory when not // configured; the WriteBufferManager is opt-in (0 disables). // + // envGet may return values straight from process.env as strings (configUtils + // doesn't cast every code path), so we explicitly coerce numeric/boolean values + // here before passing them to the native binding. + // // Note: writeBufferManagerCostToCache and writeBufferManagerAllowStall are fixed at WBM // creation time inside rocksdb-js (the underlying RocksDB API doesn't support changing // costToCache on a live manager, and allowStall is only re-applied when explicitly changed). // In practice that's fine — these come from process-level config that doesn't change. - const configuredBlockCacheSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE); + const toBool = (v: unknown): boolean | undefined => { + if (v == null) return undefined; + if (typeof v === 'boolean') return v; + if (typeof v === 'string') return v.toLowerCase() === 'true'; + return Boolean(v); + }; + const configuredBlockCacheSize = Number(envGet(CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE)); const blockCacheSize = configuredBlockCacheSize > 0 ? configuredBlockCacheSize : Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; - const writeBufferManagerSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE); - const writeBufferManagerCostToCache = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE); - const writeBufferManagerAllowStall = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL); + const writeBufferManagerSize = Number(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE)); + const writeBufferManagerCostToCache = toBool(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE)); + const writeBufferManagerAllowStall = toBool(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL)); RocksDatabase.config({ blockCacheSize, ...(writeBufferManagerSize > 0 ? { writeBufferManagerSize } : {}), - ...(writeBufferManagerCostToCache != null ? { writeBufferManagerCostToCache } : {}), - ...(writeBufferManagerAllowStall != null ? { writeBufferManagerAllowStall } : {}), + ...(writeBufferManagerCostToCache !== undefined ? { writeBufferManagerCostToCache } : {}), + ...(writeBufferManagerAllowStall !== undefined ? { writeBufferManagerAllowStall } : {}), }); if (!existsSync(path)) { mkdirSync(path, { recursive: true }); From 0f75049b312d186f82fb5cfa6988a92c760dc126 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 26 May 2026 06:02:04 -0600 Subject: [PATCH 3/3] Enforce types from envGet rather than coerce MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts 064501bf8's Number()/toBool() coercion in favor of strict type checks. Config values flow through configUtils.castConfigValue which produces proper numbers/booleans from YAML, env vars, and CLI args — anything else arriving here is misconfiguration that should fall through to the default rather than be silently rescued. - blockCacheSize / writeBufferManagerSize: only honored when `typeof === 'number' && value > 0`. Non-numbers (including unparseable strings) fall back to the 25% default for the cache and disable the WBM. - writeBufferManagerCostToCache / allowStall: only honored when `typeof === 'boolean'`. The 'false' string is no longer accepted — it's a YAML-quoting mistake the operator should fix. Per feedback on PR #780 review thread. Co-Authored-By: Claude Sonnet 4.7 (cherry picked from commit 46109e6c144475fb970c3b604faf4a2620cc414f) --- resources/databases.ts | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index b7667f850..1421f3cde 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -116,33 +116,33 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo // respected. The block cache falls back to 25% of constrained (cgroup) memory when not // configured; the WriteBufferManager is opt-in (0 disables). // - // envGet may return values straight from process.env as strings (configUtils - // doesn't cast every code path), so we explicitly coerce numeric/boolean values - // here before passing them to the native binding. + // We enforce types rather than coerce — values from YAML config and env vars flow + // through configUtils.castConfigValue which produces proper numbers/booleans/null, + // so anything else is misconfiguration and should fall through to the default. // // Note: writeBufferManagerCostToCache and writeBufferManagerAllowStall are fixed at WBM // creation time inside rocksdb-js (the underlying RocksDB API doesn't support changing // costToCache on a live manager, and allowStall is only re-applied when explicitly changed). // In practice that's fine — these come from process-level config that doesn't change. - const toBool = (v: unknown): boolean | undefined => { - if (v == null) return undefined; - if (typeof v === 'boolean') return v; - if (typeof v === 'string') return v.toLowerCase() === 'true'; - return Boolean(v); - }; - const configuredBlockCacheSize = Number(envGet(CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE)); + const configuredBlockCacheSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_BLOCKCACHESIZE); const blockCacheSize = - configuredBlockCacheSize > 0 + typeof configuredBlockCacheSize === 'number' && configuredBlockCacheSize > 0 ? configuredBlockCacheSize : Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; - const writeBufferManagerSize = Number(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE)); - const writeBufferManagerCostToCache = toBool(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE)); - const writeBufferManagerAllowStall = toBool(envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL)); + const writeBufferManagerSize = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERSIZE); + const writeBufferManagerCostToCache = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERCOSTTOCACHE); + const writeBufferManagerAllowStall = envGet(CONFIG_PARAMS.STORAGE_ROCKS_WRITEBUFFERMANAGERALLOWSTALL); RocksDatabase.config({ blockCacheSize, - ...(writeBufferManagerSize > 0 ? { writeBufferManagerSize } : {}), - ...(writeBufferManagerCostToCache !== undefined ? { writeBufferManagerCostToCache } : {}), - ...(writeBufferManagerAllowStall !== undefined ? { writeBufferManagerAllowStall } : {}), + ...(typeof writeBufferManagerSize === 'number' && writeBufferManagerSize > 0 + ? { writeBufferManagerSize } + : {}), + ...(typeof writeBufferManagerCostToCache === 'boolean' + ? { writeBufferManagerCostToCache } + : {}), + ...(typeof writeBufferManagerAllowStall === 'boolean' + ? { writeBufferManagerAllowStall } + : {}), }); if (!existsSync(path)) { mkdirSync(path, { recursive: true });