Merge pull request #496 from HarperFast/fix/rocksdb-entry-count

Use RocksDB's `estimate-num-keys` instead of non-existent `entryCount`

Author: kriszyp

resources/Table.ts

@@ -957,7 +957,8 @@ export function makeTable(options) {
 					records: './', // an href to the records themselves
 					name: tableName,
 					database: databaseName,
-					auditSize: auditStore?.getStats().entryCount,
+					auditSize:
+						auditStore instanceof RocksDatabase ? auditStore.getKeysCount() : auditStore?.getStats().entryCount,
 					attributes,
 					recordCount: undefined,
 					estimatedRecordRange: undefined,
@@ -3173,13 +3174,13 @@ export function makeTable(options) {
 		}
 		static async getRecordCount(options?: any) {
 			// iterate through the metadata entries to exclude their count and exclude the deletion counts
+			const exactCount = options?.exactCount;
 			const entryCount = isRocksDB
-				? (primaryStore.getDBIntProperty('rocksdb.estimate-num-keys') ?? 0)
+				? primaryStore.getKeysCount({ start: exactCount ? null : undefined })
 				: primaryStore.getStats().entryCount;
 			const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
 			const start = performance.now();
 			const halfway = Math.floor(entryCount / 2);
-			const exactCount = options?.exactCount;
 			let recordCount = 0;
 			let entriesScanned = 0;
 			let limit: number;

resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -3,6 +3,7 @@ import { FLOAT32_OPTIONS } from 'msgpackr';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
 import { ClientError } from '../../utility/errors/hdbError.js';
 import type { Id } from '../../resources/ResourceInterface.ts';
+import { RocksDatabase } from '@harperfast/rocksdb-js';
 
 const logger = loggerWithTag('HNSW');
 /**
@@ -636,7 +637,9 @@ export class HierarchicalNavigableSmallWorld {
 	 * @returns
 	 */
 	estimateCountAsSort() {
-		return Math.sqrt(this.indexStore.getStats().entryCount * this.efConstructionSearch);
+		const count =
+			this.indexStore instanceof RocksDatabase ? this.indexStore.getKeysCount() : this.indexStore.getStats().entryCount;
+		return Math.sqrt(count * this.efConstructionSearch);
 	}
 
 	/**

resources/search.ts

@@ -7,6 +7,7 @@ import type { DirectCondition, Id } from './ResourceInterface.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import { lastMetadata } from './RecordEncoder.ts';
 import { recordAction } from './analytics/write';
+import { RocksDatabase } from '@harperfast/rocksdb-js';
 
 // these are ratios/percentages of overall table size
 const OPEN_RANGE_ESTIMATE = 0.3;
@@ -1294,7 +1295,7 @@ function estimatedEntryCount(store) {
 	const now = Date.now();
 	if ((store.estimatedEntryCountExpires || 0) < now) {
 		// use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own
-		store.estimatedEntryCount = store.readerCheck ? store.getStats().entryCount : store.getKeysCount();
+		store.estimatedEntryCount = store instanceof RocksDatabase ? store.getKeysCount() : store.getStats().entryCount;
 		store.estimatedEntryCountExpires = now + 10000;
 	}
 	return store.estimatedEntryCount;