diff --git a/src/embedder/index.ts b/src/embedder/index.ts index 8fbce55..9d3ba7b 100644 --- a/src/embedder/index.ts +++ b/src/embedder/index.ts @@ -15,7 +15,7 @@ export type { Embedder } from './types'; // Language detection & CJK utilities — re-exported from utils for backward compatibility -export { isCJK, splitMixed, detectLanguage, tokenizerForLanguage, detectTokenizer } from '../utils/tokenizer'; +export { isCJK, detectLanguage, tokenizerForLanguage, detectTokenizer } from '../utils/tokenizer'; export type { LanguageHint } from '../utils/tokenizer'; // TransformersEmbedder — production-quality model embedder @@ -26,4 +26,4 @@ export { EmbedderManager, getEmbedder, resetEmbedder } from './manager'; // Embedder resolution export { resolveEmbedder } from './resolve'; -export type { EmbedderInfo, EmbedderKind, ResolveResult } from './resolve'; \ No newline at end of file +export type { EmbedderInfo, EmbedderKind } from './resolve'; \ No newline at end of file diff --git a/src/embedder/manager.ts b/src/embedder/manager.ts index 83a4787..d66212a 100644 --- a/src/embedder/manager.ts +++ b/src/embedder/manager.ts @@ -30,19 +30,16 @@ export interface EmbedderManagerOptions { export class EmbedderManager { private _defaultEmbedder: Embedder | null = null; private readonly _loader: TransformersLoader; - private readonly _ownsLoader: boolean; constructor(options?: EmbedderManagerOptions) { if (options?.transformersLoader) { this._loader = options.transformersLoader; - this._ownsLoader = false; } else { // Default: use global shared loader (backward-compatible) this._loader = { load: loadTransformersModule, reset: resetTransformersModule, }; - this._ownsLoader = false; } } @@ -70,12 +67,12 @@ export class EmbedderManager { if (!t) { throw new Error( '@huggingface/transformers is not installed. Semantic search requires a model-based embedder.\n' + - ' Install it with:\n' + - ' npm install @huggingface/transformers\n' + - ' Then download the model:\n' + - ' node scripts/download-model.mjs\n' + - ' Or set mirror for China:\n' + - ' export HF_ENDPOINT=https://hf-mirror.com' + ' Install it with:\n' + + ' npm install @huggingface/transformers\n' + + ' Then download the model:\n' + + ' node scripts/download-model.mjs\n' + + ' Or set mirror for China:\n' + + ' export HF_ENDPOINT=https://hf-mirror.com', ); } @@ -86,9 +83,9 @@ export class EmbedderManager { } catch (err) { throw new Error( `Failed to load embedding model (bge-small-zh-v1.5): ${(err as Error).message?.split('\n')[0] ?? 'unknown'}\n` + - ' To fix model download:\n' + - ' 1. Set mirror: export HF_ENDPOINT=https://hf-mirror.com\n' + - ' 2. Manual download: node scripts/download-model.mjs' + ' To fix model download:\n' + + ' 1. Set mirror: export HF_ENDPOINT=https://hf-mirror.com\n' + + ' 2. Manual download: node scripts/download-model.mjs', ); } return this._defaultEmbedder; diff --git a/src/index.ts b/src/index.ts index 0567fd0..57866cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,22 +37,19 @@ export { tokenizerForLanguage, detectTokenizer, } from './embedder'; -export type { Embedder } from './embedder'; -export type { LanguageHint } from './embedder'; -export type { EmbedderInfo, EmbedderKind } from './embedder'; +export type { Embedder, LanguageHint, EmbedderInfo, EmbedderKind } from './embedder'; // Zvec store — custom vector storage backends export { - IZvecStore, ActualZvecStore, createZvecStore, openZvecStore, openZvecStoreSync, isZvecAvailable, buildZvecSchema, - cosineSimilarity, } from './storage/zvec-store'; export type { + IZvecStore, ZvecDoc, ZvecQueryResult, ZvecSearchParams, @@ -66,3 +63,4 @@ export type { export { Store } from './storage/store'; export type { StoreQueryParams } from './storage/store'; + diff --git a/src/loaders/util.ts b/src/loaders/util.ts index 2c1fb8a..1c17ced 100644 --- a/src/loaders/util.ts +++ b/src/loaders/util.ts @@ -1,5 +1,5 @@ -import * as crypto from 'crypto'; import * as path from 'path'; +import { computeContentHash } from '../utils/hash'; /** * Convert a file path to a safe, collision-resistant ID for zvec. @@ -20,7 +20,7 @@ export function pathToId(filePath: string): string { const normalized = path.normalize(filePath).replace(/\\/g, '/').replace(/^\/+/, ''); // Generate a compact hash of the full path (16 hex chars = 64-bit) - const hash = crypto.createHash('sha256').update(normalized).digest('hex').slice(0, 16); + const hash = computeContentHash(normalized); // Derive a short readable suffix from the filename (without extension) const basename = normalized.split('/').pop() ?? ''; diff --git a/src/storage/actual-store.ts b/src/storage/actual-store.ts deleted file mode 100644 index e9dfaac..0000000 --- a/src/storage/actual-store.ts +++ /dev/null @@ -1,407 +0,0 @@ -/** - * ActualZvecStore — wraps @zvec/zvec native bindings (full schema + FTS). - */ - -import type { - ZvecDoc, - ZvecQueryResult, - ZvecSearchParams, - ZvecHybridParams, - IZvecStore, - ZvecStoreConfig, - ActualZvecStoreOptions, - ZvecFieldSchema, -} from './types'; - -// --------------------------------------------------------------------------- -// zvec module loading -// --------------------------------------------------------------------------- - -let _zvecModule: unknown = undefined; -let _zvecLoadFailed = false; - -/** Zvec SDK type — minimal interface we rely on. */ -interface ZvecSDK { - ZVecCollectionSchema: new (config: unknown) => unknown; - ZVecDataType: Record; - ZVecIndexType: Record; - ZVecMetricType: Record; - ZVecCreateAndOpen(path: string, schema: unknown): ZvecCollection; - ZVecOpen(path: string, options?: Record): ZvecCollection; -} - -/** Zvec collection type — minimal interface we rely on. */ -interface ZvecCollection { - insertSync(records: unknown[]): void; - upsertSync(records: unknown[]): void; - fetchSync(params: { ids: string | string[]; outputFields?: string[]; includeVector?: boolean }): Record; - querySync(params: unknown): ZvecQueryRaw[]; - multiQuerySync(params: unknown): ZvecQueryRaw[]; - closeSync(): void; -} - -interface ZvecQueryRaw { - id: string; - score: number; - fields?: Record; -} - -function loadZvecSync(): ZvecSDK | undefined { - if (_zvecModule) return _zvecModule as ZvecSDK; - if (_zvecLoadFailed) return undefined; - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - _zvecModule = require('@zvec/zvec'); - } catch { - _zvecLoadFailed = true; - return undefined; - } - return _zvecModule as ZvecSDK; -} - -function requireZvecSync(): ZvecSDK { - const z = loadZvecSync(); - if (!z) { - throw new Error( - '@zvec/zvec is not installed. Install it with:\n' + - ' pnpm add @zvec/zvec' - ); - } - return z; -} - -// --------------------------------------------------------------------------- -// Schema builder -// --------------------------------------------------------------------------- - -/** - * Build a ZVecCollectionSchema from a generic ZvecStoreConfig. - * - * This is a helper for callers that want to create zvec collections without - * dealing with the low-level zvec SDK schema API directly. - */ -export function buildZvecSchema(z: ZvecSDK, config: ZvecStoreConfig): unknown { - const { ZVecCollectionSchema, ZVecDataType, ZVecIndexType, ZVecMetricType } = z; - - const vectorSchema = { - name: config.vectorField, - dataType: ZVecDataType.VECTOR_FP32, - dimension: config.vectorDims, - indexParams: { - indexType: ZVecIndexType.HNSW, - metricType: ZVecMetricType.COSINE, - m: 32, - efConstruction: 200 - } - }; - - const fieldTypeMap: Record = { - STRING: ZVecDataType.STRING, - INT64: ZVecDataType.INT64, - FLOAT: ZVecDataType.FLOAT, - VECTOR_FP32: ZVecDataType.VECTOR_FP32, - }; - - const indexTypeMap: Record = { - FTS: ZVecIndexType.FTS, - INVERT: ZVecIndexType.INVERT, - HNSW: ZVecIndexType.HNSW, - }; - - const fieldSchemas = config.fields.map((f: ZvecFieldSchema) => { - const schema: Record = { - name: f.name, - dataType: fieldTypeMap[f.dataType], - }; - - if (f.indexType && f.indexType !== 'NONE' && indexTypeMap[f.indexType]) { - schema.indexParams = { - indexType: indexTypeMap[f.indexType], - ...(f.indexOptions ?? {}), - }; - } - - return schema; - }); - - return new ZVecCollectionSchema({ - name: config.collectionName, - vectors: vectorSchema, - fields: fieldSchemas, - }); -} - -// --------------------------------------------------------------------------- -// ActualZvecStore implementation -// --------------------------------------------------------------------------- - -const DEFAULT_OPEN_OPTIONS: ActualZvecStoreOptions = { - vectorField: 'embedding', - ftsFields: [], - rankConstant: 60, -}; - -/** - * ZvecStore backed by the native @zvec/zvec library. - * - * Provides true hybrid search: zvec's FTS path accepts raw text (no embedding), - * the vector path uses pre-computed embeddings, and `multiQuerySync` with RRF - * fuses both in the engine. - */ -export class ActualZvecStore implements IZvecStore { - private _collection: ZvecCollection; - private _closed = false; - private _vectorField: string; - private _ftsFields: string[]; - private _rankConstant: number; - - constructor(collection: ZvecCollection, options: ActualZvecStoreOptions) { - this._collection = collection; - this._vectorField = options.vectorField; - this._ftsFields = options.ftsFields; - this._rankConstant = options.rankConstant ?? 60; - } - - /** - * Create a new zvec collection with a generic schema config. - */ - static async create(path: string, config: ZvecStoreConfig): Promise { - const z = requireZvecSync(); - const schema = buildZvecSchema(z, config); - const collection = z.ZVecCreateAndOpen(path, schema); - return new ActualZvecStore(collection, { - vectorField: config.vectorField, - ftsFields: config.ftsFields, - }); - } - - /** Synchronous version of `create`. */ - static createSync(path: string, config: ZvecStoreConfig): ActualZvecStore { - const z = requireZvecSync(); - const schema = buildZvecSchema(z, config); - const collection = z.ZVecCreateAndOpen(path, schema); - return new ActualZvecStore(collection, { - vectorField: config.vectorField, - ftsFields: config.ftsFields, - }); - } - - static async open( - path: string, - options?: ActualZvecStoreOptions - ): Promise { - const z = requireZvecSync(); - const collection = z.ZVecOpen(path); - return new ActualZvecStore(collection, options ?? DEFAULT_OPEN_OPTIONS); - } - - /** Synchronous version of `open`. Uses read-only to allow concurrent readers. */ - static openSync( - path: string, - options?: ActualZvecStoreOptions - ): ActualZvecStore { - const z = requireZvecSync(); - const collection = z.ZVecOpen(path, { readOnly: true }); - return new ActualZvecStore(collection, options ?? DEFAULT_OPEN_OPTIONS); - } - - async insert(docs: ZvecDoc[]): Promise { - if (this._closed) throw new Error('Store is closed'); - if (docs.length === 0) return; - - const records = docs.map((d) => ({ - id: d.id, - vectors: { [this._vectorField]: d.vector }, - fields: d.fields - })); - // upsertSync handles both new docs and content-changed updates in one call. - this._collection.upsertSync(records); - } - - async fetch(ids: string[], outputFields?: string[]): Promise> { - if (this._closed) throw new Error('Store is closed'); - if (ids.length === 0) return {}; - - const raw = this._collection.fetchSync({ - ids, - outputFields, - includeVector: false, - }); - - const result: Record = {}; - for (const [id, doc] of Object.entries(raw)) { - result[id] = { - id, - score: 0, - fields: (doc as ZvecQueryRaw).fields ?? {}, - }; - } - return result; - } - - async search(params: ZvecSearchParams): Promise { - if (this._closed) throw new Error('Store is closed'); - - const rawResults = this._collection.querySync( - this._buildVectorQuery(params) - ); - - return rawResults.map((r: ZvecQueryRaw) => ({ - id: r.id, - score: r.score, - fields: r.fields ?? {} - })); - } - - async searchHybrid(params: ZvecHybridParams): Promise { - if (this._closed) throw new Error('Store is closed'); - - const rawResults = this._collection.multiQuerySync( - this._buildHybridQuery(params) - ); - - return rawResults.map((r: ZvecQueryRaw) => ({ - id: r.id, - score: r.score, - fields: r.fields ?? {} - })); - } - - searchSync(params: ZvecSearchParams): ZvecQueryResult[] { - if (this._closed) throw new Error('Store is closed'); - - const rawResults = this._collection.querySync( - this._buildVectorQuery(params) - ); - - return rawResults.map((r: ZvecQueryRaw) => ({ - id: r.id, - score: r.score, - fields: r.fields ?? {} - })); - } - - searchHybridSync(params: ZvecHybridParams): ZvecQueryResult[] { - if (this._closed) throw new Error('Store is closed'); - - const rawResults = this._collection.multiQuerySync( - this._buildHybridQuery(params) - ); - - return rawResults.map((r: ZvecQueryRaw) => ({ - id: r.id, - score: r.score, - fields: r.fields ?? {} - })); - } - - /** Build a query params object for vector search, omitting filter when undefined. */ - private _buildVectorQuery(params: ZvecSearchParams): Record { - const q: Record = { - fieldName: this._vectorField, - vector: params.vector, - topk: params.topK, - }; - if (params.filter) { - q.filter = params.filter; - } - return q; - } - - /** Build multi-field FTS query paths for hybrid search from configured ftsFields. */ - private _buildFtsQueries(queryText: string, topK: number): Record[] { - const z = loadZvecSync(); - const ftsParams = z - ? { indexType: z.ZVecIndexType.FTS, defaultOperator: 'OR' as const } - : undefined; - - return this._ftsFields.map((fieldName) => ({ - fieldName, - fts: { matchString: queryText }, - numCandidates: topK * 2, - params: ftsParams, - })); - } - - /** Build multiQuery params for hybrid search, omitting filter when undefined. */ - private _buildHybridQuery(params: ZvecHybridParams): Record { - const q: Record = { - queries: [ - { - fieldName: this._vectorField, - vector: params.queryVector, - numCandidates: params.topK * 2 - }, - ...this._buildFtsQueries(params.queryText, params.topK), - ], - topk: params.topK, - rerank: { type: 'rrf', rankConstant: this._rankConstant } - }; - if (params.filter) { - q.filter = params.filter; - } - return q; - } - - async close(): Promise { - if (this._closed) return; - this._closed = true; - try { - this._collection.closeSync(); - } catch { - // best-effort - } - } -} - -// --------------------------------------------------------------------------- -// Convenience factories -// --------------------------------------------------------------------------- - -/** - * Create a zvec store with the given config. - * - * Requires @zvec/zvec to be installed. Throws an error when it is not available. - */ -export async function createZvecStore( - path: string, - config: ZvecStoreConfig, -): Promise { - return ActualZvecStore.create(path, config); -} - -/** - * Asynchronously open a zvec store. - * - * Requires @zvec/zvec to be installed. - */ -export async function openZvecStore( - path: string, - options?: ActualZvecStoreOptions -): Promise { - return openZvecStoreSync(path, options); -} - -/** - * Synchronously open a zvec store. - * - * Requires @zvec/zvec to be installed. Throws an error when it is not available. - */ -export function openZvecStoreSync( - path: string, - options?: ActualZvecStoreOptions -): IZvecStore { - const z = loadZvecSync(); - if (z) { - return ActualZvecStore.openSync(path, options); - } - throw new Error( - '@zvec/zvec is not installed. Install it with:\n' + - ' pnpm add @zvec/zvec' - ); -} - -/** Synchronous check: is @zvec/zvec available? */ -export function isZvecAvailable(): boolean { - return loadZvecSync() !== undefined; -} diff --git a/src/storage/store.ts b/src/storage/store.ts index f12f065..ab5a7cf 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -222,17 +222,6 @@ export class Store { } } - /** - * Delete a library's store file from disk (after closing it). - */ - async deleteStore(library: string): Promise { - await this.close(library); - const filePath = this._getStorePath(library); - if (fs.existsSync(filePath)) { - fs.unlinkSync(filePath); - } - } - /** * Close all cached stores and release resources. * diff --git a/src/storage/utils.ts b/src/storage/utils.ts deleted file mode 100644 index c7aa273..0000000 --- a/src/storage/utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * ZvecStore utility functions. - */ - -/** Cosine similarity between two vectors of equal length. */ -export function cosineSimilarity(a: number[], b: number[]): number { - let dot = 0, - na = 0, - nb = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - na += a[i] * a[i]; - nb += b[i] * b[i]; - } - const denom = Math.sqrt(na) * Math.sqrt(nb); - return denom === 0 ? 0 : dot / denom; -} diff --git a/src/storage/zvec-store.ts b/src/storage/zvec-store.ts index cc8b07a..299817c 100644 --- a/src/storage/zvec-store.ts +++ b/src/storage/zvec-store.ts @@ -1,12 +1,15 @@ /** - * zvec-store — aggregate entry point for all storage modules. + * zvec-store — zvec storage module: types, implementation, and factories. * - * Re-exports from the split files for backward compatibility: - * types.ts → public types & interfaces - * actual-store.ts → ActualZvecStore, factories, schema builder - * utils.ts → cosineSimilarity + * Combines the former split files into a single module: + * types.ts → public types & interfaces + * actual-store.ts → ActualZvecStore, factories, schema builder */ +// --------------------------------------------------------------------------- +// Public types (re-exported from types.ts) +// --------------------------------------------------------------------------- + export type { ZvecDoc, ZvecQueryResult, @@ -18,13 +21,406 @@ export type { ActualZvecStoreOptions, } from './types'; -export { - ActualZvecStore, - createZvecStore, - openZvecStore, - openZvecStoreSync, - isZvecAvailable, - buildZvecSchema, -} from './actual-store'; +// --------------------------------------------------------------------------- +// Internal types — zvec SDK & collection interfaces +// --------------------------------------------------------------------------- + +import type { + ZvecDoc, + ZvecQueryResult, + ZvecSearchParams, + ZvecHybridParams, + IZvecStore, + ZvecStoreConfig, + ActualZvecStoreOptions, + ZvecFieldSchema, +} from './types'; + +let _zvecModule: unknown = undefined; +let _zvecLoadFailed = false; + +/** Zvec SDK type — minimal interface we rely on. */ +interface ZvecSDK { + ZVecCollectionSchema: new (config: unknown) => unknown; + ZVecDataType: Record; + ZVecIndexType: Record; + ZVecMetricType: Record; + ZVecCreateAndOpen(path: string, schema: unknown): ZvecCollection; + ZVecOpen(path: string, options?: Record): ZvecCollection; +} + +/** Zvec collection type — minimal interface we rely on. */ +interface ZvecCollection { + insertSync(records: unknown[]): void; + upsertSync(records: unknown[]): void; + fetchSync(params: { ids: string | string[]; outputFields?: string[]; includeVector?: boolean }): Record; + querySync(params: unknown): ZvecQueryRaw[]; + multiQuerySync(params: unknown): ZvecQueryRaw[]; + closeSync(): void; +} + +interface ZvecQueryRaw { + id: string; + score: number; + fields?: Record; +} + +function loadZvecSync(): ZvecSDK | undefined { + if (_zvecModule) return _zvecModule as ZvecSDK; + if (_zvecLoadFailed) return undefined; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + _zvecModule = require('@zvec/zvec'); + } catch { + _zvecLoadFailed = true; + return undefined; + } + return _zvecModule as ZvecSDK; +} + +function requireZvecSync(): ZvecSDK { + const z = loadZvecSync(); + if (!z) { + throw new Error( + '@zvec/zvec is not installed. Install it with:\n' + + ' pnpm add @zvec/zvec' + ); + } + return z; +} + +// --------------------------------------------------------------------------- +// Schema builder +// --------------------------------------------------------------------------- + +/** + * Build a ZVecCollectionSchema from a generic ZvecStoreConfig. + * + * This is a helper for callers that want to create zvec collections without + * dealing with the low-level zvec SDK schema API directly. + */ +export function buildZvecSchema(z: ZvecSDK, config: ZvecStoreConfig): unknown { + const { ZVecCollectionSchema, ZVecDataType, ZVecIndexType, ZVecMetricType } = z; + + const vectorSchema = { + name: config.vectorField, + dataType: ZVecDataType.VECTOR_FP32, + dimension: config.vectorDims, + indexParams: { + indexType: ZVecIndexType.HNSW, + metricType: ZVecMetricType.COSINE, + m: 32, + efConstruction: 200 + } + }; + + const fieldTypeMap: Record = { + STRING: ZVecDataType.STRING, + INT64: ZVecDataType.INT64, + FLOAT: ZVecDataType.FLOAT, + VECTOR_FP32: ZVecDataType.VECTOR_FP32, + }; + + const indexTypeMap: Record = { + FTS: ZVecIndexType.FTS, + INVERT: ZVecIndexType.INVERT, + HNSW: ZVecIndexType.HNSW, + }; + + const fieldSchemas = config.fields.map((f: ZvecFieldSchema) => { + const schema: Record = { + name: f.name, + dataType: fieldTypeMap[f.dataType], + }; + + if (f.indexType && f.indexType !== 'NONE' && indexTypeMap[f.indexType]) { + schema.indexParams = { + indexType: indexTypeMap[f.indexType], + ...(f.indexOptions ?? {}), + }; + } + + return schema; + }); + + return new ZVecCollectionSchema({ + name: config.collectionName, + vectors: vectorSchema, + fields: fieldSchemas, + }); +} + +// --------------------------------------------------------------------------- +// ActualZvecStore implementation +// --------------------------------------------------------------------------- + +const DEFAULT_OPEN_OPTIONS: ActualZvecStoreOptions = { + vectorField: 'embedding', + ftsFields: [], + rankConstant: 60, +}; + +/** + * ZvecStore backed by the native @zvec/zvec library. + * + * Provides true hybrid search: zvec's FTS path accepts raw text (no embedding), + * the vector path uses pre-computed embeddings, and `multiQuerySync` with RRF + * fuses both in the engine. + */ +export class ActualZvecStore implements IZvecStore { + private _collection: ZvecCollection; + private _closed = false; + private _vectorField: string; + private _ftsFields: string[]; + private _rankConstant: number; + + constructor(collection: ZvecCollection, options: ActualZvecStoreOptions) { + this._collection = collection; + this._vectorField = options.vectorField; + this._ftsFields = options.ftsFields; + this._rankConstant = options.rankConstant ?? 60; + } + + /** + * Create a new zvec collection with a generic schema config. + */ + static async create(path: string, config: ZvecStoreConfig): Promise { + const z = requireZvecSync(); + const schema = buildZvecSchema(z, config); + const collection = z.ZVecCreateAndOpen(path, schema); + return new ActualZvecStore(collection, { + vectorField: config.vectorField, + ftsFields: config.ftsFields, + }); + } + + /** Synchronous version of `create`. */ + static createSync(path: string, config: ZvecStoreConfig): ActualZvecStore { + const z = requireZvecSync(); + const schema = buildZvecSchema(z, config); + const collection = z.ZVecCreateAndOpen(path, schema); + return new ActualZvecStore(collection, { + vectorField: config.vectorField, + ftsFields: config.ftsFields, + }); + } + + static async open( + path: string, + options?: ActualZvecStoreOptions + ): Promise { + const z = requireZvecSync(); + const collection = z.ZVecOpen(path); + return new ActualZvecStore(collection, options ?? DEFAULT_OPEN_OPTIONS); + } + + /** Synchronous version of `open`. Uses read-only to allow concurrent readers. */ + static openSync( + path: string, + options?: ActualZvecStoreOptions + ): ActualZvecStore { + const z = requireZvecSync(); + const collection = z.ZVecOpen(path, { readOnly: true }); + return new ActualZvecStore(collection, options ?? DEFAULT_OPEN_OPTIONS); + } + + async insert(docs: ZvecDoc[]): Promise { + if (this._closed) throw new Error('Store is closed'); + if (docs.length === 0) return; + + const records = docs.map((d) => ({ + id: d.id, + vectors: { [this._vectorField]: d.vector }, + fields: d.fields + })); + // upsertSync handles both new docs and content-changed updates in one call. + this._collection.upsertSync(records); + } + + async fetch(ids: string[], outputFields?: string[]): Promise> { + if (this._closed) throw new Error('Store is closed'); + if (ids.length === 0) return {}; + + const raw = this._collection.fetchSync({ + ids, + outputFields, + includeVector: false, + }); + + const result: Record = {}; + for (const [id, doc] of Object.entries(raw)) { + result[id] = { + id, + score: 0, + fields: (doc as ZvecQueryRaw).fields ?? {}, + }; + } + return result; + } + + async search(params: ZvecSearchParams): Promise { + if (this._closed) throw new Error('Store is closed'); + + const rawResults = this._collection.querySync( + this._buildVectorQuery(params) + ); + + return rawResults.map((r: ZvecQueryRaw) => ({ + id: r.id, + score: r.score, + fields: r.fields ?? {} + })); + } + + async searchHybrid(params: ZvecHybridParams): Promise { + if (this._closed) throw new Error('Store is closed'); + + const rawResults = this._collection.multiQuerySync( + this._buildHybridQuery(params) + ); + + return rawResults.map((r: ZvecQueryRaw) => ({ + id: r.id, + score: r.score, + fields: r.fields ?? {} + })); + } + + searchSync(params: ZvecSearchParams): ZvecQueryResult[] { + if (this._closed) throw new Error('Store is closed'); + + const rawResults = this._collection.querySync( + this._buildVectorQuery(params) + ); + + return rawResults.map((r: ZvecQueryRaw) => ({ + id: r.id, + score: r.score, + fields: r.fields ?? {} + })); + } + + searchHybridSync(params: ZvecHybridParams): ZvecQueryResult[] { + if (this._closed) throw new Error('Store is closed'); + + const rawResults = this._collection.multiQuerySync( + this._buildHybridQuery(params) + ); + + return rawResults.map((r: ZvecQueryRaw) => ({ + id: r.id, + score: r.score, + fields: r.fields ?? {} + })); + } + + /** Build a query params object for vector search, omitting filter when undefined. */ + private _buildVectorQuery(params: ZvecSearchParams): Record { + const q: Record = { + fieldName: this._vectorField, + vector: params.vector, + topk: params.topK, + }; + if (params.filter) { + q.filter = params.filter; + } + return q; + } + + /** Build multi-field FTS query paths for hybrid search from configured ftsFields. */ + private _buildFtsQueries(queryText: string, topK: number): Record[] { + const z = loadZvecSync(); + const ftsParams = z + ? { indexType: z.ZVecIndexType.FTS, defaultOperator: 'OR' as const } + : undefined; + + return this._ftsFields.map((fieldName) => ({ + fieldName, + fts: { matchString: queryText }, + numCandidates: topK * 2, + params: ftsParams, + })); + } + + /** Build multiQuery params for hybrid search, omitting filter when undefined. */ + private _buildHybridQuery(params: ZvecHybridParams): Record { + const q: Record = { + queries: [ + { + fieldName: this._vectorField, + vector: params.queryVector, + numCandidates: params.topK * 2 + }, + ...this._buildFtsQueries(params.queryText, params.topK), + ], + topk: params.topK, + rerank: { type: 'rrf', rankConstant: this._rankConstant } + }; + if (params.filter) { + q.filter = params.filter; + } + return q; + } + + async close(): Promise { + if (this._closed) return; + this._closed = true; + try { + this._collection.closeSync(); + } catch { + // best-effort + } + } +} + +// --------------------------------------------------------------------------- +// Convenience factories +// --------------------------------------------------------------------------- + +/** + * Create a zvec store with the given config. + * + * Requires @zvec/zvec to be installed. Throws an error when it is not available. + */ +export async function createZvecStore( + path: string, + config: ZvecStoreConfig, +): Promise { + return ActualZvecStore.create(path, config); +} + +/** + * Asynchronously open a zvec store. + * + * Requires @zvec/zvec to be installed. + */ +export async function openZvecStore( + path: string, + options?: ActualZvecStoreOptions +): Promise { + return openZvecStoreSync(path, options); +} + +/** + * Synchronously open a zvec store. + * + * Requires @zvec/zvec to be installed. Throws an error when it is not available. + */ +export function openZvecStoreSync( + path: string, + options?: ActualZvecStoreOptions +): IZvecStore { + const z = loadZvecSync(); + if (z) { + return ActualZvecStore.openSync(path, options); + } + throw new Error( + '@zvec/zvec is not installed. Install it with:\n' + + ' pnpm add @zvec/zvec' + ); +} -export { cosineSimilarity } from './utils'; +/** Synchronous check: is @zvec/zvec available? */ +export function isZvecAvailable(): boolean { + return loadZvecSync() !== undefined; +} diff --git a/src/types.ts b/src/types.ts index 12873fb..34d99fe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,20 +1,14 @@ +import type { RerankOptions } from './utils/reranker'; +import type { EmbedderKind } from './embedder/resolve'; + /** * Document structure */ export interface Document { - /** Document unique identifier — assigned by Context.load() (hash-based). - * Not set by Loaders; Context.load() derives it from the file path - * relative to basePath for cross-machine consistency. */ - id?: string; /** Document content */ content: string; /** Markdown front-matter metadata */ meta?: Record; - /** - * SHA-256 content hash (16 chars) — used internally for change detection. - * Populated by `Context.load()` and stored in zvec as the single source of truth. - */ - contentHash?: string; } /** Configuration for query expansion. */ @@ -27,27 +21,12 @@ export interface QueryExpansionOptions { * when the key is found. Useful for CN↔EN terminology bridging * or domain-specific synonym expansion. * - * Oomit or pass an empty map for no expansion (effectively a no-op). + * Omit or pass an empty map for no expansion (effectively a no-op). * Set `queryExpansion: false` to disable expansion entirely. */ synonyms?: Record; } -/** Configuration for reranking. */ -export interface RerankOptions { - /** - * How many extra candidates to pull from the coarse search stage. - * e.g. with topK=5 and rerankFactor=3, 15 candidates are reranked. - * Default: 3 - */ - rerankFactor?: number; - /** - * Minimum number of candidates to rerank (floor). - * Default: 10 - */ - minCandidates?: number; -} - /** * Context initialization options */ @@ -135,15 +114,14 @@ export interface ContextOptions { * KeywordReranker scoring weights — tune for your domain. * * These weights control how the second-stage reranker scores candidates. - * See `RerankOptions` for available tuning parameters. When omitted, - * sensible defaults are used. + * When omitted, sensible defaults are used. * * Example: boost heading matches for API docs: * ```ts * rerankWeights: { headingTermBonus: 4.0, phraseWeight: 5.0 } * ``` */ - rerankWeights?: Omit; + rerankWeights?: Omit; } /** @@ -223,7 +201,7 @@ export interface QueryResult { * * - `'transformers'`: high-quality bilingual model (bge-small-zh-v1.5) */ - embedderKind?: import('./embedder/resolve').EmbedderKind; + embedderKind?: EmbedderKind; } /** @@ -240,13 +218,3 @@ export interface LoadProgress { /** Total documents to process in this phase */ total: number; } - -// Re-export zvec types for convenience (also available from storage/zvec-store) -export type { - ZvecDoc, - ZvecQueryResult, - ZvecSearchParams, - ZvecHybridParams, - ZvecFieldSchema, - ZvecStoreConfig, -} from './storage/zvec-store'; diff --git a/src/utils/common.ts b/src/utils/common.ts index 59becc5..c72dd3e 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,5 +1,3 @@ -import * as crypto from 'crypto'; - // --------------------------------------------------------------------------- // JSON / meta helpers // --------------------------------------------------------------------------- @@ -16,20 +14,6 @@ export function safeParseMeta(metaStr: string | undefined): Record { const filePath = path.join(FIXTURES_DIR, 'getting-started.md'); const doc = await loader.load(filePath); - // Loader does not assign id — the canonical hash-based ID is - // assigned by Context.load() for cross-machine consistency. - expect(doc.id).toBeUndefined(); expect(doc.content).toContain('Getting Started'); expect(doc.content).toContain('npm install'); expect(doc.meta).toEqual({ @@ -38,7 +35,6 @@ describe('Loaders', () => { const filePath = path.join(FIXTURES_DIR, 'api.json'); const doc = await loader.load(filePath); - expect(doc.id).toBeUndefined(); expect(doc.content).toContain('API Reference'); expect(doc.content).toContain('/users'); }); @@ -56,7 +52,6 @@ describe('Loaders', () => { const filePath = path.join(FIXTURES_DIR, 'notes.txt'); const doc = await loader.load(filePath); - expect(doc.id).toBeUndefined(); expect(doc.content).toContain('notes'); }); diff --git a/test/utils.test.ts b/test/utils.test.ts deleted file mode 100644 index c7ff7af..0000000 --- a/test/utils.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { cosineSimilarity } from '../src/storage/utils'; - -describe('cosineSimilarity', () => { - it('should return 1 for identical vectors', () => { - const vec = [1, 0, 0]; - expect(cosineSimilarity(vec, vec)).toBeCloseTo(1); - }); - - it('should return 0 for orthogonal vectors', () => { - expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0); - }); - - it('should return -1 for opposite vectors', () => { - expect(cosineSimilarity([1, 0], [-1, 0])).toBeCloseTo(-1); - }); - - it('should return 0 for zero vectors', () => { - expect(cosineSimilarity([0, 0, 0], [1, 2, 3])).toBe(0); - }); - - it('should handle high-dimensional vectors', () => { - const a = Array.from({ length: 512 }, (_, i) => i % 2 === 0 ? 0.1 : 0.2); - const b = Array.from({ length: 512 }, (_, i) => i % 2 === 0 ? 0.2 : 0.1); - const sim = cosineSimilarity(a, b); - expect(sim).toBeGreaterThan(0); - expect(sim).toBeLessThan(1); - }); -}); \ No newline at end of file