From 75c05ef60d5bd018ea8f936799c287126d759cd4 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 13:51:20 +0800 Subject: [PATCH 1/7] test: add testcases and coverage --- README.md | 9 +- src/utils/index.ts | 7 +- src/utils/sample.ts | 24 ++--- src/utils/str.ts | 7 -- src/utils/tokenizer.ts | 10 ++ test/context.test.ts | 18 ++++ test/embedder/embedder.test.ts | 18 ++++ test/expander/expand.test.ts | 135 +++++++++++++++++++++++++++ test/fixtures/docs/string.json | 1 + test/language.test.ts | 86 ----------------- test/loaders/index.test.ts | 39 ++++++++ test/{ => loaders}/loaders.test.ts | 16 +++- test/reranker/index.test.ts | 55 +++++++++++ test/{ => reranker}/reranker.test.ts | 4 +- test/storage/schema.test.ts | 51 ++++++++++ test/storage/store.test.ts | 112 ++++++++++++++++++++++ test/synonym-expander.test.ts | 60 ------------ test/utils/common.test.ts | 22 +++++ test/utils/doc.test.ts | 44 +++++++++ test/utils/hash.test.ts | 26 ++++++ test/utils/index.test.ts | 36 +++++++ test/utils/sample.test.ts | 90 ++++++++++++++++++ test/utils/tokenizer.test.ts | 68 ++++++++++++++ 23 files changed, 759 insertions(+), 179 deletions(-) delete mode 100644 src/utils/str.ts create mode 100644 test/embedder/embedder.test.ts create mode 100644 test/expander/expand.test.ts create mode 100644 test/fixtures/docs/string.json delete mode 100644 test/language.test.ts create mode 100644 test/loaders/index.test.ts rename test/{ => loaders}/loaders.test.ts (79%) create mode 100644 test/reranker/index.test.ts rename test/{ => reranker}/reranker.test.ts (96%) create mode 100644 test/storage/schema.test.ts create mode 100644 test/storage/store.test.ts delete mode 100644 test/synonym-expander.test.ts create mode 100644 test/utils/common.test.ts create mode 100644 test/utils/doc.test.ts create mode 100644 test/utils/hash.test.ts create mode 100644 test/utils/index.test.ts create mode 100644 test/utils/sample.test.ts create mode 100644 test/utils/tokenizer.test.ts diff --git a/README.md b/README.md index 9d19665..b3a7466 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # @antv/context +[![Build](https://github.com/antvis/context/actions/workflows/build.yml/badge.svg)](https://github.com/antvis/context/actions/workflows/build.yml) +[![npm version](https://img.shields.io/npm/v/@antv/context)](https://www.npmjs.com/package/@antv/context) +[![npm downloads](https://img.shields.io/npm/dm/@antv/context)](https://www.npmjs.com/package/@antv/context) +[![License](https://img.shields.io/npm/l/@antv/context)](./LICENSE) + A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. > [!TIP] @@ -48,7 +53,7 @@ await ctx.close(); | `vectorsDir` | `string` | `.context/vectors` | Directory to store vector files | | `basePath` | `string` | `process.cwd()` | Base path for resolving document IDs. Set for cross-machine consistent IDs. | | `onProgress` | `(phase, detail) => void` | — | Progress callback for `load()` phases: `'load'` → `'embed'` → `'insert'`. | -| `queryExpansion` | `QueryExpansionOptions | false` | `false` (no-op) | Query expansion with user-provided synonym map. `false` disables. Without `synonyms`, expansion is a no-op. | +| `queryExpansion` | `QueryExpansionOptions` | `false` | Query expansion with user-provided synonym map. `false` disables. Without `synonyms`, expansion is a no-op. | | `ftsFields` | `string[]` | `['content']` | Fields to index for Full Text Search in hybrid mode | | `ftsFieldWeights` | `Record` | `{ content: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. | | `rankConstant` | `number` | `60` | RRF rank constant for hybrid search fusion. Lower = "winner-takes-all", higher = more even. | @@ -96,7 +101,7 @@ Document IDs are derived from file paths relative to `basePath` for cross-machin | Parameter | Type | Description | |-----------|------|-------------| | `library` | `string` | Library name for organizing documents | -| `pattern` | `string | string[]` | Glob pattern(s) matching files to load | +| `pattern` | `string \| string[]` | Glob pattern(s) matching files to load | ```typescript await ctx.load('g2', './docs/**/*.md'); diff --git a/src/utils/index.ts b/src/utils/index.ts index 0da8493..6ddd02c 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,7 +1,6 @@ export { pathToId } from './doc'; export { computeContentHash } from './hash'; export { safeJsonParse } from './common'; -export { loadSampleText } from './sample'; -export { isCJK, detectLanguage, tokenizerForLanguage, detectTokenizer } from './tokenizer'; -export type { LanguageHint } from './tokenizer'; -export { containsCJK } from './str'; \ No newline at end of file +export { loadSampleText, selectSampleFiles } from './sample'; +export { isCJK, containsCJK, detectLanguage, tokenizerForLanguage, detectTokenizer } from './tokenizer'; +export type { LanguageHint } from './tokenizer'; \ No newline at end of file diff --git a/src/utils/sample.ts b/src/utils/sample.ts index d773bb7..d8af6c1 100644 --- a/src/utils/sample.ts +++ b/src/utils/sample.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; /** * Select a representative sample of files for tokenizer detection. */ -function selectSampleFiles(files: string[], maxCount: number): string[] { +export function selectSampleFiles(files: string[], maxCount: number): string[] { if (files.length <= maxCount) return files; const result: string[] = []; @@ -27,19 +27,15 @@ function selectSampleFiles(files: string[], maxCount: number): string[] { export async function loadSampleText(files: string[], sampleCount = 5): Promise { if (files.length === 0) return undefined; - try { - const sampleFiles = selectSampleFiles(files, sampleCount); - const samples = await Promise.allSettled( - sampleFiles.map((f) => fs.promises.readFile(f, 'utf-8')), - ); - const validSamples = samples - .filter((r): r is PromiseFulfilledResult => r.status === 'fulfilled') - .map((r) => r.value); - if (validSamples.length > 0) { - return validSamples.join('\n'); - } - } catch { - // Sample failure is non-fatal + const sampleFiles = selectSampleFiles(files, sampleCount); + const samples = await Promise.allSettled( + sampleFiles.map((f) => fs.promises.readFile(f, 'utf-8')), + ); + const validSamples = samples + .filter((r): r is PromiseFulfilledResult => r.status === 'fulfilled') + .map((r) => r.value); + if (validSamples.length > 0) { + return validSamples.join('\n'); } return undefined; } \ No newline at end of file diff --git a/src/utils/str.ts b/src/utils/str.ts deleted file mode 100644 index 8a05a2c..0000000 --- a/src/utils/str.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * String utilities. - */ - -export function containsCJK(text: string): boolean { - return /[一-鿿㐀-䶿豈-﫿]/.test(text); -} \ No newline at end of file diff --git a/src/utils/tokenizer.ts b/src/utils/tokenizer.ts index 3ddd494..2464b6d 100644 --- a/src/utils/tokenizer.ts +++ b/src/utils/tokenizer.ts @@ -25,6 +25,16 @@ export function isCJK(ch: string): boolean { ); } +/** + * Check whether a string contains any CJK characters. + */ +export function containsCJK(text: string): boolean { + for (const ch of text) { + if (isCJK(ch)) return true; + } + return false; +} + /** * Detect the dominant language category of a text sample. * diff --git a/test/context.test.ts b/test/context.test.ts index ce60855..773ee3f 100644 --- a/test/context.test.ts +++ b/test/context.test.ts @@ -41,6 +41,24 @@ describe('Context', () => { }); describe('load', () => { + it('should call onProgress callback during load', async () => { + const progressCalls: { phase: string; detail: { loaded: number; total: number } }[] = []; + + const ctxWithProgress = await Context.create({ + vectorsDir: TEST_DIR + '-progress', + onProgress: (phase, detail) => { + progressCalls.push({ phase, detail }); + }, + }); + + await ctxWithProgress.load('md', path.join(FIXTURES_DIR, '*.md')); + + expect(progressCalls.length).toBeGreaterThan(0); + expect(progressCalls.map(c => c.phase)).toContain('load'); + + await ctxWithProgress.close(); + }); + it('should load markdown files', async () => { await ctx.load('md', path.join(FIXTURES_DIR, 'getting-started.md')); diff --git a/test/embedder/embedder.test.ts b/test/embedder/embedder.test.ts new file mode 100644 index 0000000..a24cb71 --- /dev/null +++ b/test/embedder/embedder.test.ts @@ -0,0 +1,18 @@ +import { describe, it, expect } from 'vitest'; +import { Embedder } from '../../src/embedder/embedder'; + +describe('Embedder', () => { + it('should have default dimensions of 512', () => { + const embedder = new Embedder(); + expect(embedder.dimensions).toBe(512); + }); + + it('should have static pipeline property', () => { + expect(Embedder.pipeline).toBeNull(); + }); + + it('should be instantiable', () => { + const embedder = new Embedder(); + expect(embedder).toBeInstanceOf(Embedder); + }); +}); \ No newline at end of file diff --git a/test/expander/expand.test.ts b/test/expander/expand.test.ts new file mode 100644 index 0000000..17d3643 --- /dev/null +++ b/test/expander/expand.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect } from 'vitest'; +import { expand } from '../../src/expander'; + +describe('expand', () => { + const queryExpansion = { + synonyms: { + '折线图': ['line chart', '折线'], + 'tooltip': ['提示框', '提示', 'hover'], + 'animation': ['动效', 'animate', 'transition'], + 'config': ['配置', 'configuration', '设置'], + }, + }; + + it('should expand CN chart type to EN equivalent', () => { + const result = expand('折线图', queryExpansion); + expect(result).toContain('line chart'); + }); + + it('should expand EN term to CN equivalent', () => { + const result = expand('tooltip', queryExpansion); + expect(result).toContain('提示框'); + }); + + it('should not duplicate terms already in query', () => { + const result = expand('tooltip 提示框', queryExpansion); + const promptCount = result.split('提示框').length - 1; + expect(promptCount).toBe(1); + }); + + it('should expand multiple terms in one query', () => { + const result = expand('tooltip config', queryExpansion); + expect(result).toContain('提示框'); + expect(result).toContain('配置'); + }); + + it('should preserve original query text', () => { + const result = expand('animation settings', queryExpansion); + expect(result.startsWith('animation settings')).toBe(true); + }); + + it('should return original query when no synonyms match', () => { + const result = expand('random unrelated terms', queryExpansion); + expect(result).toBe('random unrelated terms'); + }); + + it('should handle empty query', () => { + const result = expand('', queryExpansion); + expect(result).toBe(''); + }); + + it('should return original query with no synonyms', () => { + const result = expand('tooltip configuration'); + expect(result).toBe('tooltip configuration'); + }); + + it('should return original query with empty synonyms', () => { + const result = expand('tooltip configuration', {}); + expect(result).toBe('tooltip configuration'); + }); + + it('should return query unchanged when queryExpansion is false', () => { + const result = expand('折线图配置', false); + expect(result).toBe('折线图配置'); + }); + + it('should handle CJK terms with substring match', () => { + // Line 14: containsCJK(term) returns true, so substring match is used + const result = expand('折线图', queryExpansion); + expect(result).toContain('折线'); + }); + + it('should handle term at start of query', () => { + // Line 18-28: while loop with word boundary at start + const result = expand('tooltip chart', queryExpansion); + expect(result).toContain('提示框'); + }); + + it('should handle term at end of query', () => { + // Test term matching at end of query + const result = expand('chart tooltip', queryExpansion); + expect(result).toContain('提示框'); + }); + + it('should handle term in middle of query', () => { + const result = expand('show tooltip here', queryExpansion); + expect(result).toContain('提示框'); + }); + + it('should not add synonym already in query', () => { + // Line 45: containsTerm check prevents duplication + const result = expand('tooltip 提示框', queryExpansion); + const parts = result.split(' '); + const promptCount = parts.filter(p => p === '提示框').length; + expect(promptCount).toBe(1); + }); + + it('should handle synonyms with already added terms', () => { + // Multiple terms matching same synonym + const result = expand('折线图 折线', queryExpansion); + expect(result).toContain('line chart'); + }); + + it('should handle term at word boundary returning true immediately', () => { + // This tests the path where containsTerm finds a match and returns true (line 26) + // Direct term at word boundary + const result = expand('test tooltip config', queryExpansion); + expect(result).toContain('提示'); + }); + + it('should not match partial Latin substring in word', () => { + // This tests the path where term is found but not at word boundaries (lines 27-29) + // Searching for "config" in "configured" - gets past include check but not boundary check + const testExpansion = { + synonyms: { 'config': ['setting'] } + }; + const result = expand('configured', testExpansion); + // "config" is part of "configured", not a standalone word - should not match + expect(result).toBe('configured'); + }); + + it('should handle term found but never at word boundary', () => { + // This tests line 29-30: term found but all occurrences fail boundary check + // Using a term that appears only as part of a longer word + const testExpansion = { + synonyms: { 'abc': ['xyz'] } + }; + // "abc" does NOT appear in this text at all - so line 11 returns false early + // Need text that contains "abc" but never as a standalone word + const result = expand('xabcy', testExpansion); + // "abc" is in "xabcy" but not at word boundary (surrounded by letters) + // This will go through the loop, never find a boundary match, then return false + // This triggers lines 29-30 + expect(result).toBe('xabcy'); + }); +}); \ No newline at end of file diff --git a/test/fixtures/docs/string.json b/test/fixtures/docs/string.json new file mode 100644 index 0000000..e20d921 --- /dev/null +++ b/test/fixtures/docs/string.json @@ -0,0 +1 @@ +"This is a JSON string value" \ No newline at end of file diff --git a/test/language.test.ts b/test/language.test.ts deleted file mode 100644 index e249309..0000000 --- a/test/language.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { isCJK, detectLanguage, tokenizerForLanguage, detectTokenizer } from '../src/utils/tokenizer'; - -describe('isCJK', () => { - it('should detect Chinese characters', () => { - expect(isCJK('中')).toBe(true); - expect(isCJK('图')).toBe(true); - }); - - it('should detect Hiragana', () => { - expect(isCJK('あ')).toBe(true); - }); - - it('should detect Katakana', () => { - expect(isCJK('ア')).toBe(true); - }); - - it('should detect Hangul', () => { - expect(isCJK('한')).toBe(true); - }); - - it('should not detect ASCII letters', () => { - expect(isCJK('a')).toBe(false); - expect(isCJK('Z')).toBe(false); - }); - - it('should not detect digits', () => { - expect(isCJK('0')).toBe(false); - }); - - it('should not detect punctuation', () => { - expect(isCJK('.')).toBe(false); - expect(isCJK('!')).toBe(false); - }); -}); - -describe('detectLanguage', () => { - it('should detect CJK-dominant text', () => { - expect(detectLanguage('折线图配置方法详解')).toBe('cjk'); - }); - - it('should detect Latin text', () => { - expect(detectLanguage('hello world configuration')).toBe('latin'); - }); - - it('should detect mixed text with >15% CJK', () => { - expect(detectLanguage('Line Chart 折线图')).toBe('mixed'); - }); - - it('should classify low CJK ratio as latin', () => { - // Only 1 CJK char out of ~20 chars — < 15% threshold - expect(detectLanguage('this is a long English sentence with one 字')).toBe('latin'); - }); - - it('should return latin for empty text', () => { - expect(detectLanguage('')).toBe('latin'); - }); -}); - -describe('tokenizerForLanguage', () => { - it('should return jieba for cjk', () => { - expect(tokenizerForLanguage('cjk')).toBe('jieba'); - }); - - it('should return jieba for mixed', () => { - expect(tokenizerForLanguage('mixed')).toBe('jieba'); - }); - - it('should return standard for latin', () => { - expect(tokenizerForLanguage('latin')).toBe('standard'); - }); -}); - -describe('detectTokenizer', () => { - it('should auto-detect jieba for Chinese text', () => { - expect(detectTokenizer('折线图')).toBe('jieba'); - }); - - it('should auto-detect standard for English text', () => { - expect(detectTokenizer('hello world')).toBe('standard'); - }); - - it('should auto-detect jieba for mixed text', () => { - expect(detectTokenizer('chart 折线图')).toBe('jieba'); - }); -}); diff --git a/test/loaders/index.test.ts b/test/loaders/index.test.ts new file mode 100644 index 0000000..a4c4460 --- /dev/null +++ b/test/loaders/index.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest'; +import { getLoader } from '../../src/loaders'; + +describe('getLoader', () => { + it('should return MarkdownLoader for .md files', () => { + const loader = getLoader('test.md'); + expect(loader).toBeDefined(); + expect(loader?.canHandle('test.md')).toBe(true); + }); + + it('should return MarkdownLoader for .markdown files', () => { + const loader = getLoader('test.markdown'); + expect(loader).toBeDefined(); + }); + + it('should return JsonLoader for .json files', () => { + const loader = getLoader('test.json'); + expect(loader).toBeDefined(); + expect(loader?.canHandle('test.json')).toBe(true); + }); + + it('should return TextLoader for .txt files', () => { + const loader = getLoader('test.txt'); + expect(loader).toBeDefined(); + expect(loader?.canHandle('test.txt')).toBe(true); + }); + + it('should return undefined for unsupported file types', () => { + expect(getLoader('test.csv')).toBeUndefined(); + expect(getLoader('test.xml')).toBeUndefined(); + expect(getLoader('test.unknown')).toBeUndefined(); + }); + + it('should return correct loader for various paths', () => { + expect(getLoader('/path/to/file.json')).toBeDefined(); + expect(getLoader('/path/to/file.md')).toBeDefined(); + expect(getLoader('./relative/path.txt')).toBeDefined(); + }); +}); \ No newline at end of file diff --git a/test/loaders.test.ts b/test/loaders/loaders.test.ts similarity index 79% rename from test/loaders.test.ts rename to test/loaders/loaders.test.ts index f7542f0..0a1757c 100644 --- a/test/loaders.test.ts +++ b/test/loaders/loaders.test.ts @@ -1,10 +1,10 @@ import { describe, it, expect } from 'vitest'; import * as path from 'path'; -import { MarkdownLoader } from '../src/loaders/markdown'; -import { JsonLoader } from '../src/loaders/json'; -import { TextLoader } from '../src/loaders/text'; +import { MarkdownLoader } from '../../src/loaders/markdown'; +import { JsonLoader } from '../../src/loaders/json'; +import { TextLoader } from '../../src/loaders/text'; -const FIXTURES_DIR = path.join(__dirname, 'fixtures/docs'); +const FIXTURES_DIR = path.join(__dirname, '../fixtures/docs'); describe('Loaders', () => { describe('MarkdownLoader', () => { @@ -39,6 +39,14 @@ describe('Loaders', () => { expect(doc.content).toContain('/users'); }); + it('should handle json string value', async () => { + const loader = new JsonLoader(); + const filePath = path.join(FIXTURES_DIR, 'string.json'); + const doc = await loader.load(filePath); + + expect(doc.content).toBe('This is a JSON string value'); + }); + it('should handle .json extension', () => { const loader = new JsonLoader(); expect(loader.canHandle('file.json')).toBe(true); diff --git a/test/reranker/index.test.ts b/test/reranker/index.test.ts new file mode 100644 index 0000000..f50786e --- /dev/null +++ b/test/reranker/index.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from 'vitest'; +import { applyRerank } from '../../src/reranker'; +import type { QueryResult } from '../../src/types'; + +describe('applyRerank', () => { + it('should return results unchanged when fewer than topK', async () => { + const results: QueryResult[] = [ + { id: '1', content: 'test', score: 0.5 }, + { id: '2', content: 'test2', score: 0.3 }, + ]; + const topK = 10; + + const rerankOptions = { rerankFactor: 2, minCandidates: 5 }; + const reranked = await applyRerank(rerankOptions, 'test', results, topK); + + expect(reranked).toEqual(results); + }); + + it('should sort results by reranked score when more than topK', async () => { + const results: QueryResult[] = [ + { id: '1', content: 'tooltip config', score: 0.9 }, + { id: '2', content: 'chart config', score: 0.8 }, + { id: '3', content: 'axis config', score: 0.7 }, + { id: '4', content: 'other content', score: 0.1 }, + ]; + const topK = 2; + + const rerankOptions = { rerankFactor: 2, minCandidates: 5 }; + const reranked = await applyRerank(rerankOptions, 'config', results, topK); + + // Results should be sorted by the new score + expect(reranked[0].score).toBeGreaterThanOrEqual(reranked[1].score); + }); + + it('should handle empty results', async () => { + const results: QueryResult[] = []; + const topK = 10; + + const rerankOptions = { rerankFactor: 2, minCandidates: 5 }; + const reranked = await applyRerank(rerankOptions, 'test', results, topK); + + expect(reranked).toEqual([]); + }); + + it('should handle undefined rerankOptions', async () => { + const results: QueryResult[] = [ + { id: '1', content: 'test', score: 0.5 }, + ]; + const topK = 10; + + const reranked = await applyRerank(undefined, 'test', results, topK); + + expect(reranked).toEqual(results); + }); +}); \ No newline at end of file diff --git a/test/reranker.test.ts b/test/reranker/reranker.test.ts similarity index 96% rename from test/reranker.test.ts rename to test/reranker/reranker.test.ts index e8567a5..07e0264 100644 --- a/test/reranker.test.ts +++ b/test/reranker/reranker.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; -import { rerank } from '../src/reranker/reranker'; -import type { RerankCandidate } from '../src/reranker/types'; +import { rerank } from '../../src/reranker/reranker'; +import type { RerankCandidate } from '../../src/reranker/types'; describe('rerank', () => { it('should return empty array for empty candidates', async () => { diff --git a/test/storage/schema.test.ts b/test/storage/schema.test.ts new file mode 100644 index 0000000..0088f87 --- /dev/null +++ b/test/storage/schema.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect } from 'vitest'; +import { buildZvecSchema } from '../../src/storage/schema'; + +describe('buildZvecSchema', () => { + it('should create schema with correct vector dimension', () => { + const schema = buildZvecSchema(384); + const schemaStr = String(schema); + + expect(schemaStr).toContain('dimension: 384'); + }); + + it('should create schema with HNSW index and COSINE metric', () => { + const schema = buildZvecSchema(256); + const schemaStr = String(schema); + + expect(schemaStr).toContain('HnswIndexParams'); + expect(schemaStr).toContain('metric:COSINE'); + }); + + it('should include content field with FTS index and jieba tokenizer (default)', () => { + const schema = buildZvecSchema(128, 'jieba'); + const schemaStr = String(schema); + + expect(schemaStr).toContain("name: 'content'"); + expect(schemaStr).toContain('FtsIndexParams'); + expect(schemaStr).toContain('tokenizer_name:jieba'); + }); + + it('should accept custom tokenizer (standard)', () => { + const schema = buildZvecSchema(128, 'standard'); + const schemaStr = String(schema); + + expect(schemaStr).toContain('tokenizer_name:standard'); + }); + + it('should include meta, path, contentHash fields', () => { + const schema = buildZvecSchema(256); + const schemaStr = String(schema); + + expect(schemaStr).toContain("name: 'meta'"); + expect(schemaStr).toContain("name: 'path'"); + expect(schemaStr).toContain("name: 'contentHash'"); + }); + + it('should use VECTOR_FP32 data type', () => { + const schema = buildZvecSchema(256); + const schemaStr = String(schema); + + expect(schemaStr).toContain('data_type: VECTOR_FP32'); + }); +}); \ No newline at end of file diff --git a/test/storage/store.test.ts b/test/storage/store.test.ts new file mode 100644 index 0000000..7b4355b --- /dev/null +++ b/test/storage/store.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import { Store } from '../../src/storage/store'; +import { Embedder } from '../../src/embedder/embedder'; + +describe('Store', () => { + const testDir = path.join(__dirname, '.test-vectors-temp'); + let store: Store; + let embedder: Embedder; + + beforeEach(() => { + embedder = new Embedder(); + store = new Store(testDir, embedder); + // Clean up any existing test files + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + fs.mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + store.close(); + // Clean up test files + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('should be instantiable', () => { + expect(store).toBeInstanceOf(Store); + }); + + describe('acquireZvec', () => { + it('should acquire zvec for a library', () => { + const zvec = store.acquireZvec('test-lib', 'jieba'); + expect(zvec).toBeDefined(); + }); + + it('should return same zvec for same library', () => { + const zvec1 = store.acquireZvec('test-lib', 'jieba'); + const zvec2 = store.acquireZvec('test-lib', 'jieba'); + expect(zvec1).toBe(zvec2); + }); + }); + + describe('addDoc', () => { + it('should add documents with required fields', () => { + const docs = [ + { + id: 'doc1', + vector: new Array(embedder.dimensions).fill(0.1), + fields: { content: 'test content', meta: '{}', path: '/test/doc1', contentHash: 'abc123' }, + }, + ]; + expect(() => store.addDoc('test-lib', docs)).not.toThrow(); + }); + + it('should not throw for empty docs array', () => { + expect(() => store.addDoc('test-lib', [])).not.toThrow(); + }); + }); + + describe('fetchDocs', () => { + it('should return empty object for empty ids', () => { + const result = store.fetchDocs('test-lib', []); + expect(result).toEqual({}); + }); + + it('should fetch documents by ids', () => { + const docId = 'doc1'; + const docs = [ + { + id: docId, + vector: new Array(embedder.dimensions).fill(0.1), + fields: { content: 'test content', meta: '{}', path: '/test/doc1', contentHash: 'abc123' }, + }, + ]; + store.addDoc('test-lib', docs); + + const result = store.fetchDocs('test-lib', [docId], ['content']); + expect(result).toHaveProperty(docId); + }); + }); + + describe('queryDoc', () => { + it('should return empty array for vector mode with no data', () => { + const result = store.queryDoc('test-lib', { + mode: 'vector', + queryVector: new Array(embedder.dimensions).fill(0.1), + topK: 10, + }); + expect(result).toEqual([]); + }); + + it('should return empty array for hybrid mode with no data', () => { + const result = store.queryDoc('test-lib', { + mode: 'hybrid', + queryText: 'test', + queryVector: new Array(embedder.dimensions).fill(0.1), + topK: 10, + }); + expect(result).toEqual([]); + }); + }); + + describe('close', () => { + it('should close without error', () => { + expect(() => store.close()).not.toThrow(); + }); + }); +}); \ No newline at end of file diff --git a/test/synonym-expander.test.ts b/test/synonym-expander.test.ts deleted file mode 100644 index ae0e75c..0000000 --- a/test/synonym-expander.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { expand } from '../src/expander'; - -describe('expand', () => { - const queryExpansion = { - synonyms: { - '折线图': ['line chart', '折线'], - 'tooltip': ['提示框', '提示', 'hover'], - 'animation': ['动效', 'animate', 'transition'], - 'config': ['配置', 'configuration', '设置'], - }, - }; - - it('should expand CN chart type to EN equivalent', () => { - const result = expand('折线图', queryExpansion); - expect(result).toContain('line chart'); - }); - - it('should expand EN term to CN equivalent', () => { - const result = expand('tooltip', queryExpansion); - expect(result).toContain('提示框'); - }); - - it('should not duplicate terms already in query', () => { - const result = expand('tooltip 提示框', queryExpansion); - const promptCount = result.split('提示框').length - 1; - expect(promptCount).toBe(1); - }); - - it('should expand multiple terms in one query', () => { - const result = expand('tooltip config', queryExpansion); - expect(result).toContain('提示框'); - expect(result).toContain('配置'); - }); - - it('should preserve original query text', () => { - const result = expand('animation settings', queryExpansion); - expect(result.startsWith('animation settings')).toBe(true); - }); - - it('should return original query when no synonyms match', () => { - const result = expand('random unrelated terms', queryExpansion); - expect(result).toBe('random unrelated terms'); - }); - - it('should handle empty query', () => { - const result = expand('', queryExpansion); - expect(result).toBe(''); - }); - - it('should return original query with no synonyms', () => { - const result = expand('tooltip configuration'); - expect(result).toBe('tooltip configuration'); - }); - - it('should return original query with empty synonyms', () => { - const result = expand('tooltip configuration', {}); - expect(result).toBe('tooltip configuration'); - }); -}); \ No newline at end of file diff --git a/test/utils/common.test.ts b/test/utils/common.test.ts new file mode 100644 index 0000000..fc8fd8a --- /dev/null +++ b/test/utils/common.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect } from 'vitest'; +import { safeJsonParse } from '../../src/utils/common'; + +describe('safeJsonParse', () => { + it('should parse valid JSON', () => { + expect(safeJsonParse('{"a": 1}')).toEqual({ a: 1 }); + expect(safeJsonParse('[1, 2, 3]')).toEqual([1, 2, 3]); + expect(safeJsonParse('"string"')).toBe('string'); + expect(safeJsonParse('123')).toBe(123); + expect(safeJsonParse('true')).toBe(true); + }); + + it('should return undefined for invalid JSON', () => { + expect(safeJsonParse('invalid')).toBeUndefined(); + expect(safeJsonParse('{a: 1}')).toBeUndefined(); + expect(safeJsonParse('')).toBeUndefined(); + }); + + it('should return undefined for undefined input', () => { + expect(safeJsonParse(undefined)).toBeUndefined(); + }); +}); \ No newline at end of file diff --git a/test/utils/doc.test.ts b/test/utils/doc.test.ts new file mode 100644 index 0000000..33ae5e7 --- /dev/null +++ b/test/utils/doc.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from 'vitest'; +import { pathToId } from '../../src/utils/doc'; + +describe('pathToId', () => { + it('should generate consistent IDs for same path', () => { + const id1 = pathToId('/some/path/file.ts'); + const id2 = pathToId('/some/path/file.ts'); + expect(id1).toBe(id2); + }); + + it('should generate different IDs for different files', () => { + const id1 = pathToId('/path/to/file1.ts'); + const id2 = pathToId('/path/to/file2.ts'); + expect(id1).not.toBe(id2); + }); + + it('should include hash in ID', () => { + const id = pathToId('/some/path/important.ts'); + expect(id).toMatch(/^[0-9a-f]{16}__/); + }); + + it('should handle Windows-style paths', () => { + const id = pathToId('C:\\Users\\test\\file.ts'); + expect(id).toBeDefined(); + expect(id).toContain('__'); + }); + + it('should normalize paths consistently', () => { + const id1 = pathToId('/a/b/../c/file.ts'); + const id2 = pathToId('/a/c/file.ts'); + expect(id1).toBe(id2); + }); + + it('should handle paths with special characters in filename', () => { + const id = pathToId('/path/to/file-name_123.ts'); + expect(id).toBeDefined(); + }); + + it('should replace special characters in suffix', () => { + const id = pathToId('/path/to/file.test.ts'); + // .ts should be removed, special chars replaced with _ + expect(id).toContain('file_test'); + }); +}); \ No newline at end of file diff --git a/test/utils/hash.test.ts b/test/utils/hash.test.ts new file mode 100644 index 0000000..7491d31 --- /dev/null +++ b/test/utils/hash.test.ts @@ -0,0 +1,26 @@ +import { describe, it, expect } from 'vitest'; +import { computeContentHash } from '../../src/utils/hash'; + +describe('computeContentHash', () => { + it('should return a 16-character hex string', () => { + const hash = computeContentHash('test content'); + expect(hash).toMatch(/^[0-9a-f]{16}$/); + }); + + it('should produce consistent results', () => { + const hash1 = computeContentHash('test content'); + const hash2 = computeContentHash('test content'); + expect(hash1).toBe(hash2); + }); + + it('should produce different hashes for different content', () => { + const hash1 = computeContentHash('content a'); + const hash2 = computeContentHash('content b'); + expect(hash1).not.toBe(hash2); + }); + + it('should handle empty string', () => { + const hash = computeContentHash(''); + expect(hash).toMatch(/^[0-9a-f]{16}$/); + }); +}); \ No newline at end of file diff --git a/test/utils/index.test.ts b/test/utils/index.test.ts new file mode 100644 index 0000000..c1ab162 --- /dev/null +++ b/test/utils/index.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect } from 'vitest'; +import * as utils from '../../src/utils'; + +describe('utils', () => { + it('should export safeJsonParse', () => { + expect(utils.safeJsonParse).toBeDefined(); + expect(typeof utils.safeJsonParse).toBe('function'); + }); + + it('should export computeContentHash', () => { + expect(utils.computeContentHash).toBeDefined(); + expect(typeof utils.computeContentHash).toBe('function'); + }); + + it('should export containsCJK', () => { + expect(utils.containsCJK).toBeDefined(); + expect(typeof utils.containsCJK).toBe('function'); + }); + + it('should export pathToId', () => { + expect(utils.pathToId).toBeDefined(); + expect(typeof utils.pathToId).toBe('function'); + }); + + it('should export tokenizer functions', () => { + expect(utils.isCJK).toBeDefined(); + expect(utils.detectLanguage).toBeDefined(); + expect(utils.tokenizerForLanguage).toBeDefined(); + expect(utils.detectTokenizer).toBeDefined(); + }); + + it('should export loadSampleText', () => { + expect(utils.loadSampleText).toBeDefined(); + expect(typeof utils.loadSampleText).toBe('function'); + }); +}); \ No newline at end of file diff --git a/test/utils/sample.test.ts b/test/utils/sample.test.ts new file mode 100644 index 0000000..1ac6e62 --- /dev/null +++ b/test/utils/sample.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { loadSampleText, selectSampleFiles } from '../../src/utils/sample'; +import * as fs from 'fs'; + +describe('loadSampleText', () => { + const testDir = '/tmp/test-sample-files'; + + beforeEach(() => { + // Create test files + fs.mkdirSync(testDir, { recursive: true }); + fs.writeFileSync(`${testDir}/file1.txt`, 'Hello World'); + fs.writeFileSync(`${testDir}/file2.txt`, 'Test Content'); + fs.writeFileSync(`${testDir}/file3.txt`, 'Sample Text'); + }); + + afterEach(() => { + fs.rmSync(testDir, { recursive: true, force: true }); + }); + + it('should return undefined for empty files array', async () => { + const result = await loadSampleText([]); + expect(result).toBeUndefined(); + }); + + it('should load sample text from files', async () => { + const files = [`${testDir}/file1.txt`, `${testDir}/file2.txt`]; + const result = await loadSampleText(files); + expect(result).toBeDefined(); + expect(result).toContain('Hello World'); + expect(result).toContain('Test Content'); + }); + + it('should limit sample count', async () => { + const files = [ + `${testDir}/file1.txt`, + `${testDir}/file2.txt`, + `${testDir}/file3.txt`, + ]; + const result = await loadSampleText(files, 2); + expect(result).toBeDefined(); + }); + + it('should handle non-existent files gracefully', async () => { + const files = [`${testDir}/nonexistent.txt`, `${testDir}/file1.txt`]; + const result = await loadSampleText(files); + expect(result).toBeDefined(); + }); + + it('should return undefined if all files fail to load', async () => { + const files = ['/nonexistent1.txt', '/nonexistent2.txt']; + const result = await loadSampleText(files); + expect(result).toBeUndefined(); + }); +}); + +describe('selectSampleFiles', () => { + it('should return all files if count is less than max', () => { + const files = ['a.txt', 'b.txt']; + const result = selectSampleFiles(files, 5); + expect(result).toEqual(['a.txt', 'b.txt']); + }); + + it('should include first and last file', () => { + const files = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt']; + const result = selectSampleFiles(files, 3); + expect(result[0]).toBe('a.txt'); + expect(result[result.length - 1]).toBe('e.txt'); + }); + + it('should sample evenly distributed files', () => { + const files = Array.from({ length: 10 }, (_, i) => `file${i}.txt`); + const result = selectSampleFiles(files, 4); + // Should have first, some middle, and last + expect(result.length).toBeGreaterThanOrEqual(3); + expect(result[0]).toBe('file0.txt'); + expect(result[result.length - 1]).toBe('file9.txt'); + }); + + it('should handle single file', () => { + const files = ['only.txt']; + const result = selectSampleFiles(files, 5); + expect(result).toEqual(['only.txt']); + }); + + it('should handle exact max count', () => { + const files = ['a.txt', 'b.txt', 'c.txt']; + const result = selectSampleFiles(files, 3); + expect(result).toEqual(files); + }); +}); \ No newline at end of file diff --git a/test/utils/tokenizer.test.ts b/test/utils/tokenizer.test.ts new file mode 100644 index 0000000..889d75c --- /dev/null +++ b/test/utils/tokenizer.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect } from 'vitest'; +import { isCJK, detectLanguage, tokenizerForLanguage, detectTokenizer } from '../../src/utils/tokenizer'; + +describe('tokenizer', () => { + describe('isCJK', () => { + it('should detect Chinese characters', () => { + expect(isCJK('中')).toBe(true); + expect(isCJK('国')).toBe(true); + }); + + it('should detect Japanese hiragana/katakana', () => { + expect(isCJK('あ')).toBe(true); + expect(isCJK('ア')).toBe(true); + }); + + it('should detect Korean hangul', () => { + expect(isCJK('한')).toBe(true); + expect(isCJK('글')).toBe(true); + }); + + it('should return false for Latin characters', () => { + expect(isCJK('a')).toBe(false); + expect(isCJK('A')).toBe(false); + }); + }); + + describe('detectLanguage', () => { + it('should detect Chinese-dominant text', () => { + expect(detectLanguage('你好世界')).toBe('cjk'); + }); + + it('should detect Latin-dominant text', () => { + expect(detectLanguage('hello world')).toBe('latin'); + expect(detectLanguage('The quick brown fox')).toBe('latin'); + }); + + it('should detect mixed CJK/Latin as mixed', () => { + expect(detectLanguage('Hello 世界')).toBe('mixed'); + expect(detectLanguage('测试 test')).toBe('mixed'); + }); + + it('should return latin for empty string', () => { + expect(detectLanguage('')).toBe('latin'); + }); + }); + + describe('tokenizerForLanguage', () => { + it('should return jieba for cjk', () => { + expect(tokenizerForLanguage('cjk')).toBe('jieba'); + }); + + it('should return jieba for mixed', () => { + expect(tokenizerForLanguage('mixed')).toBe('jieba'); + }); + + it('should return standard for latin', () => { + expect(tokenizerForLanguage('latin')).toBe('standard'); + }); + }); + + describe('detectTokenizer', () => { + it('should return tokenizer based on detected language', () => { + expect(detectTokenizer('hello')).toBe('standard'); + expect(detectTokenizer('你好')).toBe('jieba'); + expect(detectTokenizer('Hello 世界')).toBe('jieba'); + }); + }); +}); \ No newline at end of file From cbb65b67a1be4876e792865ae19841aa0a972330 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 13:55:27 +0800 Subject: [PATCH 2/7] docs: add HF_ENDPOINT tip --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b3a7466..4d9099b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ A local context retrieval library that enables semantic search over your documen npm install @antv/context ``` +> [!TIP] +> If you encounter model download timeout when first creating a Context, set the environment variable: +> ```bash +> HF_ENDPOINT=https://hf-mirror.com node your-script.js +> ``` + ```typescript import { Context } from '@antv/context'; From 413302fc20d840e5e94afe1ff959cbcccbff0575 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 13:57:00 +0800 Subject: [PATCH 3/7] Update test/storage/store.test.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- test/storage/store.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/storage/store.test.ts b/test/storage/store.test.ts index 7b4355b..7644556 100644 --- a/test/storage/store.test.ts +++ b/test/storage/store.test.ts @@ -11,12 +11,13 @@ describe('Store', () => { beforeEach(() => { embedder = new Embedder(); - store = new Store(testDir, embedder); - // Clean up any existing test files + // Clean up any existing test files first if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true, force: true }); } fs.mkdirSync(testDir, { recursive: true }); + + store = new Store(testDir, embedder); }); afterEach(() => { From 6a7e6a9be9992c9e2fe94f5db3707a7fbcf2fc04 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 13:57:06 +0800 Subject: [PATCH 4/7] Update README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d9099b..2d7b03e 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ await ctx.close(); | `vectorsDir` | `string` | `.context/vectors` | Directory to store vector files | | `basePath` | `string` | `process.cwd()` | Base path for resolving document IDs. Set for cross-machine consistent IDs. | | `onProgress` | `(phase, detail) => void` | — | Progress callback for `load()` phases: `'load'` → `'embed'` → `'insert'`. | -| `queryExpansion` | `QueryExpansionOptions` | `false` | Query expansion with user-provided synonym map. `false` disables. Without `synonyms`, expansion is a no-op. | +| queryExpansion | QueryExpansionOptions | false | false | Query expansion with user-provided synonym map. false disables. Without synonyms, expansion is a no-op. | | `ftsFields` | `string[]` | `['content']` | Fields to index for Full Text Search in hybrid mode | | `ftsFieldWeights` | `Record` | `{ content: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. | | `rankConstant` | `number` | `60` | RRF rank constant for hybrid search fusion. Lower = "winner-takes-all", higher = more even. | From 9a6ac154dc89c8280f43ba0302827b46e1938a57 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 13:57:12 +0800 Subject: [PATCH 5/7] Update test/utils/sample.test.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- test/utils/sample.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/utils/sample.test.ts b/test/utils/sample.test.ts index 1ac6e62..74ca821 100644 --- a/test/utils/sample.test.ts +++ b/test/utils/sample.test.ts @@ -1,9 +1,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { loadSampleText, selectSampleFiles } from '../../src/utils/sample'; +import * as path from 'path'; import * as fs from 'fs'; +import { loadSampleText, selectSampleFiles } from '../../src/utils/sample'; describe('loadSampleText', () => { - const testDir = '/tmp/test-sample-files'; + const testDir = path.join(__dirname, '.test-sample-files'); beforeEach(() => { // Create test files From 7ef68a686936a079f01b782bb1eeca5b4ef947c2 Mon Sep 17 00:00:00 2001 From: hustcc Date: Wed, 8 Jul 2026 14:01:26 +0800 Subject: [PATCH 6/7] fix: tmp file leak --- test/context.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/context.test.ts b/test/context.test.ts index 773ee3f..a9c7b59 100644 --- a/test/context.test.ts +++ b/test/context.test.ts @@ -43,9 +43,10 @@ describe('Context', () => { describe('load', () => { it('should call onProgress callback during load', async () => { const progressCalls: { phase: string; detail: { loaded: number; total: number } }[] = []; + const progressDir = TEST_DIR + '-progress'; const ctxWithProgress = await Context.create({ - vectorsDir: TEST_DIR + '-progress', + vectorsDir: progressDir, onProgress: (phase, detail) => { progressCalls.push({ phase, detail }); }, @@ -57,6 +58,11 @@ describe('Context', () => { expect(progressCalls.map(c => c.phase)).toContain('load'); await ctxWithProgress.close(); + + // Cleanup + if (fs.existsSync(progressDir)) { + fs.rmSync(progressDir, { recursive: true, force: true }); + } }); it('should load markdown files', async () => { From a7f8d8a739f0f994700114078d4af03d7fe2acfc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Jul 2026 06:34:12 +0000 Subject: [PATCH 7/7] fix: increase test timeouts and prevent concurrent model downloads in CI --- src/embedder/embedder.ts | 25 ++++++++++++++++++------- vitest.config.ts | 4 ++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/embedder/embedder.ts b/src/embedder/embedder.ts index a5cbc0f..58b3235 100644 --- a/src/embedder/embedder.ts +++ b/src/embedder/embedder.ts @@ -9,21 +9,32 @@ type TransformersPipeline = (texts: string[], options: Record) export class Embedder { private static pipeline: TransformersPipeline | null = null; + private static pipelinePromise: Promise | null = null; readonly dimensions = DEFAULT_DIMENSIONS; private static async getPipeline(): Promise { if (Embedder.pipeline) return Embedder.pipeline; - const mod = await import('@huggingface/transformers'); - const hfEndpoint = process.env.HF_ENDPOINT; - if (hfEndpoint && (mod as any).env) { - (mod as any).env.remoteHost = hfEndpoint; + if (!Embedder.pipelinePromise) { + Embedder.pipelinePromise = (async () => { + const mod = await import('@huggingface/transformers'); + const hfEndpoint = process.env.HF_ENDPOINT; + if (hfEndpoint && (mod as any).env) { + (mod as any).env.remoteHost = hfEndpoint; + } + + const pipe = await mod.pipeline('feature-extraction', DEFAULT_MODEL_ID) as TransformersPipeline; + Embedder.pipeline = pipe; + return Embedder.pipeline; + })().catch((err) => { + Embedder.pipeline = null; + Embedder.pipelinePromise = null; + throw err; + }); } - const pipe = await mod.pipeline('feature-extraction', DEFAULT_MODEL_ID) as TransformersPipeline; - Embedder.pipeline = pipe; - return Embedder.pipeline; + return Embedder.pipelinePromise; } async embed(text: string): Promise { diff --git a/vitest.config.ts b/vitest.config.ts index 75d5680..85e29b9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,8 +3,8 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { include: ['test/**/*.test.ts'], - testTimeout: 30000, - hookTimeout: 30000, + testTimeout: 120000, + hookTimeout: 120000, pool: 'forks', poolOptions: { forks: { singleFork: true },