feat: add transformer implementation

.eslintrc.json

@@ -0,0 +1,25 @@
+{
+  "env": {
+    "node": true,
+    "es2020": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2020,
+    "sourceType": "module"
+  },
+  "parser": "@typescript-eslint/parser",
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "plugins": ["@typescript-eslint"],
+  "rules": {
+    "@typescript-eslint/no-explicit-any": "warn",
+    "@typescript-eslint/no-unused-vars": ["warn", { "argsIgnorePattern": "^_" }],
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/no-non-null-assertion": "warn",
+    "no-console": "off"
+  },
+  "ignorePatterns": ["dist/", "node_modules/", "test/fixtures/"]
+}

.gitignore

@@ -11,3 +11,5 @@ package-lock.json
 yarn.lock
 pnpm-lock.yaml
 pnpm-workspace.yaml
+.test-tmp-*
+.vscode

.prettierrc.json

@@ -0,0 +1,8 @@
+{
+  "semi": true,
+  "singleQuote": true,
+  "trailingComma": "all",
+  "printWidth": 100,
+  "tabWidth": 2,
+  "arrowParens": "always"
+}

README.md

@@ -3,15 +3,15 @@
 A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. 
 
 > [!TIP]
-> Base it, We provide an official context HTTP server simlar with context7, used to provide AI code generation context services in MCP, Skill, and CLI, for free!
+> Based on this library, we provide an official context HTTP server similar to context7, used to provide AI code generation context services in MCP, Skill, and CLI, for free!
 
 
 ## Features
 
-- 📄 **Multi-format Support**: Supports Markdown, JSON, Text, and other file formats
-- 📚 **Multi-library Support**: Manage documents by library
-- ⚡ **Auto-indexing**: Automatic vectorization on load
-- 🔍 **Semantic Retrieval**: Retrieve relevant documents based on vector similarity (file-level)
+- 📄 **Multi-format Support**: Markdown, JSON, Text 文档自动加载与向量化
+- 🔍 **Hybrid Retrieval**: 向量语义 + FTS 全文检索双路召回，RRF 融合排序
+- 🔁 **Two-stage Reranking**: KeywordReranker 精排，关键词命中优先
+- 🌐 **Query Expansion**: 用户自定义同义词表，CN↔EN 跨语言召回增强
 
 
 ## Quick Start
@@ -23,48 +23,132 @@ npm install @antv/context
 ```typescript
 import { Context } from '@antv/context';
 
+// Standard creation — specify vectorsDir
 const ctx = await Context.create({ vectorsDir: './vectors' });
 
 // Load documents into a specific library with automatic vectorization
 await ctx.load('g2', './g2-docs/**/*.md');
 await ctx.load('f2', './f2-docs/**/*.json');
 
-// Query
+// Query a library (default: hybrid search + reranking)
 const results = await ctx.query('How to configure a line chart', { library: 'g2', topK: 5 });
-// => [{ content: '...', score: 0.92, id: 'g2-docs/line.md' }, ...]
+// => [{ content: '...', score: 0.92, scoreMode: 'reranked', id: 'g2-docs/line.md' }, ...]
+
+// Close when done (releases resources)
+await ctx.close();
 ```
 
 
 ## API
 
 ### `Context.create(options)`
 
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `vectorsDir` | `string` | Directory to store vector files |
-| `model` | `string` | Transformers model name, default `sentence-transformers/all-MiniLM-L6-v2` |
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `vectorsDir` | `string` | — | **Required**. Directory to store vector files |
+| `basePath` | `string` | `process.cwd()` | Base path for resolving document IDs. Set for cross-machine consistent IDs. |
+| `onProgress` | `(phase, detail) => void` | — | Progress callback for `load()` phases: `'load'` → `'embed'` → `'insert'`. |
+| `queryExpansion` | `QueryExpansionOptions | false` | `false` (no-op) | Query expansion with user-provided synonym map. `false` disables. Without `synonyms`, expansion is a no-op. |
+| `ftsFields` | `string[]` | `['content']` | Fields to index for Full Text Search in hybrid mode |
+| `ftsFieldWeights` | `Record<string, number>` | `{ content: 1 }` | Per-field boost weights for FTS text path. Higher = more influence. |
+| `rankConstant` | `number` | `60` | RRF rank constant for hybrid search fusion. Lower = "winner-takes-all", higher = more even. |
 
-### `ctx.load(library, glob)`
+#### Weight Configuration Example
 
-Load files into a specified library with automatic vectorization. Document ID defaults to the file path.
+```typescript
+const ctx = await Context.create({
+  vectorsDir: './vectors',
+  // Boost title matches 3x over content matches
+  ftsFieldWeights: { content: 1, title: 3 },
+  // More "winner-takes-all" ranking
+  rankConstant: 20,
+});
+```
+
+#### Query Expansion Configuration Example
+
+```typescript
+const ctx = await Context.create({
+  vectorsDir: './vectors',
+  // Define your own CN↔EN synonym bridges (no built-in defaults)
+  queryExpansion: {
+    synonyms: {
+      '折线图': ['line chart', '折线'],
+      '雷达图': ['radar chart', '蜘蛛图'],
+      'tooltip': ['提示框', 'hover', '悬浮'],
+    },
+  },
+});
+
+// Disable query expansion entirely
+const ctxNoExpand = await Context.create({
+  vectorsDir: './vectors',
+  queryExpansion: false,
+});
+```
+
+### `ctx.load(library, pattern)`
+
+Load files into a specified library with automatic batch vectorization. Documents are embedded in batches and inserted into the vector store. A content-hash change detection mechanism re-embeds files whose content has changed since the last load.
+
+Document IDs are derived from file paths relative to `basePath` for cross-machine consistency.
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `library` | `string` | Library name for organizing documents |
+| `pattern` | `string | string[]` | Glob pattern(s) matching files to load |
 
 ```typescript
 await ctx.load('g2', './docs/**/*.md');
 await ctx.load('g2', ['./docs/**/*.md', './docs/**/*.json']);
 ```
 
+Load phases emit progress via the `onProgress` callback:
+
+```typescript
+const ctx = await Context.create({
+  vectorsDir: './vectors',
+  onProgress: (phase, detail) => {
+    console.log(`${phase}: ${detail.loaded}/${detail.total}`);
+  },
+});
+// Phases: 'load' → 'embed' → 'insert'
+```
+
 ### `ctx.query(text, options)`
 
-Vector similarity retrieval.
+Two-stage retrieval: coarse search (vector / hybrid) → reranking → final topK results.
 
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `library` | `string` | Required, library to query |
-| `topK` | `number` | Number of results to return, default 5 |
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `library` | `string` | — | Library name to query. |
+| `topK` | `number` | `5` | Number of results to return |
+
+```typescript
+// Semantic search — hybrid (vector + FTS) + reranking by default
+const results = await ctx.query('sankey diagram', { library: 'g2', topK: 5 });
+```
+
+#### Query Result Fields
+
+Each result includes:
+
+| Field | Type | Description |
+|------|------|-------------|
+| `id` | `string` | Document ID |
+| `content` | `string` | Document content |
+| `score` | `number` | Similarity score (0–1) |
+| `scoreMode` | `'vector' | 'hybrid' | 'reranked'` | How the score was computed |
+| `meta` | `Record<string, unknown>` | Front-matter metadata (if present) |
+| `sourceFilePath` | `string` | Original file path relative to `basePath` |
+
+
+### `ctx.close()`
+
+Close all stores and release resources. Call this when you are done using the Context instance.
 
 ```typescript
-const results = await ctx.query('How to configure a line chart', { library: 'g2', topK 5 });
-// => [{ id: 'g2-docs/line.md', content: '...', score: 0.92 }, ...]
+await ctx.close();
 ```
 
 
@@ -85,20 +169,52 @@ const results = await ctx.query('How to configure a line chart', { library: 'g2'
        +--------------+--------------+                    |
                       v                                   v
             +-----------------+                   +-----------------+
-            |   FileLoader    |                   |  Transformers   |
-            +--------+--------+                   +--------+--------+
-                     |                                     |
-            +--------v--------+                            |
-            |  Transformers   |                            |
-            +--------+--------+                            |
-                     |                                     |
+            |   FileLoader    |                   | QueryExpander   |
+            +--------+--------+                   | (SynonymExpander)|
+                     |                            +--------+--------+
             +--------v--------+                            |
-            |      .zvec      |<---------------Query-------+
-            +-----------------+
+            |  EmbedBatch     |                            v
+            +--------+--------+                   +--------v--------+
+                     |                            |    Embedder     |
+            +--------v--------+                   +--------+--------+
+            |      .zvec      |                            |
+            +-----------------+                   +--------v--------+
+                                                  |   Vectorize     |
+                                                  +--------+--------+
+                                                           |
+                                                  +--------+-----------+
+                                                           |
+                                               +-----------v-----------+
+                                               |                       |
+                                       +-------v-------+       +-------v-------+
+                                       | FTS Text Path  |       |  Vector Path  |
+                                       |(ftsFieldWeights|       |               |
+                                       +-------+-------+       +-------+-------+
+                                               |                       |
+                                               +-----------+-----------+
+                                                           |
+                                               +-----------v-----------+
+                                               |    RRF Fusion          |
+                                               |   (rankConstant)       |
+                                               +-----------+-----------+
+                                                           |
+                                               +-----------v-----------+
+                                               |   KeywordReranker      |
+                                               |  (optional, 2nd stage) |
+                                               +-----------+-----------+
+                                               |
+                                         Query Result
 
 +------------------------------------------------------------------------+
 ```
 
+### Module Structure
+
+- **Public API**: `Context`, `QueryOptions`, `QueryResult`, `Document`, `Loader`, `MarkdownLoader`, `JsonLoader`, `TextLoader`, `pathToId`
+- **Reranking**: `KeywordReranker`, `createReranker`, `Reranker`, `RerankCandidate`, `RerankResult`, `RerankOptions`
+- **Query Expansion**: `SynonymExpander`, `NoopExpander`, `QueryExpander`, `QueryExpansionOptions`
+- **Advanced API**: `Embedder`, `TransformersEmbedder`, `EmbedderManager`, `IZvecStore`, `ActualZvecStore`, `DocumentRegistry`, `Store`
+
 
 ## License
 

package.json

@@ -4,10 +4,12 @@
   "description": "A local context retrieval library that enables semantic search over your documentation. It loads documents (Markdown, JSON, Text), vectorizes them using [Transformers.js](https://huggingface.co/transformers.js), and stores vectors locally in `.zvec` files for fast semantic querying. ",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
-  "type": "module",
   "scripts": {
     "build": "tsc",
-    "test": "vitest run --coverage"
+    "test": "vitest run --coverage",
+    "lint": "eslint src/ --ext .ts",
+    "lint:fix": "eslint src/ --ext .ts --fix",
+    "format": "prettier --write 'src/**/*.ts' 'test/**/*.ts'"
   },
   "keywords": [
     "context",
@@ -28,7 +30,12 @@
   },
   "devDependencies": {
     "@types/node": "^20.0.0",
+    "@typescript-eslint/eslint-plugin": "^7.0.0",
+    "@typescript-eslint/parser": "^7.0.0",
     "@vitest/coverage-v8": "^3.2.6",
+    "eslint": "^8.0.0",
+    "eslint-config-prettier": "^9.0.0",
+    "prettier": "^3.0.0",
     "typescript": "^5.0.0",
     "vitest": "^3.2.6"
   },