refactor: antv skill refactor

AGENTS.md

@@ -39,7 +39,7 @@ Eval + Harness 通过自动化评测发现问题，再由 LLM 优化 Skill 文
 
 ### 1. Skill Authoring (`skills/`)
 
-Skills are markdown files with YAML frontmatter, organized by library. Each skill documents a chart type or visualization pattern with metadata (category, tags, difficulty, use cases) and content (best practices, API usage, code examples).
+Skills are markdown files with YAML frontmatter, organized by library. Each skill documents a chart type or visualization pattern with metadata (category, tags, use cases) and content (best practices, API usage, code examples).
 
 ```
 skills/
@@ -69,15 +69,28 @@ The CLI (`antv` command) provides four commands:
 - `antv info <library>` - Show library core constraints (SKILL.md Section 1-2)
 
 The retrieval engine uses **zvec** (in-process vector database) with two strategies:
-- **`hybrid`** (default): zvec native multiQuery — FTS (jieba tokenizer, raw text) + Vector (HNSW ANN, 384d) + RRF fusion
+- **`hybrid`** (default): zvec native multiQuery — FTS (jieba tokenizer, raw text) + Vector (HNSW ANN, 512d) + RRF fusion
 - **`vector`**: Pure ANN vector similarity search
 
-Embedding is handled by `SimpleEmbedder` (tokenizer + multi-hash, 384d, synchronous). No model download required.
+Embedding is handled by `SimpleEmbedder` (tokenizer + multi-hash, 512d, synchronous). No model download required.
 
 Public API (`src/api.ts`) exports `retrieve()`, `info()`, `getSkillById()`, `libraries()`, and `listSkills()` for programmatic use.
 
-Additional components:
-- **HTTP Server** (`http-server/`): Standalone REST API deployment package
+### HTTP Server (`http-server/`)
+
+Standalone REST API server (Express) providing POST endpoints for skill retrieval. Used by SKILL.md files for content retrieval via HTTP calls.
+
+```bash
+cd http-server && npm run dev    # starts on http://localhost:3100 (configurable via PORT env)
+```
+
+| Endpoint | Method | Description |
+|---|---|---|
+| `/retrieve` | POST | Retrieve skills by query (hybrid/vector search). Body: `{query, library, topK, content, includeConstraints, strategy, maxTokens, progressiveLevel}` |
+| `/info` | POST | Get library core constraints. Body: `{library}` |
+| `/get` | POST | Get single skill by exact ID. Body: `{id, library}` |
+| `/list` | POST | List/filter skills. Body: `{library, category, tags}` |
+| `/libraries` | GET | List available library names |
 
 ### 3. Evaluation (`eval/`)
 

README.md

@@ -151,32 +151,31 @@ Options for retrieve:
 ```typescript
 import { retrieve } from '@antv/chart-visualization-skills';
 
-// Metadata only (no content)
+// With full markdown content and constraints (defaults)
 const skills = retrieve('bar chart', { library: 'g2', topK: 5 });
 
-// With full markdown content (core constraints auto-prepended as first result)
-const skills = retrieve('bar chart', { library: 'g2', topK: 5, content: true });
+// Metadata only (no content body or constraints)
+const skills = retrieve('bar chart', { library: 'g2', topK: 5, content: false });
 
-// Hybrid search (default) — FTS + Vector + RRF fusion
-const skills = retrieve('bar chart', { library: 'g2', strategy: 'hybrid' });
+// Explicit constraints control
+const skills = retrieve('bar chart', { library: 'g2', content: true, includeConstraints: false });
 
 // With token budget — content trimmed to fit 4000 tokens, summary + code only
 const skills = retrieve('bar chart', {
   library: 'g2',
-  content: true,
   maxTokens: 4000,
   progressiveLevel: 1,
 });
 ```
 
 ```typescript
-retrieve(query: string, options?: RetrieveOptions): Skill[]
+retrieve(query: string, options?: RetrieveOptions): Promise<Skill[]>
 
 interface RetrieveOptions {
   library?: string;    // Library filter, e.g. 'g2' or 'g6'
   topK?: number;       // Number of results (default: 7)
-  content?: boolean;   // Include markdown content body (default: false)
-  includeInfo?: boolean; // Prepend SKILL.md core constraints (default: same as content)
+  content?: boolean;   // Include markdown content body (default: true)
+  includeConstraints?: boolean; // Prepend constraints as first result (default: same as content)
   strategy?: 'hybrid' | 'vector';  // Retrieval strategy (default: 'hybrid')
   maxTokens?: number;  // Token budget — content trimmed to fit when set
   progressiveLevel?: 0 | 1 | 2;  // 0=full, 1=summary+code, 2=summary (default: 1)
@@ -187,8 +186,8 @@ interface RetrieveOptions {
 | :--- | :--- | :--- | :--- |
 | `library` | `string` | all | Library filter (`g2` / `g6` / `x6`) |
 | `topK` | `number` | `7` | Number of results |
-| `content` | `boolean` | `false` | Include markdown content body |
-| `includeInfo` | `boolean` | same as `content` | Prepend SKILL.md core constraints as first result |
+| `content` | `boolean` | `true` | Include markdown content body |
+| `includeConstraints` | `boolean` | same as `content` | Prepend constraints as first result |
 | `strategy` | `'hybrid' \| 'vector'` | `'hybrid'` | Retrieval strategy |
 | `maxTokens` | `number` | — | Token budget; content trimmed to fit when set |
 | `progressiveLevel` | `0 \| 1 \| 2` | `1` | 0=full, 1=summary+code, 2=summary only |
@@ -197,7 +196,7 @@ interface RetrieveOptions {
 > - Default retrieval uses **hybrid** strategy: zvec native FTS (jieba) + Vector (HNSW ANN) + RRF fusion.
 > - `strategy: 'vector'` uses pure ANN vector similarity search.
 > - `content: true` returns markdown content body (frontmatter metadata is excluded).
-> - When `includeInfo` is true, the core constraints block is injected as the first element (id prefixed with `__info__`).
+> - When `includeConstraints` is true, the core constraints block is injected as the first element (id prefixed with `__info__`).
 > - When `maxTokens` is set, skill content is formatted and trimmed to fit the budget according to `progressiveLevel`.
 
 ```typescript
@@ -214,7 +213,7 @@ interface SkillInfo {
   name: string;
   description: string;
   content: string;            // Full SKILL.md body (after frontmatter)
-  constraintsContent: string; // SKILL.md body up to <!-- CONSTRAINTS:END --> marker; injected by retrieve when includeInfo: true
+  constraintsContent: string; // SKILL.md body up to <!-- CONSTRAINTS:END --> marker; injected by retrieve when includeConstraints: true
 }
 ```
 

__tests__/embedder.test.ts

@@ -0,0 +1,120 @@
+import { describe, it, expect } from 'vitest';
+import { SimpleEmbedder, isCJK } from '../src/core/retrieval/embedder';
+
+describe('embedder', () => {
+  describe('isCJK', () => {
+    it('should identify CJK Unified characters', () => {
+      expect(isCJK('桑')).toBe(true);
+      expect(isCJK('图')).toBe(true);
+      expect(isCJK('树')).toBe(true);
+    });
+
+    it('should identify CJK Extension A characters', () => {
+      // CJK Ext-A range: 0x3400–0x4DBF
+      expect(isCJK('\u3447')).toBe(true);
+    });
+
+    it('should identify Hiragana and Katakana', () => {
+      expect(isCJK('\u3042')).toBe(true); // Hiragana あ
+      expect(isCJK('\u30AB')).toBe(true); // Katakana カ
+    });
+
+    it('should identify Hangul', () => {
+      expect(isCJK('\uAC00')).toBe(true); // Hangul 가
+    });
+
+    it('should not identify ASCII as CJK', () => {
+      expect(isCJK('a')).toBe(false);
+      expect(isCJK('Z')).toBe(false);
+      expect(isCJK('0')).toBe(false);
+    });
+
+    it('should not identify common punctuation as CJK', () => {
+      expect(isCJK('.')).toBe(false);
+      expect(isCJK('-')).toBe(false);
+      expect(isCJK(' ')).toBe(false);
+    });
+  });
+
+  describe('SimpleEmbedder', () => {
+    const embedder = new SimpleEmbedder();
+
+    it('should produce vectors of the correct dimension', async () => {
+      const vec = await embedder.embed('test query');
+      expect(vec.length).toBe(512);
+    });
+
+    it('should produce normalized vectors (L2 norm ≈ 1)', async () => {
+      const vec = await embedder.embed('test query');
+      let norm = 0;
+      for (const v of vec) norm += v * v;
+      norm = Math.sqrt(norm);
+      expect(Math.abs(norm - 1)).toBeLessThan(0.01);
+    });
+
+    it('should produce non-zero vectors for meaningful text', async () => {
+      const vec = await embedder.embed('桑基图 sankey');
+      const nonZeroCount = vec.filter(v => v !== 0).length;
+      expect(nonZeroCount).toBeGreaterThan(0);
+    });
+
+    it('should produce similar vectors for synonymous terms', async () => {
+      const vec1 = await embedder.embed('桑基图');
+      const vec2 = await embedder.embed('sankey');
+      // Both should expand to include each other's tokens via synonyms
+      // so their vectors should have some overlap
+      let dot = 0;
+      for (let i = 0; i < vec1.length; i++) dot += vec1[i] * vec2[i];
+      // Cosine similarity should be positive (they share synonym-expanded tokens)
+      expect(dot).toBeGreaterThan(0);
+    });
+
+    it('should produce different vectors for unrelated terms', async () => {
+      const vec1 = await embedder.embed('桑基图');
+      const vec2 = await embedder.embed('热力图');
+      let dot = 0;
+      for (let i = 0; i < vec1.length; i++) dot += vec1[i] * vec2[i];
+      // They may share some unigram overlap but should not be identical
+      expect(Math.abs(dot - 1)).toBeGreaterThan(0.01);
+    });
+
+    it('should support synchronous embedding via embedSync', () => {
+      const vec = embedder.embedSync('test query');
+      expect(vec.length).toBe(512);
+    });
+
+    it('should support batch embedding', async () => {
+      const vecs = await embedder.embedBatch(['query one', 'query two']);
+      expect(vecs.length).toBe(2);
+      expect(vecs[0].length).toBe(512);
+      expect(vecs[1].length).toBe(512);
+    });
+
+    it('should handle empty text gracefully', async () => {
+      const vec = await embedder.embed('');
+      // Empty text should still produce a 512-d vector (all zeros or near-zero)
+      expect(vec.length).toBe(512);
+    });
+
+    it('should weight CJK trigrams higher than unigrams', async () => {
+      // A query with trigrams should produce a vector with higher peak values
+      // than one with only unigrams (due to trigram weight = 2.0 vs unigram = 0.15)
+      const vecTrigram = await embedder.embed('矩形树图');
+      const maxTrigram = Math.max(...vecTrigram.map(Math.abs));
+      const vecUnigram = await embedder.embed('图型');
+      const maxUnigram = Math.max(...vecUnigram.map(Math.abs));
+      // Trigrams get higher weight, so peak values should be larger
+      expect(maxTrigram).toBeGreaterThan(maxUnigram * 0.5);
+    });
+
+    it('should expand synonyms during embedding', async () => {
+      // '树图' should get synonym expansion to 'treemap'
+      // So the embedding of '树图' should share hash buckets with 'treemap'
+      const vec1 = await embedder.embed('树图');
+      const vec2 = await embedder.embed('treemap');
+      let dot = 0;
+      for (let i = 0; i < vec1.length; i++) dot += vec1[i] * vec2[i];
+      expect(dot).toBeGreaterThan(0);
+    });
+  });
+});

__tests__/retrieve.test.ts

@@ -16,44 +16,50 @@ describe('retrieve API', () => {
     }
   };
 
-  it('should retrieve skills with default parameters', () => {
-    const results = retrieve('折线图');
-    // Default strategy is now 'hybrid'
+  it('should retrieve skills with default parameters', async () => {
+    const results = await retrieve('折线图');
+    // Defaults: content=true, includeConstraints=true
     if (zvecReady) {
       expect(results.length).toBeGreaterThan(0);
-      expect(results.length).toBeLessThanOrEqual(7);
       expect(results[0]).toHaveProperty('id');
       expect(results[0]).toHaveProperty('title');
-      expect(results[0].content).toBeUndefined();
+      expect(typeof results[0].content).toBe('string');
+      expect((results[0].content || '').length).toBeGreaterThan(0);
     } else {
       expect(results.length).toBe(0);
     }
   });
 
-  it('should respect topk parameter', () => {
-    const results = retrieve('bar chart', { library: 'g2', topK: 3 });
+  it('should respect topk parameter', async () => {
+    const results = await retrieve('bar chart', {
+      library: 'g2', topK: 3, includeConstraints: false,
+    });
     if (zvecReady) {
       expect(results.length).toBeGreaterThan(0);
       expect(results.length).toBeLessThanOrEqual(3);
     }
   });
 
-  it('should support g6 library parameter', () => {
-    const results = retrieve('graph layout', { library: 'g6' });
+  it('should support g6 library parameter', async () => {
+    const results = await retrieve('graph layout', {
+      library: 'g6', includeConstraints: false,
+    });
     expect(Array.isArray(results)).toBe(true);
   });
 
-  it('should handle mixed Chinese/English query', () => {
-    const results = retrieve('饼图 tooltip', { library: 'g2', topK: 5 });
+  it('should handle mixed Chinese/English query', async () => {
+    const results = await retrieve('饼图 tooltip', {
+      library: 'g2', topK: 5, includeConstraints: false,
+    });
     if (zvecReady) {
       expect(results.length).toBeGreaterThan(0);
       expect(results.length).toBeLessThanOrEqual(5);
     }
   });
 
-  it('should load markdown content on demand', () => {
-    // Use legacy positional overload for backward compat test
-    const results = retrieve('折线图', 'g2', 1, true);
+  it('should load markdown content on demand', async () => {
+    // Legacy positional overload — still works
+    const results = await retrieve('折线图', 'g2', 1, true);
     if (zvecReady) {
       expect(results.length).toBeGreaterThan(0);
       expect(typeof results[0].content).toBe('string');