Merge pull request #47 from brandonhthiere/pr46-pinyin-fuzzy

feat(search): 增加拼音与模糊(编辑距离)搜索及前缀联想

Author: EchoFish

pinyin_index.py

@@ -0,0 +1,74 @@
+"""Lightweight pinyin + fuzzy matching (no heavy dependencies).
+
+A small built-in CJK->pinyin map covers common characters; unknown characters
+fall back to themselves. This powers pinyin / pinyin-initial recall and a
+bounded edit-distance fuzzy fallback for typo tolerance.
+"""
+from typing import List
+
+# Compact map of common Chinese characters to pinyin (extend as needed).
+_PINYIN = {
+    "北": "bei", "京": "jing", "上": "shang", "海": "hai", "广": "guang",
+    "州": "zhou", "深": "shen", "圳": "zhen", "中": "zhong", "国": "guo",
+    "学": "xue", "习": "xi", "笔": "bi", "记": "ji", "工": "gong",
+    "作": "zuo", "生": "sheng", "活": "huo", "项": "xiang", "目": "mu",
+    "想": "xiang", "法": "fa", "今": "jin", "天": "tian", "明": "ming",
+    "编": "bian", "程": "cheng", "教": "jiao", "搜": "sou", "索": "suo",
+    "任": "ren", "务": "wu", "文": "wen", "件": "jian", "数": "shu",
+    "据": "ju", "时": "shi", "间": "jian", "标": "biao", "题": "ti",
+}
+
+
+def to_pinyin(text: str) -> str:
+    """Full pinyin of text (known chars mapped, others kept)."""
+    out = [_PINYIN.get(ch, ch if ch.isascii() else "") for ch in text]
+    return "".join(out).lower()
+
+
+def to_initials(text: str) -> str:
+    """First letters of each character's pinyin (e.g. 北京 -> bj)."""
+    out = []
+    for ch in text:
+        py = _PINYIN.get(ch)
+        if py:
+            out.append(py[0])
+        elif ch.isascii() and ch.isalnum():
+            out.append(ch.lower())
+    return "".join(out)
+
+
+def matches_pinyin(query: str, text: str) -> bool:
+    """True if query matches text's full pinyin or pinyin initials."""
+    q = query.lower()
+    return bool(q) and (q in to_pinyin(text) or q in to_initials(text))
+
+
+def edit_distance(a: str, b: str) -> int:
+    """Levenshtein distance (iterative DP)."""
+    a, b = a.lower(), b.lower()
+    if a == b:
+        return 0
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a, 1):
+        cur = [i]
+        for j, cb in enumerate(b, 1):
+            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
+        prev = cur
+    return prev[-1]
+
+
+def fuzzy_match(query: str, term: str, max_distance: int = 1) -> bool:
+    """True if query is within max_distance edits of term (or a prefix region)."""
+    if not query:
+        return False
+    if query in term:
+        return True
+    return edit_distance(query, term) <= max_distance
+
+
+def suggest(prefix: str, terms: List[str], limit: int = 5) -> List[str]:
+    """Prefix suggestions across literal terms and their pinyin/initials."""
+    p = prefix.lower()
+    hits = [t for t in terms
+            if t.lower().startswith(p) or to_pinyin(t).startswith(p) or to_initials(t).startswith(p)]
+    return hits[:limit]

search_engine.py

@@ -236,6 +236,22 @@ def search_bm25(self, query: str, notes: Dict, limit: int = 20,
                 scores[note_id] *= weights["tags"]
         return sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]
 
+    def search_fuzzy(self, query: str, notes: Dict, limit: int = 20) -> List[Tuple[str, float]]:
+        """Augment exact results with pinyin and edit-distance fallbacks at
+        lower weight, improving recall for Chinese and typos."""
+        import pinyin_index  # noqa: PLC0415
+        results = dict(self.search_bm25(query, notes, limit))
+        q = query.lower()
+        for note_id, note in notes.items():
+            if note is None or note_id in results:
+                continue
+            hay = f"{note.title} {note.content} {' '.join(note.tags)}"
+            if pinyin_index.matches_pinyin(q, hay):
+                results[note_id] = results.get(note_id, 0.0) + 2.0   # pinyin < exact
+            elif any(pinyin_index.fuzzy_match(q, tok) for tok in self.tokenize(hay)):
+                results[note_id] = results.get(note_id, 0.0) + 1.0   # fuzzy lowest
+        return sorted(results.items(), key=lambda x: x[1], reverse=True)[:limit]
+
     def search(self, query: str, notes: Dict, limit: int = 20) -> List[Tuple[str, float]]:
         if not query or not notes:
             return []

tests/test_pinyin_fuzzy.py

@@ -0,0 +1,39 @@
+"""Pinyin + fuzzy matching helpers and integration with search_fuzzy."""
+import pinyin_index
+from note_model import NoteManager
+from search_engine import SearchEngine
+
+
+def test_pinyin_full_and_initials():
+    assert pinyin_index.to_pinyin("北京") == "beijing"
+    assert pinyin_index.to_initials("北京") == "bj"
+
+
+def test_matches_pinyin():
+    assert pinyin_index.matches_pinyin("beijing", "北京游记")
+    assert pinyin_index.matches_pinyin("bj", "北京游记")
+    assert not pinyin_index.matches_pinyin("shanghai", "北京游记")
+
+
+def test_edit_distance_and_fuzzy():
+    assert pinyin_index.edit_distance("python", "pythn") == 1
+    assert pinyin_index.fuzzy_match("pythn", "python", max_distance=1)
+    assert not pinyin_index.fuzzy_match("xyz", "python", max_distance=1)
+
+
+def test_search_fuzzy_recall_pinyin(tmp_path):
+    nm = NoteManager(tmp_path / "n.db", tmp_path)
+    n = nm.create_note("北京游记", "记录北京的行程")
+    se = SearchEngine(tmp_path / "i.json")
+    se.build_index(nm.notes)
+    hits = [nid for nid, _ in se.search_fuzzy("beijing", nm.notes)]
+    assert n.id in hits
+
+
+def test_search_fuzzy_typo(tmp_path):
+    nm = NoteManager(tmp_path / "n.db", tmp_path)
+    n = nm.create_note("Python notes", "python python python")
+    se = SearchEngine(tmp_path / "i.json")
+    se.build_index(nm.notes)
+    hits = [nid for nid, _ in se.search_fuzzy("pythn", nm.notes)]  # one typo
+    assert n.id in hits