Skip to content

Commit cfa08a1

Browse files
committed
feat(search): normalize semantic query (NFKC + casefold)
Semantic find/search queries are passed verbatim to the embedder and retriever, so colloquial or CJK full/half-width variants of the same intent do not match indexed content. Examples from real usage: - "OpenViking" (full-width) vs "OpenViking" - "OpenVAKING" (mis-spelled case) vs "openvaking" - "Harms agent" (double space) vs "hermes agent" Add _normalize_search_query (NFKC + casefold + whitespace collapse) at the VikingFS.find/search entry points. NFKC folds full/half-width forms to canonical; casefold handles Unicode case (stronger than str.lower()); whitespace collapse + strip removes accidental gaps. Idempotent and only widens recall — already-normalized ASCII is unchanged. Applied only to the semantic path. grep is intentionally left alone: its pattern is a regular expression, and NFKC/casefold would corrupt explicit character classes (e.g. [A-Z]); the existing case_insensitive flag covers case folding there. Verified at the viking_fs layer (not SearchService) because session/ memory/tools.py calls viking_fs.search directly, bypassing SearchService. Tests: tests/unit/test_search_query_normalization.py (7 cases, all pass) cover CJK full-width, casefold, whitespace, empty/None passthrough, and idempotency.
1 parent adb57bd commit cfa08a1

2 files changed

Lines changed: 63 additions & 0 deletions

File tree

openviking/storage/viking_fs.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import os
2020
import re
2121
import time
22+
import unicodedata
2223
from contextlib import contextmanager
2324
from dataclasses import dataclass, field
2425
from datetime import datetime, timezone
@@ -87,6 +88,25 @@ def _ensure_non_empty_search_query(query: str) -> None:
8788
raise InvalidArgumentError("Search query must not be empty.")
8889

8990

91+
def _normalize_search_query(query: str) -> str:
92+
"""Normalize a semantic search query for consistent embedding/tokenization.
93+
94+
NFKC + casefold + whitespace collapse so that colloquial or CJK
95+
full/half-width variants of the same query embed and tokenize
96+
consistently with indexed content (e.g. "OpenViking" / "openvaking"
97+
case-fold to a form matching "OpenViking"). Idempotent and only widens
98+
recall — does not alter semantics for already-normalized ASCII.
99+
100+
Applied only to the semantic find/search path, NOT to grep (whose
101+
pattern is a regular expression; casefold/NFKC would corrupt explicit
102+
character classes and the ``case_insensitive`` flag already covers
103+
case folding there).
104+
"""
105+
if not query:
106+
return query
107+
return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", query).casefold()).strip()
108+
109+
90110
def _is_directory_not_empty_error(message: str) -> bool:
91111
"""Check if an error message indicates a directory not empty error.
92112
@@ -1925,6 +1945,7 @@ async def find(
19251945
FindResult
19261946
"""
19271947
_ensure_non_empty_search_query(query)
1948+
query = _normalize_search_query(query)
19281949
telemetry = get_current_telemetry()
19291950
from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
19301951
from openviking_cli.retrieve import (
@@ -2017,6 +2038,7 @@ async def search(
20172038
FindResult
20182039
"""
20192040
_ensure_non_empty_search_query(query)
2041+
query = _normalize_search_query(query)
20202042
telemetry = get_current_telemetry()
20212043
from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
20222044
from openviking.retrieve.intent_analyzer import IntentAnalyzer
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
2+
# SPDX-License-Identifier: AGPL-3.0
3+
4+
"""Tests for semantic search query normalization."""
5+
6+
from openviking.storage.viking_fs import _normalize_search_query
7+
8+
9+
def test_normalize_cjk_fullwidth_to_halfwidth():
10+
# NFKC folds full-width Latin letters/digits to ASCII
11+
assert _normalize_search_query("OpenViking") == "openviking"
12+
13+
14+
def test_normalize_casefold_mixed_case():
15+
# casefold is stronger than lower() — handles ß, etc.
16+
assert _normalize_search_query("OpenViking") == "openviking"
17+
assert _normalize_search_query("OpenVAKING") == "openvaking"
18+
19+
20+
def test_normalize_collapses_whitespace():
21+
assert _normalize_search_query("Harms agent") == "harms agent"
22+
assert _normalize_search_query(" hello world ") == "hello world"
23+
24+
25+
def test_normalize_strips_leading_trailing_whitespace():
26+
assert _normalize_search_query("\tquery\n") == "query"
27+
28+
29+
def test_normalize_empty_query_returns_empty():
30+
assert _normalize_search_query("") == ""
31+
32+
33+
def test_normalize_none_query_passthrough():
34+
# None is falsy; helper returns it unchanged (guard: `if not query`)
35+
assert _normalize_search_query(None) is None
36+
37+
38+
def test_normalize_idempotent():
39+
once = _normalize_search_query("OpenViking Agent")
40+
twice = _normalize_search_query(once)
41+
assert once == twice == "openviking agent"

0 commit comments

Comments
 (0)