fix(dialect): whole-word entity match + wire lang_regex into extraction (#50, #53)

mempalace/dialect.py

@@ -296,6 +296,30 @@
     "need",
 }
 
+# Default (English) decision/insight words used to score key sentences. A
+# locale can override these via its regex.decision_words entry (finding #50);
+# locales without it fall back to this set.
+_DECISION_WORDS = {
+    "decided",
+    "because",
+    "instead",
+    "prefer",
+    "switched",
+    "chose",
+    "realized",
+    "important",
+    "key",
+    "critical",
+    "discovered",
+    "learned",
+    "conclusion",
+    "solution",
+    "reason",
+    "why",
+    "breakthrough",
+    "insight",
+}
+
 
 class Dialect:
     """
@@ -347,6 +371,18 @@ def __init__(
         self.aaak_instruction = t("aaak.instruction")
         self.lang_regex = get_regex()
 
+        # Derive locale-aware extraction inputs from lang_regex, falling back to
+        # the English module defaults when a locale omits a key (finding #50:
+        # these were loaded but never consulted, so non-English content ran
+        # through ASCII-only English patterns).
+        _sw = self.lang_regex.get("stop_words")
+        self._stop_words = set(_sw.split()) if _sw else _STOP_WORDS
+        # Tokenizer: locale topic_pattern captures the script's letters (incl.
+        # umlauts); the ASCII default [a-zA-Z]... split German words at umlauts.
+        self._topic_pattern = self.lang_regex.get("topic_pattern") or r"[^\W\d_][\w-]{2,}"
+        _dw = self.lang_regex.get("decision_words")
+        self._decision_words = set(_dw.split()) if _dw else _DECISION_WORDS
+
     @classmethod
     def from_config(cls, config_path: str) -> "Dialect":
         """Load entity mappings from a JSON config file.
@@ -394,8 +430,14 @@ def encode_entity(self, name: str) -> Optional[str]:
             return self.entity_codes[name]
         if name.lower() in self.entity_codes:
             return self.entity_codes[name.lower()]
+        # Whole-word match only: a registered name must appear as a full word
+        # in the query, not as a bare substring inside a longer word. Without
+        # this, "Ann" wrongly claims "Annabelle"'s code (finding #53), while a
+        # legitimate multi-word query like "Alice Cooper" must still match
+        # "Alice".
+        name_lower = name.lower()
         for key, code in self.entity_codes.items():
-            if key.lower() in name.lower():
+            if re.search(rf"\b{re.escape(key.lower())}\b", name_lower):
                 return code
         # Auto-code: first 3 chars uppercase
         return name[:3].upper()
@@ -451,20 +493,21 @@ def _detect_flags(self, text: str) -> List[str]:
 
     def _extract_topics(self, text: str, max_topics: int = 3) -> List[str]:
         """Extract key topic words from plain text."""
-        # Tokenize: alphanumeric words, lowercase
-        words = re.findall(r"[a-zA-Z][a-zA-Z_-]{2,}", text)
-        # Count frequency, skip stop words
+        # Tokenize with the locale topic pattern (keeps umlauts/non-ASCII letters
+        # whole; the ASCII default split German words at the umlaut — finding #50).
+        words = re.findall(self._topic_pattern, text)
+        # Count frequency, skip locale stop words
         freq = {}
         for w in words:
             w_lower = w.lower()
-            if w_lower in _STOP_WORDS or len(w_lower) < 3:
+            if w_lower in self._stop_words or len(w_lower) < 3:
                 continue
             freq[w_lower] = freq.get(w_lower, 0) + 1
 
         # Also boost words that look like proper nouns or technical terms
         for w in words:
             w_lower = w.lower()
-            if w_lower in _STOP_WORDS:
+            if w_lower in self._stop_words:
                 continue
             if w[0].isupper() and w_lower in freq:
                 freq[w_lower] += 2
@@ -485,32 +528,15 @@ def _extract_key_sentence(self, text: str) -> str:
             return ""
 
         # Score each sentence
-        decision_words = {
-            "decided",
-            "because",
-            "instead",
-            "prefer",
-            "switched",
-            "chose",
-            "realized",
-            "important",
-            "key",
-            "critical",
-            "discovered",
-            "learned",
-            "conclusion",
-            "solution",
-            "reason",
-            "why",
-            "breakthrough",
-            "insight",
-        }
         scored = []
         for s in sentences:
             score = 0
             s_lower = s.lower()
-            for w in decision_words:
-                if w in s_lower:
+            for w in self._decision_words:
+                # Whole-word match: "key" must not score on "monkey", and this
+                # is what lets locale decision words (self._decision_words) count
+                # for non-English text (finding #50).
+                if re.search(rf"\b{re.escape(w)}\b", s_lower):
                     score += 2
             # Prefer shorter, punchier sentences
             if len(s) < 80:
@@ -549,7 +575,7 @@ def _detect_entities_in_text(self, text: str) -> List[str]:
                 and clean[0].isupper()
                 and clean[1:].islower()
                 and i > 0
-                and clean.lower() not in _STOP_WORDS
+                and clean.lower() not in self._stop_words
             ):
                 code = clean[:3].upper()
                 if code not in found:

mempalace/i18n/de.json

@@ -36,10 +36,11 @@
     "instruction": "Auf Deutsch komprimieren. Bindestriche zwischen Wörtern, Pipes zwischen Konzepten. Artikel und Füllwörter weglassen. Eigennamen und Zahlen exakt beibehalten."
   },
   "regex": {
-    "topic_pattern": "[A-ZÄÖÜß][a-zäöüß]{2,}|[A-Za-zÄÖÜäöüß]{3,}",
+    "topic_pattern": "[^\\W\\d_][\\w-]{2,}",
     "stop_words": "der die das ein eine eines einer einem einen den dem des und oder aber denn weil wenn als ob auch noch schon sehr viel nur nicht mehr kann wird hat ist sind war waren sein haben wurde mit von zu für auf in an um über nach durch",
     "quote_pattern": "\\u201E([^\\u201C]{10,200})\\u201C|\"([^\"]{10,200})\"",
-    "action_pattern": "(?:gebaut|behoben|geschrieben|hinzugefügt|gepusht|gemessen|getestet|überprüft|erstellt|gelöscht|aktualisiert|konfiguriert|bereitgestellt|migriert)\\s+[\\wÄÖÜäöüß\\s]{3,30}"
+    "action_pattern": "(?:gebaut|behoben|geschrieben|hinzugefügt|gepusht|gemessen|getestet|überprüft|erstellt|gelöscht|aktualisiert|konfiguriert|bereitgestellt|migriert)\\s+[\\wÄÖÜäöüß\\s]{3,30}",
+    "decision_words": "entschieden entschied weil stattdessen bevorzugt gewechselt wählte gewählt erkannt wichtig kritisch entscheidend entdeckt gelernt schlussfolgerung lösung grund warum durchbruch erkenntnis beschlossen deshalb daher"
   },
   "entity": {
     "candidate_pattern": "[A-ZÄÖÜ][a-zäöüß]{1,19}",

mempalace/i18n/en.json

@@ -36,10 +36,11 @@
     "instruction": "Compress to index format. Hyphens between words, pipes between concepts. Drop articles and filler. Keep names and numbers exact."
   },
   "regex": {
-    "topic_pattern": "[A-Z][a-z]{2,}|[A-Za-z][A-Za-z0-9_]{2,}",
+    "topic_pattern": "[^\\W\\d_][\\w-]{2,}",
     "stop_words": "the this that these those some many most each every other only such very will would could should must shall yeah okay also even then now already still back done make take give know think want need going come find work added saved session summary conversation topics source about once just really actually here there where good great better thank please sorry right wrong true false",
     "quote_pattern": "\"([^\"]{20,200})\"",
-    "action_pattern": "(?:built|fixed|wrote|added|pushed|measured|tested|reviewed|created|deleted|updated|configured|deployed|migrated)\\s+[\\w\\s]{3,30}"
+    "action_pattern": "(?:built|fixed|wrote|added|pushed|measured|tested|reviewed|created|deleted|updated|configured|deployed|migrated)\\s+[\\w\\s]{3,30}",
+    "decision_words": "decided because instead prefer switched chose realized important key critical discovered learned conclusion solution reason why breakthrough insight"
   },
   "entity": {
     "candidate_pattern": "[A-Z][a-z]+(?:[A-Z][a-z]+|[A-Z]{2,})+|[A-Z][a-z]{1,19}",

tests/test_dialect_extraction_accuracy.py

@@ -0,0 +1,85 @@
+"""test_dialect_extraction_accuracy.py — regression coverage for audit Group 8.
+
+Two independent findings in mempalace/dialect.py:
+
+  #53  encode_entity's fallback loop matched a registered name as a bare
+       substring of the query (`"ann" in "annabelle"`), so a short registered
+       name wrongly claimed a longer, unrelated name's code. The fix requires a
+       whole-word match: "Alice" must still match "Alice Cooper", but "Ann"
+       must NOT match "Annabelle".
+
+  #50  Dialect loads per-locale regex via get_regex() into self.lang_regex but
+       _extract_topics / _detect_entities_in_text / _extract_key_sentence never
+       consult it, so non-English (German) content is run through ASCII-only
+       English patterns: the tokenizer [a-zA-Z]... splits on umlauts, and the
+       English decision-word list scores German text at ~0.
+"""
+
+import pytest
+
+from mempalace import i18n
+from mempalace.dialect import Dialect
+
+
+@pytest.fixture(autouse=True)
+def _restore_lang():
+    """Restore the module-global current language after each test.
+
+    Dialect(lang="de") mutates i18n's process-global _current_lang; without
+    this, a later test that constructs Dialect() with no lang would inherit
+    German and lose the English decision words (test-isolation leak).
+    """
+    saved = i18n.current_lang()
+    yield
+    i18n.load_lang(saved)
+
+
+class TestEncodeEntityWholeWord:
+    def test_short_name_does_not_match_inside_longer_word(self):
+        """#53: 'Ann' must not claim 'Annabelle' via substring."""
+        d = Dialect(entities={"Ann": "X99"})
+        # Annabelle is a different person; must fall through to an auto-code,
+        # not steal Ann's distinctive X99.
+        assert d.encode_entity("Annabelle") != "X99", (
+            "substring collision: 'Ann' wrongly matched inside 'Annabelle'"
+        )
+
+    def test_registered_name_still_matches_as_whole_word(self):
+        """#53: legitimate whole-word match must survive the fix."""
+        d = Dialect(entities={"Alice": "ALC"})
+        assert d.encode_entity("Alice Cooper") == "ALC"
+        assert d.encode_entity("Alice") == "ALC"
+
+
+class TestGermanExtraction:
+    def test_topic_tokenizer_keeps_umlaut_words_intact(self):
+        """#50: German words with umlauts must not be split at the umlaut."""
+        d = Dialect(lang="de")
+        topics = d._extract_topics("Die Größe der Änderung war wichtig für Müller.")
+        # Under the ASCII tokenizer, 'Größe' -> 'Gr'/'e', 'Änderung' -> 'nderung'.
+        # A locale-aware tokenizer keeps them whole.
+        assert not any(t in ("gr", "nderung", "berpr") for t in topics), (
+            f"umlaut word was split by ASCII tokenizer: {topics!r}"
+        )
+
+    def test_german_decision_words_outrank_filler(self):
+        """#50: German decision words must score, not just sentence length.
+
+        The decision sentence is deliberately LONGER than the filler, so under
+        the English-only decision-word list the short filler wins on the length
+        bonus. Only if German decision words ('entschieden', 'weil', ...) score
+        does the decision sentence win — so this genuinely fails pre-fix.
+        """
+        d = Dialect(lang="de")
+        decision = (
+            "Wir haben uns nach langer Diskussion letztlich für die zweite "
+            "Variante entschieden weil sie deutlich sicherer war"
+        )
+        filler = "Es regnete gestern"
+        key = d._extract_key_sentence(decision + ". " + filler + ".")
+        # _extract_key_sentence truncates to ~55 chars, so assert on the sentence
+        # START (proves the decision sentence was SELECTED over the shorter
+        # filler), not on a trailing decision word that truncation would drop.
+        assert key.startswith("Wir haben"), (
+            f"German decision sentence lost to filler (no German scoring): {key!r}"
+        )