comments: src/languages/g* complete — galician through gujarati, all 22 files

src/languages/galician/galician.ts

@@ -86,10 +86,10 @@ const CLAUSE_MARK = MANIFEST.clausePunctuation;
 const TOKEN = new RegExp(`(${LATIN_RUN})|(\\d+(?:\\.\\d+)*(?:,\\d+)?)|([.!?…,;:])`, "giu");
 
 /**
- * This language's OWN inventory — the TOKEN word class as it stood before the widening above, lifted
- * verbatim, so nothing about the orthography is invented here. A token this REJECTS carries a letter the
- * language does not use, i.e. a foreign name. See core/hostWord.ts: this is the INVENTORY question, and it
- * is no longer also deciding where the script boundary falls.
+ * This language's OWN inventory. ⚠ TWO DIFFERENT QUESTIONS, KEPT APART: the TOKEN class above decides where
+ * the SCRIPT boundary falls (routing), while this one decides whether the g2p has rules for these letters. A
+ * token this class REJECTS carries a letter the language does not use — i.e. a foreign name. See
+ * core/hostWord.ts.
  */
 const NATIVE_CLASS = "[a-záéíóúüñ]";
 const nat = makeNativiser(NATIVE_CLASS, "iu");

src/languages/gan/gan.ts

@@ -8,7 +8,8 @@
  * (segmental IPA + a superscript pitch-number tone, narrow Nanchang diacritics kept verbatim), so the front-end is
  * the shared Han-dict engine (hanDictIpa.ts): greedy longest-match segmentation, superscript-tone → Chao contour
  * letters (SURFACE tone after a sandhi arrow ⁻), Han numerals. SINGLE authoritative source (Wiktionary/kaikki
- * Nanchang Sinological-IPA), no independent referee → 🔷.
+ * Nanchang Sinological-IPA). ⚠ SINGLE-SOURCE: there is no independent referee, so nothing here is
+ * cross-checked against a second transcription.
  */
 import type { Phonemizer } from "../../registry.ts";
 import { loadManifest } from "../../core/loadManifest.ts";

src/languages/georgian/numbers.ts

@@ -1,10 +1,10 @@
 /**
- * ★ Georgian (ka) VIGESIMAL cardinal number compositor. Returns composed Georgian TEXT that georgian.ts runs
+ * ⚠ Georgian (ka) VIGESIMAL cardinal number compositor. Returns composed Georgian TEXT that georgian.ts runs
  * through the g2p, so the IPA stays consistent with the word engine. Pattern B is mandatory here: Georgian is
  * NOT decimal-Western below 100, so the shared `westernNumberWords` (units + teens + round TENS) has no round
  * tens to read — there are none. Georgian counts 20–99 in SCORES of twenty.
  *
- *   ★ THE SCORE CONSTRUCTION (20–99). The four score words are 20 ოცი, 40 ორმოცი (ორ- "2" × ოც- "20"),
+ *   ⚠ THE SCORE CONSTRUCTION (20–99). The four score words are 20 ოცი, 40 ორმოცი (ორ- "2" × ოც- "20"),
  *     60 სამოცი (3×20), 80 ოთხმოცი (4×20). An exact multiple of 20 is just that word. Anything else is the
  *     score's stem + -და- ("and") + the PLAIN 1–19 numeral, written as ONE word — so the whole 1–19 series
  *     (including the teens) attaches into the same slot:
@@ -14,7 +14,7 @@
  *     90 is ოთხმოცდაათი (4×20 + 10). The score index is floor(n/20) and the attached remainder is n − 20·index,
  *     which lands in 0–19 by construction.
  *
- *   ★ TRUNCATION (≥ 100). From 100 up, groups are written as SEPARATE words and a numeral that is followed by a
+ *   ⚠ TRUNCATION (≥ 100). From 100 up, groups are written as SEPARATE words and a numeral that is followed by a
  *     smaller number drops its final ⟨ი⟩ (ასი→ას, ორასი→ორას, ათასი→ათას, მილიონი→მილიონ). This is LOCAL: the
  *     hundred truncates iff its own sub-hundred remainder is non-zero, and a magnitude noun truncates iff any
  *     remainder follows it. So 1300 = ათას სამასი (the thousand truncates, the hundred does not — it is final),
@@ -41,7 +41,7 @@ function sub20(n: number): string {
     return n < 10 ? UNITS[n]! : TEENS[n - 10]!;
 }
 
-/** ★ 0–99 → ONE Georgian word. 20–99 is score·20 + remainder, joined by the score stem's -და-. */
+/** 0–99 → ONE Georgian word. 20–99 is score·20 + remainder, joined by the score stem's -და-. */
 function sub100(n: number): string {
     if (n < 20) return sub20(n);
     const s = Math.floor(n / 20), // 1–4 → ოც / ორმოც / სამოც / ოთხმოც

src/languages/german/german.ts

@@ -388,43 +388,37 @@ const CLAUSE_MARK = MANIFEST.clausePunctuation;
 const TOKEN = new RegExp(`(${LATIN_RUN})|(\\d{1,3}(?:\\.\\d{3})+|\\d+(?:,\\d+)?)|([.!?…,;:])`, "gu");
 
 /**
- * This language's OWN inventory — the TOKEN word class as it stood before the widening above, lifted
- * verbatim, so nothing about the orthography is invented here. A token this REJECTS carries a letter the
- * language does not use, i.e. a foreign name. See core/hostWord.ts: this is the INVENTORY question, and it
- * is no longer also deciding where the script boundary falls.
+ * This language's OWN inventory. ⚠ TWO DIFFERENT QUESTIONS, KEPT APART: the TOKEN class above decides where
+ * the SCRIPT boundary falls (routing), while this one decides whether the g2p has rules for these letters. A
+ * token this class REJECTS carries a letter the language does not use — i.e. a foreign name. See
+ * core/hostWord.ts.
  */
 const NATIVE_CLASS = "[a-zäöüßA-ZÄÖÜ]";
 const nat = makeNativiser(NATIVE_CLASS, "u");
 
-// #562 symbol normalization — German words (Prozent/Euro/Kilometer are invariant plurals).
+// German measure and currency words are INVARIANT plurals (Prozent, Euro, Kilometer).
 const SYMBOLS = makeSymbolNormalizer({
-    // `&` was DROPPED outright: the corpus's `B&B` and `Arts & Sciences` lost the sign.
-    // `und` ×1135 in this corpus. The tier spaces it on both sides, because `B&B` is two
-    // initialisms and joining them would make one token.
-    // #586 `multiply` — this language DROPPED the sign outright. ⚠ STANDARD MATHEMATICAL REGISTER, not a corpus
-    // attestation: the sweep failed exactly as the exponent sweep did, because the plausible hits are homographs
-    // of PREPOSITIONS — es `por` ×23, it `per` ×25, ru `на` ×31 are all the preposition, never the operator.
-    // One word, so `by` defaults to it; this language does not split dimension from product.
+    // ⚠ The tier spaces `und` on both sides, because `B&B` is two initialisms and joining them would make one
+    // token.
+    // ⚠ `multiply` is STANDARD MATHEMATICAL REGISTER, not a corpus attestation: a corpus sweep for the
+    // operator returns homographs of PREPOSITIONS in every language tried. One word, so `by` defaults to it —
+    // German does not split dimension from product.
     multiply: { times: "mal" },
     ampersand: "und",
     percent: ["Prozent"],
     currency: { "€": ["Euro"], "$": ["Dollar"], "£": ["Pfund"], "¥": ["Yen"] },
-    // `m` — Meter ×6, and every digit-adjacent bare `m` in this corpus is a metre: `4892 m Höhe`,
-    // `100 m und 200 m Freistil`, `133 m/s`. Without it `Kubik`/`Quadrat` below could not reach a bare
-    // metre, so `5 m³` read as the raw letter while `5 km³` read correctly.
+    // `m` is declared because a digit-adjacent bare `m` in German is a metre (`4892 m Höhe`, `133 m/s`).
+    // ⚠ Without it `Kubik`/`Quadrat` below cannot reach a bare metre, so `5 m³` reads as the raw letter while
+    // `5 km³` reads correctly.
     units: { km: ["Kilometer"], cm: ["Zentimeter"], mm: ["Millimeter"], kg: ["Kilogramm"], mg: ["Milligramm"],
         m: ["Meter"] },
-    // #586. `5 km²` read as *fʏnf km* — the abbreviation reaching the phoneme sink verbatim, the QUANTITY lost
-    // and not merely its power, because an undeclared measure word made the tier abandon the whole match. The
-    // core now emits the unit and hands the exponent back, so this became a VISIBLE `DROP:exponent` (de went
-    // 3 → 6 drops while three sentences' units were repaired); this declaration is what closes it.
-    // German FUSES the measure word onto the front, which is `compound`: corpus ×2
-    // "2,2 Millionen Quadratkilometer Ozeanfläche", and Kubik ×2 for the cube.
+    // ⚠ AN UNDECLARED MEASURE WORD MAKES THE TIER ABANDON THE WHOLE MATCH, so `5 km²` reads as *fʏnf km* —
+    // the abbreviation reaching the phoneme sink verbatim and the QUANTITY lost, not merely its power.
+    // German FUSES the measure word onto the front, which is `compound`: *Quadratkilometer*, *Kubikmeter*.
     exponentWords: { squared: ["Quadrat"], cubed: ["Kubik"], position: "compound" },
-    // #586 BARE EXPONENT — the reading for a power with NO unit to modify (`20²`, `mc²`), which every language
-    // in the fleet was dropping silently. See `bareExponent` in core/normalizeSymbols.ts for why this cannot
-    // reuse `exponentWords` above: that is the unit MODIFIER and this is the PREDICATE, and in most languages
-    // they are different words (Quadratkilometer but zwanzig zum Quadrat).
+    // BARE EXPONENT — the reading for a power with NO unit to modify (`20²`, `mc²`).
+    // ⚠ THIS CANNOT REUSE `exponentWords` ABOVE: that is the unit MODIFIER and this is the PREDICATE, and in
+    // most languages they are different words — Quadratkilometer, but zwanzig zum Quadrat.
     // ⚠ PROVENANCE, stated because it is weaker than most data in this repo: these are STANDARD MATHEMATICAL
     // REGISTER, not corpus attestations. The power words are ×0 in this language's artifact, and the apparent
     // hits for other languages were substring traps of exactly the kind tools/normalization/attest.ts warns

src/languages/german/normalize.ts

@@ -1,30 +1,23 @@
 /**
- * German (de) TEXT NORMALIZATION — the pre-tokenizer pass that rewrites everything which is not already a
- * pronounceable word into words the existing pipeline speaks. Pure text→text; no IPA.
+ * German (de) text normalization — the pre-tokenizer pass that rewrites everything which is not already a
+ * pronounceable word into words the pipeline speaks. Pure text→text; no IPA.
  *
- * Twelfth language, and the one the ORDINAL work has been building towards. German writes the ordinal as a
- * numeral plus a bare PERIOD — `16. Jahrhundert`, `am 17. September` — which is the class Run 1 flagged as
- * needing a detector, because a regex cannot tell it from a sentence-final digit. It is the largest single
- * defect here (×100), and previously every one read as a cardinal followed by a PAUSE: "im 16. Jahrhundert"
- * came out as *sechzehn . Jahrhundert*.
+ * ⚠ GERMAN WRITES THE ORDINAL AS A NUMERAL PLUS A BARE PERIOD — `16. Jahrhundert`, `am 17. September` — which
+ * a regex cannot tell from a sentence-final digit. It is the largest single defect class here, and unhandled
+ * every one reads as a cardinal followed by a PAUSE: "im 16. Jahrhundert" → *sechzehn . Jahrhundert*.
  *
- * THE DETECTOR IS BUILT FROM THE CORPUS, not from intuition. Tabulating what surrounds `N.` over the 2,987
- * de_de utterances:
- *   AFTER   Jahrhundert(s) ×34, month names ×66, a few regiment names — and 79 instances with NOTHING
- *           after, which are the sentence-final periods that must NOT be claimed.
- *   BEFORE  am ×54, im ×14, des ×9, dem ×8, das ×7, zum ×5, vom ×2, bis ×2, ins ×1, den ×1.
- * So the rule fires on the FOLLOWING word (a month or Jahrhundert) — which alone covers ~100 of the 109 —
- * or on a preceding date/ordinal-licensing article plus a capitalised noun, which picks up the regiments.
- * A sentence-final `N.` matches neither, because nothing follows it and the word before is a content word.
+ * ⚠ THE DETECTOR IS BUILT FROM WHAT SURROUNDS `N.`, NOT FROM INTUITION:
+ *   AFTER   Jahrhundert(s), month names, a few regiment names — and, critically, the sentence-final periods
+ *           with NOTHING after them, which must NOT be claimed.
+ *   BEFORE  am, im, des, dem, das, zum, vom, bis, ins, den.
+ * So the rule fires on the FOLLOWING word (a month or Jahrhundert), or on a preceding date/ordinal-licensing
+ * article plus a capitalised noun, which picks up the regiments. A sentence-final `N.` matches neither,
+ * because nothing follows it and the word before is a content word.
  *
- * DECLENSION comes from the same evidence. The governing preposition or article decides the ending:
- * `am/im/vom/zum/dem/des/den/ins/seit/bis` take the weak **-en** (*am siebzehnten September*, *des
- * sechzehnten Jahrhunderts*), while `das/der/die` and a bare ordinal take **-e** (*das sechzehnte
- * Jahrhundert*). That is not full case agreement — it is the two forms the corpus actually needs.
- *
- * Measured over de_de (2,987 utterances): ordinals ×109, dates ×66, units ×60, dotted abbreviations ×58,
- * dot-thousands ×55, percent ×30, `Uhr` ×30, comma-decimals ×29, times ×23, `v. Chr.` ×11, all-caps ×222
- * (US ×30, USA ×14, AOL ×8), signs ×5.
+ * ⚠ DECLENSION COMES FROM THE SAME EVIDENCE, and it is TWO forms rather than full case agreement. The
+ * governing preposition or article decides the ending: `am/im/vom/zum/dem/des/den/ins/seit/bis` take the weak
+ * **-en** (*am siebzehnten September*, *des sechzehnten Jahrhunderts*), while `das/der/die` and a bare ordinal
+ * take **-e** (*das sechzehnte Jahrhundert*).
  */
 import { makeInitialismNormalizer, makeUnreadableTest } from "../../core/initialisms.ts";
 import { MANIFEST } from "./manifest.ts";

src/languages/greek/normalize.ts

@@ -1,55 +1,32 @@
 /**
- * Modern Greek (el) TEXT NORMALIZATION — the pre-tokenizer pass that rewrites everything which is not
- * already a pronounceable word into words the existing pipeline speaks. Pure text→text; no IPA.
+ * Modern Greek (el) text normalization — the pre-tokenizer pass that rewrites everything which is not already
+ * a pronounceable word into words the pipeline speaks. Pure text→text; no IPA.
  *
- * Twenty-second language. MEASURED over the el_gr FLEURS corpus (1,969 unique utterances, column 3 —
- * the cased one):
+ * ⚠ GREEK CARRIES THE HIGHEST RATE OF EMBEDDED LATIN OF ANY LANGUAGE MEASURED — roughly one utterance in
+ * seven. Most of it is brand and place names left alone deliberately (see below), but the all-caps
+ * initialisms have to be claimed, or `το FBI` reads with ENGLISH phonemes in a Greek stream
+ * (to ˈɛfbˈiːʲˈaᶦ) and `η UNESCO` comes out carrying ɪ ʊ ɹ ʃ d͡ʒ æ ɫ.
  *
- *   embedded Latin        291 utterances (14.8%) — the HIGHEST rate of any language measured
- *                         · all-caps initialisms  78 instances / 58 distinct in 68 utterances
- *                         · single Latin letters   9 after the unit/rate rules take theirs
- *                         · Latin↔Greek HOMOGLYPHS 12 (see step 1) — a corpus defect, not a loanword
- *                         · mixed-case names      the rest (Apple, Xinhua, Northern Rock) — LEFT ALONE
- *   ordinal notation       55  (15ο, 1η, 18ου, 9ης — the ending is the CASE, see step 7)
- *   grouped numbers        50  (1.000, 783.562 — Greek groups with a PERIOD)
- *   times                  21  (8:30, 07:19, 11:00) + 3 SPORTS times that must NOT be claimed
- *   ranges                 23  (3-5, 35-40) — deliberately NOT touched, see the footnote
- *   percent                12
- *   decimal comma          12  (2,3 — Greek's decimal mark is the COMMA)
- *   dotted abbreviations   27  (π.Χ. ×7, π.μ. ×7, π.χ. ×4, κ.λπ. ×4, μ.Χ. ×3, μ.μ. ×2, βλ. ×1)
- *   χλμ                     8  (3 of them dotted, 2 of them as a rate `χλμ / ώρα`)
- *   units after a number   ~20 (mm ×6, km ×3, km/h ×3, mi² ×2, km² ×1, m/s ×1, m, cm, g, °C ×1)
- *   Greek ALPHABETIC numerals 6 (Α΄ Β΄ ΙΣΤ΄ — with U+0384, not the keraia; see step 2)
- *   currency $              2  · vulgar fractions ½ ¾ 2 · `+` sign 1
+ * ⚠ LATIN↔GREEK HOMOGLYPHS ARE A REAL CLASS HERE, not a curiosity: a Latin `o` typed for the article ο makes
+ * the word vanish into the foreign path (`και o αγριόκουρκος` → ce ˈoᶷ aɣɾʝokuɾkos). See step 1.
  *
- * WHAT THE ENGINE PRODUCED BEFORE (each probed through `phonemize(form, "el")`):
- *   "το FBI και το GPS"  → to ˈɛfbˈiːʲˈaᶦ ce to d͡ʒˈiː pʰˈiː ˈɛs   ENGLISH phonemes (core/foreign.ts)
- *   "η UNESCO"           → i juːnˈɛskoᶷ                            in a Greek stream: ɪ ʊ ɹ ʃ d͡ʒ æ ɫ
- *   "τον 15ο αιώνα"      → ton ðeka pende o eona                   cardinal + a stray vowel
- *   "1.000 άτομα"        → ena . miðen atoma                       the grouping dot became a PAUSE
- *   "στις 8:30 μ.μ."     → stis oxto , tɾʝanda m . m .             the colon became a pause; μ.μ. spelled
- *   "300 π.Χ."           → tɾiakosia p . x .                       three phrase breaks out of one word
- *   "Β΄ Παγκόσμιο"       → v paŋɡozmio                             the numeral read as the LETTER beta
- *   "Το 3-5% των"        → to tɾia pende ton                       % dropped outright
- *   "783.562 km²"        → eftakosia oɣðonda tɾia . pendakosça …ˈʊkm   grouping dot + the [ˈʊkm] cluster
- *   "30 °C"              → tɾʝanda sˈiː                            the English letter C
- *   "35-40 km/h"         → … saɾanda ˈʊkm ˈeᶦt͡ʃ                    and the English letter H
- *   "το 2,3 τοις"        → to ðio , tɾia tis                       the decimal comma became a pause
- *   "και o αγριόκουρκος" → ce ˈoᶷ aɣɾʝokuɾkos                      a LATIN o typed for the article ο
+ * ⚠ GREEK GROUPS THOUSANDS WITH A PERIOD AND TAKES A COMMA DECIMAL, so both separators reach
+ * `clausePunctuation` as pauses unless claimed: `1.000 άτομα` → *ena . miðen atoma*.
  *
- * NEVER `\b` in this file: it is ASCII-defined and finds no boundary against Greek script, so a rule
- * written with it silently matches nothing (the trap that made core/initialisms.ts a no-op for Cyrillic).
- * Every boundary here is an explicit `(?<![\p{L}\p{M}])` / `(?![\p{L}\p{M}])` lookaround.
+ * ⚠ THE ORDINAL ENDING IS THE CASE (15ο, 1η, 18ου, 9ης), not a fixed suffix — see step 7.
+ *
+ * ⚠ NEVER `\b` IN THIS FILE. It is ASCII-defined and finds no boundary against Greek script, so a rule
+ * written with it silently matches NOTHING. Every boundary here is an explicit `(?<![\p{L}\p{M}])` /
+ * `(?![\p{L}\p{M}])` lookaround.
  *
  * NOT DONE, deliberately:
- *   • NUMERIC RANGES (23) — `3-5%`, `35-40 μίλια`. A Greek reader supplies a connective (έως / με), but
- *     which one is a register choice the transcript cannot settle, and the dash also occurs in `COVID-19`,
- *     `Super-G`, `1984-1985` and `Il-76`. Left alone rather than guessed at.
- *   • MIXED-CASE Latin (the bulk of the 291) — brand and place names. Same call Japanese and Thai made:
- *     letter-spelling is not an available reading for `Xinhua`, and transliterating a name is invention.
- *   • `53χρονης` (1) — an age compound whose reading is one fused word (πενηντατριάχρονης). One instance;
- *     a wrong compound is worse than the space it currently gets.
- *   • `m;esa` (1) — a Greek word typed entirely in Latin, with a stray `;`. A corpus typo, not a class.
+ *   · NUMERIC RANGES — `3-5%`, `35-40 μίλια`. A Greek reader supplies a connective (έως / με), but which one
+ *     is a register choice the text cannot settle, and the dash also occurs in `COVID-19`, `Super-G`,
+ *     `1984-1985` and `Il-76`.
+ *   · MIXED-CASE LATIN, the bulk of the embedded runs — brand and place names. Same call Japanese and Thai
+ *     made: letter-spelling is not an available reading for `Xinhua`, and transliterating a name is invention.
+ *   · AGE COMPOUNDS like `53χρονης`, whose reading is one fused word (πενηντατριάχρονης). A wrong compound is
+ *     worse than the space it currently gets.
  */
 import { makeSymbolNormalizer } from "../../core/normalizeSymbols.ts";