From 6a6a845e66143f552b64694c5213de141a7084bc Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 6 Aug 2026 22:17:08 -0600 Subject: [PATCH 1/2] =?UTF-8?q?comments:=20the=20m*=20sweep=20=E2=80=94=20?= =?UTF-8?q?a=20Russian=20word=20in=20a=20Macedonian=20rule,=20and=20two=20?= =?UTF-8?q?truncated=20sentences=20(#708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues the alphabetical comment sweep. 42 files across macedonian, madurese, magahi, maithili, malagasy, malay, malayalam, maltese, mandarin, maori, marathi, mindong, minnan, mongolian, mossi. Comment-only: the emitted code is byte-identical with --removeComments. Real defects, not just tightening: · macedonian/normalize.ts documented its DATE rule as `N месяц` — месяц is RUSSIAN. Macedonian is месец, and the rule does not match the word for "month" at all: it matches a MONTH NAME from the MONTHS set. Two errors in four characters. · the same file numbered its steps 0-5, 7-16, with 6 MISSING and 11 used TWICE — once for a one-line note saying the range rule lives at step 5, once for REGNAL ORDINALS. Renumbered 0-15, sequential, and the one internal cross-reference re-pointed. · malay/normalize.ts ended a header sentence mid-clause: "What was NOT duplicated, and why, is in." · the same file spliced two half-sentences with an orphan close-paren: "reads the point as `perpuluhan`; round-tripped through this repo's g2p), and the fraction digit by digit". · minnan.ts had a sentence restarting lowercase after a full stop, inside the note about which capitals the inventory must list. Phase chronology for work that shipped: mandarin.ts described itself as "Phase 1 … Phase 2 adds … Phase 3", and minnan.ts as "Phase 1", though all of it is present. Both now describe what the file does. The rest is the standing standard: drop decode transcripts (malayalam carried raw phoneme-recognizer output), cross-language surveys, corpus counts that record rather than license a decision, and issue numbers; keep the invariants and the traps. The ★/🔷/⛔ glyphs and the ellipsis-truncated `trap N (…)` references are gone from m* as they are from a-l. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012Lc3WnUgogC7okV7n53vjr --- src/languages/macedonian/macedonian.ts | 6 +- src/languages/macedonian/normalize.ts | 29 ++-- src/languages/madurese/madurese.ts | 8 +- src/languages/magahi/magahi.ts | 35 ++--- src/languages/maithili/maithili.ts | 35 +++-- src/languages/malagasy/malagasy.ts | 8 +- src/languages/malay/normalize.ts | 37 ++--- src/languages/malayalam/normalize.ts | 199 ++++++++++--------------- src/languages/malayalam/numbers.ts | 11 +- src/languages/maltese/maltese.ts | 8 +- src/languages/maltese/numbers.ts | 6 +- src/languages/mandarin/mandarin.ts | 16 +- src/languages/mandarin/normalize.ts | 9 +- src/languages/maori/maori.ts | 16 +- src/languages/marathi/marathi.ts | 11 +- src/languages/marathi/normalize.ts | 95 +++++------- src/languages/mindong/mindong.ts | 29 ++-- src/languages/minnan/minnan.ts | 21 +-- 18 files changed, 260 insertions(+), 319 deletions(-) diff --git a/src/languages/macedonian/macedonian.ts b/src/languages/macedonian/macedonian.ts index 1da4fec47..bd8aed601 100644 --- a/src/languages/macedonian/macedonian.ts +++ b/src/languages/macedonian/macedonian.ts @@ -44,12 +44,12 @@ const mkCountForm = (n: number): number => { return m % 10 === 1 && m !== 11 ? 0 : 1; }; -// #562 symbol normalization — Macedonian. Percent/currency/units/rates carry the count form above +// symbol normalization — Macedonian. Percent/currency/units/rates carry the count form above // (1 процент, 2 проценти; 1 километар, 2 километри). Units are written BOTH ways — Cyrillic кm and Latin // km are equally common in the corpus — so both scripts are declared. The rate denominator is "на час" // (per hour) but "во секунда" (per second), which the keyed unitPer expresses. const SYMBOLS = makeSymbolNormalizer({ - // #586 `multiply` — the word is this language's OWN, harvested from its existing `×` rule, so nothing new + // `multiply` — the word is this language's OWN, harvested from its existing `×` rule, so nothing new // is sourced. Declaring it HERE is what makes ASCII `x` read like `×`: `6x6 cm` was reading the `x` as a // LETTER NAME, and `NxN` forms outnumber `×` roughly 85 to 20 across the corpora. One word, so `by` is // omitted and defaults to it — this language does not split dimension from product. @@ -173,7 +173,7 @@ const TOKEN = /([а-шА-ШѓѕјљњќџЃЅЈЉЊЌЏѐѝЀЍ]+)|(\d+(?:,\d+)? class MacedonianPhonemizer implements Phonemizer { text(input: string): string { - // #562 order: Macedonian rewrites (grouping, ordinals, century/date, clock, ranges, signs) → + // order: Macedonian rewrites (grouping, ordinals, century/date, clock, ranges, signs) → // INITIALISMS (after abbreviations, so `Д-р` is not spelled DE-ER) → the shared symbol tier last // (it needs the number still adjacent to its unit/sign). Roman numerals arrive already converted // at the registry seam, so regnal "Лиалофи III" is "3" by the time normalize runs. diff --git a/src/languages/macedonian/normalize.ts b/src/languages/macedonian/normalize.ts index e8b07143c..c7832c62c 100644 --- a/src/languages/macedonian/normalize.ts +++ b/src/languages/macedonian/normalize.ts @@ -188,7 +188,8 @@ export function normalizeMacedonian(input: string): string { return `${a} до ${b}`; }); - // 7) CLOCK. The colon was clause punctuation, so `06:30` read as "шест , триесет". Macedonian reads + // 6) CLOCK. The colon is otherwise clause punctuation, so `06:30` reads as "шест , триесет". + // Macedonian reads // the time CARDINALLY with "и": 6:30 = "шест и триесет"; :00 drops the minutes (10:00 = "десет"). // The trailing "часот"/"ч" (the hour) is kept/expanded — "ч" is the abbreviation for "часот". A // comma after the minutes marks a sports time, not a clock. The range rule ran first, so @@ -204,12 +205,12 @@ export function normalizeMacedonian(input: string): string { return typeof ч === "string" ? `${body} часот` : body; }, ); - // 7b) `am`/`pm` after a time → претпладне/попладне. Standalone words (the clock already consumed the + // 6b) `am`/`pm` after a time → претпладне/попладне. Standalone words (the clock already consumed the // digits), matched on word boundaries so "Сами" is untouched. s = s.replace(/\bam\b/giu, "претпладне"); s = s.replace(/\bpm\b/giu, "попладне"); - // 8) THE ORDINAL SUFFIX — the big one. `(\d+)[- ]?` + the written suffix (the last letters of the + // 7) THE ORDINAL SUFFIX, the largest class in this file. `(\d+)[- ]?` + the written suffix (the last letters of the // spoken ordinal). The `-те` suffix is ambiguous between a count ("the N") and a decade: a 4-digit // number followed by "години" is a decade, otherwise it is "the N". s = s.replace( @@ -241,9 +242,9 @@ export function normalizeMacedonian(input: string): string { }, ); - // 9) CENTURY — `N век` → ordinal + век (10 век → десетти век), including the compound list + // 8) CENTURY — `N век` → ordinal + век (10 век → десетти век), including the compound list // `10 и 11 век`. Also the one Germanic remnant `8. век` (with a dot). The suffix forms (17-ти век, - // 18-тиот век) were already claimed by step 8. + // 18-тиот век) are already claimed by step 7. s = s.replace(/(\d+)\s+и\s+(\d+)\s*век(?![\p{L}\p{M}])/gu, (_m, a: string, b: string) => { const oa = mkOrdinal(Number(a)), ob = mkOrdinal(Number(b)); return oa !== undefined && ob !== undefined ? `${oa} и ${ob} век` : _m; @@ -257,7 +258,7 @@ export function normalizeMacedonian(input: string): string { return o === undefined ? m0 : `${o} век`; }); - // 10) DATES — `N месяц` → ordinal + месяц (на 6 октомври → на шести октомври), including a date range + // 9) DATES — `N ` → ordinal + month (на 6 октомври → на шести октомври), including a date range // `24 август - 5 септември` and the Germanic-dot form `4. јули 1776`. The day must be 1–31. const monthAlt = [...MONTHS].join("|"); s = s.replace( @@ -277,9 +278,7 @@ export function normalizeMacedonian(input: string): string { }, ); - // 11) (the range rule lives at step 5, before the clock.) - - // 11) REGNAL ORDINALS. All three corpus Romans (Лиалофи III, Елизабета II, Луј XVI) are regnal proper + // 10) REGNAL ORDINALS. All three corpus Romans (Лиалофи III, Елизабета II, Луј XVI) are regnal proper // names, and the shared Roman pass has already rewritten them to CARDINAL digits before this engine // runs (Macedonian has no ROMAN_POLICIES entry). The digit after a capitalized NAME is read as an // ordinal: Лиалофи 3 → Лиалофи трети, Луј 16 → Луј шеснаесетти. Guarded by the same ≤39 monarch @@ -297,7 +296,7 @@ export function normalizeMacedonian(input: string): string { return `${name} ${name.endsWith("а") ? femIndef(o) : o}`; }); - // 12) RATE UNITS the shared tier cannot compose. `милји/час` (miles/hour — милји is a full word, not an + // 11) RATE UNITS the shared tier cannot compose. `милји/час` (miles/hour — милји is a full word, not an // abbreviation), the Latin `mph`/`kph`, `Mbit/s` (megabits per second). The Cyrillic squared units // `мм2`/`км2` are also local: the tier's exponent lookbehind `(?<=[a-zA-Z])` is ASCII-only and the // corpus writes `3136 мм2`. @@ -308,7 +307,7 @@ export function normalizeMacedonian(input: string): string { s = s.replace(/(\d+)\s*мм\s*[²2](?!\d)/gu, "$1 квадратни милиметри"); s = s.replace(/(\d+)\s*км\s*[²2](?!\d)/gu, "$1 квадратни километри"); - // 13) DEGREES — `90°F`, `35° W`, and a bare `N°`. The corpus's own spelled-out form is + // 12) DEGREES — `90°F`, `35° W`, and a bare `N°`. The corpus's own spelled-out form is // "30 степени целзиусови", so °C/°F use the "по" construction; `° W`/`° E` are coordinates. s = s.replace(/(\d+)\s*°\s*C(?![\p{L}\p{M}])/gu, "$1 степени по Целзиус"); s = s.replace(/(\d+)\s*°\s*F(?![\p{L}\p{M}])/gu, "$1 степени по Фаренхаjт"); @@ -316,7 +315,7 @@ export function normalizeMacedonian(input: string): string { s = s.replace(/(\d+)\s*°\s*E(?![\p{L}\p{M}])/gu, "$1 степени исток"); s = s.replace(/(\d+)\s*°/gu, "$1 степени"); - // 14) SIGNS. `+30` (the corpus's "над +30 степени") reads "плус". Minus, ×, ÷, =, <, > and the + // 13) SIGNS. `+30` (the corpus's "над +30 степени") reads "плус". Minus, ×, ÷, =, <, > and the // ampersand are the handoff's sign classes — none occurs in the corpus but a dropped sign is // inaudible, so each is read. s = s.replace(/(^|[\s(])[-−]\s?(?=\d)/gu, "$1минус "); @@ -334,12 +333,12 @@ export function normalizeMacedonian(input: string): string { s = s.replace(/\s*>\s*/gu, " поголемо од "); s = s.replace(/\s*[&＆]\s*/gu, " и "); - // 15) pH → "пе ха" (letter names) and `Ghz` → "гигахерци" — the corpus's lowercase-tech tokens that + // 14) pH → "пе ха" (letter names) and `Ghz` → "гигахерци" — the corpus's lowercase-tech tokens that // would otherwise read as consonant clusters or Latin foreign. s = s.replace(/(?> = { /** Compass letters after a degree sign: `35°W` was read as the glued `dərˈad͡ʒatw`. */ const COMPASS: Readonly> = { n: "utara", s: "selatan", e: "timur", w: "barat" }; -const L = "(? `${h} ${meridiemWord(Number(h), mark)}`); // …and `pg`, the corpus's abbreviation for `pagi`, which read as the bare letters `pɡ`. Only after a @@ -282,8 +283,8 @@ export function normalizeMalay(input: string): string { s = s.replace(/(?`/`×` are claimed ONLY between digits, because this corpus's text still carries HTML @@ -297,7 +298,7 @@ export function normalizeMalay(input: string): string { // silently dropped, so `10-60 minit` read *sepuluh enam puluh minit*. Fires ONLY for a pair that a // measure/period noun follows, or a pair of four-digit years — which is what keeps it out of the // four sports scores (5-3, 6-6, 7-2, 26 - 00), none of which has either. Operands are re-emitted - // verbatim and the class ends in a digit, so a following clause comma cannot be eaten (trap 14 (agreement cannot be applied to digits)). + // verbatim and the class ends in a digit, so a following clause comma cannot be eaten. const num = "\\d+(?:\\.\\d+)?"; s = s.replace(new RegExp(`(?> = { * malayalam.ts because its position in the ordering matters and the ordering is this file's job. */ const SYMBOLS = makeSymbolNormalizer({ - // #586 `multiply` — this language had NO word for the sign at all. ⚠ STANDARD MATHEMATICAL REGISTER, not a + // `multiply` — this language had NO word for the sign at all. ⚠ STANDARD MATHEMATICAL REGISTER, not a // corpus attestation: the sweep's plausible hits were homographs of PREPOSITIONS (es `por` ×23, it `per` ×25, // ru `на` ×31 are all the preposition), the same trap that defeated the exponent sourcing. One word, so `by` // defaults to it — this language does not split dimension from product. multiply: { times: "ഗുണം" }, - // `&` was DROPPED outright, losing the sign from `കോളേജ് ഓഫ് ആർട്സ് & സയൻസസ്`. `ആൻഡ്` is the - // TRANSLITERATED English "and", and it is attested in exactly this construction rather than merely as a - // word: every wiki hit is an English institution name rendered in Malayalam — `ഒബ്സ്റ്റട്രിക്ക്സ് ആൻഡ് - // ഗൈനക്കോളജി`, `അമേരിക്കൻ ബോർഡ് ഓഫ് ഒബ്സ്റ്റട്രിക്സ് ആൻഡ് ഗൈനക്കോളജി`, `റോയൽ കോളേജ് ഓഫ് ഒബ്സ്റ്റട്രീഷ്യൻസ് - // ആൻഡ് ഗൈനക്കോളജിസ്റ്റ്സ്` — and one of those sentences writes the sign itself, `ഒ& ജി`. 18 wiki tokens, - // 8 in the corpus. Chosen over the native options because they do not fit the slot: `ഉം` is a BOUND - // suffix (it attaches to both coordinands, so it cannot stand between two initialisms), and `ഒപ്പം` / - // `കൂടാതെ` mean "along with" / "besides" — 1 and 2 tokens, and the wrong register for a proper name. - // Trap 8, checked: EVERY `&` in all five corpora treated in this batch is the universal `B&B` / - // `Arts & Sciences` pair — no `AT&T`, no URL query — so "and" is right in every attested context. + // `&` is otherwise DROPPED outright, losing the sign from `കോളേജ് ഓഫ് ആർട്സ് & സയൻസസ്`. `ആൻഡ്` is the + // TRANSLITERATED English "and", attested in exactly this construction rather than merely as a word: + // the hits are English institution names rendered in Malayalam (`ഒബ്സ്റ്റട്രിക്ക്സ് ആൻഡ് ഗൈനക്കോളജി`), and + // one such sentence writes the sign itself, `ഒ& ജി`. + // ⚠ THE NATIVE OPTIONS DO NOT FIT THE SLOT: `ഉം` is a BOUND suffix — it attaches to BOTH coordinands, so + // it cannot stand between two initialisms — and `ഒപ്പം` / `കൂടാതെ` mean "along with" / "besides", the + // wrong register for a proper name. ampersand: "ആൻഡ്", percent: ["ശതമാനം"], currency: { "US$": ["ഡോളർ"], "$": ["ഡോളർ"] }, @@ -92,9 +73,9 @@ const SYMBOLS = makeSymbolNormalizer({ }); /** - * The clitics this corpus actually welds onto a numeral, each with the stem it selects. A CLOSED list, - * closed to what is ATTESTED — an open "digits + any Malayalam run" rule would swallow ordinary nouns - * that merely follow a number ("100 അടി", "56 വ്യത്യസ്ത"), which is trap #2. + * The clitics Malayalam welds onto a numeral, each with the stem it selects. ⚠ A CLOSED list, closed to + * what is ATTESTED — an open "digits + any Malayalam run" rule swallows ordinary nouns that merely follow + * a number ("100 അടി", "56 വ്യത്യസ്ത"). * * Longest first, so ത്തിലെ is not shadowed by ലെ. ത്തിൽ/ത്തിലെ carry their own -ത്തി- linker, which is * precisely the oblique stem of a ം-final magnitude, so they are folded to ൽ/ലെ and take the same path @@ -112,8 +93,8 @@ const FOLD_CLITIC: Readonly> = { }; /** - * The Malayalam normalizer. A numbered, ORDER-DEPENDENT sequence; the coupling is stated at each step - * because a future reader cannot recover it from the code. + * The Malayalam normalizer. ⚠ A numbered, ORDER-DEPENDENT sequence — the coupling is stated at each step, + * because it is not recoverable from the code. */ export function normalizeMalayalam(input: string): string { // 1) ZERO-WIDTH characters — FIRST, because every later rule asserts letter/digit adjacency and an @@ -178,19 +159,18 @@ export function normalizeMalayalam(input: string): string { s = SYMBOLS(s); // 6) TIMES BEFORE the decimal step: a bare-number rule must not claim 06:30, and this corpus writes - // the sports times 2:11.60, 1:09.02 and 4:41.30 where a restart inside the number would be - // exactly the Indonesian defect the playbook records. - // (a) :00 minutes are DROPPED, not read — "11:00 (യുടിസി" was giving പതിനൊന്ന് പൂജ്യം. - // (b) every remaining digit-colon-digit becomes a SPACE: `:` is clause punctuation in this - // engine (malayalam.jsonc maps it to ","), so it inserted a pause inside 8:30 and 9:30. - // NO മണി is added. The noun is already in the text where it belongs ("11:00 ന് കഴിഞ്ഞപ്പോൾ", - // "8:30 ന് ആരംഭിക്കും") — the same call Tamil, Telugu and Kannada made on the same evidence. + // the sports times 2:11.60, 1:09.02 and 4:41.30, where a bare-number rule restarting INSIDE the + // number is the classic failure. + // (a) :00 minutes are DROPPED, not read, or "11:00" gives പതിനൊന്ന് പൂജ്യം. + // (b) every remaining digit-colon-digit becomes a SPACE: ⚠ `:` is clause punctuation in this engine + // (malayalam.jsonc maps it to ","), so left alone it inserts a pause INSIDE 8:30. + // NO മണി is added — the noun is already in the text where it belongs ("11:00 ന് കഴിഞ്ഞപ്പോൾ"). s = s.replace(/(?` here is a range, a score or a closed designation, and there are ZERO instances of the - // one shape no guard can reject, `word · space · hyphen · digit`. That test is what decides this class: - // mr, nl, ta, gu, kn and yue all have such an instance and all decline the rule - // (ACCEPTED_SIGN_SILENCE); this corpus does not, so a guarded rule is safe HERE — a fact about the - // corpus, not about the guard. + // THE PLUS is പ്ലസ്, and Malayalam voices it in BOTH positions (`UTC+1`, `+30°C`). ⚠ This is the + // MEASUREMENT plus — the reading a language uses for a signed quantity — which not every language + // shares; some read a word meaning "above" instead. MUST PRECEDE the degree rule, or `+30°C` has + // lost its sign by the time the plus rule looks. // - // THREE GUARDS: a digit immediately after the sign (rejects `- 2`), a letter or digit immediately before - // (rejects closed designations), and a digit ANYWHERE to the left (rejects a SPACED range or score, which - // the fleet's usual guard misses — the gap that cost a real defect in th). + // THE MINUS AND ±. ⚠ WHETHER A MINUS RULE IS SAFE IS A FACT ABOUT THE TEXT, NOT ABOUT THE GUARD. The + // shape no guard can reject is `word · space · hyphen · digit`, which is indistinguishable from a + // spaced range or a dashed designation; a language whose text contains that shape must decline the + // rule outright. This one does not contain it, so a guarded rule is safe HERE and nowhere by default. // - // SOURCED, AND THE SAME SENTENCE GIVES ± DIRECTLY: ml.wikipedia writes "പ്ലസ്-മൈനസ് ചിഹ്നം, ±, - // ഒന്നിലധികം അർത്ഥങ്ങളുള്ള ഒരു ഗണിത…" — the PLUS-MINUS SIGN, ±, a mathematical symbol with several - // meanings. So both the minus word (x16 / 8 articles) and the ± pairing come from one quote naming the - // glyph, which is the strongest shape tier 4 takes. + // THREE GUARDS: a digit immediately after the sign (rejects `- 2`), a letter or digit immediately + // before (rejects closed designations), and a digit ANYWHERE to the left — that last one rejects a + // SPACED range or score, which the usual guard misses. // - // ± is then this language's own two words juxtaposed, both lifted from rules in this file. + // ± IS SOURCED BY THE SAME SENTENCE AS THE MINUS: ml.wikipedia names the glyph directly — + // "പ്ലസ്-മൈനസ് ചിഹ്നം, ±, ഒന്നിലധികം അർത്ഥങ്ങളുള്ള ഒരു ഗണിത…" (the plus-minus sign, ±, a mathematical + // symbol with several meanings). A citation that names the WORD against the GLYPH is the strongest + // form this kind of sourcing takes. ± is then the two words juxtaposed, both already in this file. s = s.replace(/±/gu, " പ്ലസ് മൈനസ് "); s = s.replace(/(? /\d\s*$/u.test(whole.slice(0, off)) ? m0 : "മൈനസ് "); s = s.replace(/(\S)\+\s?(?=\d)/gu, "$1 പ്ലസ് "); s = s.replace(/(^|\s)\+\s?(?=\d)/gu, "$1പ്ലസ് "); - // THE DIVISION AND COMPARISON SIGNS. The EQUALITY IS DELIBERATELY LEFT DROPPED — see the end. - // - // ⚠ THE PARALLEL-CORPUS FORM IS A SUBORDINATE CLAUSE, NOT A READING, and Malayalam is the clearest case of - // why a hit in FLEURS's aspect-ratio sentence is a lead rather than an answer. That sentence gives - // "പന്ത്രണ്ട് ഉപയോഗിച്ച് ഹരിക്കുമ്പോൾ", and both halves are inflected FOR THAT SENTENCE: + // THE DIVISION AND COMPARISON SIGNS. // - // ഹരിക്കുമ്പോൾ = ഹരിക്ക്- (divide) + -ുമ്പോൾ, and -ുമ്പോൾ IS the word "when" (historically -ഉം + പോൾ - // "time"). So the form means "when dividing" — a temporal SUBORDINATOR, subordinated here to the main - // predicate ആണെന്ന് പറയാം ("can be said to be"). Between two operands there is no main clause for it to - // attach to, so `6 ÷ 3` would read "six when-divided three", a fragment awaiting a predicate. + // ⚠ A VERB FORM FOUND IN RUNNING TEXT IS USUALLY INFLECTED FOR THAT SENTENCE, not for the slot between + // two operands. Malayalam is the clearest case: the obvious source phrase is + // "പന്ത്രണ്ട് ഉപയോഗിച്ച് ഹരിക്കുമ്പോൾ", and BOTH halves are sentence-bound — // - // ഉപയോഗിച്ച് ("using") is a converb, not the instrumental case -കൊണ്ട് that Malayalam puts on a divisor. - // The translator wrote a two-clause paraphrase of the operation, not a reading of the notation. + // ഹരിക്കുമ്പോൾ = ഹരിക്ക്- (divide) + -ുമ്പോൾ, and -ുമ്പോൾ IS "when" (historically -ഉം + പോൾ "time"), + // so the form means "when dividing" — a temporal SUBORDINATOR needing a main predicate. Between two + // operands there is none, so `6 ÷ 3` would read "six when-divided three", a fragment. // - // What the sentence DOES establish, with certainty, is the ROOT: ഹരി-. The form comes from ml.wikipedia's - // arithmetic article, which names the sign against the glyph in a section heading — + // ഉപയോഗിച്ച് ("using") is a converb, not the instrumental -കൊണ്ട് that Malayalam puts on a divisor. // - // === ഹരണം (÷ or /) === then "ഗുണനത്തിന്റെ വിപരീത ക്രിയയാണ് ഹരണം" + // What such a phrase DOES establish is the ROOT, ഹരി-. The form shipped is the sign's own NAME, taken + // from an arithmetic article that names it against the glyph — `=== ഹരണം (÷ or /) ===`. ⚠ The naming + // citation is the EVIDENCE; placing that name INFIX is an INFERENCE, and is marked as one. // - // — so ഹരണം (×12 token / 2 articles) is the sign's own NAME, and a sign name reads infix: `el` does exactly - // this with ίσον, `ja` with イコール, and `ta`'s wiki writes the parallel Dravidian nominal out in the slot - // ("a வகுத்தல் b"). ⚠ That last step is a fleet PATTERN rather than an attested "a ഹരണം b" string, and is - // marked as such: the naming citation is the evidence, the infix placement is the inference. + // ⚠ AND ഹരണം'S CORPUS HITS ARE ALL INSIDE അപഹരണം, "ABDUCTION" — a substring trap. Counting tokens for a + // short root without checking what encloses them measures the wrong word. // - // ⚠ AND ഹരണം'S SIX CORPUS HITS ARE ALL INSIDE അപഹരണം, "ABDUCTION" — the substring trap, and the funniest - // instance of it so far. The corpus contributes the root and nothing else. + // ⚠ A BOUND MORPHEME CANNOT BE TOKEN-COUNTED AT ALL. -എക്കാൾ ("than") appears only fused + // (കൾച്ചർ ഷോക്കിനെക്കാൾ, പരമ്പരാഗത ഭാഷകളെക്കാൾ), so its token count is ×0 BY CONSTRUCTION, not by + // absence. What IS countable is the head it governs: കൂടുതൽ, കുറവ്. Postposed, so a comparison here + // cannot read backwards. // - // ⚠ THE COMPARATIVE MORPHEME CANNOT BE TOKEN-COUNTED AT ALL. -എക്കാൾ ("than") is BOUND: it appears only - // fused (കൾച്ചർ ഷോക്കിനെക്കാൾ, പരമ്പരാഗത ഭാഷകളെക്കാൾ), so its token count is ×0 by construction, not by - // absence — the agglutinative counterpart of the ZWNJ false negative Persian produced. What IS countable is - // the head it governs: കൂടുതൽ ×182 token, കുറവ് ×12. Postposed, so the comparison cannot read backwards. - // - // Emitted UNFUSED, as ta's accusative is, and for the same reason: fusing needs the numeral spelled here - // plus its sandhi. The phones are unchanged; the tokenizer sees a boundary Malayalam would not write. + // Emitted UNFUSED: fusing would need the numeral spelled out here plus its sandhi. The phones are + // unchanged; the tokenizer merely sees a boundary Malayalam would not write. s = postposedSign(s, "<", "എക്കാൾ കുറവ്"); s = postposedSign(s, ">", "എക്കാൾ കൂടുതൽ"); s = s.replace(/\s?÷\s?/gu, " ഹരണം "); - // THE EQUALITY, and ⚠ IT WAS THE REGISTER RESTRICTION THAT HID IT — the Polish lesson, repeated exactly. - // This rule was first left DROPPED on the finding that സമം, തുല്യം and ഹരിച്ചാൽ were ×0 in both corpus and - // wiki. That was measured with `attest.ts --context "ഗണിതം അങ്കഗണിതം ഹരണം"`, i.e. inside maths articles, and - // those articles write the notation instead of reading it. Dropping the restriction found the word at once: - // - // `തുല്യം` ×11 token / 11 ARTICLES "കിലോഗ്രാമിന്റെ പിണ്ഡത്തിന് തുല്യം" — EQUAL TO the mass of the - // kilogram; "രാജസൂയത്തിനു തുല്യം ഫലം" — a result EQUAL TO the Rajasuya - // `സമമാണ്` ×19 the predicative form, "is equal" + // THE EQUALITY. ⚠ SEARCHING ONLY INSIDE MATHS ARTICLES HIDES THE EQUALITY WORD, because those articles + // WRITE the notation rather than read it aloud. The word turns up immediately in ordinary prose: + // തുല്യം in "കിലോഗ്രാമിന്റെ പിണ്ഡത്തിന് തുല്യം" (equal TO the mass of the kilogram), and the predicative + // സമമാണ് ("is equal"). // - // ⚠ AND `സമം` IS THE WRONG WORD FOR THIS SLOT, which is why probing it first was misleading. It is ×22 but - // means "in equal MEASURE" adverbially ("ഇവ സമം കഷായം" — these in equal parts, "10 ഗ്രാം സമം നെയ്യും") and is - // separately the name of a rhetorical figure ("സമം എന്ന അലങ്കാരം"). Its Tamil, Kannada and Telugu cognates - // (சமம், ಸಮ, సమానం) ARE the equality word in those languages — all four are the same Sanskrit loan *sama* — - // so a sister-language inference would have picked exactly the wrong member of the set. The cognate tells - // you where to look; it does not tell you which sense the borrowing settled into. + // ⚠ AND `സമം` IS THE WRONG WORD FOR THIS SLOT, though it is the commonest candidate. It means "in equal + // MEASURE" adverbially ("ഇവ സമം കഷായം" — these in equal parts) and is separately the name of a rhetorical + // figure. Its Tamil, Kannada and Telugu cognates (சமம், ಸಮ, సమానం) ARE the equality word there — all four + // are the same Sanskrit loan *sama* — so a sister-language inference picks exactly the wrong member of + // the set. A cognate tells you where to look, not which sense the borrowing settled into. // // ⚠ POSTPOSED, because the attested construction is DATIVE + തുല്യം: the standard comes first - // (`പിണ്ഡത്തിന് തുല്യം`), so `A = B` is "A B-ന് തുല്യം". The dative is emitted unfused for the same reason - // the comparative's -എക്കാൾ is, and with the same known limitation. + // (`പിണ്ഡത്തിന് തുല്യം`), so `A = B` is "A B-ന് തുല്യം". The dative is emitted unfused, as above. s = postposedSign(s, "=", "ന് തുല്യം"); s = s.replace(/(\d)\s?°\s?/gu, "$1 ഡിഗ്രി "); diff --git a/src/languages/malayalam/numbers.ts b/src/languages/malayalam/numbers.ts index 4626868a7..0c9190a95 100644 --- a/src/languages/malayalam/numbers.ts +++ b/src/languages/malayalam/numbers.ts @@ -1,12 +1,11 @@ /** * Malayalam cardinal number → words, plus the ordinal and oblique morphology normalize.ts needs. * - * THE COMPOSITION IS THE SHARED `dravidianNumberWords` (core/numbers.ts). Malayalam is the fourth - * Dravidian language in this repo and the one the playbook named as the trigger to consolidate the - * three private composers rather than write a fourth: it needs every capability Tamil, Telugu and - * Kannada each needed separately — a fused 21-99, suppletive round hundreds, suppletive round - * thousands, and a combining magnitude form before a remainder. All of it is DATA in malayalam.jsonc, - * with its provenance; this file is only the Malayalam-facing wrapper plus the two morphologies. + * THE COMPOSITION IS THE SHARED `dravidianNumberWords` (core/numbers.ts). Malayalam exercises every + * capability that composer has — a fused 21–99, suppletive round hundreds, suppletive round thousands, + * and a combining magnitude form before a remainder — which is why it is worth reading as the reference + * case for the shared path. All of it is DATA in malayalam.jsonc, with its provenance; this file is only + * the Malayalam-facing wrapper plus the two morphologies. * * What `indicNumberWords` produced before, measured on this corpus's own numerals: * 21 ഇരുപത് ഒന്ന് → ഇരുപത്തിയൊന്ന് diff --git a/src/languages/maltese/maltese.ts b/src/languages/maltese/maltese.ts index a0d19daa7..cf04e32a5 100644 --- a/src/languages/maltese/maltese.ts +++ b/src/languages/maltese/maltese.ts @@ -101,10 +101,10 @@ export function phonemizeWord(word: string): string { const TOKEN = new RegExp(`(${hostWordRun(["Latin"], "'")})|(\\d+)|([.!?…,;:])`, "gu"); /** - * This language's OWN inventory — the TOKEN word class as it stood before the widening above, lifted - * verbatim, so nothing about the orthography is invented here. A token this REJECTS carries a letter the - * language does not use, i.e. a foreign name. See core/hostWord.ts: this is the INVENTORY question, and it - * is no longer also deciding where the script boundary falls. + * This language's OWN inventory. ⚠ TWO DIFFERENT QUESTIONS, KEPT APART: the TOKEN class above decides where + * the SCRIPT boundary falls (routing), while this one decides whether the g2p has rules for these letters. A + * token this class REJECTS carries a letter the language does not use — i.e. a foreign name. See + * core/hostWord.ts. * * ⚠ NOT QUITE VERBATIM: à ò À Ò were REMOVED, because the g2p has no rule for them and DROPPED them outright. * The old token class listed them anyway, and the word-level fold hid the mismatch — a word containing one was diff --git a/src/languages/maltese/numbers.ts b/src/languages/maltese/numbers.ts index f26f7113c..bfc0e02d8 100644 --- a/src/languages/maltese/numbers.ts +++ b/src/languages/maltese/numbers.ts @@ -4,7 +4,7 @@ * only loans are the magnitude words `miljun`/`biljun` (Italian milione/bilione), which are the ordinary Maltese * words for those magnitudes — there is no rival Semitic form in use. * - * ★ THE ONE REAL JUDGMENT CALL — **counting form, not attributive form.** Maltese has two numeral series and the + * ⚠ THE ONE REAL JUDGMENT CALL — **counting form, not attributive form.** Maltese has two numeral series and the * split is the classic Maltese textbook point: the ABSOLUTE/COUNTING form (`tnejn` 2, `tlieta` 3, `erbgħa` 4 …) * used when counting aloud, doing arithmetic, or citing a bare figure, versus the ATTRIBUTIVE/construct form * (`żewġ` 2, `tliet` 3, `erba'` 4 …) used immediately before a counted noun. A digit in running text is a bare @@ -15,7 +15,7 @@ * follows. Reading the counting form there is the standard TTS compromise (and is how a Maltese speaker reads a * figure aloud out of context). * - * ★ SOURCES + * SOURCES * - **GF Resource Grammar Library, `NumeralMlt.gf`** (John J. Camilleri, 2011–2013, LGPL) — * https://raw.githubusercontent.com/GrammaticalFramework/gf-rgl/master/src/maltese/NumeralMlt.gf. A complete * published generator for Maltese 1–999 999; every composition rule below is read off it: the unit/attributive/ @@ -27,7 +27,7 @@ * - Attested composed forms used as goldens: `tliet mija u ħdax-il elf` (311 000, Maltese broadcast text) and * Camilleri's own `elf, erba' mija u għoxrin` (1420) — both reproduced exactly by this composer. * - * ★ STRUCTURE — three things make Maltese not a Western decimal: + * STRUCTURE — three things make Maltese not a Western decimal: * 1. **Units-first inside 21–99** with the connector `u`: 21 = `wieħed u għoxrin` ("one and twenty"), 45 = * `ħamsa u erbgħin`. (Semitic order, same as Arabic waːħid wa ʕiʃruːn and German einundzwanzig.) * 2. **DUAL forms** for exactly 2× a magnitude: 200 `mitejn`, 2000 `elfejn` — not *żewġ mija, not *żewġt elef. diff --git a/src/languages/mandarin/mandarin.ts b/src/languages/mandarin/mandarin.ts index fb703fe50..aff2aec07 100644 --- a/src/languages/mandarin/mandarin.ts +++ b/src/languages/mandarin/mandarin.ts @@ -1,8 +1,10 @@ /** - * Mandarin Chinese (cmn) phonemizer — canonical IPA. Phase 1: the pinyin input path (tokenized pinyin with - * tone digits → IPA with Chao tones + third-tone sandhi). Phase 2 adds the Hanzi front-end (pypinyin char + - * phrase dicts → polyphone-aware pinyin), Phase 3 numbers + normalization. The data (syllable→IPA table, - * tone system, sandhi) lives beside this file; this module wires it into the Phonemizer interface. + * Mandarin Chinese (cmn) phonemizer — canonical IPA. Two input paths, one converter: + * · HANZI → pinyin via the pypinyin char + phrase dicts (polyphone-aware), then pinyin → IPA; + * · direct tokenized pinyin with tone digits → IPA. + * The converter emits Chao tone letters and applies third-tone sandhi within a Han run, plus 一/不 sandhi. + * Numbers and text normalization run ahead of it. The data (syllable→IPA table, tone system, sandhi) lives + * beside this file; this module wires it into the Phonemizer interface. */ import type { Phonemizer } from "../../registry.ts"; import { makeSymbolNormalizer } from "../../core/normalizeSymbols.ts"; @@ -42,9 +44,9 @@ const PINYIN_INPUT = /^[a-zü:]+[1-5]?(?:\s+[a-zü:]+[1-5]?)*$/i; /** Embedded Latin → foreign (en) phonemizer, injected by the registry (lazy, like Hindi). */ export type ForeignPhonemizer = (latin: string) => string; -// #562 symbol normalization — Mandarin: 百分之 PRECEDES the number (百分之九十三); units follow. +// symbol normalization — Mandarin: 百分之 PRECEDES the number (百分之九十三); units follow. const SYMBOLS = makeSymbolNormalizer({ - // #586 `multiply` — this language's OWN word, harvested from its existing `×` rule, so nothing new is + // `multiply` — this language's OWN word, harvested from its existing `×` rule, so nothing new is // sourced. Declaring it here is what makes ASCII `x` read like `×`: `6x6 cm` read the `x` as a LETTER NAME, // and `NxN` forms outnumber `×` roughly 85 to 20 across the corpora. One word, so `by` defaults to it. multiply: { times: "乘以" }, @@ -67,7 +69,7 @@ const SYMBOLS = makeSymbolNormalizer({ // Attested in the artifact itself: 公园占地 19500 平方公里 · 783,562 平方公里（300,948 平方英里）. // One form each, because a Chinese measure word does not agree with its count. exponentWords: { squared: ["平方"], cubed: ["立方"], position: "compound" }, - // #586 BARE EXPONENT — the reading for a power with NO unit to modify (`20²`, `mc²`), which every language + // BARE EXPONENT — the reading for a power with NO unit to modify (`20²`, `mc²`), which every language // in the fleet was dropping silently. See `bareExponent` in core/normalizeSymbols.ts for why this cannot // reuse `exponentWords` above: that is the unit MODIFIER and this is the PREDICATE, and in most languages // they are different words (平方公里 but 二十的平方). diff --git a/src/languages/mandarin/normalize.ts b/src/languages/mandarin/normalize.ts index 465cc0fd3..9a6a1cd1e 100644 --- a/src/languages/mandarin/normalize.ts +++ b/src/languages/mandarin/normalize.ts @@ -21,10 +21,9 @@ * * ── THE SIGN CLASSES ──────────────────────────────────────────────────────────────────────────── * - * The #562 pass left every math sign DROPPED, which the review gate reported as - * `DROPPED: minus plus equals less-than times` and the artifact scan as `DROP math-sign ×1`. The minus was - * the one that mattered: `-5 度` read as 五度, **positive** five degrees, so a below-freezing temperature - * was silently reported as above it. + * ⚠ AN UNCLAIMED MATH SIGN IS DROPPED, NOT MISREAD, which is what makes it dangerous: `-5 度` reads as + * 五度, **positive** five degrees, so a below-freezing temperature is silently reported as above it. A + * dropped sign is invisible to every check that looks for a wrong reading. * * EVERY WORD BELOW IS ATTESTED IN ITS OWN NOTATION SLOT, from zh.wikipedia via * `tools/normalization/attest.ts` (cached in `tools/corpus/attest/cmn.jsonc`) and, for 乘以 and 平方, from @@ -94,7 +93,7 @@ const SIGNS: readonly (readonly [RegExp, string])[] = [ ]; /** - * THE AMPERSAND, which is the one drop left after the signs above and the class #586 opens with. + * THE AMPERSAND, the one drop left after the signs above. * * The artifact's only `&` is `一众 B&B 公司` — "a number of B&B companies" — and it read as *bˈiː bˈiː*, * "B B". Between LATIN letters the ampersand stays inside the Latin run and is spelled ` and `, because the diff --git a/src/languages/maori/maori.ts b/src/languages/maori/maori.ts index dcdd70c11..bb458ce0e 100644 --- a/src/languages/maori/maori.ts +++ b/src/languages/maori/maori.ts @@ -46,10 +46,10 @@ export function phonemizeWord(word: string): string { const TOKEN = new RegExp(`(${hostWordRun(["Latin"], "'ʻ-")})|(\\d+)|([.!?…,;:])`, "gu"); /** - * This language's OWN inventory — the TOKEN word class as it stood before the widening above, lifted - * verbatim, so nothing about the orthography is invented here. A token this REJECTS carries a letter the - * language does not use, i.e. a foreign name. See core/hostWord.ts: this is the INVENTORY question, and it - * is no longer also deciding where the script boundary falls. + * This language's OWN inventory. ⚠ TWO DIFFERENT QUESTIONS, KEPT APART: the TOKEN class above decides where + * the SCRIPT boundary falls (routing), while this one decides whether the g2p has rules for these letters. A + * token this class REJECTS carries a letter the language does not use — i.e. a foreign name. See + * core/hostWord.ts. */ const NATIVE_CLASS = "[a-zāēīōūA-ZĀĒĪŌŪ'ʻ-]"; const nat = makeNativiser(NATIVE_CLASS, "u"); @@ -58,8 +58,8 @@ const nat = makeNativiser(NATIVE_CLASS, "u"); * * ⚠ `NATIVE_CLASS` IS THE TOKEN CLASS, NOT THE ALPHABET — it spans `a-zA-Z`, because that is what the tokenizer * needs in order to claim a word at all. Using it to decide routing routes NOTHING: `Safari` is entirely ASCII, so - * it tests as native and never reaches the reader. The two have been distinct all along and the names hid it; for - * the #657 fold the difference was invisible, because that fold only ever fires on a NON-ASCII letter. + * it tests as native and never reaches the reader. The two questions are distinct and the shared name hides it; + * a fold that only ever fires on a NON-ASCII letter cannot expose the difference. * * ⚠ IT WALKS THE WORD THE WAY THE G2P DOES — longest digraph first, then a single grapheme — rather than testing * membership in a flat letter set. A flat set has to admit `g` for the sake of ⟨ng⟩, and then a standalone `g` @@ -95,8 +95,8 @@ class MaoriPhonemizer implements Phonemizer { // // Until that exists, an English reading is the honest answer for a word Māori cannot spell: the wrong // voice, but legal phones and a recognisable name. THIRTEEN of twenty-six letters are outside this - // alphabet, which makes Māori the fleet's most extreme case rather than a typical one — most of the 45 - // engines measured in #663 are missing one or two letters and need no routing. + // alphabet, which makes Māori an extreme case rather than a typical one — most Latin-script + // engines are missing one or two letters and need no routing at all. // // The floor stays underneath: `nat` still applies on the native branch, and `latinPhone` still backs // the g2p, for the case where no reader is injected (direct engine use, or a test). diff --git a/src/languages/marathi/marathi.ts b/src/languages/marathi/marathi.ts index 237fab771..796da48fe 100644 --- a/src/languages/marathi/marathi.ts +++ b/src/languages/marathi/marathi.ts @@ -23,14 +23,11 @@ function engine(foreign?: ForeignPhonemizer) { ); } -/** #562 normalization. Marathi shares Hindi's ENGINE but not Hindi's orthographic conventions, so it - * supplies its OWN normalizer and its OWN symbol words through `makeNativeHindi`'s overrides rather than - * inheriting Hindi's. Before that parameter existed the Marathi pass had to run AHEAD of `text()` and be - * written to consume its input completely, so the Hindi pass inside would find nothing left to claim — - * a coupling that was load-bearing in three separate steps. Verified byte-identical over the whole - * mr_in corpus when moved onto the seam. */ +/** Normalization. Marathi shares Hindi's ENGINE but not Hindi's orthographic conventions, so it supplies + * its OWN normalizer and its OWN symbol words through `makeNativeHindi`'s overrides rather than + * inheriting Hindi's. */ const MR_SYMBOLS = makeSymbolNormalizer({ - // #586 `multiply` — this language had NO word for the sign at all. ⚠ STANDARD MATHEMATICAL REGISTER, not a + // `multiply` — this language had NO word for the sign at all. ⚠ STANDARD MATHEMATICAL REGISTER, not a // corpus attestation: the sweep's plausible hits were homographs of PREPOSITIONS (es `por` ×23, it `per` ×25, // ru `на` ×31 are all the preposition), the same trap that defeated the exponent sourcing. One word, so `by` // defaults to it — this language does not split dimension from product. diff --git a/src/languages/marathi/normalize.ts b/src/languages/marathi/normalize.ts index 6d2fa08a2..a4c4fd428 100644 --- a/src/languages/marathi/normalize.ts +++ b/src/languages/marathi/normalize.ts @@ -2,38 +2,21 @@ * Marathi (mr) TEXT NORMALIZATION — the pre-tokenizer pass that rewrites everything which is not already * a pronounceable word into words the existing pipeline speaks. Pure text→text; no IPA. * - * Marathi shares Hindi's script and Hindi's abugida engine (`makeNativeHindi`), but NOT Hindi's - * orthographic conventions, and the two diverge in every tier this file touches: Marathi writes the - * percent as टक्के (not प्रतिशत), the clock as वाजून/वाजता (not बजकर/बजे), the ordinal suffix as - * -व्या/-वा/-वी/-वे with its own suppletive 1-4, and — the one that mattered most — it writes a large - * fraction of its numbers in DEVANAGARI DIGITS, which every ASCII-defined rule in the fleet misses. + * ⚠ MARATHI SHARES HINDI'S SCRIPT AND HINDI'S ABUGIDA ENGINE (`makeNativeHindi`) BUT NOT HINDI'S + * ORTHOGRAPHIC CONVENTIONS, and the two diverge in every tier this file touches: Marathi writes the + * percent as टक्के (not प्रतिशत), the clock as वाजून/वाजता (not बजकर/बजे), and the ordinal suffix as + * -व्या/-वा/-वी/-वे with its own suppletive 1–4. It also writes a large fraction of its numbers in + * DEVANAGARI DIGITS, which every ASCII-defined rule misses — see step 2, and note that the same lead + * FAILS for several sibling Indic languages, so it must be measured per language rather than assumed. * - * Measured over the mr_in FLEURS corpus (1,992 unique utterances, column 3): - * Devanagari digits ०-९ .......... 597 (the "native digits" lead HOLDS here, unlike hi/bn/ur) - * two-digit numbers .............. 233 (every one of them mis-read — see marathi.jsonc `compound`) - * four-digit years ............... 171 - * three-digit numbers ............ 76 - * ordinal suffixes (व्या/वा/वे) ... 45 - * grouped numbers (1,234) ........ 46 - * ZWJ inside a word .............. 66 (्‍य ×45, अ‍ॅ ×21 — split the word in the tokenizer) - * ASCII ':' written for visarga .. ~42 (विशेषत: ×12, स्वत:* ×17, सामान्यत: ×9 …) - * decimals ....................... 29 - * Devanagari unit abbreviations .. 32 (किमी, मिमी, मी, किमी/तास, किमी²) - * clock times h:mm ............... 17 (3 of which are SPORTS times 4:41.30 — not clocks) - * ranges N-M ..................... 17 (4 of which are SPORTS SCORES — see step 12) - * currency signs ................. 11 ($ ×7, ¥ ×3, € ×1 — 7 of them on Devanagari digits) - * percent ........................ 5 - * fractions ...................... 3 - * era markers इ.स.पू. ............ 4 - * डॉ. ............................ 6 - * degrees ° ...................... 2 - * danda ।/॥ ...................... 0 (NEGATIVE RESULT: the Marathi corpus uses the ASCII period) + * ⚠ THE DANDA ।/॥ DOES NOT APPEAR — Marathi text of this kind uses the ASCII period. A negative result + * worth recording, because it is the opposite of what the script suggests. * - * A NOTE ON THE SHARED ENGINE, which shapes several rules below. `makeNativeHindi` applies HINDI's - * normalizer and HINDI's symbol tier unconditionally to all seven languages that reuse it, and there is - * no seam to pass a different one. So this pass runs FIRST and must leave nothing behind that Hindi's - * pass can claim — every rule here is written to consume its input completely. Where that forced a rule - * into a shape it would not otherwise take, it is called out at the step (5, 7, 12). + * ⚠ A NOTE ON THE SHARED ENGINE, which shapes several rules below. `makeNativeHindi` applies HINDI's + * normalizer and HINDI's symbol tier to every language that reuses it, and there is no seam to pass a + * different one. So this pass runs FIRST and must leave nothing behind that Hindi's pass can claim — + * every rule here is written to consume its input completely. Where that forced a rule into a shape it + * would not otherwise take, it is called out at the step (5, 7, 12). */ import { indicNumberWords, type NumbersDef } from "../../core/numbers.ts"; import { postposedSign } from "../../core/postposedSign.ts"; @@ -89,8 +72,8 @@ const UNIT_WORD: Readonly> = { // Marathi's as well, but the bare-hundred rewrite in step 14 turns `100 km` into `शंभर km`, which // that tier can no longer match (its NUM is a digit run) — the Latin would have been stranded and // read out as letter names. Owning them here settles the ordering. Single-letter `m` is deliberately - // NOT here: the playbook's `rateDenominators` note records a one-letter unit matching an - // alphanumeric designation, and this corpus's only `100m`/`200m` are swim events, 2 instances. + // NOT here: ⚠ a one-letter unit key matches alphanumeric designations, and the `100m`/`200m` + // that occur are swim events, not measurements. "km": "किलोमीटर", "cm": "सेंटीमीटर", "mm": "मिलीमीटर", "kg": "किलोग्रॅम", }; const UNIT_ALT = Object.keys(UNIT_WORD) @@ -253,7 +236,7 @@ export function makeMarathiNormalizer( // Hindi clock rule claims them (its `(?![\d:])` permits a following dot) and produced // "चार बजकर एकेचाळीस मिनट . तीस" — a bogus clock, a Hindi word, and a spurious phrase // break. Dropping the colon leaves two plain numbers, which is the honest reading and - // which nothing downstream can re-claim. (Same failure the playbook records for ru/id.) + // which nothing downstream can re-claim. // (the trailing guard is `(?![\d:])`, NOT `(?![\d.,:])` — the corpus writes these in a // comma-separated list, "4:41.30, 2:11.60", and excluding a following comma made the rule // miss the first of the two.) @@ -316,7 +299,7 @@ export function makeMarathiNormalizer( // step 13, whose solidus would otherwise have to compete with किमी/तास. The shared tier // matches a unit only when a NUMBER is adjacent and these keys are Devanagari, so this stays // local. `(?![\p{L}\p{M}])` after the key is what keeps मी (metre) out of मीटर, मिनिटे and - // the pronoun मी — the same over-counting trap that bit Bengali's ম. + // the pronoun मी — the same over-counting trap any short unit key has in an abugida. s = s.replace( new RegExp(`(\\d)\\s?(${UNIT_ALT})(?![\\p{L}\\p{M}])`, "gu"), (_m, d: string, u: string) => `${d} ${UNIT_WORD[u]!}`, @@ -355,40 +338,38 @@ export function makeMarathiNormalizer( // `शंभरm` from the swim event "100m आणि 200m" — the guard leaves any digits+Latin pair alone. s = s.replace(/(?", "पेक्षा जास्त"); diff --git a/src/languages/mindong/mindong.ts b/src/languages/mindong/mindong.ts index d31a4d200..95613ae40 100644 --- a/src/languages/mindong/mindong.ts +++ b/src/languages/mindong/mindong.ts @@ -1,19 +1,20 @@ /** - * Min Dong / Eastern Min (cdo) — Fuzhou dialect (Fuzhounese), Sinitic, tonal (~9M speakers), the only major Sinitic - * branch otherwise absent from the fleet. This phonemizer consumes **Bàng-uâ-cê (BUC / Foochow Romanized)** — the - * phonemic missionary Latin orthography (used by the cdo Wikipedia + historical Bible/press) — and converts it to - * canonical IPA, mirroring the Min Nan (nan) direct-Tâi-lô path. The converter (mindong.jsonc): strip the tone - * diacritic (identifies the tone) → [initial] + rime → IPA + Chao tone letters. BUC follows the missionary - * convention where the plain stop letters are ASPIRATED: ⟨p t k⟩ = [pʰ tʰ kʰ], ⟨b d g⟩ = [p t k]; ⟨c⟩ = [t͡s], - * ⟨ch⟩ = [t͡sʰ]; ⟨ng⟩ = [ŋ]. + * Min Dong / Eastern Min (cdo) — Fuzhou dialect (Fuzhounese), Sinitic, tonal (~9M speakers). This phonemizer + * consumes **Bàng-uâ-cê (BUC / Foochow Romanized)** — the phonemic missionary Latin orthography used by the cdo + * Wikipedia and historical Bible/press — and converts it to canonical IPA, mirroring the Min Nan direct-Tâi-lô + * path. The converter (mindong.jsonc): strip the tone diacritic (which identifies the tone) → [initial] + rime + * → IPA + Chao tone letters. * - * SEGMENTAL + CITATION tone, with the 韻變 (rime alternation) MODELLED: each rime has a TIGHT form and, where it - * alternates, a LOOSE form, selected by the tone register (LOOSE under the acute/circumflex tones 陰去/陰入/陽去, - * tight otherwise); the ⟨io⟩-family additionally picks its medial [y]/[u] by the initial place. Still DEFERRED: the - * Han front-end (no independent Han→reading dict exists — the only source is Wiktionary, the referee's source → - * circular), tone sandhi (連讀變調), and initial assimilation (聲母類化). The eval folds tones (Chao letters stripped - * both sides) and validates the segmental backbone. Referee: BUC↔IPA pairs from the kaikki Chinese dump (Wiktionary - * Module:cdo-pron output) → 🔷 reference-implementation parity, not independent human attestation. + * ⚠ BUC FOLLOWS THE MISSIONARY CONVENTION, so the plain stop letters are ASPIRATED: ⟨p t k⟩ = [pʰ tʰ kʰ] and + * ⟨b d g⟩ = [p t k]. ⟨c⟩ = [t͡s], ⟨ch⟩ = [t͡sʰ], ⟨ng⟩ = [ŋ]. + * + * SEGMENTAL + CITATION tone, with the 韻變 (rime alternation) MODELLED: each rime has a TIGHT form and, where + * it alternates, a LOOSE form, selected by the tone register (LOOSE under the acute/circumflex tones 陰去/陰入/ + * 陽去, tight otherwise); the ⟨io⟩-family additionally picks its medial [y]/[u] by the initial place. + * + * DEFERRED: the Han front-end (no independent Han→reading dictionary exists — the only source is Wiktionary, + * which is also the referee's source, so it would be circular), tone sandhi (連讀變調), and initial + * assimilation (聲母類化). */ import type { Phonemizer } from "../../registry.ts"; import { assembleClauses, clauseSink } from "../../core/clauses.ts"; diff --git a/src/languages/minnan/minnan.ts b/src/languages/minnan/minnan.ts index 68b10ff2e..47b171886 100644 --- a/src/languages/minnan/minnan.ts +++ b/src/languages/minnan/minnan.ts @@ -6,7 +6,9 @@ * The Tâi-lô→IPA converter (minnan.jsonc, initial/final/tone maps from the epitran nan-Latn-tl spec): strip the * tone diacritic (identifies the tone) → [initial] + final → IPA + Chao tone letter. Sibilants PALATALISE before * i (ts/tsh/s/j+i → t͡ɕ/t͡ɕʰ/ɕ/d͡ʑ); checked finals -p̚/-t̚/-k̚, -h→ʔ; nasalised -nn vowels; syllabic m̩/ŋ̍. - * Phase 1: segmental + CITATION tone (the tone-sandhi circle is deferred). + * + * Segmental + CITATION tone. The tone-sandhi circle is DEFERRED — it is phrase-level and not recoverable + * from a syllable-at-a-time conversion. */ import type { Phonemizer } from "../../registry.ts"; import { assembleClauses, clauseSink } from "../../core/clauses.ts"; @@ -239,16 +241,15 @@ export function phonemizeWord(word: string): string { } /** - * This language's OWN inventory — the TOKEN word class as it stood before the widening above, lifted verbatim, so - * nothing about the orthography is invented here. A token this REJECTS carries a letter the language does not - * use, i.e. a foreign name. See core/hostWord.ts: this is the INVENTORY question, and it is no longer also - * deciding where the script boundary falls. + * This language's OWN inventory. ⚠ TWO DIFFERENT QUESTIONS, KEPT APART: the TOKEN class above decides where + * the SCRIPT boundary falls (routing), while this one decides whether the g2p has rules for these letters. A + * token this class REJECTS carries a letter the language does not use — i.e. a foreign name. See + * core/hostWord.ts. * - * ⚠ ONE ADDITION BEYOND THE VERBATIM LIFT: the capitals `ÀÁÂĀǍ`. the Tâi-lô tone vowels were listed in lower case only, so `TÂI` lost the tone `tâi` kept — so a - * CAPITALISED native word failed the inventory test and the fold stripped its diacritic. Harmless while the - * class was only deciding tokenization (the letter fell out of the token and fragmented, which is the defect this - * issue is about); it becomes a silent DELETION the moment the class also drives the fold. Found by checking every - * class against the upper case of its own letters, not by a corpus. + * ⚠ THE CAPITALS `ÀÁÂĀǍ` MUST BE LISTED ALONGSIDE THE LOWER CASE. Listing only the lower-case Tâi-lô tone + * vowels means `TÂI` fails the inventory test where `tâi` passes, so a CAPITALISED native word is treated as + * foreign and the fold strips its tone diacritic — a silent DELETION, since this class drives the fold and + * not merely the tokenization. Any class that omits the upper case of its own letters has the same bug. */ const NATIVE_CLASS = "[A-Za-zàáâāǎÀÁÂĀǍ̀-̍]"; const nat = makeNativiser(NATIVE_CLASS, "u"); \ No newline at end of file From d625259f37c2593bb2913d096e5c9c263a42ec26 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Thu, 6 Aug 2026 22:20:29 -0600 Subject: [PATCH 2/2] review: a false ordering claim in the macedonian header, and a wrong example word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both found by checking the comments against the RUNNING ENGINE rather than re-reading them. · macedonian/normalize.ts's ORDERING COUPLINGS claimed "the ordinal-suffix rule BEFORE ... the range rule". It never was: ranges were step 5 and the ordinal suffix step 8 on main, so the header asserted the opposite of the code, and asserted it in the same list as the (true) claim that ranges run before the clock. Pre-existing — the renumbering in the previous commit did not move either rule. There is no real coupling to state: the range regex requires digits on BOTH sides, so `1970-тите` cannot match it. The century/date half of the claim IS true and is kept. · malay/normalize.ts illustrated its digit-by-digit decimal with "the tokenizer says *lima kosong*". The engine says *lima nol* — both are Malay for zero, but the inherited number path emits nol. The other five macedonian ordering claims were verified by running them: de-grouping (400.000 → четиристотини илјади), era before year (356 г. п.н.е. → … година пред нашата ера), range before clock (22:00-23:00 → … до …), ordinal suffix (17-ти век → седумнаесетти век), personal initials before the initialism pass (Н. Вејн → ен вејн). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012Lc3WnUgogC7okV7n53vjr --- src/languages/macedonian/normalize.ts | 2 +- src/languages/malay/normalize.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/languages/macedonian/normalize.ts b/src/languages/macedonian/normalize.ts index c7832c62c..d6b380025 100644 --- a/src/languages/macedonian/normalize.ts +++ b/src/languages/macedonian/normalize.ts @@ -41,7 +41,7 @@ * ORDERING COUPLINGS, each a bug that happened: * · de-grouping FIRST — a period/space is otherwise a token boundary or clause mark. * · multi-dot era markers BEFORE the single-dot year rule, and before the `N г.` expansion. - * · the ordinal-suffix rule BEFORE the century/date rules and the range rule. + * · the ordinal-suffix rule BEFORE the century/date rules, which would otherwise claim the digits. * · the range rule BEFORE the clock rule, so `22:00-23:00` becomes "22:00 до 23:00" first. * · personal-initial single capitals BEFORE the initialism pass (which would otherwise see the dot). */ diff --git a/src/languages/malay/normalize.ts b/src/languages/malay/normalize.ts index de8483468..be02847cd 100644 --- a/src/languages/malay/normalize.ts +++ b/src/languages/malay/normalize.ts @@ -307,8 +307,8 @@ export function normalizeMalay(input: string): string { // 15) DECIMALS, LAST — every rule that consumes a glued unit has run, so a letter still stuck to the // fraction now means a VERSION (`802.11a/b/g/n`), not a unit. Malay reads the point as - // `perpuluhan` and the fraction DIGIT BY DIGIT — hence the space between the digits, so the - // tokenizer says *lima kosong* and not *lima puluh*. + // `perpuluhan` and the fraction DIGIT BY DIGIT — hence the space between the digits, so `.50` + // reads *lima nol* and not *lima puluh* (fifty). // `\d{1,3}.000` is EXCLUDED: the corpus's one `9.000 orang` is Indonesian-convention thousands // grouping in a translated sentence, and the inherited tokenizer already reads it as *sembilan // ribu*. The exclusion is narrowed to an ALL-ZERO group rather than any three digits, because the