feat(text): add script detection, normalisation, bidi shaping and glyph coverage

Arabic ground truth has two representations that must never be confused: the
logical string the model predicts and the visual string the rasteriser draws.
ocrsmith.text makes that split explicit.

- script.py: script classification and first-strong-character base direction.
- normalization.py: NormalizationPolicy with opt-in, idempotent transforms
  (diacritics, tatweel, alef/ya/ta-marbuta unification, numeral systems,
  whitespace). Every transform changes the label, so none is applied silently.
- shaping.py: two interchangeable backends behind one protocol. Pillow builds
  with Raqm shape and reorder themselves (TransparentShaper); builds without it
  get presentation forms and visual reordering from arabic-reshaper +
  python-bidi (ReshaperBidiShaper). Both keep the logical label identical, so
  the dataset does not depend on how Pillow was compiled.
- coverage.py: exact cmap coverage via fontTools, cached per font file. A font
  that cannot draw a character is now rejectable instead of silently rendering
  tofu that contradicts the label.

Adds fonttools as a dependency and drops two empty placeholder modules.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: hBouanane

CHANGELOG.md

@@ -8,6 +8,18 @@ All notable changes to this project are documented here. The format follows
 
 ### Added
 
+- `ocrsmith.text`: a text subsystem that owns everything happening to a string before it
+  becomes pixels.
+  - Script and base-direction detection (`detect_script`, `detect_direction`).
+  - `NormalizationPolicy` — opt-in, idempotent transforms for diacritics, tatweel,
+    alef/ya/ta-marbuta unification, numeral systems and whitespace. Each one changes the
+    label, so each one is explicit and recorded.
+  - Bidi/shaping with two interchangeable backends: `TransparentShaper` when Pillow has
+    Raqm, `ReshaperBidiShaper` (arabic-reshaper + python-bidi) otherwise. Both keep the
+    logical string as the label so datasets are identical across machines.
+  - Font glyph coverage via fontTools `cmap` (`supports_text`, `fonts_supporting`), so a
+    font that cannot draw a character is rejected instead of silently emitting tofu.
+
 - Development tooling: ruff lint/format configuration, pytest configuration with
   `pythonpath = ["src"]` (so a clean clone is testable without installing), coverage
   settings, and a GitHub Actions CI matrix over Linux/Windows and Python 3.10/3.12.

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
     "numpy>=1.24",
     "arabic-reshaper>=3.0",
     "python-bidi>=0.4.2",
+    "fonttools>=4.40",
     "typer>=0.12",
     "rich>=13.0",
     "tqdm>=4.66",

src/ocrsmith/text/__init__.py

@@ -0,0 +1,58 @@
+"""Text handling: script detection, normalisation, shaping and font coverage.
+
+This package owns everything that happens to a string *before* it becomes pixels.
+"""
+
+from .coverage import (
+    CoverageReport,
+    fonts_supporting,
+    has_glyph,
+    missing_glyphs,
+    supports_text,
+)
+from .normalization import (
+    NormalizationPolicy,
+    NumeralSystem,
+    normalize_text,
+    strip_diacritics,
+    strip_tatweel,
+    to_numeral_system,
+)
+from .script import Direction, Script, detect_direction, detect_script, is_arabic_char
+from .shaping import (
+    ReshaperBidiShaper,
+    ShapedText,
+    TextShaper,
+    TransparentShaper,
+    raqm_available,
+    resolve_shaper,
+)
+
+__all__ = [
+    # script
+    "Direction",
+    "Script",
+    "detect_direction",
+    "detect_script",
+    "is_arabic_char",
+    # normalisation
+    "NormalizationPolicy",
+    "NumeralSystem",
+    "normalize_text",
+    "strip_diacritics",
+    "strip_tatweel",
+    "to_numeral_system",
+    # shaping
+    "ReshaperBidiShaper",
+    "ShapedText",
+    "TextShaper",
+    "TransparentShaper",
+    "raqm_available",
+    "resolve_shaper",
+    # coverage
+    "CoverageReport",
+    "fonts_supporting",
+    "has_glyph",
+    "missing_glyphs",
+    "supports_text",
+]

src/ocrsmith/text/coverage.py

@@ -0,0 +1,121 @@
+"""Font glyph coverage.
+
+A font that lacks a glyph does not fail — it draws a blank or a tofu box while the label
+still claims the character is there. At scale that quietly teaches a model to hallucinate.
+Coverage is therefore checked up front and unsupported (font, text) pairs are rejected
+before they can reach the dataset.
+
+Coverage is read from the font's `cmap` via fontTools, which is exact and needs no
+rendering. Results are cached per font file because a generation run asks the same
+question millions of times.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+__all__ = [
+    "CoverageReport",
+    "fonts_supporting",
+    "has_glyph",
+    "missing_glyphs",
+    "supports_text",
+]
+
+# Characters no font is expected to carry a glyph for and which never need one.
+_IGNORED = set(" \t\n\r​‌‍‎‏")
+
+
+@dataclass(frozen=True, slots=True)
+class CoverageReport:
+    """Which characters of a text a font can actually draw."""
+
+    font_path: str
+    total: int
+    missing: tuple[str, ...]
+
+    @property
+    def covered(self) -> int:
+        return self.total - len(self.missing)
+
+    @property
+    def ratio(self) -> float:
+        """Fraction of checkable characters the font can draw; 1.0 for empty text."""
+        return 1.0 if self.total == 0 else self.covered / self.total
+
+    @property
+    def is_complete(self) -> bool:
+        return not self.missing
+
+    def __bool__(self) -> bool:
+        return self.is_complete
+
+
+@lru_cache(maxsize=512)
+def _codepoints(font_path: str) -> frozenset[int]:
+    """Every code point mapped by the font's character map."""
+    from fontTools.ttLib import TTFont
+
+    covered: set[int] = set()
+    with TTFont(font_path, fontNumber=0, lazy=True) as font:
+        for table in font["cmap"].tables:
+            covered.update(table.cmap.keys())
+    return frozenset(covered)
+
+
+def has_glyph(font_path: str | Path, char: str) -> bool:
+    """Whether the font at `font_path` maps `char` to a glyph."""
+    if not char:
+        return True
+    if char in _IGNORED:
+        return True
+    return ord(char[0]) in _codepoints(str(Path(font_path)))
+
+
+def missing_glyphs(font_path: str | Path, text: str) -> tuple[str, ...]:
+    """Unique characters of `text` the font cannot draw, in first-seen order."""
+    covered = _codepoints(str(Path(font_path)))
+    missing: list[str] = []
+    seen: set[str] = set()
+    for char in text:
+        if char in _IGNORED or char in seen:
+            continue
+        seen.add(char)
+        if ord(char) not in covered:
+            missing.append(char)
+    return tuple(missing)
+
+
+def supports_text(font_path: str | Path, text: str) -> CoverageReport:
+    """Full coverage report for rendering `text` with the font at `font_path`."""
+    checkable = {char for char in text if char not in _IGNORED}
+    return CoverageReport(
+        font_path=str(font_path),
+        total=len(checkable),
+        missing=missing_glyphs(font_path, text),
+    )
+
+
+def fonts_supporting(
+    font_paths: Iterable[str | Path],
+    text: str,
+    *,
+    min_ratio: float = 1.0,
+) -> Sequence[str]:
+    """Subset of `font_paths` that can draw `text` to at least `min_ratio` coverage.
+
+    Used to pick a font per sample instead of discovering mid-render that the chosen
+    face has no Arabic in it.
+    """
+    eligible: list[str] = []
+    for path in font_paths:
+        try:
+            report = supports_text(path, text)
+        except Exception:
+            continue  # unreadable or exotic font container: treat as ineligible
+        if report.ratio >= min_ratio:
+            eligible.append(str(path))
+    return eligible

src/ocrsmith/text/normalization.py

@@ -0,0 +1,141 @@
+"""Text normalisation policies.
+
+Every transform here changes the ground-truth label, not just the pixels, so each one is
+opt-in and recorded on the sample. A dataset that silently strips diacritics teaches a
+model to drop them; that has to be a deliberate choice.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass
+from enum import Enum
+
+__all__ = [
+    "NormalizationPolicy",
+    "NumeralSystem",
+    "normalize_text",
+    "strip_diacritics",
+    "strip_tatweel",
+    "to_numeral_system",
+]
+
+TATWEEL = "ـ"
+
+# Tashkeel, Quranic annotation marks and the superscript alef.
+_DIACRITICS = re.compile(
+    "["
+    "ؐ-ؚ"
+    "ً-ٟ"
+    "ٰ"
+    "ۖ-ۜ"
+    "۟-ۨ"
+    "۪-ۭ"
+    "࣓-ࣿ"
+    "]"
+)
+
+_ALEF_VARIANTS = str.maketrans({"أ": "ا", "إ": "ا", "آ": "ا", "ٱ": "ا"})
+_YA_VARIANTS = str.maketrans({"ى": "ي"})
+_TA_MARBUTA = str.maketrans({"ة": "ه"})
+
+_WESTERN_DIGITS = "0123456789"
+_ARABIC_INDIC_DIGITS = "٠١٢٣٤٥٦٧٨٩"
+_EASTERN_DIGITS = "۰۱۲۳۴۵۶۷۸۹"
+
+_HORIZONTAL_WS = re.compile(r"[^\S\n]+")
+_ALL_WS = re.compile(r"\s+")
+
+
+class NumeralSystem(str, Enum):
+    """Which digit shapes appear in the rendered text and in the label."""
+
+    KEEP = "keep"
+    WESTERN = "western"
+    ARABIC_INDIC = "arabic_indic"
+    EASTERN_ARABIC_INDIC = "eastern_arabic_indic"
+
+
+_DIGIT_TABLES: dict[NumeralSystem, str] = {
+    NumeralSystem.WESTERN: _WESTERN_DIGITS,
+    NumeralSystem.ARABIC_INDIC: _ARABIC_INDIC_DIGITS,
+    NumeralSystem.EASTERN_ARABIC_INDIC: _EASTERN_DIGITS,
+}
+
+_ALL_DIGITS = _WESTERN_DIGITS + _ARABIC_INDIC_DIGITS + _EASTERN_DIGITS
+
+
+def strip_diacritics(text: str) -> str:
+    """Remove Arabic tashkeel and Quranic annotation marks, leaving the skeleton."""
+    return _DIACRITICS.sub("", text)
+
+
+def strip_tatweel(text: str) -> str:
+    """Remove kashida (tatweel) elongation characters."""
+    return text.replace(TATWEEL, "")
+
+
+def to_numeral_system(text: str, system: NumeralSystem) -> str:
+    """Rewrite every digit in `text` using `system`'s digit shapes."""
+    if system is NumeralSystem.KEEP:
+        return text
+    target = _DIGIT_TABLES[system]
+    table = {ord(digit): target[index % 10] for index, digit in enumerate(_ALL_DIGITS)}
+    return text.translate(table)
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizationPolicy:
+    """Declarative description of how raw source text becomes a label.
+
+    Defaults are deliberately conservative: only runs of whitespace are collapsed, which
+    no OCR label format preserves anyway.
+    """
+
+    collapse_whitespace: bool = True
+    preserve_line_breaks: bool = False
+    strip_diacritics: bool = False
+    strip_tatweel: bool = False
+    unify_alef: bool = False
+    unify_ya: bool = False
+    unify_ta_marbuta: bool = False
+    numerals: NumeralSystem = NumeralSystem.KEEP
+    unicode_form: str | None = "NFC"
+
+    def apply(self, text: str) -> str:
+        return normalize_text(text, self)
+
+
+def normalize_text(text: str, policy: NormalizationPolicy | None = None) -> str:
+    """Apply `policy` to `text`.
+
+    The order is fixed and the result is idempotent: composing characters are folded
+    first so that later character-level rules see a canonical form.
+    """
+    policy = policy or NormalizationPolicy()
+
+    if policy.unicode_form:
+        text = unicodedata.normalize(policy.unicode_form, text)
+    if policy.strip_diacritics:
+        text = strip_diacritics(text)
+    if policy.strip_tatweel:
+        text = strip_tatweel(text)
+    if policy.unify_alef:
+        text = text.translate(_ALEF_VARIANTS)
+    if policy.unify_ya:
+        text = text.translate(_YA_VARIANTS)
+    if policy.unify_ta_marbuta:
+        text = text.translate(_TA_MARBUTA)
+    text = to_numeral_system(text, policy.numerals)
+
+    if policy.collapse_whitespace:
+        if policy.preserve_line_breaks:
+            text = _HORIZONTAL_WS.sub(" ", text)
+            # Keep one break between non-empty lines and no leading/trailing padding,
+            # so a paragraph's line structure survives without ragged indentation.
+            text = "\n".join(line.strip() for line in text.split("\n") if line.strip())
+        else:
+            text = _ALL_WS.sub(" ", text)
+        text = text.strip()
+    return text