|
| 1 | +"""Fetching fonts from Google Fonts. |
| 2 | +
|
| 3 | +Font diversity is the single highest-impact lever in synthetic text data — engines that |
| 4 | +scaled to six figures of typefaces report it as the dominant factor, well ahead of layout |
| 5 | +or degradation tricks. A repository cannot ship that many fonts, and should not: they are |
| 6 | +other people's work under their own licences. So OCRSmith fetches them on demand and |
| 7 | +records exactly what it took, from where, under which licence. |
| 8 | +
|
| 9 | +Only permissively licensed directories of the `google/fonts` repository are used |
| 10 | +(`ofl/`, `apache/`, `ufl/`), and the licence file is downloaded alongside every family. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import contextlib |
| 16 | +import json |
| 17 | +import urllib.error |
| 18 | +import urllib.request |
| 19 | +from collections.abc import Iterator, Sequence |
| 20 | +from dataclasses import dataclass, field |
| 21 | +from pathlib import Path |
| 22 | + |
| 23 | +__all__ = [ |
| 24 | + "FontFamilyRecord", |
| 25 | + "LICENCE_DIRECTORIES", |
| 26 | + "fetch_families", |
| 27 | + "list_families", |
| 28 | + "load_manifest", |
| 29 | +] |
| 30 | + |
| 31 | +_METADATA_URL = "https://fonts.google.com/metadata/fonts" |
| 32 | +_CONTENTS_API = "https://api.github.com/repos/google/fonts/contents/{directory}/{slug}" |
| 33 | +_RAW_URL = "https://raw.githubusercontent.com/google/fonts/main/{directory}/{slug}/{name}" |
| 34 | +_FONT_SUFFIXES = (".ttf", ".otf") |
| 35 | +_USER_AGENT = "ocrsmith-font-fetcher" |
| 36 | + |
| 37 | +#: Directories of google/fonts whose licences permit redistribution and modification. |
| 38 | +#: `apache` is Apache-2.0, `ofl` is the SIL Open Font Licence, `ufl` the Ubuntu Font |
| 39 | +#: Licence. Anything outside these is skipped rather than guessed at. |
| 40 | +LICENCE_DIRECTORIES = ("ofl", "apache", "ufl") |
| 41 | + |
| 42 | +MANIFEST_NAME = "fonts-manifest.json" |
| 43 | + |
| 44 | + |
| 45 | +@dataclass(frozen=True, slots=True) |
| 46 | +class FontFamilyRecord: |
| 47 | + """One typeface family as Google Fonts describes it.""" |
| 48 | + |
| 49 | + family: str |
| 50 | + category: str |
| 51 | + subsets: tuple[str, ...] |
| 52 | + designers: tuple[str, ...] = () |
| 53 | + is_noto: bool = False |
| 54 | + #: Populated once the family has been located in the repository. |
| 55 | + directory: str | None = None |
| 56 | + files: tuple[str, ...] = () |
| 57 | + licence: str | None = None |
| 58 | + |
| 59 | + @property |
| 60 | + def slug(self) -> str: |
| 61 | + """Directory name in the google/fonts repository.""" |
| 62 | + return self.family.lower().replace(" ", "") |
| 63 | + |
| 64 | + def to_dict(self) -> dict: |
| 65 | + return { |
| 66 | + "family": self.family, |
| 67 | + "category": self.category, |
| 68 | + "subsets": list(self.subsets), |
| 69 | + "designers": list(self.designers), |
| 70 | + "directory": self.directory, |
| 71 | + "licence": self.licence, |
| 72 | + "files": list(self.files), |
| 73 | + } |
| 74 | + |
| 75 | + |
| 76 | +def _get(url: str, *, as_json: bool = False, timeout: float = 30.0): |
| 77 | + request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) |
| 78 | + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - fixed hosts |
| 79 | + payload = response.read() |
| 80 | + return json.loads(payload.decode("utf-8")) if as_json else payload |
| 81 | + |
| 82 | + |
| 83 | +def list_families(subset: str = "arabic", *, open_source_only: bool = True) -> tuple[FontFamilyRecord, ...]: |
| 84 | + """Every Google Fonts family carrying `subset`, e.g. "arabic" or "latin".""" |
| 85 | + metadata = _get(_METADATA_URL, as_json=True) |
| 86 | + families = [] |
| 87 | + for entry in metadata.get("familyMetadataList", []): |
| 88 | + subsets = tuple(entry.get("subsets", ())) |
| 89 | + if subset and subset.lower() not in {s.lower() for s in subsets}: |
| 90 | + continue |
| 91 | + if open_source_only and not entry.get("isOpenSource", False): |
| 92 | + continue |
| 93 | + families.append( |
| 94 | + FontFamilyRecord( |
| 95 | + family=entry["family"], |
| 96 | + category=entry.get("category", "unknown"), |
| 97 | + subsets=subsets, |
| 98 | + designers=tuple(entry.get("designers", ())), |
| 99 | + is_noto=bool(entry.get("isNoto", False)), |
| 100 | + ) |
| 101 | + ) |
| 102 | + return tuple(sorted(families, key=lambda record: record.family)) |
| 103 | + |
| 104 | + |
| 105 | +def _locate(record: FontFamilyRecord) -> FontFamilyRecord | None: |
| 106 | + """Find a family in the permissively licensed directories, listing its files.""" |
| 107 | + for directory in LICENCE_DIRECTORIES: |
| 108 | + try: |
| 109 | + listing = _get(_CONTENTS_API.format(directory=directory, slug=record.slug), as_json=True) |
| 110 | + except urllib.error.HTTPError as error: |
| 111 | + if error.code == 404: |
| 112 | + continue |
| 113 | + raise |
| 114 | + if not isinstance(listing, list): |
| 115 | + continue |
| 116 | + files = tuple(item["name"] for item in listing if item["name"].lower().endswith(_FONT_SUFFIXES)) |
| 117 | + licence = next( |
| 118 | + (item["name"] for item in listing if "license" in item["name"].lower() or "OFL" in item["name"]), |
| 119 | + None, |
| 120 | + ) |
| 121 | + if files: |
| 122 | + return FontFamilyRecord( |
| 123 | + family=record.family, |
| 124 | + category=record.category, |
| 125 | + subsets=record.subsets, |
| 126 | + designers=record.designers, |
| 127 | + is_noto=record.is_noto, |
| 128 | + directory=directory, |
| 129 | + files=files, |
| 130 | + licence=licence, |
| 131 | + ) |
| 132 | + return None |
| 133 | + |
| 134 | + |
| 135 | +def fetch_families( |
| 136 | + records: Sequence[FontFamilyRecord], |
| 137 | + destination: str | Path, |
| 138 | + *, |
| 139 | + skip_existing: bool = True, |
| 140 | + on_progress=None, |
| 141 | +) -> Iterator[FontFamilyRecord]: |
| 142 | + """Download each family into `destination`, yielding what was actually retrieved. |
| 143 | +
|
| 144 | + Families that cannot be located under a permissive licence are skipped rather than |
| 145 | + guessed at, and every family's licence file is downloaded next to its fonts. |
| 146 | + """ |
| 147 | + destination = Path(destination) |
| 148 | + destination.mkdir(parents=True, exist_ok=True) |
| 149 | + |
| 150 | + for record in records: |
| 151 | + located = _locate(record) |
| 152 | + if located is None: |
| 153 | + if on_progress: |
| 154 | + on_progress(record.family, "skipped (no permissive licence found)") |
| 155 | + continue |
| 156 | + |
| 157 | + family_dir = destination / located.slug |
| 158 | + family_dir.mkdir(parents=True, exist_ok=True) |
| 159 | + written = 0 |
| 160 | + for name in located.files: |
| 161 | + target = family_dir / name |
| 162 | + if skip_existing and target.exists(): |
| 163 | + written += 1 |
| 164 | + continue |
| 165 | + url = _RAW_URL.format(directory=located.directory, slug=located.slug, name=name) |
| 166 | + try: |
| 167 | + target.write_bytes(_get(url)) |
| 168 | + written += 1 |
| 169 | + except (urllib.error.URLError, OSError) as error: |
| 170 | + if on_progress: |
| 171 | + on_progress(located.family, f"failed on {name}: {error}") |
| 172 | + |
| 173 | + if located.licence: |
| 174 | + licence_path = family_dir / located.licence |
| 175 | + if not licence_path.exists(): |
| 176 | + # The fonts remain usable if this fails; the manifest records the gap. |
| 177 | + with contextlib.suppress(urllib.error.URLError, OSError): |
| 178 | + licence_path.write_bytes( |
| 179 | + _get( |
| 180 | + _RAW_URL.format( |
| 181 | + directory=located.directory, |
| 182 | + slug=located.slug, |
| 183 | + name=located.licence, |
| 184 | + ) |
| 185 | + ) |
| 186 | + ) |
| 187 | + |
| 188 | + if on_progress: |
| 189 | + on_progress(located.family, f"{written} file(s) [{located.directory}]") |
| 190 | + if written: |
| 191 | + yield located |
| 192 | + |
| 193 | + |
| 194 | +def write_manifest(destination: str | Path, records: Sequence[FontFamilyRecord], subset: str) -> Path: |
| 195 | + """Record what was fetched, so a dataset can be regenerated with the same faces.""" |
| 196 | + destination = Path(destination) |
| 197 | + path = destination / MANIFEST_NAME |
| 198 | + payload = { |
| 199 | + "source": "https://github.com/google/fonts", |
| 200 | + "subset": subset, |
| 201 | + "licence_directories": list(LICENCE_DIRECTORIES), |
| 202 | + "families": [record.to_dict() for record in records], |
| 203 | + } |
| 204 | + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") |
| 205 | + return path |
| 206 | + |
| 207 | + |
| 208 | +def load_manifest(destination: str | Path) -> dict: |
| 209 | + path = Path(destination) / MANIFEST_NAME |
| 210 | + if not path.exists(): |
| 211 | + return {} |
| 212 | + return json.loads(path.read_text(encoding="utf-8")) |
| 213 | + |
| 214 | + |
| 215 | +@dataclass(frozen=True, slots=True) |
| 216 | +class FetchSummary: |
| 217 | + """What a fetch run produced.""" |
| 218 | + |
| 219 | + families: int = 0 |
| 220 | + files: int = 0 |
| 221 | + skipped: tuple[str, ...] = field(default_factory=tuple) |
0 commit comments