OCRSmith/src/ocrsmith/config/default_config.yaml at 203a6a4bf3d45d79f1163933ec51368052dacf43 · atlasia-ma/OCRSmith · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# OCRSmith default corpus specification.
#
# Everything that varies per sample is a distribution, not a value: a corpus is defined by
# its distributions, and pinning one narrows the dataset in a way that is invisible until
# a model fails on whatever was left out.

seed: 1234

fonts:
  paths: ["assets/fonts"]
  size_range: [18, 30]
  # Reject a font that cannot draw a character rather than emitting a tofu box that
  # contradicts the label.
  require_full_coverage: true

text:
  source:
    # Defaults to the built-in sentences so a fresh clone generates something immediately.
    # Point this at a real corpus before generating anything you intend to train on -
    # synthetic layout over six sentences teaches a vocabulary that does not exist.
    # See examples/configs/darija_scan.yaml for a Hugging Face source.
    type: inline
    column: content
    title_column: title
    split: train
    limit: 20000
    sentences:
      - "المغرب بلد يقع في شمال إفريقيا ويطل على المحيط الأطلسي والبحر الأبيض المتوسط."
      - "تهدف أطلسيا إلى بناء نماذج ذكاء اصطناعي مفتوحة المصدر للغة الدارجة المغربية."
      - "يحتوي هذا التقرير على جداول وأرقام ومعلومات إضافية مفيدة للقارئ المهتم."
      - "في سنة 2024 أطلقت المجموعة نموذج OCR جديد يدعم اللغتين العربية والفرنسية."
      - "تعتمد الطريقة المقترحة على توليد بيانات اصطناعية متنوعة وواقعية قدر الإمكان."
  diacritics:
    # Marks are only ever removed, never invented: adding them would need a diacritiser
    # model and would make the label assert vowels nobody wrote. Point this at a
    # diacritised corpus and use "mixed" to reproduce how real Arabic actually varies -
    # fully marked religious and pedagogical texts, lightly marked news, bare prose.
    mode: keep          # keep | strip | partial | mixed
    keep_range: [0.1, 0.6]
    mixed_weights: [0.15, 0.25, 0.60]
  normalization:
    # Each of these rewrites the ground truth, so all default to off.
    strip_diacritics: false
    strip_tatweel: false
    unify_alef: false
    unify_ya: false
    numerals: keep
    collapse_whitespace: true
  direction: auto

page:
  papers:
    a4: 4.0
    a5: 1.0
    letter: 1.0
  dpi_range: [110, 200]
  margin_mm_range: [12.0, 28.0]
  columns:
    1: 3.0
    2: 1.0
  landscape_probability: 0.05
  header_probability: 0.35
  footer_probability: 0.45
  max_pages: 3
  background:
    kinds:
      paper: 3.0
      solid: 1.0
      gradient: 0.3
      image: 0.0
    image_paths: []
    tint_range: [[238, 234, 226], [255, 255, 255]]

templates:
  weights:
    article: 3.0
    report: 2.0
    newspaper: 1.5
    letter: 1.0
    form: 1.0
    invoice: 1.0

degradations:
  # Capture conditions, weighted so the corpus is balanced deliberately rather than
  # accidentally. "clean" keeps a share of undegraded pages for curriculum training.
  presets:
    clean: 1.0
    scan: 4.0
    photo: 3.0
    fax: 0.5
    archive: 1.0

quality:
  # Validate every page before it reaches the dataset. Synthetic data fails quietly:
  # blank pages, washed-out text and boxes off the canvas do not raise, they just teach
  # a model something false.
  enabled: true
  max_rejection_rate: 0.5

output:
  dir: outputs/dataset
  format: jsonl
  image_format: png
  image_quality: 92
  images_subdir: images
  shard_size: 500
  eval_fraction: 0.0

run:
  num_samples: 100
  workers: 1
  start_index: 0