config.txt

[DOCUMENT]
# atrium-project#13 (paradata pair): directory of <doc_id>.document.json records,
# shared across every stage below. Empty = disabled (default); every stage then
# behaves exactly as it did before this section existed. Overridable per-run via
# the DOCUMENT_JSON_DIR env var (see document_hook.resolve_document_json_dir).
JSON_DIR =
# atrium-project#10 (D5): how the ORIGINAL input was acquired, written once by
# page_split.py as `source.origin`. Since Issue #18 §1a that value is what
# AUTHORISES this repo to write the positional blocks (pages/content/lines/tables),
# so it must start with an ORIGIN_ORIGINATORS prefix this repo owns — ABBYY-ALTO,
# ocr:<engine> or vlm:<engine>; anything else makes the mixed-plane check abstain
# in silence, which is the state D5 found it in.
# Empty = use the per-format default (ABBYY-ALTO for ALTO XML input, ocr:generic for
# generic JSON OCR input). Set it when the engine is known, e.g. `ocr:pero`.
# Overridable per-run via --source-origin or the DOCUMENT_SOURCE_ORIGIN env var.
SOURCE_ORIGIN =

[EXTRACT]
INPUT_CSV          = ./data_samples/test_alto_stats.csv
OUTPUT_TXT         = ./data_samples/PAGE_TXT
WORKERS_MAX        = 16
OUTPUT_TXT_LR      = ./data_samples/PAGE_TXT_LR
WORKERS_MAX_LR     = 1
LR_MODEL           = hantian/layoutreader
LR_CHUNK_SIZE      = 350
LR_MIN_CHUNK_SIZE  = 50
OUTPUT_TXT_LLM     = ./data_samples/PAGE_TXT_LLM
WORKERS_MAX_LLM    = 1
LLM_MODEL          = THUDM/glm-4v-9b
LLM_MAX_RESOLUTION = 1344
LLM_MAX_NEW_TOKENS = 4096
LLM_PROMPT         = OCR: Transcribe all text on this page exactly as it appears.
OUTPUT_TXT_JSON    = ./data_samples/PAGE_TXT_JSON
WORKERS_MAX_JSON   = 16
# (#37) Document-assembly policy for extract_JSON_2_TXT.py's accretion into
# <doc_id>.document.json (only takes effect when [DOCUMENT].JSON_DIR is set):
# false (default) writes one pages[] row per source page; true collapses every
# source page for a document into a single pages[] row. Overridable per-run via
# the --force-single-page CLI flag, which takes precedence over this value.
FORCE_SINGLE_PAGE_JSON = false

[CLASSIFY]
INPUT_CSV = data_samples/test_alto_stats.csv
TEXT_DIR = data_samples/PAGE_TXT_LR
OUTPUT_LINES_LOG = data_samples/DOC_LINE_CATEG
BATCH_SIZE = 128
WORKERS_MAX = 32
GPU_WAIT_TIMEOUT = 600.0
EXPECTED_LANGS = ces,deu,eng
TRUSTED_FOREIGN_LANGS = deu,eng,fra,pol,ita,slk
MODEL_NAME = Qwen/Qwen2.5-0.5B
# (#7 Tier 1) FastText language-ID weights path (was hardcoded in
# init_cpu_worker).
FASTTEXT_MODEL = lid.176.bin
# (#7 Tier 1) Two-tier trust multipliers on the FastText confidence before it
# feeds the quality score: known-but-unexpected language / unknown language.
TRUST_TIER_TRUSTED = 0.85
TRUST_TIER_UNKNOWN = 0.50
# (#7 Tier 1) Languages that keep their original FastText confidence when
# remapped to the default language (Slovak ~ Czech).
REMAP_KEEP_SCORE_LANGS = slk

[AGGREGATE]
RAW_LINES_CSV = data_samples/DOC_LINE_CATEG
OUTPUT_STATS = samples_page_stats.csv
OUTPUT_DOC_DIR = data_samples/DOC_LINE_STATS
STANDARD_COLS = Clear,Noisy,Trash,Non-text,Empty

[TEXT_UTILS]
LANG_SCORE_ROUGH = 0.45
ALLOWED_INTERNAL = .-,+()"'—–:%;?!/
STRIP_CHARS = .,;:!?()[]"'\/\
METADATA_MARKERS = Tb.,č.neg,neg.,obr.,obr ,neg ,Tb ,č. neg,č.neg.,č.,str.,Datum
CZ_DIACS = áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ
# (#7 Tier 1) German diacritic glyphs; _LANG_DIACRITICS is rebuilt from
# CZ_DIACS + DEU_DIACS at import time.
DEU_DIACS = äöüßÄÖÜ
# (#7 Tier 1) Minimum diacritic share among alphabetic chars for
# infer_lang_from_diacritics (was a hardcoded 0.07 default argument).
DIACRITIC_INFER_THRESHOLD = 0.07
VOWEL_CHARS = aeiouyáéíóúýěůäöüAEIOUYÁÉÍÓÚÝĚŮÄÖÜ
ROTATABLE_CHARS = pbqdnuwmoxszeyv
# (#7 Tier 1) Letters rare in Czech — wqx-heavy tokens signal OCR noise
# (score_word / score_words_in_line / determine_category).
WQX_CHARS = wqxWQX
# (#7 Tier 1) Czech function words recognisable upright; their mirror/rotation
# ghost images (ROT_GHOSTLIST) are derived at import time. Changing this key
# requires re-import — override_constants() does not rebuild the ghostlist.
ROT_WHITELIST = po,pod,do,od,on,ony,by,bez,ne,nebo,ven,den,zde,se,ve,mez,pouze,bude
# (#7 Tier 1) Ghost images colliding with real words — never count as ghosts.
GHOST_WORD_COLLISIONS = no,bo
# (#7 Tier 1) Trailing filler chars stripped before headline/short-line
# checks. Unicode-escape decoded (configparser strips leading whitespace, so
# the leading space is written as \x20).
TRAILING_FILL_CHARS = \x20._:-<\u2013\u2014
# (#7 Tier 1) Collection-specific literal markers forcing the Non-text route
# in pre_filter_line (ARUP/B "IVerc" stamp).
NONTEXT_MARKERS = IVerc

QS_LENGTH_MAX         = 100
PERPLEXITY_THRESHOLD_MAX  = 1000.0

SHORT_PPL_CAP               = 850.0
PPL_INVERTED_MIN            = 200.0

QS_WEIGHT_VALID_WORD  = 0.35
QS_WEIGHT_WEIRD       = 0.18
QS_WEIGHT_PERPLEXITY  = 0.08
QS_WEIGHT_LENGTH      = 0.02
QS_WEIGHT_GARBAGE     = 0.18
QS_WEIGHT_VOWEL       = 0.07
QS_WEIGHT_LANG        = 0.05
QS_WEIGHT_GIBBERISH   = 0.04
QS_WEIGHT_FUSED       = 0.03

CATEG_NOISY_SCORE_MAX       = 0.80
CATEG_TRASH_SCORE_MAX       = 0.55

CATEG_GARBAGE_DENSITY_HIGH  = 0.35
# (B2) QS normalisation scale, decoupled from the hard gate above.
# Defaults equal CATEG_GARBAGE_DENSITY_HIGH => bit-identical output unless changed.
QS_GARBAGE_NORM_MAX         = 0.35

ROT_RATIO_INVERTED_MIN      = 0.55
# (#7 Phase 0) WEIRD_RATIO_INVERTED_MIN is loaded and registered in the sweep
# space but currently used by NO decision path — a decision-inert knob and a
# wasted sweep dimension. Kept pending team ack (it sits in the published
# sweep space); flag to DanaKriv before removal.
WEIRD_RATIO_INVERTED_MIN    = 0.35
ROT_HIGH_LANG_CONF          = 0.90

LANG_SCORE_REMAP             = 0.75
LANG_SCORE_REMAP_FAR        = 0.50
# (#3 2026-07-02 calibration - Extra) true: remap_lang() always assigns
# LANG_SCORE_REMAP/LANG_SCORE_REMAP_FAR to a remapped row, regardless of the
# original FastText confidence (DanaKriv: "the original lang score should not
# matter"). false: only assigns it when the original score is ABOVE the cap
# (the prior #3 A1 "cap, don't inflate a weak guess" behaviour) -- a config
# toggle rather than a code change, since this exact call has flipped twice.
LANG_REMAP_ALWAYS           = true
SINGLE_CHAR_ALLOWED         = aAiIuUvVzZkKsS
SHORT_VALID_WORDS           = a,i,k,o,s,u,v,z,se,si,po,na,za,ze,do,od,ke,ku,ve,ní,mi,ti,by,je,to,co,ač,my,ty,on,ji,jí,už,až
REPEAT_ALLOWED_CHARS        = oOuU
REPEATED_DOUBLE_MIN         = 2
VOWEL_RATIO_LOW             = 0.20
VOWEL_RATIO_HIGH            = 0.70
ACADEMIC_TITLES             = PhDr,MUDr,JUDr,MVDr,RNDr,PaedDr,CSc,DrSc,Ing,Mgr,Bc,PhD,DiS,prof,doc
LDL_ALLOWED_FOLLOW          = .,/:%-;?)=
LDL_UNITS                   = m,cm,mm,g,kg,km,ha,l,ml
# (#3 2026-07-02 calibration) is_forgiven_headline: bare unit/abbreviation
# tokens and short numbered headlines/captions get a Trash/Non-text -> Noisy
# floor instead of falling through the ordinary valid-word-ratio scoring.
SHORT_EXCEPTION_TOKENS       = mm,cm,m,g,kg,km,ha,l,ml,tb,neg,obr,str,č,čneg
HEADLINE_MAX_WORDS          = 8
# Pure-digit tokens longer than this are treated as GARBAGE, not harmless
# structural numbering — keeps archive/stamp codes (e.g. "6694", "1920")
# from being swept up as forgiven captions.
HEADLINE_MAX_DIGITS         = 2
GARBAGE_KEEP_CHARS          =
FUSED_VOWEL_RUN_MIN         = 3
WX_REPEAT_MIN               = 2
ISOLATED_CHAR_RATIO_MAX     = 0.40
ISOLATED_CHAR_MIN_TOKENS    = 3
SYM_LET_DIG_NONTEXT         = true
MOSTLY_READABLE_VALID_MIN   = 0.85
SHORT_NOISY_QS_PENALTY      = 0.20

LOWPPL_CLEAR_MAX            = 50.0
HARD_SWEEP_LANG_MAX        = 0.45
HARD_SWEEP_PPL_MIN         = 1000.0
GHOST_DOMINATED_MIN_RATIO  = 0.5
WORD_W_PENALTY             = 0.20

INVERTED_RUN_MIN           = 4
INVERTED_PAGE_MAJORITY     = 0.60

# (#3) Extreme-perplexity trash route (catches confident-labeled garbage the
# lang-gated hard sweep misses). orig-score gate spares readable OCR-degraded text.
PPL_EXTREME_MIN            = 3000.0
EXTREME_LANG_CONF          = 0.85
# (#3) LM-confident upright-Czech recovery past the fragile valid_ratio cap.
LOWPPL_CZECH_CLEAR_MAX     = 180.0
CZECH_CLEAR_GARBAGE_MAX    = 0.15

# Fix 1: Linguistic Anchor Bypass
ANCHOR_MIN_WORDS            = 2
ANCHOR_WORD_LEN             = 3
ANCHOR_VOWEL_RATIO          = 0.10

# Fix 2: Inverted / Rotated Text Organic Penalties
SUSPICIOUS_ROT_RATIO        = 0.65
SUSPICIOUS_WQX_RATIO        = 0.15
INVERTED_WEIRD_PENALTY      = 0.45
# (#3 remaining hard cases)
# P3 absolute-perplexity garbage route (ignores lang confidence; non-upright only).
PPL_GARBAGE_ABSOLUTE        = 30000.0
# P1 minimum ghost-token share for the broadened per-line inverted-scan route.
GHOST_HITS_INVERTED_MIN     = 1

[PIPELINE]
METHOD        = layoutreader
INPUT_DIR     = data_samples/ALTO
PAGE_ALTO_DIR = data_samples/PAGE_ALTO
# (#31) Per-page output dir for generic JSON input (--method json-keys),
# parallel to PAGE_ALTO_DIR.
PAGE_JSON_DIR = data_samples/PAGE_JSON
PARADATA_DIR  = paradata
# Per-stage skip flags (#6): skip a stage whose outputs already exist. CLI
# --skip-<stage> / --start-from override these. A skipped stage's outputs must
# already be present on disk for the downstream stages to run.
SKIP_SPLIT     = false
SKIP_STATS     = false
SKIP_EXTRACT   = false
SKIP_CLASSIFY  = false
SKIP_AGGREGATE = false