atrium_document.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://github.com/ufal/atrium-project/docs/templates/shared/atrium_document.schema.json",
  "title": "ATRIUM document record",
  "description": "Per-document aggregate record built by accretion across the ATRIUM pipeline. Each tool writes only its own block and passes every other block through untouched. References are restricted to the original input and persistent step outputs; disposable derivations (Markdown, page images) appear in 'regenerable' as recipes, never as stored paths. Produced by atrium_document.py; sibling of the atrium_paradata.py provenance record.",
  "type": "object",
  "required": ["schema_version", "doc_id"],
  "additionalProperties": true,
  "properties": {
    "schema_version": {
      "type": "string",
      "pattern": "^[0-9]+\\.[0-9]+$",
      "description": "Record schema version. A newer MAJOR than the reading tool supports must be refused; an older one is migrated transparently.",
      "examples": ["1.0"]
    },
    "record_type": {
      "type": "string",
      "enum": ["atrium-document", "atrium-document-merged"],
      "description": "'atrium-document' for the normal linear accretion; '-merged' when parallel partial records were folded."
    },
    "doc_id": {
      "type": "string",
      "minLength": 1,
      "description": "Stable document identifier, carried unchanged by every tool.",
      "examples": ["CTX000000001"]
    },

    "source": {
      "type": "object",
      "description": "The ORIGINAL input. First writer wins; later tools must not overwrite it. The durable key is doc_id + sha256 — originals are archive-managed, not pipeline-local, so no path is required.",
      "additionalProperties": true,
      "properties": {
        "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
        "filename": { "type": "string" },
        "media_type": { "type": "string", "examples": ["application/alto+xml", "application/pdf"] },
        "page_count": { "type": "integer", "minimum": 0 },
        "language": { "type": "array", "items": { "type": "string" } },
        "origin": {
          "type": "string",
          "description": "How the text was obtained. Load-bearing since Issue #18: for the positional blocks (pages/content/lines/tables) this value selects which originator is authorised to write them — see ORIGIN_ORIGINATORS in atrium_document.py. First writer wins, so it is fixed for the life of the record.",
          "examples": ["ABBYY-ALTO", "digital-born-pdf", "docx", "ocr:tesseract-ces", "ocr:pero", "vlm:glm-4v"]
        }
      }
    },

    "derived_from": {
      "type": "object",
      "description": "PERSISTENT step outputs this record was built from. Keys are stage-scoped; values are references to stored artifacts. Never a transient artifact.",
      "additionalProperties": { "type": "string" },
      "examples": [
        {
          "alto_categ": "DOC_LINE_CATEG/CTX000000001.csv",
          "teitok": "TEITOK/CTX000000001.teitok.xml",
          "enriched": "KW_PER_DOC_LLM/CTX000000001_enriched.json",
          "paradata": "paradata/260724-101112_pipeline-run.json"
        }
      ]
    },

    "regenerable": {
      "type": "object",
      "description": "DISPOSABLE derivations recorded as reproducible recipes rather than stored paths — the annotated Markdown, and by the same rule any rendered page image.",
      "additionalProperties": {
        "type": "object",
        "required": ["from"],
        "additionalProperties": true,
        "properties": {
          "from": { "type": "string", "description": "Persistent artifact the derivation is regenerated from." },
          "converter": { "type": "string", "examples": ["xml_to_md@0.3.0"] },
          "detail": { "type": "string", "enum": ["full", "standard", "minimal"] }
        }
      }
    },

    "provenance": {
      "type": "object",
      "description": "Accreting license union plus the list of contributing tool runs. Mirrors the paradata license block; references paradata rather than duplicating it.",
      "additionalProperties": true,
      "properties": {
        "license": { "type": "string", "examples": ["CC BY-NC-SA 4.0"] },
        "license_url": { "type": "string" },
        "license_detail": {
          "type": "object",
          "description": "Output of para_licenses.merge_effective_licenses over every contributing tool — most restrictive wins.",
          "additionalProperties": true
        },
        "contributors": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": true,
            "properties": {
              "program": { "type": "string" },
              "run_id": { "type": "string" },
              "paradata_ref": { "type": "string" },
              "blocks": { "type": "string", "description": "Comma-separated blocks this run wrote." },
              "at": { "type": "string", "format": "date-time" }
            }
          }
        }
      }
    },

    "assembled": {
      "type": "object",
      "description": "Per-block provenance — the source of granularity. Re-running one tool rewrites exactly one entry here.",
      "additionalProperties": true,
      "properties": {
        "blocks": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "additionalProperties": true,
            "properties": {
              "program": { "type": "string" },
              "run_id": { "type": "string" },
              "paradata_ref": { "type": "string" },
              "updated_at": { "type": "string", "format": "date-time" }
            }
          }
        },
        "had_baseline": {
          "type": "boolean",
          "description": "False when the tool ran without a previous record and emitted only its own part."
        },
        "note": { "type": "string" }
      }
    },

    "page_categories": {
      "description": "Owner: page-classification. Predicted category per page label.",
      "type": "object",
      "additionalProperties": { "type": "string" },
      "examples": [{ "1": "Text", "2": "Plate" }]
    },

    "pages": {
      "description": "Originator: alto-postprocess (OCR/ALTO path) or digital-convert (digital-born PDF/DOCX path), selected per document by source.origin — never both. page-classification may write category fields; nlp-enrich may write teitok_surface. Field-level ownership — never co-mutated.",
      "type": "array",
      "items": {
        "type": "object",
        "required": ["page"],
        "additionalProperties": true,
        "properties": {
          "page": { "type": "string", "description": "Page label as a string, so 'iv' or 'A-1' survive. Consumers must NOT derive ordering from it — use page_index." },
          "page_index": {
            "type": "integer",
            "minimum": 1,
            "description": "1-based physical position of this page in the document — the ordering key, and the only one that works when `page` is a roman numeral or an appendix label. Writable by both originators: an ALTO record needs it as much as a digital-born one, since roman-numeral front matter is common in both."
          },
          "quality_score": { "type": "number", "minimum": 0, "maximum": 1 },
          "quality_band": { "type": "string", "enum": ["Trash", "Noisy", "Clear"] },
          "category": { "type": "string" },
          "category_confidence": { "type": "number", "minimum": 0, "maximum": 1 },
          "needs_ocr": {
            "type": "boolean",
            "description": "Set by whichever tool originated this page. On the digital-born path it means the embedded text layer exists but does not decode to trustworthy text (Issue #10: non-embedded WinAnsi with no /ToUnicode yields systematic diacritic corruption), i.e. this page should be re-acquired by OCR despite not being a scan. The converter reports; routing policy lives outside it. Because the two paths mean OPPOSITE things by it (no text layer vs. an untrustworthy one), always write needs_ocr_reason alongside. Setting this is also what authorises alto-postprocess to re-originate the positional plane of a digital-born record — see _assert_origin_consistent() in atrium_document.py.",
            "examples": [true]
          },
          "needs_ocr_reason": {
            "type": "string",
            "description": "Why this page needs OCR, in words, because needs_ocr alone is ambiguous between the two paths and the Markdown renderer emits it as a layout cue the model reads. Without it, api_util/xml_to_md.rows_to_layout_markdown() fell back to 'no extractable text layer' — factually inverted for every digital-born page, which by definition HAS a text layer.",
            "examples": [
              "no extractable text layer",
              "text layer decodes to corrupt diacritics (WinAnsi declared, no /ToUnicode)"
            ]
          },
          "ocr": {
            "type": ["object", "null"],
            "additionalProperties": true,
            "properties": {
              "engine": { "type": "string" },
              "lang": { "type": "string" }
            }
          },
          "canvas": {
            "type": "object",
            "description": "Page dimensions, and the unit every bbox on this page is expressed in. Write `unit` whenever any bbox is present — a digital-born PDF page is in points, an ALTO page is usually in pixels, and $defs/bbox has no other way to say which.",
            "additionalProperties": true,
            "properties": {
              "width": { "type": "number" },
              "height": { "type": "number" },
              "unit": { "type": "string", "default": "px", "examples": ["pt", "px"] }
            }
          },
          "teitok_surface": {
            "type": "string",
            "description": "Logical TEITOK <surface> id — an identifier, never an image path."
          }
        }
      }
    },

    "content": {
      "description": "Originator: alto-postprocess or digital-convert, selected per document by source.origin (Issue #18) — never both. Reading-order text for search, or null when externalised by reference.",
      "type": "object",
      "additionalProperties": true,
      "properties": {
        "text": { "type": ["string", "null"] },
        "reading_order": { "type": "string", "examples": ["ltr-columns"] }
      }
    },

    "lines": {
      "description": "Originator: alto-postprocess (categ/quality/lang/text) or digital-convert (text/bbox/group_id/lang/quality_score/categ), selected per document by source.origin (Issue #18) — never both. nlp-enrich writes morphology and teitok_ref. Field-level ownership. NOTE for validators: only page+line are required, so a row stripped of its text by a field-ownership filter still validates here — the round-trip assertion belongs in the writing tool, not in this schema.",
      "type": "array",
      "items": {
        "type": "object",
        "required": ["page", "line"],
        "additionalProperties": true,
        "properties": {
          "page": {
            "type": "string",
            "description": "Page LABEL, matching a pages[].page. A string for the same reason as there — 'iv' and 'A-1' must survive — so a writer must not pass an integer here: merge_block() keys rows on it, and a consumer sorting on int(page) needs pages[].page_index to order non-numeric labels."
          },
          "line": { "type": "integer", "minimum": 0 },
          "text": { "type": "string" },
          "categ": {
            "type": "string",
            "description": "Line category. Open string, but two values are LOAD-BEARING and must be spelled exactly: 'Garbage' and 'Inverted' are what api_util/json_to_md.py's DROP_CATEGORIES filters out before the text reaches the model. A digital-born converter reporting undecodable text must emit one of those two rather than inventing a synonym — a typo here does not fail validation, it silently disables the filter.",
            "examples": ["Text", "Heading", "Garbage", "Inverted"]
          },
          "lang": { "type": "string" },
          "quality_score": { "type": "number", "minimum": 0, "maximum": 1 },
          "lemma": { "type": "string", "description": "nlp-enrich: UD lemma of the line's dominant token/lead form." },
          "upos": { "type": "string", "description": "nlp-enrich: UD universal POS tag." },
          "feats": { "type": "string", "description": "nlp-enrich: UD morphological features string." },
          "bbox": {
            "$ref": "#/$defs/bbox",
            "description": "On the ALTO path, derived by nlp-enrich while aligning to TEITOK. On the digital-born path, the PDF adapter's native coordinates — the only bbox such a record will ever have, since there is no TEITOK to align to. Two declared writers, mutually exclusive per document by source.origin."
          },
          "teitok_ref": { "type": "string" },
          "group_id": {
            "type": ["string", "null"],
            "description": "Issue #18: groups lines belonging to the same source-structural unit — a DOCX paragraph or table CELL, or a PDF text block from the structural extractor. Lines sharing a group_id are contiguous and in the same reading order; null when the source has no such grouping (e.g. OCR/ALTO lines from alto-postprocess, which stays line-only). Not a schema block of its own — deliberately kept at line-granularity to avoid a second nesting level competing with 'lines' as the position-addressable unit the rest of the schema (bbox, entities' page/line keys) already keys off. CONSUMER CONTRACT: a group boundary renders as a paragraph break. Markdown already encodes paragraphs natively as a blank line, so this needs no new layout_md cue — implemented in api_util/xml_to_md.rows_to_layout_markdown() (which emits the blank line) and api_util/json_to_md._rows_from_lines() (which carries the field through); pinned by tests/test_json_to_md.py. 'table CELL', not 'table row': a per-row value cannot address the cell a line belongs to, which is what tables[].cells[].group_id joins on."
          },
          "style": {
            "type": "object",
            "description": "Issue #18 §1 (the 'font row', decided as option (c) — semantic style only). Carries what a downstream reader can ACT on and deliberately drops typeface and point size: the LLM cannot use 'Helvetica 12pt', and heading-ness is already carried by categ. Absent on the ALTO path, where no style signal exists. Writer: digital-convert only.",
            "additionalProperties": true,
            "properties": {
              "bold": { "type": "boolean" },
              "italic": { "type": "boolean" },
              "heading_level": {
                "type": ["integer", "null"],
                "minimum": 1,
                "maximum": 6,
                "description": "1-6 for a heading (DOCX outlineLvl + 1, or a PDF size/weight heuristic); null or absent for body text."
              }
            }
          }
        }
      }
    },

    "tables": {
      "description": "Originator: alto-postprocess (ALTO spatial-block tables) or digital-convert (DOCX/PDF structural extraction), selected per document by source.origin (Issue #18, resolved) — never both. Grid SHAPE only: cell text is never duplicated here, it lives once in lines[] and is linked back via group_id, so this stays the lossless structured twin of the disposable Markdown table in 'regenerable'.",
      "type": "array",
      "items": {
        "type": "object",
        "required": ["table_id", "page"],
        "additionalProperties": true,
        "properties": {
          "table_id": { "type": "string" },
          "page": { "type": "string" },
          "caption": { "type": "string" },
          "n_rows": { "type": "integer", "minimum": 1 },
          "n_cols": { "type": "integer", "minimum": 1 },
          "group_id": {
            "type": ["string", "null"],
            "description": "Namespace prefix shared by this table's cell group_ids (e.g. 't1' for cells 't1.r0c0', 't1.r0c1'), so a consumer can find a table's lines without scanning cells[]. NOT the join key — that is per-cell, on cells[].group_id: one scalar for a whole table cannot address which lines carry which cell, so the 'linked back via group_id' contract was unimplementable while this was the only reference."
          },
          "cells": {
            "type": "array",
            "items": {
              "type": "object",
              "required": ["row", "col"],
              "additionalProperties": true,
              "properties": {
                "row": { "type": "integer", "minimum": 0 },
                "col": { "type": "integer", "minimum": 0 },
                "rowspan": { "type": "integer", "minimum": 1 },
                "colspan": { "type": "integer", "minimum": 1 },
                "is_header": { "type": "boolean" },
                "bbox": { "$ref": "#/$defs/bbox" },
                "group_id": {
                  "type": ["string", "null"],
                  "description": "THE join key: the lines[].group_id whose lines carry this cell's text. Never a copy of the text. Positional inference is not an alternative — rowspan/colspan mean lines-per-row is not n_cols, and a multi-line cell breaks any ordering assumption — so without this the grid cannot be reassembled from the record at all."
                },
                "lines": {
                  "type": "array",
                  "description": "Optional explicit line list for a cell whose text spans more than one group (rare). When present it wins over group_id.",
                  "items": { "$ref": "#/$defs/line_ref" }
                }
              }
            }
          }
        }
      }
    },

    "entities": {
      "description": "Owner: nlp-enrich; translator writes translation_en; llm-enrich writes pid. Carries all three tagsets so no mapping is lost.",
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": true,
        "properties": {
          "surface": { "type": "string" },
          "lemma": { "type": "string" },
          "type_onto": { "type": "string", "description": "OntoNotes v5 — the default configured NameTag tagset." },
          "type_cnec": { "type": "string", "description": "CNEC 2.0 fine-grained code." },
          "type_teitok": { "type": "string", "description": "Coarse TEITOK type (PER/ORG/LOC/MISC)." },
          "page": { "type": "string" },
          "line": { "type": "integer" },
          "char_span": {
            "type": "array",
            "items": { "type": "integer", "minimum": 0 },
            "minItems": 2,
            "maxItems": 2
          },
          "bbox": { "$ref": "#/$defs/bbox" },
          "teitok_ref": { "type": "string" },
          "translation_en": { "type": "string" },
          "pid": {
            "type": "object",
            "description": "FAIR ontology cross-references — the ARIADNE/GoTriple hook. Null until resolved.",
            "additionalProperties": true,
            "properties": {
              "wikidata": { "type": ["string", "null"] },
              "geonames": { "type": ["string", "null"] },
              "aat": { "type": ["string", "null"] },
              "amcr": { "type": ["string", "null"] }
            }
          }
        }
      }
    },

    "translations": {
      "description": "Owner: translator. Language pair and the persistent translated artifact it produced.",
      "type": "object",
      "additionalProperties": true,
      "properties": {
        "source_lang": { "type": "string" },
        "target_lang": { "type": "string" },
        "backend": { "type": "string", "examples": ["lindat", "ctranslate2", "llm"] }
      }
    },

    "enrichment": {
      "description": "Owner: llm-enrich. Superset of the LLM document schema, plus the explicit page citation.",
      "type": "object",
      "additionalProperties": true,
      "properties": {
        "items": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": true,
            "properties": {
              "locator": { "type": "string", "description": "Short verbatim snippet copied from the document." },
              "page": { "type": ["string", "null"], "description": "Page label from the nearest PAGE_BREAK / '## Page N' marker." },
              "extracted_keywords_cs": { "type": "array", "items": { "type": "string" } },
              "extracted_keywords_en": { "type": "array", "items": { "type": "string" } },
              "teater_category": { "type": "string", "description": "Single most relevant TEATER/AMCR vocabulary category." },
              "confidence_score": { "type": "number", "minimum": 0, "maximum": 1 },
              "citation": { "type": "string", "examples": ["[Source: CTX000000001, Page 1]"] }
            }
          }
        },
        "summary": { "type": ["string", "null"] },
        "topics": { "type": "array", "items": { "type": "string" } }
      }
    },

    "forms": {
      "description": "Owner: llm-enrich. VLM/LLM-driven labeled fixed-field extraction (e.g. archival intake sheets: name/date/plot-number/signature). No deterministic ALTO/DOCX structural signal marks a region as a form field the way it does for tables, so this carries a confidence score per field and is always single-owner regardless of document origin.",
      "type": "array",
      "items": {
        "type": "object",
        "required": ["field_id", "label"],
        "additionalProperties": true,
        "properties": {
          "field_id": { "type": "string" },
          "label": { "type": "string" },
          "value": { "type": ["string", "null"] },
          "value_type": { "type": "string", "enum": ["text", "date", "number", "checkbox", "signature_present"] },
          "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
          "page": { "type": "string" },
          "bbox": { "$ref": "#/$defs/bbox" },
          "entity_ref": {
            "type": ["integer", "null"],
            "description": "DEPRECATED in favour of entity_key. A positional index into entities[], and therefore valid only within a single unmerged record: merge_document_records() resolves each top-level block independently by assembled.blocks[*].updated_at, so a `forms` block from one fan-out branch can end up beside an `entities` block from another and the index then addresses whatever happens to sit at that position — with the record still validating."
          },
          "entity_key": {
            "oneOf": [{ "$ref": "#/$defs/entity_ref" }, { "type": "null" }],
            "description": "The entities[] row this field's value is, addressed by the natural key BLOCK_KEY_FIELDS['entities'] merges on. Survives a fan-in merge; prefer this over entity_ref."
          }
        }
      }
    }
  },

  "$defs": {
    "bbox": {
      "type": "array",
      "description": "[x_min, y_min, x_max, y_max] with the origin at the page TOP-LEFT and y increasing DOWNWARDS, in the unit named by that page's pages[].canvas.unit. This convention is normative and format-independent: it is what ALTO already uses, so the OCR path is unchanged, but it is NOT PDF user space, which puts the origin bottom-left. A digital-born PDF writer MUST convert before writing — with pdfplumber that means using 'top'/'bottom' (top-down, points) and never 'y0'/'y1' (bottom-up). Stated explicitly since Issue #18 gave this $def a second writer: the description used to say 'the coordinate space of the ALTO/TEITOK page', which a digital-born record does not have, leaving the y-axis direction and the unit undefined for exactly the writer whose selling point is exact coordinates.",
      "items": { "type": "number" },
      "minItems": 4,
      "maxItems": 4
    },
    "entity_ref": {
      "type": "object",
      "description": "A reference to one entities[] row by its natural key — BLOCK_KEY_FIELDS['entities'].",
      "required": ["page", "line", "char_span"],
      "additionalProperties": false,
      "properties": {
        "page": { "type": "string" },
        "line": { "type": "integer" },
        "char_span": {
          "type": "array",
          "items": { "type": "integer", "minimum": 0 },
          "minItems": 2,
          "maxItems": 2
        }
      }
    },
    "line_ref": {
      "type": "object",
      "description": "A reference to one lines[] row by its natural key — the same key BLOCK_KEY_FIELDS['lines'] merges on. Survives merge_document_records(), which an array index does not.",
      "required": ["page", "line"],
      "additionalProperties": false,
      "properties": {
        "page": { "type": "string" },
        "line": { "type": "integer", "minimum": 0 }
      }
    }
  }
}