paper_schema.json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://github.com/Transconnectome/paperorchestra-data/metadata/paper_schema.json",
  "title": "PaperWritingBench (Brain Science) — Benchmark Entry",
  "description": "Schema for one entry in the brain-science PaperWritingBench. Each entry provides reverse-engineered raw materials (I, E, T, G, F) from the PaperOrchestra formulation P = W(I, E, T, G, F), plus ground-truth metadata for automated evaluation. See docs/benchmark_plan.md for full rationale.",
  "type": "object",
  "required": [
    "id",
    "venue",
    "year",
    "title",
    "authors",
    "brain_science_class",
    "raw_materials",
    "ground_truth",
    "provenance"
  ],
  "properties": {
    "id": {
      "type": "string",
      "pattern": "^(neurips|iclr|icml|cvpr)_20(20|21|22|23|24|25)_[a-z0-9_]+$",
      "description": "Canonical bench ID: {venue_lower}_{year}_{slug}, e.g. 'neurips_2023_mindeye', 'cvpr_2024_mindbridge'"
    },
    "venue": {
      "type": "string",
      "enum": ["NeurIPS", "ICLR", "ICML", "CVPR"]
    },
    "venue_tier": {
      "type": "string",
      "enum": ["main", "workshop"],
      "default": "main",
      "description": "Main conference track vs associated workshop"
    },
    "venue_workshop_name": {
      "type": ["string", "null"],
      "description": "Workshop name if venue_tier=workshop, e.g. 'NeuroAI', 'SVRHM', 'DCAMI'"
    },
    "year": {
      "type": "integer",
      "minimum": 2020,
      "maximum": 2025
    },
    "title": {
      "type": "string"
    },
    "authors": {
      "type": "array",
      "items": { "type": "string" },
      "minItems": 1
    },
    "abstract": {
      "type": "string"
    },
    "arxiv_id": {
      "type": ["string", "null"],
      "description": "e.g. '2308.13234v2' — version-pinned for reproducibility. Null if no arxiv preprint."
    },
    "doi": {
      "type": ["string", "null"]
    },
    "semantic_scholar_id": {
      "type": ["string", "null"],
      "description": "Semantic Scholar corpus ID"
    },
    "openalex_id": {
      "type": ["string", "null"],
      "description": "OpenAlex work ID e.g. 'W1234567890'"
    },
    "keywords": {
      "type": "array",
      "items": { "type": "string" }
    },

    "brain_science_class": {
      "type": "string",
      "enum": [
        "neuroimaging_decoding",
        "clinical_neuroimaging",
        "bci_neural_population",
        "brain_dnn_alignment",
        "eeg_methods",
        "other"
      ],
      "description": "Topic diversity quota bucket. See docs/benchmark_plan.md §3.4"
    },
    "brain_science_subtopic": {
      "type": "string",
      "description": "Fine-grained label, e.g. 'fMRI-to-image reconstruction', 'Alzheimer connectome classification', 'intracortical BCI decoding'"
    },
    "selection": {
      "type": "object",
      "required": ["discovered_via", "filter_decision", "filter_confidence"],
      "properties": {
        "discovered_via": {
          "type": "string",
          "description": "Which seed source surfaced this paper: 'awesome-neurofm', 'dblp:fMRI venue:NeurIPS', 'refchain:MindEye2', etc."
        },
        "filter_decision": {
          "type": "string",
          "enum": ["definitely_brain", "ambiguous_reviewed_in", "manual_override_in"]
        },
        "filter_confidence": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        },
        "human_audit": {
          "type": "object",
          "properties": {
            "reviewer_id": { "type": "string" },
            "reviewed_at": { "type": "string", "format": "date-time" },
            "notes": { "type": "string" }
          }
        }
      }
    },

    "raw_materials": {
      "type": "object",
      "description": "The (I, E, T, G, F) tuple that the PaperOrchestra pipeline consumes. All paths are relative to the entry directory.",
      "required": [
        "idea_sparse_path",
        "idea_dense_path",
        "experimental_log_path",
        "template_key",
        "guidelines_key"
      ],
      "properties": {
        "idea_sparse_path": {
          "type": "string",
          "description": "Relative path to idea_sparse.md (I-Sparse variant, ~600 words, no equations, no results)"
        },
        "idea_dense_path": {
          "type": "string",
          "description": "Relative path to idea_dense.md (I-Dense variant, ~1000 words, with formal definitions + LaTeX equations, still no results)"
        },
        "experimental_log_path": {
          "type": "string",
          "description": "Relative path to experimental_log.md (E, de-contextualized setup + tabular results, 1500-2400 words)"
        },
        "template_key": {
          "type": "string",
          "description": "Pointer into shared ../../templates/ dir, e.g. 'cvpr_2024_double_column', 'iclr_single_column', 'neurips_2024'"
        },
        "guidelines_key": {
          "type": "string",
          "description": "Pointer into shared ../../guidelines/ dir, e.g. 'cvpr_2024', 'iclr_2024'"
        },
        "figures_path": {
          "type": ["string", "null"],
          "description": "Relative path to figures/ dir (F). Null or empty for autonomous generation tests (PlotOff setting)."
        },
        "generation": {
          "type": "object",
          "description": "Metadata about how raw materials were generated",
          "properties": {
            "model": { "type": "string", "description": "e.g. 'claude-opus-4-6', 'gemini-3-1-pro'" },
            "prompt_version": { "type": "string", "description": "Semver of prompt template used" },
            "temperature": { "type": "number" },
            "generated_at": { "type": "string", "format": "date-time" },
            "anti_leakage_check_passed": { "type": "boolean" },
            "word_counts": {
              "type": "object",
              "properties": {
                "idea_sparse": { "type": "integer" },
                "idea_dense": { "type": "integer" },
                "experimental_log": { "type": "integer" }
              }
            }
          }
        }
      }
    },

    "ground_truth": {
      "type": "object",
      "description": "Ground-truth paper P_GT and derived artifacts used for evaluation. PDFs/sources NOT redistributed — stored as URLs only.",
      "required": ["pdf_url", "parsed_sections_path"],
      "properties": {
        "pdf_url": {
          "type": "string",
          "format": "uri",
          "description": "Stable source URL (not redistributed copy)"
        },
        "supplementary_url": {
          "type": ["string", "null"],
          "format": "uri"
        },
        "latex_source_url": {
          "type": ["string", "null"],
          "format": "uri",
          "description": "arXiv e-print URL if LaTeX source available, e.g. 'https://arxiv.org/e-print/2308.13234v2'"
        },
        "parsed_sections_path": {
          "type": "string",
          "description": "Relative path to parsed/sections.json (abstract/intro/method/exp/results/discussion)"
        },
        "parsed_references_path": {
          "type": "string",
          "description": "Relative path to parsed/references.bib"
        },
        "parsed_figures_path": {
          "type": "string",
          "description": "Relative path to parsed/figures/ dir"
        },
        "parsed_tables_path": {
          "type": "string",
          "description": "Relative path to parsed/tables.json"
        },
        "reference_ids": {
          "type": "object",
          "description": "GT reference list partitioned per PaperOrchestra §5.2",
          "properties": {
            "P0": {
              "type": "array",
              "items": { "type": "string" },
              "description": "Must-Cite: baselines, datasets, metrics, foundational methods. Semantic Scholar corpus IDs."
            },
            "P1": {
              "type": "array",
              "items": { "type": "string" },
              "description": "Good-to-Cite: orthogonal/background context."
            },
            "partition_model": {
              "type": "string",
              "description": "LLM used for P0/P1 labeling"
            }
          }
        },
        "statistics": {
          "type": "object",
          "properties": {
            "num_figures": { "type": "integer" },
            "num_tables": { "type": "integer" },
            "num_citations": { "type": "integer" },
            "num_sections": { "type": "integer" },
            "word_count": { "type": "integer" }
          }
        }
      }
    },

    "code": {
      "type": "object",
      "description": "Code repository linked to paper (if available). URL only, NOT cloned.",
      "properties": {
        "repo_url": { "type": ["string", "null"], "format": "uri" },
        "commit_sha": { "type": ["string", "null"] },
        "discovered_via": {
          "type": "string",
          "enum": ["huggingface_papers", "arxiv_extra_services", "github_search", "manual", "none"]
        }
      }
    },

    "provenance": {
      "type": "object",
      "required": ["acquired_at"],
      "properties": {
        "acquired_at": {
          "type": "string",
          "format": "date-time",
          "description": "ISO 8601 timestamp of acquisition"
        },
        "acquired_from": {
          "type": "string",
          "description": "Actual URL fetched (may differ from ground_truth.pdf_url after redirects)"
        },
        "pdf_sha256": {
          "type": ["string", "null"],
          "description": "SHA-256 of downloaded PDF for reproducibility verification"
        },
        "latex_sha256": {
          "type": ["string", "null"],
          "description": "SHA-256 of downloaded arXiv .tar.gz"
        },
        "parser_versions": {
          "type": "object",
          "description": "Versions of parsing tools used",
          "properties": {
            "marker": { "type": "string" },
            "pdffigures2": { "type": "string" },
            "grobid": { "type": "string" },
            "pylatexenc": { "type": "string" },
            "texsoup": { "type": "string" }
          }
        }
      }
    },

    "evaluator_outputs": {
      "type": "object",
      "description": "Populated after running the PaperOrchestra pipeline + evaluators on this entry. Keyed by pipeline run ID.",
      "additionalProperties": {
        "type": "object",
        "properties": {
          "run_id": { "type": "string" },
          "pipeline_version": { "type": "string" },
          "evaluated_at": { "type": "string", "format": "date-time" },
          "citation_f1": {
            "type": "object",
            "properties": {
              "P0": {
                "type": "object",
                "properties": {
                  "precision": { "type": "number" },
                  "recall": { "type": "number" },
                  "f1": { "type": "number" }
                }
              },
              "P1": { "$ref": "#/properties/evaluator_outputs/additionalProperties/properties/citation_f1/properties/P0" },
              "combined": { "$ref": "#/properties/evaluator_outputs/additionalProperties/properties/citation_f1/properties/P0" }
            }
          },
          "lit_review_quality": {
            "type": "object",
            "description": "6-axis 0-100 scores per PaperOrchestra §5",
            "properties": {
              "citation_practices": { "type": "number", "minimum": 0, "maximum": 100 },
              "coverage_completeness": { "type": "number", "minimum": 0, "maximum": 100 },
              "critical_analysis_synthesis": { "type": "number", "minimum": 0, "maximum": 100 },
              "organization_writing": { "type": "number", "minimum": 0, "maximum": 100 },
              "positioning_novelty": { "type": "number", "minimum": 0, "maximum": 100 },
              "relevance_focus": { "type": "number", "minimum": 0, "maximum": 100 },
              "judge_model": { "type": "string" }
            }
          },
          "paper_quality_axes": {
            "type": "object",
            "description": "From SxS evaluator — see PaperOrchestra App. F",
            "properties": {
              "scientific_depth": { "type": "number" },
              "technical_execution": { "type": "number" },
              "logical_flow": { "type": "number" },
              "writing_clarity": { "type": "number" },
              "evidence_presentation": { "type": "number" },
              "academic_style": { "type": "number" }
            }
          }
        }
      }
    }
  }
}