run_experiments.py

import json
import os

USE_ANTHROPIC = False  # <-- intentional default

if USE_ANTHROPIC:
    from models.anthropic_model import query_anthropic as query_model
else:
    from models.mock_model import query_model


results = []

for file in ["factual", "ambiguous", "unanswerable"]:
    data = json.load(open(f"data/{file}.json"))
    for item in data:
        answer, confidence = query_model(item["question"])

        correct = (
            item["answer"] is not None
            and answer is not None
            and answer.lower() == item["answer"].lower()
        )

        results.append({
            "question": item["question"],
            "confidence": confidence,
            "correct": correct,
            "type": file,
            "model": "anthropic" if USE_ANTHROPIC else "mock"
        })

json.dump(results, open("results.json", "w"), indent=2)