eval.yaml

# Info Naut - Evaluation Test Cases
test_cases:
  - question: "What is Info Naut?"
    expected_answer: "Info Naut is an advanced Retrieval-Augmented Generation RAG question-answering system"
    expected_citations: ["01_infonaut_overview.md"]
    threshold: 0.85

  # Feature-related questions
  - question: "What are the key features of Info Naut?"
    expected_answer: "Info Naut includes guardrails, attribution, hallucination detection, PII redaction"
    expected_citations: ["01_infonaut_overview.md"]
    threshold: 0.80

  # Installation questions
  - question: "How do I install Info Naut?"
    expected_answer: "Install Python dependencies with pip install -r requirements.txt and frontend with npm install"
    expected_citations: ["02_installation_guide.md"]
    threshold: 0.75

# Grounding test cases
grounding_tests:
  - question: "What is the capital of France?"
    expected_grounded: false
    reason: "Out of domain question not in knowledge base"
    
  - question: "What types of guardrails does Info Naut have?"
    expected_grounded: true
    reason: "Documented feature in the knowledge base"

# Metrics configuration
metrics:
  faithfulness_threshold: 0.85
  answer_relevancy_threshold: 0.80
  citation_precision_threshold: 0.75