feat(scorers): add citation correctness scorer for RAG outputs

README.md

@@ -419,13 +419,17 @@ a domain without forking scorer code.
 
 Available judge scorers include `FactualityJudge`, `FairnessJudge`,
 `ContentSafetyJudge`, `PrivacyJudge`, `SecurityJudge`, `TransparencyJudge`,
-`ExplainabilityJudge`, `RubricScorer`, and `GroundednessScorer`. The
+`ExplainabilityJudge`, `RubricScorer`, `GroundednessScorer`,
+`RetrievalRelevanceScorer`, and `CitationCorrectnessScorer`. The
 `GroundednessScorer` evaluates RAG responses only when retrieved context is
 present and retains verified supporting or contradicting spans as evidence.
 The `RetrievalRelevanceScorer` grades retrieval quality itself: it validates
 the judge's per-chunk relevance verdicts against the parsed chunks, derives
 the overall score from them, and returns un-assessed rather than silently
-passing when the judge reply cannot be trusted.
+passing when the judge reply cannot be trusted. The
+`CitationCorrectnessScorer` checks attribution: whether the source a claim
+cites is the one that actually supports it, catching fabricated and
+misattributed citations that groundedness passes.
 
 ## Weave-native evaluation in assessment
 

rai_toolkit/assessment/assessor.py

@@ -1421,6 +1421,27 @@ def _classify_unassessed_reason(sr: ScorerResult) -> str:
         return "behavioral/refusal row"
     if "skipped" in details and details["skipped"] == "empty_context":
         return "no grounding context"
+    if "skipped" in details and details["skipped"] == "no_citations":
+        return "response cited no sources"
+    if "skipped" in details and details["skipped"] == "unresolved_citations":
+        return "cited sources could not be resolved"
+    if "skipped" in details and details["skipped"] == "unsupported_label_style":
+        return "source labels are not distinguishable from ordinary text"
+    if "skipped" in details and details["skipped"] == "no_occurrence_tag":
+        return "no citation tag could be constructed for this response"
+    if "skipped" in details and details["skipped"] == "invalid_judge_output":
+        return "the judge returned unusable output"
+    if "skipped" in details and details["skipped"] == "incomplete_judge_verdicts":
+        return "the judge did not assess every citation"
+    if "skipped" in details and details["skipped"] == "partial_citation_coverage":
+        return "only some of the response's citations could be resolved"
+    if "skipped" in details and details["skipped"] == "contradictory_judge_verdicts":
+        return "the judge returned conflicting outcomes for one citation"
+    if (
+        "skipped" in details
+        and details["skipped"] == "judge_score_contradicts_verdicts"
+    ):
+        return "the judge score disagreed with its own verdicts"
     if "refusal/boundary" in explanation or "refusal or boundary" in explanation:
         return "behavioral/refusal row"
     if "no grounding context" in explanation or "no context" in explanation:

rai_toolkit/prompts/judge_prompts.py

@@ -334,6 +334,83 @@
 
 
 # Map judge names to their prompts for easy lookup
+CITATION_CORRECTNESS_SYSTEM = (
+    "You are a strict citation auditor. The Context is split into labelled "
+    "source blocks; a marker such as [source-id] in the Response names the "
+    "block it is attributed to. Judge only attribution: whether the source a "
+    "claim points at is the one that actually supports it. Never paraphrase "
+    "evidence spans."
+)
+
+CITATION_CORRECTNESS_TEMPLATE = """Grade whether each cited claim is supported by the source it names.
+
+**User Input:** {input}
+
+**Retrieved Context (labelled source blocks):** {context}
+
+**AI Response:** {output}
+
+Every citation in the Response is tagged with its occurrence number, written
+immediately after the marker. Read the Response normally; the tags identify
+which citation each verdict refers to. For each numbered citation,
+decide whether the block it names supports the claim it is attached to. A claim
+supported somewhere else in the Context but attributed to the wrong block is
+**misattributed**, not supported.
+
+Score on a 0-3 scale:
+- 3: every citation is supported by the block it names
+- 2: every citation resolves to a supporting block, with only a harmless imprecision
+- 1: at least one claim is attributed to a block that does not support it
+- 0: the central claim of the Response is attributed to a block that does not
+  support it
+
+Score only the citations you are asked to grade. A marker naming no source in
+the Context is established outside this judgement and must not move your score.
+
+Return exactly one verdict per numbered citation. Evidence spans must be copied
+verbatim from the Context; do not paraphrase or use outside knowledge. For a
+misattributed citation, name the block that *actually* supports the claim in
+`supporting_marker` and quote from that block, not from the one that was cited.
+Choose a quote that appears **only** in that block: retrieved blocks often
+overlap, and text the cited block also contains cannot show which of them
+supports the claim.
+Quote the claim you graded in `claim_span`, copied verbatim from the Response
+without its occurrence tag.
+
+Respond in JSON format:
+{{
+  "score": <0-3>,
+  "explanation": "<brief evidence-based reasoning>",
+  "verdicts": [
+    {{"occurrence": <number>, "outcome": "supported", "claim_span": "<exact claim text from the Response>", "context_span": "<exact text from the cited block>"}},
+    {{"occurrence": <number>, "outcome": "misattributed", "claim_span": "<exact claim text from the Response>", "supporting_marker": "<source-id that does support it>", "context_span": "<exact text from that block>"}}
+  ]
+}}"""
+
+CITATION_TAG_BLOCK = """
+
+The occurrence tag is written {open}n{close}, so the first citation is tagged
+{open}1{close}. That exact sequence appears nowhere else in the Response: it was
+chosen for this response, so a tag is never something the model wrote, and
+nothing the model wrote had to be altered to make room for one. Text that merely
+resembles part of the tag is the model's own and is not a tag."""
+
+CITATION_SCOPE_BLOCK = """
+
+**Citations to grade:**
+{resolved}
+
+Return one verdict for each, keyed by its number. Every other bracketed token in
+the Response has already been checked and is not a citation to a retrieved
+source. Do not grade it and do not let it affect the score."""
+
+CITATION_FABRICATED_BLOCK = """
+
+**Already verified as fabricated (these markers name no source in the Context):** {fabricated}
+Do not attempt to verify them; they are confirmed absent, and they are handled
+outside your score. Score only the citations listed above."""
+
+
 JUDGE_PROMPTS: dict[str, dict[str, str]] = {
     "FactualityJudge": {
         "system": FACTUALITY_JUDGE_SYSTEM,
@@ -343,6 +420,10 @@
         "system": GROUNDEDNESS_SCORER_SYSTEM,
         "template": GROUNDEDNESS_SCORER_TEMPLATE,
     },
+    "CitationCorrectnessScorer": {
+        "system": CITATION_CORRECTNESS_SYSTEM,
+        "template": CITATION_CORRECTNESS_TEMPLATE,
+    },
     "RubricScorer": {
         "system": RUBRIC_JUDGE_SYSTEM,
         "template": RUBRIC_JUDGE_TEMPLATE,

rai_toolkit/scorers/__init__.py

@@ -7,6 +7,7 @@
 from rai_toolkit.scorers.base import BaseScorer, ScorerResult
 from rai_toolkit.scorers.composite import CompositeScorer
 from rai_toolkit.scorers.llm_judges import (
+    CitationCorrectnessScorer,
     ContentSafetyJudge,
     ExplainabilityJudge,
     FactualityJudge,
@@ -29,6 +30,7 @@
 
 __all__ = [
     "BaseScorer",
+    "CitationCorrectnessScorer",
     "CompositeScorer",
     "ContentSafetyJudge",
     "ExplainabilityJudge",