Verify finance reasoning provenance

Signed-off-by: Ander Alvarez Sanz <104446704+aalvsz@users.noreply.github.com>

Author: aalvsz

docs/recipes/finance/provenanceguard-benchmark.md

@@ -45,6 +45,12 @@ Evidence is limited to `retrieve_information` excerpts. Canonical SEC source
 IDs require CIK, accession, and document; incomplete or ambiguous attribution
 is unavailable and therefore fail-closed.
 
+When the native rollout retains its user question, the evaluator can also
+recompute year-over-year changes, gross margins, and highest/lowest comparisons
+from individually attributed facts. Evidence-card refusals are accepted only
+for an explicit closed-world question whose requested entity, metric, and year
+are absent. Other claims continue through the routing and NLI path.
+
 ## Run
 
 Install the project dependencies, then pin both public model revisions (the
@@ -75,6 +81,29 @@ PROVENANCEGUARD_RUN_MODELS=1 uv run pytest -q \
   tests/test_provenance_benchmark.py -k pinned_public_models --no-cov
 ```
 
+## Native held-out confirmation
+
+A preregistered, issuer-disjoint run used 180 native NVFlow finance-agent
+rollouts to freeze 120 eligible traces: 40 calculations, 40 comparisons, and
+40 closed-world refusals. Each trace contributed one grounded answer and one
+independently assigned attack.
+
+| Endpoint | Correct | Rate | Wilson 95% interval |
+|---|---:|---:|---:|
+| Grounded acceptance | 119 / 120 | 99.17% | 95.43-99.85% |
+| Attack rejection | 120 / 120 | 100% | 96.90-100% |
+
+All 120 attacks were blocked; they covered numeric fabrication, entity,
+metric, temporal, and source conflation, wrong or contradictory comparison
+winners, and unsupported refusal claims. The one false block was caused by a
+legal company suffix (`N.V.`) being split into an incomplete clause.
+
+An earlier run was retired as development evidence after returning NO-GO: its
+generator had not bound values to their year within one sentence, and the
+closed-world parser did not accept singular "evidence card". The confirmation
+fixed those generic protocol/parser defects before selecting a new set of 60
+unseen issuers. No candidate change was made after the confirmation result.
+
 ## Claim boundary
 
 Passing means the public evaluator met those gates on controlled,

nvflow/provenanceguard/decomposer.py

@@ -48,7 +48,10 @@
 )
 
 # Compound claim splitters within a sentence.
-_COMPOUND_SPLIT_RE = re.compile(r"\s*;\s*|\s+--\s+|\s+but also\s+", re.IGNORECASE)
+_COMPOUND_SPLIT_RE = re.compile(
+    r"\s*;\s*|\s+--\s+|,?\s+(?:although|but(?:\s+also)?|however|whereas)\s+",
+    re.IGNORECASE,
+)
 _REFUTED_SUFFIX_RE = re.compile(r",?\s+which\s+contradicts?\b.*$", re.IGNORECASE)
 _META_EVIDENCE_RE = re.compile(
     r"\b(?:provided|stated|found)\s+in\s+(?:the\s+)?evidence\s+card|"
@@ -114,8 +117,9 @@ def decompose(self, answer: str) -> Sequence[AtomicClaim]:
                     continue
                 if _is_meta_evidence_fragment(frag):
                     continue
-                # Skip very short fragments (noise).
-                if len(frag.split()) < 3:
+                # Keep short clauses created by an explicit compound split
+                # (for example, "Acme won"); otherwise skip short noise.
+                if len(frag.split()) < 3 and len(fragments) == 1:
                     continue
                 sec_ids, sec_urls = _extract_stated_sec_ids(frag)
                 claim_id = _stable_claim_id(frag, sentence)

nvflow/provenanceguard/evaluator.py

@@ -58,7 +58,7 @@
     EvidenceChunk,
 )
 
-ALGORITHM_VERSION = "routing-nli-sec-attribution-v4"
+ALGORITHM_VERSION = "routing-nli-sec-attribution-v5"
 
 _TOKEN_RE = re.compile(r"[A-Za-z0-9]+")
 _DIRECT_SUPPORT_STOPWORDS = {
@@ -263,6 +263,76 @@ def evaluate(
             errors=tuple(errors),
         )
 
+    def verify_against_premise(self, answer: str, premise: str) -> Decision:
+        """Require every submitted claim to be entailed by a canonical premise.
+
+        This is used for derived facts, such as arithmetic or comparisons, for
+        which callers can construct a deterministic premise from attributed
+        source facts. No source routing or lexical entity vocabulary is used.
+        """
+        claims = self._decomposer.decompose(answer)
+        if not claims:
+            return Decision(status="unavailable", reason="no_claims_extracted")
+
+        verdicts = []
+        errors = []
+        for claim in claims:
+            claim_text = " ".join(claim.text.split())
+            premise_text = " ".join(premise.split())
+            if claim_text in premise_text:
+                verdicts.append(
+                    ClaimVerdict(
+                        claim_id=claim.claim_id,
+                        claim_text=claim.text,
+                        raw_nli_label="entailment",
+                        raw_nli_probabilities=(("entailment", 1.0),),
+                        final_label="entailment",
+                        evidence_excerpt=premise[: self._config.evidence_excerpt_length],
+                    )
+                )
+                continue
+            try:
+                result = self._nli.score(premise=premise, hypothesis=claim.text)
+            except Exception as exc:
+                error = f"NLI error for claim {claim.claim_id}: {exc!s}"
+                errors.append(error)
+                verdicts.append(
+                    ClaimVerdict(
+                        claim_id=claim.claim_id,
+                        claim_text=claim.text,
+                        final_label="neutral",
+                        evidence_excerpt=premise[: self._config.evidence_excerpt_length],
+                        errors=(error,),
+                    )
+                )
+                continue
+            verdicts.append(
+                ClaimVerdict(
+                    claim_id=claim.claim_id,
+                    claim_text=claim.text,
+                    raw_nli_label=result.label,
+                    raw_nli_probabilities=result.probabilities,
+                    final_label=result.label,
+                    evidence_excerpt=premise[: self._config.evidence_excerpt_length],
+                )
+            )
+
+        if errors:
+            return Decision(
+                status="unavailable",
+                reason="semantic_verification_error",
+                verdicts=tuple(verdicts),
+                errors=tuple(errors),
+            )
+        for label in ("contradiction", "neutral"):
+            if any(verdict.final_label == label for verdict in verdicts):
+                return Decision(
+                    status="block",
+                    reason=f"semantic_{label}",
+                    verdicts=tuple(verdicts),
+                )
+        return Decision(status="allow", reason="semantic_entailment", verdicts=tuple(verdicts))
+
     def _evaluate_claim(
         self,
         claim: AtomicClaim,

nvflow/provenanceguard/protected_values.py

@@ -57,7 +57,7 @@ class ProtectedValue:
 
 # Percentage: 12.3%, 5 percent, etc.
 _PERCENTAGE_RE = re.compile(
-    r"\d[\d,]*(?:\.\d+)?\s*(?:%|percent\b)",
+    r"-?\d[\d,]*(?:\.\d+)?\s*(?:%|percent\b)",
     re.IGNORECASE,
 )
 
@@ -98,6 +98,7 @@ class ProtectedValue:
     ),
     "free_cash_flow": re.compile(r"\bfree cash flow\b", re.IGNORECASE),
     "gross_profit": re.compile(r"\bgross profit\b", re.IGNORECASE),
+    "research_and_development": re.compile(r"\b(?:research and development|R&D)\b", re.IGNORECASE),
 }
 
 
@@ -232,17 +233,23 @@ def check_protected_values(
 
 def has_financial_metric_mismatch(claim_text: str, evidence_text: str) -> bool:
     """Return whether a claim names a financial metric absent from evidence."""
-    claim_metrics = {
-        name for name, pattern in _FINANCIAL_METRICS.items() if pattern.search(claim_text)
-    }
+    claim_metrics = extract_financial_metrics(claim_text)
     if not claim_metrics:
         return False
-    evidence_metrics = {
-        name for name, pattern in _FINANCIAL_METRICS.items() if pattern.search(evidence_text)
-    }
+    evidence_metrics = extract_financial_metrics(evidence_text)
     if evidence_metrics:
         return not claim_metrics.issubset(evidence_metrics)
     # Metric-less evidence is only a deterministic mismatch when it repeats the
     # claim's protected values; otherwise NLI remains responsible for relevance.
     outcome, _ = check_protected_values(claim_text, evidence_text)
     return outcome == "pass"
+
+
+def extract_financial_metrics(text: str) -> set[str]:
+    """Return canonical financial metric names found in text."""
+    return {name for name, pattern in _FINANCIAL_METRICS.items() if pattern.search(text)}
+
+
+def protected_numeric_value(value: ProtectedValue) -> Decimal | None:
+    """Return the exact numeric value represented by a protected value."""
+    return _numeric_value(value)