Merge pull request #19 from bx33661/refactor/scoring-single-source-of-truth

refactor(contracts): single source of truth for evidence/submission scoring

Author: bx33661

contracts/evidence.v1.yaml

@@ -11,7 +11,11 @@
 #   - Use "unknown" for unverified fields; never fill from memory.
 #   - List unverified fields under provenance.unverified_fields.
 #   - omv-report must preserve all unverified markers; never silently upgrade them.
-#   - CVE readiness score 0-100 based on field completeness; threshold >=75 is necessary but not sufficient.
+#   - submission_score (computed, 0-100) = evidence_score minus penalties (blockers,
+#     missing observed_result, weak version boundaries, incomplete dedup, blocked/disproven
+#     verdict, missing repro artifacts, unverified fields, low confidence in verdict).
+#     Threshold: submission_score >= 75 plus validation ok and status confirmed is required
+#     before suggesting promotion to /omv-report.
 #   - CLI validation is the machine gate for status transitions; confirmed findings must satisfy field-level checks.
 #   - Unknown values in important fields must be listed in provenance.unverified_fields until verified.
 
@@ -104,26 +108,11 @@ provenance:
   unverified_fields: []    # list field paths that carry "unknown" or unconfirmed data; checked by CLI validation
   tool_versions: {}        # e.g. {omv-find: "1.0"}
 
-# ── Evidence and Submission Scores (computed, not stored) ─────────────────────
-# omv computes evidence_score from field completeness.
-# omv computes submission_score by penalizing unresolved blockers, unknown
-# observed_result, unknown affected range, incomplete dedup, and blocked/disproven
-# verdicts. A high evidence_score is not enough for submission-ready output.
+# ── Evidence Score (evidence_score) ──────────────────────────────────
+# evidence_score = sum of weights below; field-completeness measure, 0-100.
+# Never penalized, never clamped. Computed by CLI src/cli/findings.ts computeEvidenceScore.
 #
-# Evidence scoring guide (total 100):
-#   tested version present          +20
-#   source identified               +10
-#   sink identified                 +10
-#   guard missing confirmed         +10
-#   local reproducer written        +15
-#   observed result documented      +10
-#   cvss vector present             +10
-#   dedup search completed          +10
-#   vendor contacted                +5
-#
-# Threshold: submission_score >= 75 plus validation OK is required before
-# suggesting promotion to confirmed.
-cve_readiness_scoring:
+evidence_score_weights:
   tested_version: 20
   source_identified: 10
   sink_identified: 10
@@ -134,4 +123,24 @@ cve_readiness_scoring:
   dedup_searched: 10
   vendor_contacted: 5
   total: 100
-  threshold: 75
+
+# ── Submission Score (submission_score) ────────────────────────────────
+# submission_score = max(0, min(100, evidence_score - Σdeductions - cvss_confidence_penalty)).
+# Blocked findings: submission_score = 0.
+# Computed by CLI src/cli/findings.ts computeSubmissionScore. Must mirror findings.ts exactly.
+submission_score:
+  threshold: 75            # confirmed + validation.ok + submission_score >= 75 → report-ready
+  cvss_confidence_penalty:
+    per_unverified_field: 3   # capped at 20
+    confidence_medium: 5
+    confidence_low: 15
+    confidence_unknown: 20
+  deductions:
+    missingObservedResult: 25
+    unresolvedBlockers: 30
+    unknownAffectedRange: 10
+    incompleteDedup: 15
+    blockedOrDisproven: 50
+    plausibleExploitability: 10
+    confirmedBelowThreshold: 10
+    missingReproArtifacts: 10

skills/omv-report/SKILL.md

@@ -55,7 +55,7 @@ Use the validation result to choose output mode:
 - Validation errors: lead with the errors and blockers; do not produce a submission-ready VulDB/CVE/GHSA/OSV report.
 - `status: blocked`: explain blockers and minimum evidence needed.
 - `status: candidate`: produce only triage notes or a draft outline clearly marked not ready for submission.
-- `status: confirmed`: proceed only if required evidence is present and submission score is at least 75/100; include validation warnings in the pre-submission checklist.
+- `status: confirmed`: proceed only if required evidence is present and submission score is at least 75/100 (`submissionScore` = `submission_score` in contract — the gating score after deducting blockers/unverified fields/confidence penalties); include validation warnings in the pre-submission checklist.
 - `submissionScore` below 75 or `verdict.exploitability` not `proven`: do not produce a submission-ready report; explain what evidence or reproduction artifact is missing.
 - `evidence.repro_artifacts` present: reference the artifacts as local reviewer evidence. If absent, warn that the report depends only on inline reproducer text.
 

skills/omv-report/scripts/render_template.py

@@ -120,6 +120,7 @@ class Finding:
     # Provenance
     verification_date: str = ""
     researcher: str = ""
+    unverified_fields: list[str] = field(default_factory=list)
     # Scores (computed)
     evidence_score: int = 0
     submission_score: int = 0
@@ -194,13 +195,22 @@ def load_finding(path: Path) -> Finding:
     prov = data.get("provenance") or {}
     f.verification_date = str(prov.get("verification_date", ""))
     f.researcher = str(prov.get("researcher", ""))
+    uf = prov.get("unverified_fields")
+    if isinstance(uf, list):
+        f.unverified_fields = [str(x) for x in uf]
 
     f.evidence_score, f.submission_score = _compute_scores(f)
     return f
 
 
 def _compute_scores(f: Finding) -> tuple[int, int]:
-    """Mirror contracts/evidence.v1.yaml scoring guide."""
+    """Evidence and submission scores. MUST mirror src/cli/findings.ts computeSubmissionScore exactly.
+
+    evidence_score = field completeness only, 0-100, never penalized or clamped.
+    submission_score = max(0, min(100, evidence_score - deductions - cvss_penalty)).
+    Blocked findings have submission_score = 0.
+    """
+    # evidence_score — sum of known non-"unknown" fields
     ev = 0
     if is_set(f.tested):
         ev += 20
@@ -221,23 +231,55 @@ def _compute_scores(f: Finding) -> tuple[int, int]:
     if f.vendor_contacted:
         ev += 5
 
+    # submission_score — starts from evidence_score, subtracts deductions
     sub = ev
-    if f.blockers:
-        sub -= min(30, len(f.blockers) * 15)
     if not is_set(f.observed_result):
-        sub -= 20
-    if not is_set(f.affected_range):
-        sub -= 15
+        sub -= 25
+    if f.blockers:
+        sub -= 30
+    if not is_set(f.affected_range) or f.affected_range.strip().lower() == "unknown":
+        sub -= 10
     if not (f.nvd_searched and f.ghsa_searched and f.ecosystem_db_searched):
+        sub -= 15
+    if f.exploitability in ("blocked", "disproven"):
+        sub -= 50
+    if f.exploitability == "plausible":
         sub -= 10
+    if f.repro_artifacts and not _any_repro_artifact_exists(f):
+        sub -= 10
+
+    # cvss confidence penalty
+    unverified_count = len(getattr(f, 'unverified_fields', []))
+    penalty = min(20, unverified_count * 3)
+    if f.confidence == "medium":
+        penalty += 5
+    elif f.confidence == "low":
+        penalty += 15
+    elif f.confidence == "unknown":
+        penalty += 20
+    sub -= penalty
+
+    # blocked → 0
     if f.exploitability in ("blocked", "disproven"):
-        sub -= 30
-    if not is_set(f.tested):
-        sub -= 20
+        sub = 0
+    elif f.status == "blocked":
+        sub = 0
+
+    # extra -10 for confirmed findings still below threshold
+    if f.status == "confirmed" and sub < 75:
+        sub -= 10
 
     return ev, max(0, min(100, sub))
 
 
+def _any_repro_artifact_exists(f: Finding) -> bool:
+    """Check if any listed repro_artifact file path exists on disk (CLI's existingArtifactPaths logic)."""
+    for ap in f.repro_artifacts:
+        if ap and Path(ap).exists():
+            return True
+    return False
+
+
 def _score_line(f: Finding) -> str:
     return (
         f"Rendered by omv render_template  |  evidence: {f.evidence_score}/100"