diff --git a/contracts/evidence.v1.yaml b/contracts/evidence.v1.yaml index df02f01..f2076c3 100644 --- a/contracts/evidence.v1.yaml +++ b/contracts/evidence.v1.yaml @@ -11,7 +11,11 @@ # - Use "unknown" for unverified fields; never fill from memory. # - List unverified fields under provenance.unverified_fields. # - omv-report must preserve all unverified markers; never silently upgrade them. -# - CVE readiness score 0-100 based on field completeness; threshold >=75 is necessary but not sufficient. +# - submission_score (computed, 0-100) = evidence_score minus penalties (blockers, +# missing observed_result, weak version boundaries, incomplete dedup, blocked/disproven +# verdict, missing repro artifacts, unverified fields, low confidence in verdict). +# Threshold: submission_score >= 75 plus validation ok and status confirmed is required +# before suggesting promotion to /omv-report. # - CLI validation is the machine gate for status transitions; confirmed findings must satisfy field-level checks. # - Unknown values in important fields must be listed in provenance.unverified_fields until verified. @@ -104,26 +108,11 @@ provenance: unverified_fields: [] # list field paths that carry "unknown" or unconfirmed data; checked by CLI validation tool_versions: {} # e.g. {omv-find: "1.0"} -# ── Evidence and Submission Scores (computed, not stored) ───────────────────── -# omv computes evidence_score from field completeness. -# omv computes submission_score by penalizing unresolved blockers, unknown -# observed_result, unknown affected range, incomplete dedup, and blocked/disproven -# verdicts. A high evidence_score is not enough for submission-ready output. +# ── Evidence Score (evidence_score) ────────────────────────────────── +# evidence_score = sum of weights below; field-completeness measure, 0-100. +# Never penalized, never clamped. Computed by CLI src/cli/findings.ts computeEvidenceScore. # -# Evidence scoring guide (total 100): -# tested version present +20 -# source identified +10 -# sink identified +10 -# guard missing confirmed +10 -# local reproducer written +15 -# observed result documented +10 -# cvss vector present +10 -# dedup search completed +10 -# vendor contacted +5 -# -# Threshold: submission_score >= 75 plus validation OK is required before -# suggesting promotion to confirmed. -cve_readiness_scoring: +evidence_score_weights: tested_version: 20 source_identified: 10 sink_identified: 10 @@ -134,4 +123,24 @@ cve_readiness_scoring: dedup_searched: 10 vendor_contacted: 5 total: 100 - threshold: 75 + +# ── Submission Score (submission_score) ──────────────────────────────── +# submission_score = max(0, min(100, evidence_score - Σdeductions - cvss_confidence_penalty)). +# Blocked findings: submission_score = 0. +# Computed by CLI src/cli/findings.ts computeSubmissionScore. Must mirror findings.ts exactly. +submission_score: + threshold: 75 # confirmed + validation.ok + submission_score >= 75 → report-ready + cvss_confidence_penalty: + per_unverified_field: 3 # capped at 20 + confidence_medium: 5 + confidence_low: 15 + confidence_unknown: 20 + deductions: + missingObservedResult: 25 + unresolvedBlockers: 30 + unknownAffectedRange: 10 + incompleteDedup: 15 + blockedOrDisproven: 50 + plausibleExploitability: 10 + confirmedBelowThreshold: 10 + missingReproArtifacts: 10 diff --git a/skills/omv-report/SKILL.md b/skills/omv-report/SKILL.md index 1a87083..881387e 100644 --- a/skills/omv-report/SKILL.md +++ b/skills/omv-report/SKILL.md @@ -55,7 +55,7 @@ Use the validation result to choose output mode: - Validation errors: lead with the errors and blockers; do not produce a submission-ready VulDB/CVE/GHSA/OSV report. - `status: blocked`: explain blockers and minimum evidence needed. - `status: candidate`: produce only triage notes or a draft outline clearly marked not ready for submission. -- `status: confirmed`: proceed only if required evidence is present and submission score is at least 75/100; include validation warnings in the pre-submission checklist. +- `status: confirmed`: proceed only if required evidence is present and submission score is at least 75/100 (`submissionScore` = `submission_score` in contract — the gating score after deducting blockers/unverified fields/confidence penalties); include validation warnings in the pre-submission checklist. - `submissionScore` below 75 or `verdict.exploitability` not `proven`: do not produce a submission-ready report; explain what evidence or reproduction artifact is missing. - `evidence.repro_artifacts` present: reference the artifacts as local reviewer evidence. If absent, warn that the report depends only on inline reproducer text. diff --git a/skills/omv-report/scripts/render_template.py b/skills/omv-report/scripts/render_template.py index 68e3391..e3dd6e7 100644 --- a/skills/omv-report/scripts/render_template.py +++ b/skills/omv-report/scripts/render_template.py @@ -120,6 +120,7 @@ class Finding: # Provenance verification_date: str = "" researcher: str = "" + unverified_fields: list[str] = field(default_factory=list) # Scores (computed) evidence_score: int = 0 submission_score: int = 0 @@ -194,13 +195,22 @@ def load_finding(path: Path) -> Finding: prov = data.get("provenance") or {} f.verification_date = str(prov.get("verification_date", "")) f.researcher = str(prov.get("researcher", "")) + uf = prov.get("unverified_fields") + if isinstance(uf, list): + f.unverified_fields = [str(x) for x in uf] f.evidence_score, f.submission_score = _compute_scores(f) return f def _compute_scores(f: Finding) -> tuple[int, int]: - """Mirror contracts/evidence.v1.yaml scoring guide.""" + """Evidence and submission scores. MUST mirror src/cli/findings.ts computeSubmissionScore exactly. + + evidence_score = field completeness only, 0-100, never penalized or clamped. + submission_score = max(0, min(100, evidence_score - deductions - cvss_penalty)). + Blocked findings have submission_score = 0. + """ + # evidence_score — sum of known non-"unknown" fields ev = 0 if is_set(f.tested): ev += 20 @@ -221,23 +231,55 @@ def _compute_scores(f: Finding) -> tuple[int, int]: if f.vendor_contacted: ev += 5 + # submission_score — starts from evidence_score, subtracts deductions sub = ev - if f.blockers: - sub -= min(30, len(f.blockers) * 15) if not is_set(f.observed_result): - sub -= 20 - if not is_set(f.affected_range): - sub -= 15 + sub -= 25 + if f.blockers: + sub -= 30 + if not is_set(f.affected_range) or f.affected_range.strip().lower() == "unknown": + sub -= 10 if not (f.nvd_searched and f.ghsa_searched and f.ecosystem_db_searched): + sub -= 15 + if f.exploitability in ("blocked", "disproven"): + sub -= 50 + if f.exploitability == "plausible": sub -= 10 + if f.repro_artifacts and not _any_repro_artifact_exists(f): + sub -= 10 + + # cvss confidence penalty + unverified_count = len(getattr(f, 'unverified_fields', [])) + penalty = min(20, unverified_count * 3) + if f.confidence == "medium": + penalty += 5 + elif f.confidence == "low": + penalty += 15 + elif f.confidence == "unknown": + penalty += 20 + sub -= penalty + + # blocked → 0 if f.exploitability in ("blocked", "disproven"): - sub -= 30 - if not is_set(f.tested): - sub -= 20 + sub = 0 + elif f.status == "blocked": + sub = 0 + + # extra -10 for confirmed findings still below threshold + if f.status == "confirmed" and sub < 75: + sub -= 10 return ev, max(0, min(100, sub)) +def _any_repro_artifact_exists(f: Finding) -> bool: + """Check if any listed repro_artifact file path exists on disk (CLI's existingArtifactPaths logic).""" + for ap in f.repro_artifacts: + if ap and Path(ap).exists(): + return True + return False + + def _score_line(f: Finding) -> str: return ( f"Rendered by omv render_template | evidence: {f.evidence_score}/100"