diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0e88ae3..a20f506 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,14 +11,23 @@ - [ ] Performance / sparse-path / numerics improvement - [ ] Docs / examples / packaging only - [ ] CI / workflow / security +- [ ] Release / packaging / publish evidence ## Verification -- [ ] `pytest -q` passes locally +- [ ] `pytest -q` passes locally, or this PR is docs/workflow-only and CI is the authority - [ ] `tests/test_anchors.py` unchanged, or anchor change is documented in `CHANGELOG.md` - [ ] If touching `MANIFEST_SCHEMA.json`: bumped `schema_version` and noted migration - [ ] If new external dependency: added to `pyproject.toml` and reviewed against minimal-deps policy -- [ ] If touching `.github/workflows/`: actions remain SHA-pinned (regression-gate) +- [ ] If touching `.github/workflows/`: actions remain full-SHA pinned and token permissions remain minimal +- [ ] If touching claims/docs/release wording: new status claims are backed by release audit, tag, CI run, PyPI/DOI evidence, or marked `UNVERIFIED`/not complete +- [ ] If touching branch protection / required checks: the required workflow runs on every relevant PR and is not path-filtered into a stuck-check trap + +## Quality contract + +- [ ] This PR does not introduce unsupported production-ready, externally certified, PyPI-published, DOI/Zenodo-archived, or release-complete wording +- [ ] This PR preserves the research / pre-clinical disclaimer unless a separate evidence-locked release audit proves a status change +- [ ] False-pass and rework risk considered, not only green CI ## Reproducibility note diff --git a/.github/scripts/check_claim_safety.py b/.github/scripts/check_claim_safety.py new file mode 100644 index 0000000..4187a5f --- /dev/null +++ b/.github/scripts/check_claim_safety.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Documentation claim-safety check for LiouScope. + +The goal is not to ban strong claims forever; the goal is to prevent public-facing +status, release, certification, and deployment claims from appearing without +explicit qualifiers or linked evidence. + +Release-audit files are intentionally excluded: they are the evidence ledger where +PyPI/DOI/release gaps and evidence are discussed in detail. Public status claims +should point to those files instead of duplicating unchecked wording. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DOC_PATHS = [ROOT / "README.md", ROOT / "CHANGELOG.md", ROOT / "docs"] +SKIP_NAMES = {"QUALITY_WORKFLOW_OS.md"} +SKIP_PREFIXES = ("RELEASE_AUDIT_",) + +RISK_PATTERNS = [ + re.compile(r"\bproduction[- ]ready\b", re.IGNORECASE), + re.compile(r"\bexternally certified\b", re.IGNORECASE), + re.compile(r"\bclinical(?:ly)? validated\b", re.IGNORECASE), + re.compile(r"\boperational(?:ly)? validated\b", re.IGNORECASE), + re.compile(r"\bPyPI[- ]published\b", re.IGNORECASE), + re.compile(r"\bDOI\b.*\b(complete|published|archived|released)\b", re.IGNORECASE), + re.compile(r"\bZenodo\b.*\b(complete|published|archived|released)\b", re.IGNORECASE), + re.compile(r"\brelease[- ]complete\b", re.IGNORECASE), +] + +ALLOW_MARKERS = [ + "not ", + "no ", + "does not ", + "must not ", + "unless ", + "until ", + "unverified", + "open gap", + "missing evidence", + "not for diagnostic or operational use", + "does not certify", + "placeholder", +] + + +def _should_skip(path: Path) -> bool: + name = path.name + return name in SKIP_NAMES or any(name.startswith(prefix) for prefix in SKIP_PREFIXES) + + +def _iter_docs() -> list[Path]: + paths: list[Path] = [] + for root in DOC_PATHS: + if root.is_file() and not _should_skip(root): + paths.append(root) + elif root.is_dir(): + paths.extend( + sorted( + path + for path in root.rglob("*.md") + if path.is_file() and not _should_skip(path) + ) + ) + return paths + + +def _line_is_allowed(line: str) -> bool: + lowered = line.lower() + return any(marker in lowered for marker in ALLOW_MARKERS) + + +def main() -> int: + errors: list[str] = [] + docs = _iter_docs() + + for path in docs: + text = path.read_text(encoding="utf-8") + for line_number, line in enumerate(text.splitlines(), start=1): + if _line_is_allowed(line): + continue + for pattern in RISK_PATTERNS: + if pattern.search(line): + rel = path.relative_to(ROOT) + errors.append( + f"{rel}:{line_number}: risky unsupported claim wording: {line.strip()}" + ) + break + + if errors: + print("Claim-safety check failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + print( + "\nFix by adding evidence, linking a release audit, or marking the status as not/UNVERIFIED.", + file=sys.stderr, + ) + return 1 + + print(f"Claim-safety check passed for {len(docs)} public-facing markdown files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/check_workflow_hardening.py b/.github/scripts/check_workflow_hardening.py new file mode 100644 index 0000000..aa06693 --- /dev/null +++ b/.github/scripts/check_workflow_hardening.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Static workflow hardening checks for LiouScope. + +This intentionally avoids third-party dependencies so the gate can run in GitHub +Actions with only the standard library. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github" / "workflows" +FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) +USES_RE = re.compile(r"^\s*uses:\s*([^\s#]+)") + + +def _iter_workflow_files() -> list[Path]: + if not WORKFLOWS.exists(): + return [] + return sorted( + path + for path in WORKFLOWS.iterdir() + if path.is_file() and path.suffix in {".yml", ".yaml"} + ) + + +def _is_third_party_uses(ref: str) -> bool: + return not ( + ref.startswith("./") + or ref.startswith("docker://") + or ref.startswith("github.com/") + ) + + +def _check_uses_pin(path: Path, line_number: int, line: str, errors: list[str]) -> None: + match = USES_RE.match(line) + if not match: + return + + ref = match.group(1).strip().strip('"').strip("'") + if "@" not in ref: + if _is_third_party_uses(ref): + errors.append(f"{path}:{line_number}: third-party action is missing @ref: {ref}") + return + + action, version = ref.rsplit("@", 1) + if not _is_third_party_uses(action): + return + if not FULL_SHA_RE.fullmatch(version): + errors.append( + f"{path}:{line_number}: action must be pinned to a full 40-char SHA, got {ref}" + ) + + +def _check_privileged_trigger(path: Path, text: str, errors: list[str]) -> None: + if "pull_request_target" in text and "ALLOW_PULL_REQUEST_TARGET:" not in text: + errors.append( + f"{path}: uses pull_request_target without ALLOW_PULL_REQUEST_TARGET rationale" + ) + + +def _check_permissions_declared(path: Path, text: str, errors: list[str]) -> None: + # Minimal parser: require an explicit top-level permissions key before jobs. + before_jobs = text.split("\njobs:", 1)[0] + if "\npermissions:" not in f"\n{before_jobs}": + errors.append(f"{path}: missing explicit top-level permissions block") + + +def main() -> int: + workflow_files = _iter_workflow_files() + errors: list[str] = [] + + if not workflow_files: + errors.append("no GitHub workflow files found under .github/workflows") + + for path in workflow_files: + text = path.read_text(encoding="utf-8") + _check_permissions_declared(path, text, errors) + _check_privileged_trigger(path, text, errors) + for line_number, line in enumerate(text.splitlines(), start=1): + _check_uses_pin(path, line_number, line, errors) + + if errors: + print("Workflow hardening check failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print(f"Workflow hardening check passed for {len(workflow_files)} workflow files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/quality-contract.yml b/.github/workflows/quality-contract.yml new file mode 100644 index 0000000..c0062c9 --- /dev/null +++ b/.github/workflows/quality-contract.yml @@ -0,0 +1,33 @@ +# Quality Contract Gate — claim safety + workflow hardening. +# This is intentionally fast and dependency-free. It should be safe as a required +# check because it runs on every PR to main and has no path filters. +name: Quality Contract + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +jobs: + contract: + name: quality contract + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + - name: Check workflow hardening + run: python .github/scripts/check_workflow_hardening.py + - name: Check documentation claim safety + run: python .github/scripts/check_claim_safety.py diff --git a/docs/QUALITY_WORKFLOW_OS.md b/docs/QUALITY_WORKFLOW_OS.md new file mode 100644 index 0000000..94b0e3b --- /dev/null +++ b/docs/QUALITY_WORKFLOW_OS.md @@ -0,0 +1,151 @@ +# LiouScope Quality Workflow OS v1 + +**Status:** production-readiness contract for CI, release evidence, and claim safety. +**Scope:** repository governance and GitHub workflows only; this document does not certify scientific correctness, PyPI availability, DOI archival status, or production readiness. +**Source of truth:** code, workflows, release artifacts, tags, and release audit files beat README prose. + +## 1. Why this exists + +LiouScope already has strong engineering posture: pinned GitHub Actions, a Python CI matrix, workflow security audit, Scorecard posture scanning, guarded PyPI publish, and explicit research-use disclaimers. The remaining quality risk is **drift**: + +- README/status text can overstate what the repository has actually released. +- A workflow can stay green while no longer protecting the right failure mode. +- A security scanner score can improve while false-pass or rework risk remains hidden. +- A publish workflow can exist before PyPI Trusted Publishing is actually enabled. + +This OS turns those risks into small, repeatable gates. + +## 2. Non-negotiable gates + +### G1 — Minimal token permissions + +Every workflow must set top-level `permissions: {}` or `permissions: contents: read`, then raise permissions only at job level when needed. + +Allowed examples: + +```yaml +permissions: {} +``` + +```yaml +permissions: + contents: read +``` + +Job-level exceptions must explain the reason in a comment. + +### G2 — Full-SHA action pinning + +Every third-party `uses:` reference must pin to a full 40-character commit SHA. + +Allowed: + +```yaml +uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +``` + +Not allowed: + +```yaml +uses: actions/checkout@v4 +``` + +Local reusable workflows such as `./.github/workflows/ci-python-local.yml` are allowed. + +### G3 — No unsafe privileged PR trigger + +`pull_request_target` is disallowed unless a workflow contains a specific reviewed exception marker: + +```yaml +# ALLOW_PULL_REQUEST_TARGET: +``` + +The default solution is to use `pull_request`, `workflow_dispatch`, `release`, or `workflow_run` with clear artifact trust boundaries. + +### G4 — Claim-safety + +Documentation must not claim any of the following unless a release/evidence file explicitly supports it: + +- not evidence-backed production-ready status +- not evidence-backed clinical or operational validation +- not evidence-backed PyPI-published status +- not evidence-backed DOI/Zenodo archival status +- not evidence-backed external certification +- not evidence-backed release-complete status + +Negative disclaimers are allowed and encouraged, for example: “not for diagnostic or operational use”. + +### G5 — Release evidence lock + +A release is not considered complete until the release audit contains the actual evidence: + +- tag and commit SHA +- sdist/wheel hashes +- workflow run URL +- PyPI status if applicable +- artifact attestation/SBOM status if applicable +- changelog/version reconciliation +- explicit unresolved gaps + +### G6 — Required-check safety + +A workflow may be branch-protection-required only if it runs on every relevant pull request. Path-filtered workflows must not be required unless the branch protection rule is also path-scoped, which standard GitHub branch protection is not. + +### G7 — Paired metrics + +Quality claims must include at least one success metric and one counter-metric: + +| Success metric | Counter-metric | +|---|---| +| CI pass rate | false-pass rate | +| lead time | rework rate | +| Scorecard score | unresolved high-risk finding count | +| release frequency | failed release recovery time | +| coverage | escaped defect rate | + +## 3. Repository Quality Delta Score (RQDS v0.2) + +`RQDS = 0.20*CI_Reliability + 0.20*Security_Posture + 0.15*Evidence_Coverage + 0.15*Maintainability + 0.15*Delivery_Stability + 0.10*Observability + 0.05*Cost_Discipline - Penalty` + +### Component definitions + +- **CI_Reliability:** required checks run deterministically on relevant PRs. +- **Security_Posture:** pinned actions, minimal permissions, safe triggers, dependency updates, scanner signal. +- **Evidence_Coverage:** hard claims have linked source-of-truth evidence. +- **Maintainability:** workflow logic is simple, reusable, documented, and locally understandable. +- **Delivery_Stability:** release/publish path is guarded and recoverable. +- **Observability:** failures leave enough logs/artifacts to diagnose cause without guessing. +- **Cost_Discipline:** fast PR gates and deep nightly/release gates are separated. +- **Penalty:** unmarked uncertainty, false release claim, unpinned action, privileged trigger, missing release evidence, or branch-protection trap. + +RQDS is advisory, not a certification. A high RQDS never overrides a concrete blocker. + +## 4. RED-GREEN-EVIDENCE-LOCK + +1. **RED:** State what could be false. +2. **GREEN:** Run or inspect the relevant repo/workflow evidence. +3. **EVIDENCE:** Link the source of truth. +4. **LOCK:** Only then update release/status wording. +5. **LEARN:** Add a negative result if the failure class is reusable. + +## 5. PR acceptance checklist + +A PR touching workflows, docs, release, packaging, or claims must answer: + +- [ ] Did all third-party actions remain full-SHA pinned? +- [ ] Did token permissions remain minimal? +- [ ] Did this avoid unsafe `pull_request_target` use? +- [ ] Are new claims backed by code, tag, release audit, CI run, PyPI, Zenodo/DOI, or explicit `UNVERIFIED` status? +- [ ] If release-related: are hashes, tag, changelog, and unresolved gaps captured? +- [ ] If branch-protection-related: can the required checks run on non-workflow PRs? +- [ ] If a scanner score improved: was false-pass/rework risk also checked? + +## 6. What this document does not do + +- This document does **not** assert that LiouScope is production-ready. +- This document does **not** assert that LiouScope is externally certified. +- This document does **not** assert that PyPI or DOI publication is complete. +- This document does **not** replace scientific validation. +- This document does **not** replace human release approval. + +It defines the repo’s quality contract so those statuses can be reached without drifting into unsupported claims.