-
Notifications
You must be signed in to change notification settings - Fork 0
Add quality workflow OS and claim safety gate #67
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
e9fb225
1692aca
eb4d9e4
8ba1804
5dcf4d5
9983f60
6b6dc21
524fd12
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| #!/usr/bin/env python3 | ||
| """Documentation claim-safety check for LiouScope. | ||
|
|
||
| The goal is not to ban strong claims forever; the goal is to prevent public-facing | ||
| status, release, certification, and deployment claims from appearing without | ||
| explicit qualifiers or linked evidence. | ||
|
|
||
| Release-audit files are intentionally excluded: they are the evidence ledger where | ||
| PyPI/DOI/release gaps and evidence are discussed in detail. Public status claims | ||
| should point to those files instead of duplicating unchecked wording. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import re | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[2] | ||
| DOC_PATHS = [ROOT / "README.md", ROOT / "CHANGELOG.md", ROOT / "docs"] | ||
| SKIP_NAMES = {"QUALITY_WORKFLOW_OS.md"} | ||
| SKIP_PREFIXES = ("RELEASE_AUDIT_",) | ||
|
|
||
| RISK_PATTERNS = [ | ||
| re.compile(r"\bproduction[- ]ready\b", re.IGNORECASE), | ||
| re.compile(r"\bexternally certified\b", re.IGNORECASE), | ||
| re.compile(r"\bclinical(?:ly)? validated\b", re.IGNORECASE), | ||
| re.compile(r"\boperational(?:ly)? validated\b", re.IGNORECASE), | ||
| re.compile(r"\bPyPI[- ]published\b", re.IGNORECASE), | ||
| re.compile(r"\bDOI\b.*\b(complete|published|archived|released)\b", re.IGNORECASE), | ||
| re.compile(r"\bZenodo\b.*\b(complete|published|archived|released)\b", re.IGNORECASE), | ||
|
Comment on lines
+24
to
+25
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
These DOI/Zenodo patterns only match when Useful? React with 👍 / 👎. |
||
| re.compile(r"\brelease[- ]complete\b", re.IGNORECASE), | ||
| ] | ||
|
|
||
| ALLOW_MARKERS = [ | ||
| "not ", | ||
| "no ", | ||
| "does not ", | ||
| "must not ", | ||
| "unless ", | ||
| "until ", | ||
| "unverified", | ||
| "open gap", | ||
| "missing evidence", | ||
| "not for diagnostic or operational use", | ||
| "does not certify", | ||
| "placeholder", | ||
| ] | ||
|
|
||
|
|
||
| def _should_skip(path: Path) -> bool: | ||
| name = path.name | ||
| return name in SKIP_NAMES or any(name.startswith(prefix) for prefix in SKIP_PREFIXES) | ||
|
|
||
|
|
||
| def _iter_docs() -> list[Path]: | ||
| paths: list[Path] = [] | ||
| for root in DOC_PATHS: | ||
| if root.is_file() and not _should_skip(root): | ||
| paths.append(root) | ||
| elif root.is_dir(): | ||
| paths.extend( | ||
| sorted( | ||
| path | ||
| for path in root.rglob("*.md") | ||
| if path.is_file() and not _should_skip(path) | ||
| ) | ||
| ) | ||
| return paths | ||
|
|
||
|
|
||
| def _line_is_allowed(line: str) -> bool: | ||
| lowered = line.lower() | ||
| return any(marker in lowered for marker in ALLOW_MARKERS) | ||
|
Comment on lines
+54
to
+56
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Because Useful? React with 👍 / 👎. |
||
|
|
||
|
|
||
| def main() -> int: | ||
| errors: list[str] = [] | ||
| docs = _iter_docs() | ||
|
|
||
| for path in docs: | ||
| text = path.read_text(encoding="utf-8") | ||
| for line_number, line in enumerate(text.splitlines(), start=1): | ||
| if _line_is_allowed(line): | ||
| continue | ||
| for pattern in RISK_PATTERNS: | ||
| if pattern.search(line): | ||
|
Comment on lines
+66
to
+69
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a release status is actually proven, a line such as Useful? React with 👍 / 👎. |
||
| rel = path.relative_to(ROOT) | ||
| errors.append( | ||
| f"{rel}:{line_number}: risky unsupported claim wording: {line.strip()}" | ||
| ) | ||
| break | ||
|
|
||
| if errors: | ||
| print("Claim-safety check failed:", file=sys.stderr) | ||
| for error in errors: | ||
| print(f"- {error}", file=sys.stderr) | ||
| print( | ||
| "\nFix by adding evidence, linking a release audit, or marking the status as not/UNVERIFIED.", | ||
| file=sys.stderr, | ||
| ) | ||
| return 1 | ||
|
|
||
| print(f"Claim-safety check passed for {len(docs)} public-facing markdown files.") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| #!/usr/bin/env python3 | ||
| """Static workflow hardening checks for LiouScope. | ||
|
|
||
| This intentionally avoids third-party dependencies so the gate can run in GitHub | ||
| Actions with only the standard library. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import re | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[2] | ||
| WORKFLOWS = ROOT / ".github" / "workflows" | ||
| FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) | ||
| USES_RE = re.compile(r"^\s*uses:\s*([^\s#]+)") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The pinning gate misses the common GitHub Actions shorthand form used elsewhere in this repo, such as Useful? React with 👍 / 👎. |
||
|
|
||
|
|
||
| def _iter_workflow_files() -> list[Path]: | ||
| if not WORKFLOWS.exists(): | ||
| return [] | ||
| return sorted( | ||
| path | ||
| for path in WORKFLOWS.iterdir() | ||
| if path.is_file() and path.suffix in {".yml", ".yaml"} | ||
| ) | ||
|
|
||
|
|
||
| def _is_third_party_uses(ref: str) -> bool: | ||
| return not ( | ||
| ref.startswith("./") | ||
| or ref.startswith("docker://") | ||
| or ref.startswith("github.com/") | ||
| ) | ||
|
|
||
|
|
||
| def _check_uses_pin(path: Path, line_number: int, line: str, errors: list[str]) -> None: | ||
| match = USES_RE.match(line) | ||
| if not match: | ||
| return | ||
|
|
||
| ref = match.group(1).strip().strip('"').strip("'") | ||
| if "@" not in ref: | ||
| if _is_third_party_uses(ref): | ||
| errors.append(f"{path}:{line_number}: third-party action is missing @ref: {ref}") | ||
| return | ||
|
|
||
| action, version = ref.rsplit("@", 1) | ||
| if not _is_third_party_uses(action): | ||
| return | ||
| if not FULL_SHA_RE.fullmatch(version): | ||
| errors.append( | ||
| f"{path}:{line_number}: action must be pinned to a full 40-char SHA, got {ref}" | ||
| ) | ||
|
|
||
|
|
||
| def _check_privileged_trigger(path: Path, text: str, errors: list[str]) -> None: | ||
| if "pull_request_target" in text and "ALLOW_PULL_REQUEST_TARGET:" not in text: | ||
| errors.append( | ||
| f"{path}: uses pull_request_target without ALLOW_PULL_REQUEST_TARGET rationale" | ||
| ) | ||
|
|
||
|
|
||
| def _check_permissions_declared(path: Path, text: str, errors: list[str]) -> None: | ||
| # Minimal parser: require an explicit top-level permissions key before jobs. | ||
| before_jobs = text.split("\njobs:", 1)[0] | ||
| if "\npermissions:" not in f"\n{before_jobs}": | ||
| errors.append(f"{path}: missing explicit top-level permissions block") | ||
|
Comment on lines
+65
to
+69
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
G1 says top-level Useful? React with 👍 / 👎. |
||
|
|
||
|
|
||
| def main() -> int: | ||
| workflow_files = _iter_workflow_files() | ||
| errors: list[str] = [] | ||
|
|
||
| if not workflow_files: | ||
| errors.append("no GitHub workflow files found under .github/workflows") | ||
|
|
||
| for path in workflow_files: | ||
| text = path.read_text(encoding="utf-8") | ||
| _check_permissions_declared(path, text, errors) | ||
| _check_privileged_trigger(path, text, errors) | ||
| for line_number, line in enumerate(text.splitlines(), start=1): | ||
| _check_uses_pin(path, line_number, line, errors) | ||
|
|
||
| if errors: | ||
| print("Workflow hardening check failed:", file=sys.stderr) | ||
| for error in errors: | ||
| print(f"- {error}", file=sys.stderr) | ||
| return 1 | ||
|
|
||
| print(f"Workflow hardening check passed for {len(workflow_files)} workflow files.") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| # Quality Contract Gate — claim safety + workflow hardening. | ||
| # This is intentionally fast and dependency-free. It should be safe as a required | ||
| # check because it runs on every PR to main and has no path filters. | ||
| name: Quality Contract | ||
|
|
||
| on: | ||
| pull_request: | ||
| branches: [main] | ||
| push: | ||
| branches: [main] | ||
| workflow_dispatch: | ||
|
|
||
| permissions: {} | ||
|
|
||
| jobs: | ||
| contract: | ||
| name: quality contract | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: read | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| persist-credentials: false | ||
| - name: Set up Python | ||
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Check workflow hardening | ||
| run: python .github/scripts/check_workflow_hardening.py | ||
| - name: Check documentation claim safety | ||
| run: python .github/scripts/check_claim_safety.py |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The claim gate is advertised as protecting documentation/release/status wording, but it only scans
README.md,CHANGELOG.md, anddocs/. Public root Markdown such asSECURITY.md,CONTRIBUTING.md, or future GitHub-facing templates can still introduce unsupported production/PyPI/DOI claims without this workflow seeing them, leaving a gap in the drift protection.Useful? React with 👍 / 👎.