|
| 1 | +"""End-to-end tests against real bundles from the mpak registry. |
| 2 | +
|
| 3 | +These tests scan real published bundles to catch false positives and |
| 4 | +regressions that synthetic fixtures miss. |
| 5 | +
|
| 6 | +Setup: |
| 7 | + mpak bundle pull @nimblebraininc/finnhub -o tests/data/finnhub.mcpb |
| 8 | + mpak bundle pull @nimblebraininc/folk -o tests/data/folk.mcpb |
| 9 | + mpak bundle pull @nimblebraininc/nationalparks -o tests/data/nationalparks.mcpb |
| 10 | +
|
| 11 | +Run: |
| 12 | + uv run pytest tests/test_e2e_bundles.py -v |
| 13 | + uv run pytest -m e2e -v |
| 14 | +""" |
| 15 | + |
| 16 | +from pathlib import Path |
| 17 | + |
| 18 | +import pytest |
| 19 | + |
| 20 | +from mpak_scanner import scan_bundle |
| 21 | +from mpak_scanner.models import ControlStatus, Severity |
| 22 | + |
| 23 | +DATA_DIR = Path(__file__).parent / "data" |
| 24 | + |
| 25 | +# Bundle paths |
| 26 | +FINNHUB = DATA_DIR / "finnhub.mcpb" |
| 27 | +FOLK = DATA_DIR / "folk.mcpb" |
| 28 | +NATIONALPARKS = DATA_DIR / "nationalparks.mcpb" |
| 29 | + |
| 30 | +ALL_BUNDLES = [ |
| 31 | + pytest.param(FINNHUB, id="finnhub"), |
| 32 | + pytest.param(FOLK, id="folk"), |
| 33 | + pytest.param(NATIONALPARKS, id="nationalparks"), |
| 34 | +] |
| 35 | + |
| 36 | +PYTHON_BUNDLES = [ |
| 37 | + pytest.param(FINNHUB, id="finnhub"), |
| 38 | + pytest.param(FOLK, id="folk"), |
| 39 | +] |
| 40 | + |
| 41 | +NODE_BUNDLES = [ |
| 42 | + pytest.param(NATIONALPARKS, id="nationalparks"), |
| 43 | +] |
| 44 | + |
| 45 | + |
| 46 | +def skip_if_missing(bundle_path: Path) -> None: |
| 47 | + if not bundle_path.exists(): |
| 48 | + pytest.skip(f"Bundle not found: {bundle_path.name} (run: mpak bundle pull ... -o {bundle_path})") |
| 49 | + |
| 50 | + |
| 51 | +@pytest.mark.e2e |
| 52 | +class TestBundleCompleteness: |
| 53 | + """AI-05: Real bundles should not have false-positive unexpected executables.""" |
| 54 | + |
| 55 | + @pytest.mark.parametrize("bundle", ALL_BUNDLES) |
| 56 | + def test_ai05_passes(self, bundle: Path) -> None: |
| 57 | + """AI-05 should PASS on all published bundles (no false positives).""" |
| 58 | + skip_if_missing(bundle) |
| 59 | + report = scan_bundle(bundle) |
| 60 | + |
| 61 | + ai05 = report.all_controls.get("AI-05") |
| 62 | + assert ai05 is not None |
| 63 | + assert ai05.status == ControlStatus.PASS, f"AI-05 false positives on {bundle.name}: " + ", ".join( |
| 64 | + f.title for f in ai05.findings if f.severity in {Severity.HIGH, Severity.CRITICAL} |
| 65 | + ) |
| 66 | + |
| 67 | + @pytest.mark.parametrize("bundle", ALL_BUNDLES) |
| 68 | + def test_no_high_or_critical_in_ai05(self, bundle: Path) -> None: |
| 69 | + """AI-05 should have zero HIGH/CRITICAL findings on published bundles.""" |
| 70 | + skip_if_missing(bundle) |
| 71 | + report = scan_bundle(bundle) |
| 72 | + |
| 73 | + ai05 = report.all_controls.get("AI-05") |
| 74 | + assert ai05 is not None |
| 75 | + blocking = [f for f in ai05.findings if f.severity in {Severity.HIGH, Severity.CRITICAL}] |
| 76 | + assert blocking == [], f"Blocking findings on {bundle.name}: {[f.title for f in blocking]}" |
| 77 | + |
| 78 | + |
| 79 | +@pytest.mark.e2e |
| 80 | +class TestManifestValidation: |
| 81 | + """AI-01: Real bundles should have valid manifests.""" |
| 82 | + |
| 83 | + @pytest.mark.parametrize("bundle", ALL_BUNDLES) |
| 84 | + def test_ai01_passes(self, bundle: Path) -> None: |
| 85 | + skip_if_missing(bundle) |
| 86 | + report = scan_bundle(bundle) |
| 87 | + |
| 88 | + ai01 = report.all_controls.get("AI-01") |
| 89 | + assert ai01 is not None |
| 90 | + assert ai01.status == ControlStatus.PASS, f"AI-01 failed on {bundle.name}: {ai01.findings}" |
| 91 | + |
| 92 | + |
| 93 | +@pytest.mark.e2e |
| 94 | +class TestSafeExecution: |
| 95 | + """CQ-05: Real bundles should pass safe execution checks.""" |
| 96 | + |
| 97 | + @pytest.mark.parametrize("bundle", ALL_BUNDLES) |
| 98 | + def test_cq05_passes(self, bundle: Path) -> None: |
| 99 | + skip_if_missing(bundle) |
| 100 | + report = scan_bundle(bundle) |
| 101 | + |
| 102 | + cq05 = report.all_controls.get("CQ-05") |
| 103 | + assert cq05 is not None |
| 104 | + assert cq05.status == ControlStatus.PASS, f"CQ-05 failed on {bundle.name}: {cq05.findings}" |
| 105 | + |
| 106 | + |
| 107 | +@pytest.mark.e2e |
| 108 | +class TestFullScan: |
| 109 | + """Full scan results for each bundle.""" |
| 110 | + |
| 111 | + @pytest.mark.parametrize("bundle", PYTHON_BUNDLES) |
| 112 | + def test_python_bundles_no_critical_findings(self, bundle: Path) -> None: |
| 113 | + """Python bundles should have no CRITICAL findings across all controls.""" |
| 114 | + skip_if_missing(bundle) |
| 115 | + report = scan_bundle(bundle) |
| 116 | + |
| 117 | + critical = [] |
| 118 | + for control_id, result in report.all_controls.items(): |
| 119 | + for f in result.findings: |
| 120 | + if f.severity == Severity.CRITICAL: |
| 121 | + critical.append(f"{control_id}: {f.title}") |
| 122 | + assert critical == [], f"Critical findings on {bundle.name}: {critical}" |
| 123 | + |
| 124 | + @pytest.mark.parametrize("bundle", NODE_BUNDLES) |
| 125 | + def test_node_bundles_no_critical_findings(self, bundle: Path) -> None: |
| 126 | + """Node.js bundles should have no CRITICAL findings across all controls.""" |
| 127 | + skip_if_missing(bundle) |
| 128 | + report = scan_bundle(bundle) |
| 129 | + |
| 130 | + critical = [] |
| 131 | + for control_id, result in report.all_controls.items(): |
| 132 | + for f in result.findings: |
| 133 | + if f.severity == Severity.CRITICAL: |
| 134 | + critical.append(f"{control_id}: {f.title}") |
| 135 | + assert critical == [], f"Critical findings on {bundle.name}: {critical}" |
| 136 | + |
| 137 | + @pytest.mark.parametrize("bundle", ALL_BUNDLES) |
| 138 | + def test_scan_completes_without_errors(self, bundle: Path) -> None: |
| 139 | + """Scanner should not produce ERROR status on any control.""" |
| 140 | + skip_if_missing(bundle) |
| 141 | + report = scan_bundle(bundle) |
| 142 | + |
| 143 | + errors = [f"{cid}: {r.findings}" for cid, r in report.all_controls.items() if r.status == ControlStatus.ERROR] |
| 144 | + assert errors == [], f"Controls errored on {bundle.name}: {errors}" |
0 commit comments