|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Guards that the default test suite cannot spend money. |
| 3 | +
|
| 4 | +Regression tests for issue #109. |
| 5 | +
|
| 6 | +Background: ``tests/integration/`` originally carried no pytest markers, so the |
| 7 | +documented command ``pytest tests/ -m "not real_world"`` selected |
| 8 | +``test_aws_eks_real_provision.py`` ("this WILL create resources and incur |
| 9 | +costs!") and opened live connections to AWS. |
| 10 | +
|
| 11 | +Two files in that directory (``test_eks_permissions.py`` and |
| 12 | +``test_aws_eks_debug.py``) are standalone scripts with no ``__main__`` guard: |
| 13 | +they fetch credentials, call boto3, and invoke ``exit()`` at *module scope*. |
| 14 | +That means a marker-based skip is not sufficient -- pytest imports a module in |
| 15 | +order to collect it, so the AWS calls happen before any marker is consulted. |
| 16 | +The guard must therefore prevent *collection*, not merely execution. |
| 17 | +
|
| 18 | +These tests run pytest in a subprocess so they exercise the real collection |
| 19 | +machinery rather than asserting against a stubbed pytest. |
| 20 | +""" |
| 21 | + |
| 22 | +import os |
| 23 | +import pathlib |
| 24 | +import subprocess |
| 25 | +import sys |
| 26 | + |
| 27 | +import pytest |
| 28 | + |
| 29 | +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] |
| 30 | +INTEGRATION_DIR = REPO_ROOT / "tests" / "integration" |
| 31 | +OPT_IN_VAR = "CLUSTRIX_ALLOW_BILLABLE" |
| 32 | + |
| 33 | +# Phrases pytest uses when a run selected nothing. Kept in one place so the |
| 34 | +# "gate" and "not a deletion" tests cannot drift apart. |
| 35 | +_EMPTY_COLLECTION_PHRASES = ( |
| 36 | + "no tests collected", |
| 37 | + "no tests ran", |
| 38 | + "collected 0 items", |
| 39 | +) |
| 40 | + |
| 41 | + |
| 42 | +def _collected_nothing(output: str) -> bool: |
| 43 | + return any(phrase in output for phrase in _EMPTY_COLLECTION_PHRASES) |
| 44 | + |
| 45 | + |
| 46 | +def _collect_integration(opt_in: bool, tmp_home): |
| 47 | + """Run `pytest --collect-only` against tests/integration in a subprocess. |
| 48 | +
|
| 49 | + `-o addopts=` strips the project's default addopts so this does not depend |
| 50 | + on xdist being installed. |
| 51 | +
|
| 52 | + The subprocess gets a throwaway HOME and a scrubbed environment. This is |
| 53 | + deliberate: a test whose job is to prove the suite cannot spend money must |
| 54 | + not itself be able to spend money if the guard it is testing is absent or |
| 55 | + broken. `FlexibleCredentialManager` reads `~/.clustrix/.env` |
| 56 | + (credential_manager.py:304-305), so redirecting HOME removes the only |
| 57 | + on-disk credential source these modules would otherwise find. |
| 58 | + """ |
| 59 | + env = dict(os.environ) |
| 60 | + env.pop(OPT_IN_VAR, None) |
| 61 | + if opt_in: |
| 62 | + env[OPT_IN_VAR] = "1" |
| 63 | + env["HOME"] = str(tmp_home) |
| 64 | + |
| 65 | + # Hard network block for the subprocess. |
| 66 | + # |
| 67 | + # Scrubbing HOME is not sufficient on its own: FlexibleCredentialManager |
| 68 | + # also resolves credentials through the 1Password CLI, so a developer with |
| 69 | + # an unlocked `op` session still gets live AWS keys. If the guard under |
| 70 | + # test is broken, this subprocess would then make real API calls -- the |
| 71 | + # test would cause the very harm it exists to detect. Blocking socket |
| 72 | + # connections makes the failure mode "loud error" instead of "AWS bill". |
| 73 | + sitecustomize = tmp_home / "sitecustomize.py" |
| 74 | + sitecustomize.write_text( |
| 75 | + "import socket\n" |
| 76 | + "class _Blocked(OSError):\n" |
| 77 | + " pass\n" |
| 78 | + "def _deny(*a, **k):\n" |
| 79 | + " raise _Blocked('network disabled by test_billable_safety')\n" |
| 80 | + "socket.socket.connect = _deny\n" |
| 81 | + "socket.socket.connect_ex = _deny\n" |
| 82 | + "socket.create_connection = _deny\n" |
| 83 | + ) |
| 84 | + env["PYTHONPATH"] = os.pathsep.join( |
| 85 | + [str(tmp_home), env.get("PYTHONPATH", "")] |
| 86 | + ).rstrip(os.pathsep) |
| 87 | + for leaked in ( |
| 88 | + "AWS_ACCESS_KEY_ID", |
| 89 | + "AWS_SECRET_ACCESS_KEY", |
| 90 | + "AWS_SESSION_TOKEN", |
| 91 | + "AWS_PROFILE", |
| 92 | + "AZURE_CLIENT_SECRET", |
| 93 | + "GOOGLE_APPLICATION_CREDENTIALS", |
| 94 | + "LAMBDA_CLOUD_API_KEY", |
| 95 | + ): |
| 96 | + env.pop(leaked, None) |
| 97 | + return subprocess.run( |
| 98 | + [ |
| 99 | + sys.executable, |
| 100 | + "-m", |
| 101 | + "pytest", |
| 102 | + str(INTEGRATION_DIR), |
| 103 | + "--collect-only", |
| 104 | + "-q", |
| 105 | + "-o", |
| 106 | + "addopts=", |
| 107 | + "-p", |
| 108 | + "no:cacheprovider", |
| 109 | + ], |
| 110 | + cwd=str(REPO_ROOT), |
| 111 | + env=env, |
| 112 | + capture_output=True, |
| 113 | + text=True, |
| 114 | + timeout=300, |
| 115 | + ) |
| 116 | + |
| 117 | + |
| 118 | +def test_integration_tests_are_not_collected_by_default(tmp_path): |
| 119 | + """Without explicit opt-in, tests/integration must collect zero tests. |
| 120 | +
|
| 121 | + This is the core guarantee: the default suite is free to run. |
| 122 | + """ |
| 123 | + result = _collect_integration(opt_in=False, tmp_home=tmp_path) |
| 124 | + combined = result.stdout + result.stderr |
| 125 | + |
| 126 | + assert _collected_nothing(combined), ( |
| 127 | + "tests/integration was collected without opt-in.\n" |
| 128 | + f"exit={result.returncode}\n{combined[-3000:]}" |
| 129 | + ) |
| 130 | + |
| 131 | + |
| 132 | +def test_default_collection_does_not_import_billable_modules(tmp_path): |
| 133 | + """Collection must not *import* the unguarded AWS scripts. |
| 134 | +
|
| 135 | + `test_eks_permissions.py` and `test_aws_eks_debug.py` make real boto3 calls |
| 136 | + and call exit() at module scope, so importing them is itself the harm. |
| 137 | + """ |
| 138 | + result = _collect_integration(opt_in=False, tmp_home=tmp_path) |
| 139 | + combined = result.stdout + result.stderr |
| 140 | + |
| 141 | + for marker in ( |
| 142 | + "Testing EKS permissions", # printed at import by test_eks_permissions |
| 143 | + "Getting AWS credentials", # printed at import by test_aws_eks_debug |
| 144 | + "botocore", |
| 145 | + "NoCredentialsError", |
| 146 | + ): |
| 147 | + assert marker not in combined, ( |
| 148 | + f"Billable module was imported during default collection " |
| 149 | + f"(saw {marker!r}).\n{combined[-3000:]}" |
| 150 | + ) |
| 151 | + |
| 152 | + |
| 153 | +def test_integration_tests_are_still_reachable_with_opt_in(tmp_path): |
| 154 | + """The guard is a gate, not a deletion. |
| 155 | +
|
| 156 | + With CLUSTRIX_ALLOW_BILLABLE=1 the directory must become collectable again, |
| 157 | + otherwise we have silently dropped the tests instead of protecting them. |
| 158 | + """ |
| 159 | + result = _collect_integration(opt_in=True, tmp_home=tmp_path) |
| 160 | + combined = result.stdout + result.stderr |
| 161 | + |
| 162 | + assert not _collected_nothing(combined), ( |
| 163 | + "tests/integration collected nothing even with opt-in; the guard is " |
| 164 | + f"deleting tests rather than gating them.\n{combined[-3000:]}" |
| 165 | + ) |
| 166 | + |
| 167 | + |
| 168 | +@pytest.mark.parametrize( |
| 169 | + "path", |
| 170 | + sorted(INTEGRATION_DIR.glob("*.py")), |
| 171 | + ids=lambda p: p.name, |
| 172 | +) |
| 173 | +def test_every_integration_file_is_declared_billable(path): |
| 174 | + """Every file under tests/integration must be covered by the opt-in gate. |
| 175 | +
|
| 176 | + A new file dropped into this directory must not be able to run for free. |
| 177 | + The gate is directory-wide, so this asserts the directory conftest exists |
| 178 | + and names the file's suffix -- i.e. that nothing escapes by being added |
| 179 | + later. |
| 180 | + """ |
| 181 | + conftest = INTEGRATION_DIR / "conftest.py" |
| 182 | + assert conftest.exists(), ( |
| 183 | + "tests/integration/conftest.py is missing; without it the directory " |
| 184 | + "collects for free." |
| 185 | + ) |
| 186 | + source = conftest.read_text() |
| 187 | + assert ( |
| 188 | + OPT_IN_VAR in source |
| 189 | + ), f"tests/integration/conftest.py does not reference {OPT_IN_VAR}" |
| 190 | + assert "collect_ignore" in source, ( |
| 191 | + "tests/integration/conftest.py must prevent collection (collect_ignore), " |
| 192 | + "not merely skip at runtime -- several modules make live AWS calls at " |
| 193 | + "import time." |
| 194 | + ) |
0 commit comments