Merge issue-91-static-manifest: build runtime static manifest (#91)

Author: alexeygrigorev

.github/workflows/ci.yml

@@ -419,6 +419,49 @@ jobs:
           test "$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image")" = "$RELEASE_SHA"
           test "$(docker run --rm --entrypoint id "$image" -u)" = "10001"
           test "$(docker run --rm --entrypoint id "$image" -g)" = "10001"
+      - name: Verify the built runtime static manifest
+        if: github.event_name != 'workflow_dispatch' || inputs.reuse_existing_image == false
+        env:
+          RELEASE_SHA: ${{ needs.resolve-release.outputs.release_sha }}
+        run: |
+          set -euo pipefail
+          image="dtc-website:$RELEASE_SHA"
+          fixtures="$PWD/.tmp/static-manifest-fixtures"
+          mkdir -p "$fixtures/absent"
+          printf '{\n' > "$fixtures/malformed.json"
+          printf '{"paths": {}, "version": "1.1", "hash": "fixture"}\n' > "$fixtures/missing-entry.json"
+
+          verify_manifest() {
+            settings_module="$1"
+            shift
+            docker run --rm \
+              --entrypoint sh \
+              --env "DJANGO_SETTINGS_MODULE=$settings_module" \
+              "$@" \
+              "$image" -c 'uv run --no-sync python -m scripts.verify_static_manifest'
+          }
+
+          verify_manifest website.settings.collectstatic
+
+          if verify_manifest website.settings.collectstatic \
+            --mount "type=bind,source=$fixtures/absent,target=/app/staticfiles,readonly"; then
+            echo "Static verification accepted an absent manifest." >&2
+            exit 1
+          fi
+          if verify_manifest website.settings.collectstatic \
+            --mount "type=bind,source=$fixtures/malformed.json,target=/app/staticfiles/staticfiles.json,readonly"; then
+            echo "Static verification accepted a malformed manifest." >&2
+            exit 1
+          fi
+          if verify_manifest website.settings.collectstatic \
+            --mount "type=bind,source=$fixtures/missing-entry.json,target=/app/staticfiles/staticfiles.json,readonly"; then
+            echo "Static verification accepted a manifest without courses.css." >&2
+            exit 1
+          fi
+          if verify_manifest website.settings.test; then
+            echo "Static verification accepted an incompatible storage backend." >&2
+            exit 1
+          fi
       - name: Smoke-test liveness without publishing
         if: github.event_name != 'workflow_dispatch' || inputs.reuse_existing_image == false
         env:
@@ -433,6 +476,7 @@ jobs:
           trap 'docker logs dtc-web; docker rm --force dtc-web' EXIT
           for attempt in $(seq 1 30); do
             if test "$(curl --fail --silent http://127.0.0.1:8000/health/live)" = "{\"status\": \"ok\", \"version\": \"$RELEASE_SHA\"}"; then
+              curl --fail --silent --output /dev/null http://127.0.0.1:8000/unified/
               exit 0
             fi
             sleep 1

Dockerfile

@@ -23,7 +23,7 @@ ENV PATH="/app/.venv/bin:$PATH" \
     PYTHONUNBUFFERED=1 \
     DJANGO_SETTINGS_MODULE=website.settings.production
 RUN mkdir -p /app/.cache/uv \
-    && DJANGO_SETTINGS_MODULE=website.settings.test uv run --no-sync python manage.py collectstatic --noinput \
+    && DJANGO_SETTINGS_MODULE=website.settings.collectstatic uv run --no-sync python manage.py collectstatic --noinput \
     && chown -R dtc:dtc /app/.cache
 USER 10001:10001
 LABEL org.opencontainers.image.revision=$SOURCE_SHA

core/tests/test_deployment_workflow.py

@@ -730,7 +730,7 @@ def test_gate_b_evidence_contract_is_atomic_offline_and_workflow_isolated(self)
 
         unchanged_hashes = {
             ".github/workflows/ci.yml": (
-                "6932845a0f919c816a086bdcf5976d38bb1195ee75af4d118e6fd4b962e11ac9"
+                "0a2d97843ad8045375258af0a96464e32aba1feb19f83c3b20f61f415bb8211c"
             ),
             "deploy/oidc_probe.py": (
                 "10f38b3c3df04c763f0e09ffe6128fc9d9fe174c4f3f7f161600992fcd84e2ff"
@@ -897,7 +897,7 @@ def test_gate_b_operator_contract_is_exact_and_workflow_isolated(self) -> None:
                 "4dd65a576f3bd3d3bd2dff41170ee161f45ffe5362a4e4e1f8af0feabc081027"
             ),
             ".github/workflows/ci.yml": (
-                "6932845a0f919c816a086bdcf5976d38bb1195ee75af4d118e6fd4b962e11ac9"
+                "0a2d97843ad8045375258af0a96464e32aba1feb19f83c3b20f61f415bb8211c"
             ),
             "deploy/oidc_probe.py": (
                 "10f38b3c3df04c763f0e09ffe6128fc9d9fe174c4f3f7f161600992fcd84e2ff"
@@ -1000,6 +1000,34 @@ def test_one_image_is_built_tested_and_published_only_after_all_gates(self) -> N
         self.assertIn("published-image record independently of deployment", workflow)
         self.assertNotIn("terraform apply", workflow)
 
+    def test_release_image_builds_and_verifies_the_runtime_static_manifest(self) -> None:
+        dockerfile = (ROOT / "Dockerfile").read_text()
+        workflow = (ROOT / ".github/workflows/ci.yml").read_text()
+        manifest_gate = workflow.split(
+            "- name: Verify the built runtime static manifest", maxsplit=1
+        )[1].split("- name: Smoke-test liveness without publishing", maxsplit=1)[0]
+
+        self.assertIn("DJANGO_SETTINGS_MODULE=website.settings.collectstatic", dockerfile)
+        self.assertNotIn(
+            "DJANGO_SETTINGS_MODULE=website.settings.test "
+            "uv run --no-sync python manage.py collectstatic",
+            dockerfile,
+        )
+        self.assertIn("python -m scripts.verify_static_manifest", manifest_gate)
+        self.assertIn("website.settings.collectstatic", manifest_gate)
+        self.assertIn("website.settings.test", manifest_gate)
+        self.assertIn("malformed.json", manifest_gate)
+        self.assertIn("missing-entry.json", manifest_gate)
+        self.assertIn("target=/app/staticfiles,readonly", manifest_gate)
+        self.assertLess(
+            workflow.index("- name: Verify the built runtime static manifest"),
+            workflow.index("- name: Preserve the one tested image"),
+        )
+        self.assertIn(
+            "curl --fail --silent --output /dev/null http://127.0.0.1:8000/unified/",
+            workflow,
+        )
+
     def test_serving_entrypoint_never_runs_migrations(self) -> None:
         entrypoint = (ROOT / "entrypoint.sh").read_text()
         dockerfile = (ROOT / "Dockerfile").read_text()

core/tests/test_settings.py

@@ -187,3 +187,58 @@ def test_database_url_selects_postgresql(self) -> None:
     def test_sqlite_is_not_an_implicit_local_fallback(self) -> None:
         with patch.dict(os.environ, {}, clear=True), self.assertRaises(ImproperlyConfigured):
             database_from_environment(allow_sqlite=True)
+
+
+class CollectstaticSettingsTests(SimpleTestCase):
+    def import_collectstatic_settings(
+        self, *, environment_name: str | None = None
+    ) -> subprocess.CompletedProcess[str]:
+        environment = os.environ.copy()
+        for name in (
+            "DATABASE_URL",
+            "DJANGO_ALLOWED_HOSTS",
+            "DJANGO_CSRF_TRUSTED_ORIGINS",
+            "DJANGO_SECRET_KEY",
+            "DJANGO_SETTINGS_MODULE",
+            "DTC_ENVIRONMENT",
+        ):
+            environment.pop(name, None)
+        if environment_name is not None:
+            environment["DTC_ENVIRONMENT"] = environment_name
+        command = (
+            "import json; import website.settings.collectstatic as s; "
+            "print(json.dumps({"
+            "'backend': s.STORAGES['staticfiles']['BACKEND'], "
+            "'database': s.DATABASES['default']['ENGINE']"
+            "}))"
+        )
+        return subprocess.run(
+            [sys.executable, "-c", command],
+            cwd=os.getcwd(),
+            env=environment,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    def test_collectstatic_settings_are_self_contained_and_use_runtime_manifest_storage(
+        self,
+    ) -> None:
+        result = self.import_collectstatic_settings()
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            json.loads(result.stdout),
+            {
+                "backend": "whitenoise.storage.CompressedManifestStaticFilesStorage",
+                "database": "django.db.backends.sqlite3",
+            },
+        )
+
+    def test_collectstatic_settings_reject_deployed_environments(self) -> None:
+        for environment_name in ("development", "production"):
+            with self.subTest(environment_name=environment_name):
+                result = self.import_collectstatic_settings(environment_name=environment_name)
+
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("Collectstatic settings are build-only", result.stderr)

scripts/verify_static_manifest.py

@@ -0,0 +1,61 @@
+"""Verify that a built image contains the static manifest required at runtime."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import django
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+EXPECTED_BACKEND = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+REQUIRED_ASSET = "courses.css"
+
+
+def _fail(message: str) -> int:
+    print(f"Static manifest verification failed: {message}", file=sys.stderr)
+    return 1
+
+
+def main() -> int:
+    try:
+        django.setup()
+        backend = settings.STORAGES["staticfiles"]["BACKEND"]
+        if backend != EXPECTED_BACKEND:
+            return _fail("staticfiles storage does not use the runtime manifest backend")
+
+        static_root = Path(settings.STATIC_ROOT).resolve()
+        manifest_path = static_root / "staticfiles.json"
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError):
+            return _fail("staticfiles.json is absent or malformed")
+
+        paths = manifest.get("paths")
+        if not isinstance(paths, dict) or not isinstance(paths.get(REQUIRED_ASSET), str):
+            return _fail(f"staticfiles.json has no {REQUIRED_ASSET} entry")
+
+        resolved_url = staticfiles_storage.url(REQUIRED_ASSET)
+        resolved_path = urlsplit(resolved_url).path
+        if not resolved_path.startswith(settings.STATIC_URL):
+            return _fail(f"resolved {REQUIRED_ASSET} URL is outside STATIC_URL")
+
+        relative_path = resolved_path.removeprefix(settings.STATIC_URL)
+        if paths[REQUIRED_ASSET] != relative_path:
+            return _fail(f"manifest and storage disagree about {REQUIRED_ASSET}")
+
+        asset_path = (static_root / relative_path).resolve()
+        if not asset_path.is_relative_to(static_root) or not asset_path.is_file():
+            return _fail(f"resolved {REQUIRED_ASSET} file is absent")
+    except Exception as error:
+        return _fail(f"{type(error).__name__} while loading the runtime storage contract")
+
+    print(f"Static manifest verified: {REQUIRED_ASSET} -> {relative_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

website/settings/collectstatic.py

@@ -0,0 +1,21 @@
+"""Build-only settings for generating release static assets.
+
+This module deliberately reuses the self-contained test bootstrap while restoring
+the manifest storage contract used by deployed settings. It must never be selected
+by a deployed process.
+"""
+
+from django.core.exceptions import ImproperlyConfigured
+
+from .test import *  # noqa: F403
+
+if RUNTIME_ENVIRONMENT in {  # noqa: F405
+    RuntimeEnvironment.DEVELOPMENT,  # noqa: F405
+    RuntimeEnvironment.PRODUCTION,  # noqa: F405
+}:
+    raise ImproperlyConfigured("Collectstatic settings are build-only")
+
+STORAGES = {
+    **STORAGES,  # noqa: F405
+    "staticfiles": {"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage"},
+}