test: add Prism-based OpenAPI mock verification (Closes #452)

.github/workflows/prism-mock.yml

@@ -0,0 +1,47 @@
+name: Prism Contract Verification
+
+on:
+  pull_request:
+    paths:
+      - 'openapi/**'
+      - 'tests/pact/**'
+      - '.github/workflows/prism-mock.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'openapi/**'
+      - 'tests/pact/**'
+
+jobs:
+  prism-contract:
+    name: Verify OpenAPI contract with Prism mock
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Start Prism mock server
+        run: |
+          docker run --rm -d --name prism \
+            -p 4010:4010 \
+            -v "$PWD/openapi:/tmp/openapi:ro" \
+            stoplight/prism:5 mock -h 0.0.0.0 /tmp/openapi/openapi.yaml
+          echo "Waiting for Prism to become ready..."
+          for i in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:4010/ >/dev/null 2>&1; then
+              echo "Prism is ready"
+              break
+            fi
+            sleep 2
+            if [ "$i" = "30" ]; then
+              echo "::error::Prism failed to start within 60s"
+              docker logs prism
+              exit 1
+            fi
+          done
+
+      - name: Replay recorded request corpus against Prism
+        run: |
+          node tests/pact/replay-prism.mjs tests/pact
+        env:
+          PRISM_URL: http://127.0.0.1:4010

scripts/prism_replay.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Replay a recorded request corpus against a Prism mock server.
+#
+# Usage:
+#     python3 scripts/prism_replay.py [corpus.json] [base_url]
+#
+# Exits non-zero if any recorded request cannot be resolved by Prism.
+# Writes a JSON coverage report to prism-coverage.json.
+import json
+import sys
+import urllib.request
+import urllib.error
+
+
+def main():
+    corpus_path = sys.argv[1] if len(sys.argv) > 1 else "tests/pact/prism-replay-corpus.json"
+    base_url = (sys.argv[2] if len(sys.argv) > 2 else "http://localhost:4010").rstrip("/")
+
+    with open(corpus_path, "r", encoding="utf-8") as f:
+        corpus = json.load(f)
+
+    report = {"total": len(corpus), "resolved": 0, "unresolved": [], "results": []}
+    for entry in corpus:
+        method = entry.get("method", "GET").upper()
+        path = entry.get("path", "/")
+        url = base_url + path
+        req = urllib.request.Request(url, method=method)
+        try:
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                status = resp.status
+                body = resp.read().decode("utf-8", "replace")
+        except urllib.error.HTTPError as e:
+            status = e.code
+            body = e.read().decode("utf-8", "replace")
+        except Exception as e:
+            status = -1
+            body = str(e)
+
+        prism_error = "stoplight.io/prism/errors" in body
+        resolved = (status < 400) and not prism_error
+
+        item = {"method": method, "path": path, "status": status, "resolved": resolved}
+        report["results"].append(item)
+        if resolved:
+            report["resolved"] += 1
+        else:
+            report["unresolved"].append({"method": method, "path": path, "status": status})
+
+    with open("prism-coverage.json", "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+
+    print("Prism replay coverage: %d/%d resolved" % (report["resolved"], report["total"]))
+    for u in report["unresolved"]:
+        print("  UNRESOLVED: %s %s (status %s)" % (u["method"], u["path"], u["status"]))
+
+    if report["unresolved"]:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tests/pact/prism-replay-corpus.json

@@ -0,0 +1,6 @@
+[
+  { "method": "GET", "path": "/api/health" },
+  { "method": "GET", "path": "/api/v1/plans" },
+  { "method": "GET", "path": "/api/subscriptions" },
+  { "method": "GET", "path": "/api/subscriptions/{id}" }
+]

tests/pact/replay-prism.mjs

@@ -0,0 +1,64 @@
+// Replays the recorded Pact request corpus against a running Prism mock
+// server and fails if any request cannot be resolved by the OpenAPI spec.
+import fs from 'node:fs';
+import path from 'node:path';
+
+const PRISM = process.env.PRISM_URL || 'http://127.0.0.1:4010';
+
+function walk(dir) {
+  const out = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...walk(p));
+    else if (entry.name.endsWith('.json')) out.push(p);
+  }
+  return out;
+}
+
+function extractRequests(pact) {
+  const reqs = [];
+  const interactions = pact.interactions || pact.pacts || pact.contracts || [];
+  for (const it of interactions) {
+    const req = it.request || it;
+    if (req && req.method && req.path) {
+      reqs.push({ method: String(req.method).toUpperCase(), path: req.path, description: it.description || '' });
+    }
+  }
+  return reqs;
+}
+
+let total = 0;
+let unresolved = 0;
+const files = walk(process.argv[2] || 'tests/pact');
+if (files.length === 0) {
+  console.error('::error::No Pact JSON files found under tests/pact');
+  process.exit(1);
+}
+
+for (const file of files) {
+  const pact = JSON.parse(fs.readFileSync(file, 'utf8'));
+  for (const req of extractRequests(pact)) {
+    total += 1;
+    const url = `${PRISM}${req.path}`;
+    try {
+      const res = await fetch(url, {
+        method: req.method,
+        headers: { 'Content-Type': 'application/json', Prefer: 'code=200, example=first' },
+      });
+      const text = await res.text();
+      const notResolved = text.includes('NO_ROUTE_MATCHED') || text.includes('No route matched');
+      if (res.status === 404 && notResolved) {
+        console.error(`FAIL ${req.method} ${req.path} - not resolvable by OpenAPI spec (${req.description})`);
+        unresolved += 1;
+      } else {
+        console.log(`OK ${res.status} ${req.method} ${req.path} (${req.description})`);
+      }
+    } catch (e) {
+      console.error(`FAIL ${req.method} ${req.path} - ${e.message}`);
+      unresolved += 1;
+    }
+  }
+}
+
+console.log(`Replayed ${total} requests, ${unresolved} unresolved.`);
+if (unresolved > 0) process.exit(1);