chore(release): make staging smoke migration-owned and dependency-aware

.env.example

@@ -5,7 +5,12 @@ DB_PORT=5432
 DB_USERNAME=postgres
 DB_PASSWORD=postgres
 DB_NAME=agentverse
-DB_SYNCHRONIZE=true
+# Migrations own the schema. Docker Compose interpolates this same file, so
+# leaving it true would auto-synchronize on top of a migrated database.
+# Run `npm run migration:run` before starting outside Docker.
+DB_SYNCHRONIZE=false
+# Container entrypoint applies migrations before starting
+RUN_MIGRATIONS_ON_START=true
 DB_LOGGING=false
 DB_SEED_ON_STARTUP=true
 
@@ -41,6 +46,8 @@ MOCK_PAYMENT_FAIL=false
 AWS_REGION=us-east-1
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# Required when NODE_ENV=production; bootstrap aborts if it is unset
+AWS_KMS_KEY_ID=
 
 # S3-compatible encrypted prompt storage (MinIO defaults for Docker Compose)
 S3_ENDPOINT=http://localhost:9000

.github/workflows/ci.yml

@@ -54,11 +54,18 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
-      - name: Lint
-        run: npm run lint
-
       - name: Test
         run: npm test
 
+      # The Jest config sets rootDir to "src", so `npm test` does not pick up
+      # anything under test/. Keep the integration suite in the CI gate.
+      - name: E2E test
+        run: npm run test:e2e
+
       - name: Build
         run: npm run build
+
+      # Lint remains last so test and build results are visible while the
+      # repository-wide debt tracked in Backend #14 is remediated.
+      - name: Lint
+        run: npm run lint

.github/workflows/staging-smoke.yml

@@ -0,0 +1,205 @@
+name: staging-smoke
+
+# Provisions a throwaway, production-like environment on every run and proves the
+# market journey against it: schema built by migrations, DB_SYNCHRONIZE=false,
+# the compiled entrypoint, and the smoke suite in test/staging-smoke.smoke-spec.ts.
+#
+# This is deliberately a separate workflow from `ci`. `ci` owns lint/test/build
+# quality gates and is red on main for reasons tracked in #14; gating release
+# evidence on that would mean this never runs.
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: staging-smoke-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: agentverse_staging
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d agentverse_staging"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      DB_HOST: 127.0.0.1
+      DB_PORT: 5432
+      DB_USERNAME: postgres
+      DB_PASSWORD: postgres
+      DB_NAME: agentverse_staging
+      # The point of this workflow: the schema is owned by migrations, never by
+      # auto-synchronize. `ci` runs with DB_SYNCHRONIZE=true, which is why the
+      # drift this suite catches was invisible there.
+      DB_SYNCHRONIZE: false
+      DB_SEED_ON_STARTUP: false
+      DB_LOGGING: false
+      JWT_SECRET: staging-smoke-secret
+      STELLAR_NETWORK: testnet
+      STELLAR_RPC_URL: https://soroban-testnet.stellar.org
+      STELLAR_NETWORK_PASSPHRASE: Test SDF Network ; September 2015
+      # No long-lived signing key and no real contract id are configured here, and
+      # neither is stored as a secret. The smoke suite runs with none at all; the
+      # boot step alone exports a throwaway keypair it generates in-process and
+      # never signs with, plus PLACEHOLDER contract ids, because production env
+      # validation requires those variables to be present. See
+      # docs/release-runbook.md for what a configured environment needs.
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Verify the deploy entrypoints exist
+        run: |
+          set -euo pipefail
+          for artifact in dist/main.js dist/database/data-source.js; do
+            if [ ! -f "$artifact" ]; then
+              echo "::error::$artifact was not emitted; the container entrypoint cannot start"
+              exit 1
+            fi
+            echo "ok: $artifact"
+          done
+
+      - name: Apply migrations with the compiled data source
+        run: npm run migration:run:prod
+
+      - name: Verify the schema is owned by migrations
+        run: |
+          set -euo pipefail
+          applied=$(node -e "
+            const { Client } = require('pg');
+            const c = new Client({
+              host: process.env.DB_HOST, port: Number(process.env.DB_PORT),
+              user: process.env.DB_USERNAME, password: process.env.DB_PASSWORD,
+              database: process.env.DB_NAME,
+            });
+            c.connect()
+              .then(() => c.query('SELECT count(*)::int AS n FROM migrations'))
+              .then((r) => { console.log(r.rows[0].n); return c.end(); })
+              .catch((e) => { console.error(e.message); process.exit(1); });
+          ")
+          echo "migrations applied: $applied"
+          if [ "$applied" -lt 5 ]; then
+            echo "::error::expected at least 5 applied migrations, found $applied"
+            exit 1
+          fi
+
+      - name: Start the compiled application under production validation
+        run: |
+          set -euo pipefail
+          # Generated per run, never written to a file, never echoed, never
+          # persisted. env.validation.ts requires the variable in production;
+          # this run never signs anything with it. Masked so it cannot appear in
+          # the log even if something downstream decides to print its environment.
+          STELLAR_ADMIN_SECRET_KEY="$(node -e "console.log(require('@stellar/stellar-sdk').Keypair.random().secret())")"
+          echo "::add-mask::$STELLAR_ADMIN_SECRET_KEY"
+          export STELLAR_ADMIN_SECRET_KEY
+          # Required by the same validator. Nothing in this run calls KMS; a real
+          # environment supplies the deployed key id.
+          export AWS_KMS_KEY_ID="alias/agentverse-staging"
+          export AWS_REGION="us-east-1"
+          export CORS_ORIGINS="http://localhost:3000"
+          export SOROBAN_TOKEN_MINT_CONTRACT_ID="PLACEHOLDER"
+          export SOROBAN_TOKEN_SALE_CONTRACT_ID="PLACEHOLDER"
+          export SOROBAN_MARKETPLACE_CONTRACT_ID="PLACEHOLDER"
+          export NODE_ENV=production
+          export PORT=3000
+          # nohup so the process survives this step's shell exiting.
+          nohup node dist/main > staging-boot.log 2>&1 &
+          echo $! > app.pid
+          for attempt in $(seq 1 30); do
+            if curl -fsS http://127.0.0.1:3000/api/health/live > /dev/null 2>&1; then
+              echo "application answered liveness after ${attempt}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::application did not become live within 30s"
+          cat staging-boot.log
+          exit 1
+
+      - name: Probe readiness against the deployed schema
+        run: |
+          set -euo pipefail
+          body=$(curl -fsS http://127.0.0.1:3000/api/health)
+          echo "$body"
+          node -e "
+            const report = JSON.parse(process.argv[1]);
+            const byName = Object.fromEntries(report.checks.map((c) => [c.name, c]));
+            const failures = [];
+            if (report.status !== 'ok') failures.push('status is ' + report.status);
+            if (byName.database?.status !== 'ok') failures.push('database check failed');
+            if (byName.schema?.status !== 'ok') failures.push('schema check failed');
+            if (byName.schema?.detail !== 'AlignMigratedSchemaWithEntities1700000004000') {
+              failures.push('unexpected applied migration: ' + byName.schema?.detail);
+            }
+            if (failures.length) { console.error(failures.join('; ')); process.exit(1); }
+            console.log('readiness verified');
+          " "$body"
+
+      - name: Stop the application
+        if: always()
+        run: |
+          if [ -f app.pid ]; then kill "$(cat app.pid)" 2>/dev/null || true; fi
+
+      - name: Run the staging smoke suite
+        run: npm run test:smoke
+
+      - name: Assert no signing key reached the repository or the logs
+        if: always()
+        run: |
+          set -euo pipefail
+          # Stellar secret seeds are 56 characters starting with 'S'. Committed
+          # tracked files and this run's captured output must contain none.
+          if git grep -nIE '\bS[A-Z2-7]{55}\b' -- . ':!*.lock' ':!package-lock.json'; then
+            echo "::error::a Stellar secret seed appears in a tracked file"
+            exit 1
+          fi
+          if [ -f staging-boot.log ] && grep -qE '\bS[A-Z2-7]{55}\b' staging-boot.log; then
+            echo "::error::a Stellar secret seed was printed to the application log"
+            exit 1
+          fi
+          echo "no signing key found in tracked files or captured logs"
+
+      - name: Upload smoke evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: staging-smoke-evidence
+          path: staging-boot.log
+          if-no-files-found: warn
+          retention-days: 14

Dockerfile

@@ -19,6 +19,9 @@ RUN chmod +x /app/entrypoint.sh && \
     chown -R appuser:appgroup /app
 USER appuser
 EXPOSE 3000
+# /api/health is dependency-aware and answers 503 when the database or schema is
+# unavailable, so an unhealthy container now means something. Restart-only probes
+# should target /api/health/live instead.
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
   CMD wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1
 ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -26,9 +26,14 @@ Environment validation is centralized in `src/config/env.validation.ts`.
 ```bash
 npm ci
 cp .env.example .env
+npm run migration:run
 npm run start:dev
 ```
 
+`.env.example` sets `DB_SYNCHRONIZE=false`, so the schema comes from migrations
+here exactly as it does in a deployment. Until they are applied, `/api/health`
+reports the missing schema and answers 503.
+
 ## Docker
 
 ```bash
@@ -39,16 +44,44 @@ docker compose down
 ## Tests
 
 ```bash
-npm test
+npm test          # unit tests
+npm run test:e2e  # e2e tests, no database required
 npm run test:cov
 npm run build
 ```
 
+The staging smoke suite is separate because it needs a real, migrated database:
+
+```bash
+npm run migration:run
+npm run test:smoke
+```
+
+It refuses to run unless `DB_HOST` and `DB_NAME` are set, so it cannot pass by
+finding nothing to do.
+
 ## CI
 
 `.github/workflows/ci.yml` runs on push and pull request to `main`.
 It installs dependencies, then runs lint, tests, and build.
 
+`.github/workflows/staging-smoke.yml` provisions a throwaway production-like
+environment on the same events: it builds, applies migrations with the compiled
+data source, boots the compiled app under production validation with
+`DB_SYNCHRONIZE=false`, probes `/api/health`, and runs the smoke suite.
+
+## Health
+
+| Endpoint | Answers | Use for |
+| --- | --- | --- |
+| `GET /api/health` | Database, applied migration, Soroban RPC, marketplace contract, delivery worker. `503` when a required dependency is down | Rollout gates, load balancers |
+| `GET /api/health/live` | Process liveness only, always `200` | Restart probes |
+
+## Deployment
+
+See `docs/release-runbook.md` for the environment contract, deploy and rollback
+procedure, evidence to record, and the gaps that block a full release.
+
 ## Swagger
 
 OpenAPI docs are available in development at:
@@ -68,7 +101,7 @@ Docs use bearer auth and stay disabled in production unless `SWAGGER_ENABLED=tru
 | `DB_USERNAME` | `postgres` | Database user |
 | `DB_PASSWORD` | `postgres` | Database password |
 | `DB_NAME` | `agentverse` | Database name |
-| `DB_SYNCHRONIZE` | `true` in dev, `false` in prod | TypeORM schema sync |
+| `DB_SYNCHRONIZE` | `false` in `.env.example`; `false` in prod | TypeORM schema sync. Keep `false` wherever migrations own the schema — Compose interpolates `.env`, so this file decides what the container gets |
 | `DB_LOGGING` | `false` | TypeORM SQL logging |
 | `JWT_SECRET` | `dev-secret` in dev | JWT signing secret |
 | `JWT_EXPIRES_IN` | `24h` | JWT token lifetime |
@@ -77,24 +110,26 @@ Docs use bearer auth and stay disabled in production unless `SWAGGER_ENABLED=tru
 | `PROMPT_CONTENT_ENCRYPTION_KEY` | — | Base64-encoded 32-byte key used to encrypt prompt blobs before storage |
 | `AWS_KMS_KEY_ID` | — | KMS key used with tenant and delivery encryption context |
 | `PROMPT_DELIVERY_WORKER_ENABLED` | `false` | Enables the PostgreSQL delivery worker polling loop |
+| `STELLAR_NETWORK` | `testnet` | Stellar network name |
+| `STELLAR_RPC_URL` | `https://soroban-testnet.stellar.org` | Soroban RPC endpoint |
+| `STELLAR_NETWORK_PASSPHRASE` | `Test SDF Network ; September 2015` | Stellar network passphrase |
+| `CORS_ORIGINS` | `*` in dev | Comma-separated allowed origins |
+| `SOROBAN_TOKEN_MINT_CONTRACT_ID` | empty | Mint contract ID |
+| `SOROBAN_TOKEN_SALE_CONTRACT_ID` | empty | Sale contract ID |
+| `STELLAR_ADMIN_SECRET_KEY` | empty | Admin signing key. Optional at boot; required by env validation when `NODE_ENV=production` |
+| `SWAGGER_ENABLED` | `false` in prod | Force docs on in production |
+| `RUN_MIGRATIONS_ON_START` | `true` | Container entrypoint applies migrations before starting |
 
 ## Marketplace purchase flow
 
-Purchase intents are JWT-protected and scoped to published `PROMPT` assets. The buyer signs the returned unsigned XDR locally, then submits only the transaction hash for RPC verification. Run the purchase migration before starting a deployment:
+Purchase intents are JWT-protected and scoped to published `PROMPT` assets. The buyer signs the returned unsigned XDR locally, then submits only the transaction hash for RPC verification. Migrations must be applied before a deployment serves traffic; the container entrypoint does this automatically, and outside Docker:
 
 ```bash
-npm run migration:run
+npm run migration:run       # from source, uses ts-node
+npm run migration:run:prod  # from dist/, for the production image
 ```
 
 Production requires `SOROBAN_MARKETPLACE_CONTRACT_ID`; the Testnet contract cannot be omitted or replaced by the development mock.
-| `STELLAR_NETWORK` | `testnet` | Stellar network name |
-| `STELLAR_RPC_URL` | `https://soroban-testnet.stellar.org` | Soroban RPC endpoint |
-| `STELLAR_NETWORK_PASSPHRASE` | `Test SDF Network ; September 2015` | Stellar network passphrase |
-| `CORS_ORIGINS` | `*` in dev | Comma-separated allowed origins |
-| `SOROBAN_TOKEN_MINT_CONTRACT_ID` | empty | Mint contract ID |
-| `SOROBAN_TOKEN_SALE_CONTRACT_ID` | empty | Sale contract ID |
-| `STELLAR_ADMIN_SECRET_KEY` | empty | Optional admin key |
-| `SWAGGER_ENABLED` | `false` in prod | Force docs on in production |
 
 ## Notes
 

docker-compose.yml

@@ -19,6 +19,9 @@ services:
       timeout: 5s
       retries: 5
 
+  # Reserved for the encrypted prompt blob store. No application code imports
+  # @aws-sdk/client-s3 yet, so the api service does not wait on it; gating
+  # startup on a bucket nothing reads only adds a way for a clean deploy to hang.
   minio:
     image: minio/minio:latest
     container_name: agentverse-storage
@@ -54,7 +57,10 @@ services:
       DB_USERNAME: ${DB_USERNAME:-postgres}
       DB_PASSWORD: ${DB_PASSWORD:-postgres}
       DB_NAME: ${DB_NAME:-agentverse}
-      DB_SYNCHRONIZE: ${DB_SYNCHRONIZE:-true}
+      # The schema is applied by entrypoint.sh via migration:run, not by
+      # TypeORM auto-synchronize. Auto-sync against a migrated database silently
+      # rewrites columns and indexes.
+      DB_SYNCHRONIZE: ${DB_SYNCHRONIZE:-false}
       JWT_SECRET: ${JWT_SECRET:-dev-secret}
       PORT: 3000
       STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet}
@@ -64,8 +70,6 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-      minio:
-        condition: service_healthy
     entrypoint: [ "/app/entrypoint.sh" ]
 
 volumes: