From 192364eca28cd8cfab618986b2bbce6de204f19a Mon Sep 17 00:00:00 2001 From: kingksjo Date: Wed, 26 Aug 2026 20:48:30 +0100 Subject: [PATCH] feat: implement release checklist, incident runbook, and frontend enhancements (#1146, #1149, #979, #983) --- CHANGELOG.md | 4 + docs/DEPLOYMENT_CHECKLIST.md | 6 +- docs/FRONTEND_STATE_MANAGEMENT.md | 11 +- docs/RELEASE_CHECKLIST.md | 243 ++++++++++++++++ docs/runbooks/README.md | 1 + docs/runbooks/rollback-and-hotfix.md | 267 ++++++++++++++++-- .../AccountStatementExport.test.tsx | 57 ++++ .../components/NetworkWarningBanner.test.tsx | 86 ++++++ 8 files changed, 654 insertions(+), 21 deletions(-) create mode 100644 docs/RELEASE_CHECKLIST.md create mode 100644 frontend/src/components/NetworkWarningBanner.test.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c7d9542..a8c8b9f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Features +- Implement network mismatch detector and guided fix flow with instant Freighter recheck and accessible step-by-step guidance (#979) +- Add downloadable account statement export flow with custom date range filtering, holdings summary, and CSV/JSON serialization (#983) - Add advanced filter and sort to the transaction history table: multi-column sort (up to three keys) with shift-click tiebreakers, a keyboard-reachable sort panel, URL-shareable ordering with legacy single-column links still honoured, relative date presets, removable active-filter chips, and inline reporting of contradictory ranges (#1035) - Harden the transfer orchestration service to be idempotent and retry-safe: wallet-scoped idempotency keys, request validation and canonicalisation, submission-boundary failure classification that never blindly resubmits a transfer whose outcome is unknown, stored terminal rejections, circuit-breaker fail-fast, a submission timeout, and an operator reconciliation queue with metrics (#1043) - Harden the vault comparison screen for multi-strategy selection: numeric strategy catalog with locale-aware formatting, URL-synced shareable selection and column ordering, best-in-class marking with non-colour cues, and screen-reader announcements for selection and sort changes (#1036) @@ -27,6 +29,8 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Vault performance dynamic date filter ### Documentation +- Create release checklist for testnet and mainnet deployment covering preflight, deployment, validation, and multi-tier sign-off (#1146) +- Add runbook for incident response and rollback operations with failure scenarios, activation triggers, and operational owner matrix (#1149) - Add incident postmortem templates, publication playbook, and CI validation workflow (#769) - Add release notes playbook and changelog curation guidelines (#618) - Add API versioning and deprecation policy with sunset windows, migration guide, and breaking-change classification (#610) diff --git a/docs/DEPLOYMENT_CHECKLIST.md b/docs/DEPLOYMENT_CHECKLIST.md index 2768d8da..143b8a56 100644 --- a/docs/DEPLOYMENT_CHECKLIST.md +++ b/docs/DEPLOYMENT_CHECKLIST.md @@ -1,8 +1,8 @@ # 🚀 YieldVault-RWA: Deployment Checklist (Testnet & Mainnet) -**Issue:** [#1060](https://github.com/Junirezz/YieldVault-RWA/issues/1060) -**Purpose:** A comprehensive, step-by-step deployment checklist covering both **Testnet** and **Mainnet** environments for the YieldVault-RWA stack (smart contracts, backend, frontend, and infrastructure). -**Last Updated:** July 28, 2026 +**Issues:** [#1146](https://github.com/kingksjo/YieldVault-RWA/issues/1146), [#1060](https://github.com/Junirezz/YieldVault-RWA/issues/1060) +**Purpose:** A comprehensive, step-by-step deployment checklist covering both **Testnet** and **Mainnet** environments for the YieldVault-RWA stack (smart contracts, backend, frontend, and infrastructure). See also [Release Checklist](./RELEASE_CHECKLIST.md). +**Last Updated:** August 2026 --- diff --git a/docs/FRONTEND_STATE_MANAGEMENT.md b/docs/FRONTEND_STATE_MANAGEMENT.md index 5b42bcd0..33fa9767 100644 --- a/docs/FRONTEND_STATE_MANAGEMENT.md +++ b/docs/FRONTEND_STATE_MANAGEMENT.md @@ -112,11 +112,20 @@ Custom hooks in `frontend/src/hooks/` encapsulate all complex logic. **Mutation Hooks (React Query):** - **`useVaultMutations`**: Exposes `useDepositMutation` and `useWithdrawMutation` for executing Soroban contract calls. Applies optimistic cache updates via `optimisticVaultCache`, rolls back on failure, and invalidates related query caches on settle. -**Utility Hooks:** +**Utility Hooks & State Detectors:** +- **`useWalletNetwork`**: Polls and monitors the connected Freighter wallet network passphrase against configured network settings. Exposes `isMismatch`, `walletNetwork`, `expectedNetwork`, and `checkNow()` for instant recheck without waiting for polling intervals. - **`useClientDataTable`**: Handles client-side pagination, sorting, and text-based filtering of arrays. - **`useInfiniteScroll`**: Manages IntersectionObserver logic for loading more items in infinite-scroll views. - **`useFeeEstimate` / `useSlippage`**: Calculates estimated transaction fees and slippage parameters for deposits and withdrawals. +### Network Mismatch Detector & Guided Fix Flow +- **Components**: `NetworkWarningBanner`, `NetworkMismatchGuideModal`. +- **Behavior**: When a connected wallet is on a mismatched network (e.g. Freighter on Mainnet when app is on Testnet), a persistent, accessible banner alerts the user. Clicking "Show me how to fix this" opens a step-by-step guided modal that allows instant network verification (`checkNow()`) and automatically dismisses upon successful resolution. + +### Account Statement Export Flow +- **Components & Libs**: `AccountStatementExport`, `lib/accountStatement.ts`, `lib/exportDownload.ts`. +- **Behavior**: Generates client-side CSV or JSON account statements covering holdings and transaction histories. Supports customizable start/end date range filtering, formatted ISO timestamps, asset amounts, and secure download triggers. + --- ## Component Architecture diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md new file mode 100644 index 00000000..db686b37 --- /dev/null +++ b/docs/RELEASE_CHECKLIST.md @@ -0,0 +1,243 @@ +# 🚀 YieldVault-RWA: Release Checklist (Testnet & Mainnet) + +**Issues:** [#1146](https://github.com/kingksjo/YieldVault-RWA/issues/1146), [#1060](https://github.com/kingksjo/YieldVault-RWA/issues/1060) +**Purpose:** A formal, standardized release checklist covering preflight verification, environment-specific deployment procedures, post-deployment validation, rollback triggers, and multi-disciplinary sign-off for **Testnet (Staging)** and **Mainnet (Production)**. +**Maintained By:** Release Engineering & Operations + +--- + +## 1. How to Use This Checklist + +1. **Target Environment Selection**: Identify whether this release is for Testnet (staging) or Mainnet (production). +2. **Sequential Execution**: Complete each section in order. Check items `[x]` upon verification. +3. **Waivers**: If an item is not applicable or waived, replace `[ ]` with `[~]` and document the rationale inline. +4. **Sign-Off Gate**: Obtain all mandatory signatures from Engineering and Operations leads (Section 7) prior to tagging production or applying migrations. +5. **Post-Deployment Verification**: Complete Section 5 within 30 minutes of deployment. +6. **Filing**: Attach the filled checklist to the GitHub Release or release tracking ticket. + +### Release Metadata + +| Metadata Field | Value | +|----------------|-------| +| **Target Environment** | `[ ] Testnet` / `[ ] Mainnet` | +| **Release Version / Git Tag** | `v...` | +| **Target Commit Hash** | `...` | +| **Deployment Lead** | `@username` | +| **Secondary Reviewer** | `@username` | +| **DevOps / Ops Lead** | `@username` | +| **Deployment Window (UTC)** | `YYYY-MM-DD HH:MM UTC` | +| **Incident War Room** | `#yieldvault-war-room` / `#yieldvault-incidents` | + +--- + +## 2. Environment Matrix & Configuration Guidance + +| Configuration Item | Testnet (Staging) | Mainnet (Production) | +|--------------------|-------------------|----------------------| +| **Network Name** | `testnet` | `mainnet` | +| **Passphrase** | `Test SDF Network ; September 2015` | `Public Global Stellar Network ; September 2015` | +| **Soroban RPC URL** | `https://soroban-testnet.stellar.org` | `https://soroban-mainnet.stellar.org` (with failover) | +| **Horizon API URL** | `https://horizon-testnet.stellar.org` | `https://horizon.stellar.org` | +| **Deployer Funding** | Stellar Friendbot faucet | Dedicated cold/warm operations wallet (> 200 XLM) | +| **Admin Key Model** | Dedicated staging keypair | Multi-signature (3-of-5) or Hardware Security Module (HSM) | +| **Emergency Approvers** | Staging ops key | Multi-party distinct emergency approvers | +| **Contract Config** | `deployments/contracts.testnet.json` | `deployments/contracts.mainnet.json` | +| **Backend DB** | Staging PostgreSQL (managed) | Production HA PostgreSQL (multi-AZ, automated backups) | +| **Backend Cache** | Staging Redis | Production Redis (TLS + ACL enabled) | +| **Frontend Base URL** | `https://staging.yieldvault.finance` | `https://yieldvault.finance` | +| **Backend Base URL** | `https://staging-api.yieldvault.finance` | `https://api.yieldvault.finance` | +| **Alerting Destination** | Slack `#staging-alerts` | PagerDuty + Slack `#yieldvault-incidents` | + +--- + +## 3. Preflight Checklist (All Environments) + +### 3.1 Toolchains & Shared Packages +- [ ] **Shared Schemas Built**: `cd packages/api-schemas && npm ci && npm run build` (Ensures `@yieldvault/api-schemas` is up to date). +- [ ] **Rust Toolchain**: `rustc --version` matches pinned toolchain (`rust-toolchain.toml`). +- [ ] **WASM Target Installed**: `rustup target list --installed | grep wasm32-unknown-unknown`. +- [ ] **Soroban CLI Version**: `stellar --version` or `soroban --version` verified against pinned version. +- [ ] **Node.js Runtime**: `node --version` is Node 20.x. +- [ ] **Clean Working Tree**: `git status` clean with no untracked or modified artifacts. + +### 3.2 Smart Contracts Preflight +- [ ] **Unit Tests**: `cargo test` passes for all crates (`vault`, `mock-strategy`, `share-price-math`). +- [ ] **Share Price Math Formatting & Clippy**: + - `cargo fmt --all -- --check` passes. + - `cargo clippy -p share-price-math --all-targets -- -D warnings` passes. +- [ ] **Fuzz Testing**: `share-price-math` fuzz target runs with zero regressions. +- [ ] **Storage Layout**: Storage layout verified against `docs/storage-layout-reference.md`. +- [ ] **WASM Optimization**: Contract WASM built and optimized (`soroban contract optimize`). +- [ ] **Hash Computation**: SHA256 checksums of release WASM binaries recorded. + +### 3.3 Backend Preflight +- [ ] **Linting**: `cd backend && npm run lint` exits 0. +- [ ] **Unit & Integration Tests**: `cd backend && npm test` (with `maxWorkers: 1`) passes all test suites. +- [ ] **Governance & Snapshot Checks**: + - `npm run snapshots:check` exits 0. + - `npm run generate:openapi` produces no uncommitted diff against `backend/openapi.json`. + - `npm run check:migrations:canary` verifies schema compatibility. +- [ ] **Migration Safety**: All Prisma migrations verified to be non-destructive (additive changes only; no uncoordinated drops). +- [ ] **Rollback SQL Script**: Reversible migration rollback scripts documented at `backend/prisma/migrations//rollback.sql`. + +### 3.4 Frontend Preflight +- [ ] **Linting**: `cd frontend && npm run lint` passes with no errors. +- [ ] **Unit Tests**: `cd frontend && npm run test:run` completes with 100% passing suites. +- [ ] **Typecheck & Production Build**: `cd frontend && npm run build` (`tsc -b && vite build`) completes cleanly. +- [ ] **Bundle Size Budget**: `npm run check-size` confirms JS bundle ≤ 450 kB gzip, CSS ≤ 50 kB gzip. +- [ ] **Environment Validation**: `npm run validate:frontend-env -- --strict` passes for target environment. + +### 3.5 Security, Environment & Secrets +- [ ] **Secrets Scan**: Pre-commit / CI secret check (`scripts/secrets-check.js`) passes. +- [ ] **Security Audits**: + - `cargo audit` zero high/critical vulnerabilities. + - `npm audit --audit-level=high` zero unmitigated high findings. +- [ ] **Environment Variables Audit**: Target environment variables verified against `docs/ENV_VARIABLE_MATRIX.md`. +- [ ] **Deployer Balance**: Deployer wallet funded with sufficient native XLM for transaction fees. + +### 3.6 CI/CD Release Workflows +- [ ] **CI Pipeline Status**: All target branch workflows green: + - `backend-governance.yml` + - `frontend.yml` + - `rust-wasm.yml` + - `rust-security.yml` + - `secret-scanning.yml` + - `e2e.yml` (staging) + +--- + +## 4. Deployment Procedures + +### 4.1 Testnet (Staging) Deployment Flow +1. **Contract Deployment**: + ```bash + ./scripts/deploy_contracts.sh testnet + ``` + - Update `deployments/contracts.testnet.json` with newly deployed contract IDs. +2. **Contract Initialization**: + - Initialize vault instance with admin address and testnet USDC token contract. + - Verify: `soroban contract invoke --id --network testnet -- version`. +3. **Backend Deployment**: + - Apply Prisma migrations: `npx prisma migrate deploy`. + - Start backend service with `VAULT_CONTRACT_ID` set. + - Verify health: `curl -f https://staging-api.yieldvault.finance/health`. +4. **Frontend Deployment**: + - Deploy build artifact to staging environment (Vercel/Cloudflare). + - Verify staging URL loads with correct network configuration. + +### 4.2 Mainnet (Production) Deployment Flow +1. **Change Window & Notice**: Announce scheduled release window in `#yieldvault-incidents` and status channels. +2. **Database Backup**: Take a manual, verified snapshot of the production PostgreSQL database. +3. **Contract Upgrade / Deployment**: + - **For Upgrades**: Follow `docs/runbooks/CONTRACT_UPGRADE_PLAYBOOK.md`: + 1. Set pause: `soroban contract invoke --id --source admin --network mainnet -- set_pause --paused true`. + 2. Install new WASM: `soroban contract install --wasm --network mainnet`. + 3. Execute upgrade via multisig: `soroban contract invoke --id --source admin --network mainnet -- upgrade --new_wasm_hash `. + 4. Verify version & state: `soroban contract invoke --id --network mainnet -- version`. + 5. Unpause vault: `soroban contract invoke --id --source admin --network mainnet -- set_pause --paused false`. + - **For Fresh Deployment**: + 1. Deploy WASM: `soroban contract deploy --wasm --source deployer --network mainnet`. + 2. Initialize vault parameters (Admin multisig, Treasury address, Fee BPS, Caps). + 3. Record addresses in `deployments/contracts.mainnet.json`. +4. **Backend Production Deployment**: + - Execute database migration: `npx prisma migrate deploy`. + - Deploy backend containers/instances with rolling update strategy (zero downtime). + - Confirm healthy readiness probe (`/ready`) across all backend instances. +5. **Frontend Production Deployment**: + - Push release tag `v*.*.*` triggering `production-deploy.yml` (or trigger production Vercel deployment). + - Invalidate edge CDN cache if necessary. + +--- + +## 5. Post-Deployment Validation + +### 5.1 Immediate Checks (T+0 to T+10 min) +- [ ] **Health & Readiness Endpoints**: + - `GET /health` returns HTTP 200 with status `"healthy"`. + - `GET /ready` returns HTTP 200 with status `{"ready": true}` and all subsystem probes green. +- [ ] **Core API Responses**: + - `GET /api/v1/vault/summary` returns valid non-null numerical metrics. + - `GET /api/v1/transactions?limit=1` returns properly structured envelope. + - `GET /api/v1/vault/apy/history?days=7` responds within latency SLO. +- [ ] **Frontend Verification**: + - App loads cleanly with no unhandled console errors or missing chunk errors. + - Connected wallet correctly identifies target network (no mismatch warning). + - Share price, APY, and Total Value Locked render accurately on the dashboard. +- [ ] **Contract Smoke Invocations**: + - `total_assets()` matches expected on-chain accounting. + - `total_shares()` matches expected share issuance. + - `paused()` returns `false`. + +### 5.2 Extended Monitoring (T+10 to T+30 min) +- [ ] **End-to-End User Journeys**: + - Perform test deposit transaction (or verify telemetry on live user deposits). + - Perform test withdrawal transaction. + - Confirm transactions appear in history index within 1 ledger close (~5 seconds). +- [ ] **Telemetry & Error Budgets**: + - P95 latency on critical API endpoints remains under 200 ms. + - HTTP 5xx error rate remains < 0.1%. + - Zero unhandled exceptions in Sentry. + - Soroban RPC error rate < 1%. +- [ ] **Background Workers**: + - APY snapshot background job running on schedule. + - Event indexing worker keeping pace with the Stellar ledger tip. + +--- + +## 6. Rollback Triggers & Procedures + +### 6.1 Rollback Activation Triggers +Initiate immediate rollback if any of the following occur during the 30-minute validation window: +- `/health` or `/ready` fails continuously for > 2 minutes. +- HTTP 5xx error rate on critical write or read endpoints exceeds 5%. +- Critical smart contract logic failure, accounting invariant violation, or unexpected revert on deposits/withdrawals. +- Database data corruption or migration failure affecting core entities. +- Active security vulnerability discovered or exploited. + +### 6.2 Rollback Actions by Layer +- **Smart Contracts**: + 1. Immediately pause vault: `soroban contract invoke --id --source admin --network -- set_pause --paused true`. + 2. If upgrade corrupted logic: downgrade to previous known good WASM hash (`upgrade --new_wasm_hash `). +- **Backend Service**: + 1. Roll back container image / deployment to previous git tag. + 2. If migrations were executed, run designated down-migration: `psql -f backend/prisma/migrations//rollback.sql`. +- **Frontend SPA**: + 1. Trigger instant rollback in Vercel or point CDN traffic back to previous immutable build hash. +- **Incident Escalation**: + 1. Page Incident Commander and log event in `#yieldvault-incidents`. + 2. Open postmortem tracking ticket. + +--- + +## 7. Sign-Off & Approvals + +### Testnet (Staging) Sign-Off + +| Role | Name / GitHub Handle | Status | Date (UTC) | +|------|----------------------|--------|------------| +| **Deployment Lead** | | `[ ] Approved` | | +| **Secondary Reviewer** | | `[ ] Approved` | | +| **QA / Integration Tester** | | `[ ] Approved` | | + +### Mainnet (Production) Sign-Off + +| Role | Name / GitHub Handle | Status | Date (UTC) | +|------|----------------------|--------|------------| +| **Release Owner** | | `[ ] Approved` | | +| **Smart Contracts Lead** | | `[ ] Approved` | | +| **Backend Lead** | | `[ ] Approved` | | +| **Frontend Lead** | | `[ ] Approved` | | +| **DevOps / Operations Lead** | | `[ ] Approved` | | +| **Security Lead** | | `[ ] Approved` | | +| **QA / Release Manager** | | `[ ] Approved` | | + +--- + +## 8. Related Documentation & Workflows + +- [Deployment Guide](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/DEPLOYMENT.md) +- [Deployment Checklist (Operational)](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/DEPLOYMENT_CHECKLIST.md) +- [Release Readiness Checklist](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/RELEASE_READINESS_CHECKLIST.md) +- [Incident Response & Rollback Runbook](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/runbooks/rollback-and-hotfix.md) +- [Contract Upgrade Playbook](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/runbooks/CONTRACT_UPGRADE_PLAYBOOK.md) +- [Environment Variable Matrix](file:///C:/Users/Kamiye/Desktop/drips/YieldVault-RWA/docs/ENV_VARIABLE_MATRIX.md) diff --git a/docs/runbooks/README.md b/docs/runbooks/README.md index c0ab94f2..cda314f4 100644 --- a/docs/runbooks/README.md +++ b/docs/runbooks/README.md @@ -9,6 +9,7 @@ This directory contains operational runbooks for disaster recovery and incident | Runbook | RTO | RPO | Use When | |---------|-----|-----|----------| | [Incident Triage & Severity Classification](./INCIDENT_TRIAGE_AND_SEVERITY.md) | N/A | N/A | First responder triage and severity classification for any alert | +| [Incident Response & Rollback Operations](./rollback-and-hotfix.md) | 15 min | N/A | Emergency pause, incident escalation, and multi-tier rollback procedures | | [RTO/RPO Targets](./RTO_RPO_TARGETS.md) | N/A | N/A | Understanding recovery objectives | | [Database Restore](./DATABASE_RESTORE.md) | 1 hour | 15 min | Database corruption or failure | | [Backend Redeploy](./BACKEND_REDEPLOY.md) | 30 min | N/A | Backend service issues | diff --git a/docs/runbooks/rollback-and-hotfix.md b/docs/runbooks/rollback-and-hotfix.md index 8c9e7606..cc50b3c9 100644 --- a/docs/runbooks/rollback-and-hotfix.md +++ b/docs/runbooks/rollback-and-hotfix.md @@ -1,25 +1,258 @@ -# Production Runbook: Deployment Rollback & Hotfixes +# Incident Response & Rollback Operations Runbook -## 1. Trigger Criteria & Incident Severity +**Issue:** [#1149](https://github.com/kingksjo/YieldVault-RWA/issues/1149) +**Purpose:** Operational playbook defining incident triage, escalation protocols, rollback triggers, component-specific rollback procedures, and communication cadences for the YieldVault RWA platform. +**Audience:** Incident Commanders, On-Call Engineers (Backend, Contracts, Frontend, Platform/DevOps), Security Team +**Last Updated:** August 2026 -### Severe Incidents (Triggers Rollback or Immediate Hotfix) -- **Financial/State Risk:** Vault balance accounting mismatch, unauthorized yield distribution, or smart contract logic flaw. -- **Service Outage:** Frontend or indexing API down/unresponsive post-deployment. -- **Security Vulnerability:** Exploitable entrypoint or leaked credentials/keys. +--- + +## 1. Incident Severity & Operational Ownership + +### 1.1 Severity Classification Matrix + +| Severity | Definition & Impact | Response SLA | Mitigation Target | War Room Required? | +|---|---|---|---|---| +| **Sev0 (Critical)** | Core functionality unavailable; user funds or vault accounting at risk; active security breach; critical data loss exceeding 15m RPO. | **≤ 5 min** | Immediate / All hands | Yes (`#yieldvault-war-room`) | +| **Sev1 (High)** | Core feature degraded (e.g. deposits failing, RPC degraded >20%, P95 >5x SLO); circuit breaker open; transaction delivery stall. | **≤ 15 min** | ≤ 60 min | Yes (`#yieldvault-incidents`) | +| **Sev2 (Medium)** | Non-critical feature broken (e.g. analytics chart blank, export failure); minor latency elevation; single wallet impact with workaround. | **≤ 1 hour** | ≤ 4 business hours | No (threaded discussion) | +| **Sev3 (Low)** | Cosmetic UI glitch; documentation error; flaky non-blocking test; minor non-customer-facing anomaly. | **≤ 1 business day** | Next release cycle | No (GitHub Issue) | + +### 1.2 Operational Ownership & Incident Team Roles + +| Role | Primary Responsibilities | Current Escalation Path | +|---|---|---| +| **Incident Commander (IC)** | Leads incident lifecycle, declares severity, coordinates communications, authorizes rollbacks or emergency pauses. | PagerDuty: `YieldVault IC` | +| **Smart Contracts Lead** | Diagnoses on-chain invariants, executes emergency pause, coordinates WASM hash rollbacks or contract upgrades. | `#contracts-oncall` | +| **Backend Lead** | Triages API errors, analyzes database logs, executes service rollback and down-migrations. | `#backend-oncall` | +| **Frontend Lead** | Diagnoses client runtime errors, wallet connection regressions, triggers edge/Vercel rollbacks. | `#frontend-oncall` | +| **DevOps / Platform Lead** | Manages RPC failovers, load balancers, database snapshots/PITR, container rollbacks. | `#platform-oncall` | +| **Security Lead** | Assesses exploit vectors, coordinates key rotations, verifies integrity during breaches. | PagerDuty: `YieldVault Security` | +| **Comms / Support Lead** | Posts public status page updates, coordinates customer success notifications. | `#comms-incident` | + +--- + +## 2. Common Failure Scenarios & Triage Steps + +```mermaid +flowchart TD + A[Alert Received / Anomaly Detected] --> B{Initial Triage: T+0 to T+5m} + B --> C[Validate Health & Metrics] + C --> D{Is Funds/State at Risk?} + D -- Yes --> E[Sev0: Pause Vault & Page IC + Security] + D -- No --> F{Is Core Service Down?} + F -- Yes --> G[Sev1: Page IC & Relevant SME] + F -- No --> H[Sev2/Sev3: Log Ticket & Investigate] + E --> I[Assess Rollback Activation Conditions] + G --> I + I -- Triggers Met --> J[Execute Component Rollback] + I -- Triggers Not Met --> K[Deploy Targeted Hotfix] + J --> L[Post-Rollback Verification & Status Update] + K --> L +``` + +### Scenario A: Smart Contract Accounting Discrepancy / Fund Safety Threat +- **Symptoms**: `total_assets()` or share price deviates from expected formula; deposit/withdraw events revert on-chain; unauthorized asset movement. +- **Immediate Action**: + 1. Incident Commander & Contracts Lead immediately trigger emergency circuit breaker: + ```bash + soroban contract invoke \ + --id \ + --source admin \ + --network \ + -- set_pause --paused true + ``` + 2. Page Security Lead if unauthorized access is suspected. + 3. Freeze indexer and backend ingestion if needed: `SET INDEXER_PAUSED=true`. + +### Scenario B: Stellar RPC Provider Failure / Extreme Latency +- **Symptoms**: Horizon/Soroban RPC requests timing out (> 1500 ms); `504 Gateway Timeout` on contract calls; indexer falling behind ledger head. +- **Immediate Action**: + 1. Switch to secondary RPC provider via configuration or load balancer: + ```bash + # Update backend environment variable or Consul config + export STELLAR_RPC_URL="https://backup-rpc.stellar.org" + systemctl reload yieldvault-backend + ``` + 2. See [`docs/runbooks/RPC_FAILOVER.md`](./RPC_FAILOVER.md) for full failover procedures. + +### Scenario C: Transaction Submission & Delivery Failures +- **Symptoms**: Transactions returning `TxFailed`, fee insufficient errors, or stuck in transaction queue. +- **Immediate Action**: + 1. Check network base fee and adjust fee bump multiplier. + 2. Clear stale pending transaction intents from memory/redis queue. + 3. Verify sequence numbers and account sequence synchronization. + +### Scenario D: Database Migration Drift / Corruption Post-Deploy +- **Symptoms**: Backend crashing with Prisma schema mismatch; foreign key violation spikes; unrecoverable write errors. +- **Immediate Action**: + 1. Stop incoming API traffic to backend. + 2. Execute migration rollback SQL: + ```bash + psql $DATABASE_URL -f backend/prisma/migrations//rollback.sql + ``` + 3. Revert backend binary/container to previous release commit. + +### Scenario E: Frontend Broken Bundle / Chunk Hydration Failures +- **Symptoms**: Blank page on user visits; unhandled JavaScript runtime exceptions in Sentry; CSS/layout corruption. +- **Immediate Action**: + 1. Trigger instant rollback in hosting platform: + ```bash + vercel rollback --token $VERCEL_TOKEN + ``` + 2. Purge Cloudflare / CDN edge cache for frontend routes. + +--- + +## 3. Rollback Activation Conditions + +The Incident Commander MUST authorize a rollback if any of the following conditions are met: + +1. **Unresolved Health Probe Failure**: Backend `/health` or `/ready` failing for > 2 consecutive minutes after deployment. +2. **Critical Error Budget Exhaustion**: HTTP 5xx error rate > 5% on tier-critical API endpoints (`/api/v1/vault/*`, `/api/v1/transactions/*`) for > 5 minutes. +3. **Smart Contract Invariant Violation**: Any state corruption, math inconsistency in share pricing, or failure in deposit/withdraw execution. +4. **Data Integrity Failure**: Database migrations producing data loss or query failures that cannot be fixed within 15 minutes. +5. **Security Vulnerability**: Critical vulnerability discovered in newly deployed code that exposes funds, user data, or API authorization keys. + +--- + +## 4. Step-by-Step Rollback Operations + +### 4.1 Smart Contracts Rollback + +#### Option 1: WASM Hash Downgrade (In-place contract rollback) +When the contract proxy or upgrade pattern allows WASM replacement: +```bash +# 1. Ensure vault remains paused +soroban contract invoke \ + --id \ + --source admin \ + --network \ + -- set_pause --paused true + +# 2. Revert to previous known-good WASM hash +soroban contract invoke \ + --id \ + --source admin \ + --network \ + -- upgrade --new_wasm_hash + +# 3. Verify contract version and invariant state +soroban contract invoke --id --network -- version +soroban contract invoke --id --network -- total_assets + +# 4. Unpause once verified +soroban contract invoke \ + --id \ + --source admin \ + --network \ + -- set_pause --paused false +``` + +#### Option 2: Fallback Contract Address Re-pointing +If in-place WASM upgrade is damaged: +1. Update `VAULT_CONTRACT_ID` in backend and frontend environment configs to point to the previous immutable contract address. +2. Trigger backend redeploy and frontend CDN redeployment. + +### 4.2 Backend Service Rollback + +```bash +# 1. Stop active backend process / container +sudo systemctl stop yieldvault-backend + +# 2. Checkout previous stable release commit/tag +cd /app/yieldvault-backend +git checkout + +# 3. If database migration was executed, apply down-migration +psql $DATABASE_URL -f prisma/migrations//rollback.sql + +# 4. Rebuild & start service +npm ci --production +npm run build +sudo systemctl start yieldvault-backend + +# 5. Verify local health +curl -f http://localhost:3000/health +curl -f http://localhost:3000/ready +``` + +### 4.3 Frontend Rollback + +```bash +# Vercel Instant Rollback +vercel rollback --token=$VERCEL_TOKEN --yes + +# Or re-deploy pre-built previous artifact +vercel deploy --prebuilt --prod --token=$VERCEL_TOKEN + +# Clear CDN Edge Caches +curl -X POST "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/purge_cache" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"purge_everything":true}' +``` + +### 4.4 Database Disaster Recovery (Point-in-Time Recovery) +If database state was corrupted: +1. Stop backend writers. +2. Initiate Point-in-Time Recovery (PITR) to timestamp `T_deploy - 5 minutes` via AWS RDS / Supabase / GCP Cloud SQL console. +3. Validate row counts and table integrity. +4. Point backend to restored database instance and restart. + +--- + +## 5. Incident Communication Cadence & Status Updates + +### 5.1 Update Cadence by Severity + +| Severity | Internal Status Frequency | Public Status Page Frequency | Target Channels | +|---|---|---|---| +| **Sev0** | Every 15 minutes | Every 15–30 minutes | `#yieldvault-war-room`, Statuspage.io, PagerDuty | +| **Sev1** | Every 30 minutes | Every 30–60 minutes | `#yieldvault-incidents`, Statuspage.io | +| **Sev2** | Every 1–2 hours | As appropriate | `#yieldvault-incidents` | +| **Sev3** | Upon resolution | Not required | GitHub Issue | + +### 5.2 Communication Templates + +#### Template: Initial Acknowledgment (T+5 min) +```text +🚨 INCIDENT DECLARED: [Brief summary, e.g. Elevated 5xx error rate on vault deposit route] +Severity: [Sev0 / Sev1 / Sev2] +Incident Commander: @[Name] +Lead Responder: @[Name] +War Room: #yieldvault-war-room (Zoom bridge link) +Status: Investigating root cause. Next update in 15 minutes. +``` + +#### Template: Rollback In Progress +```text +⚠️ ROLLBACK INITIATED: [Reason for rollback, e.g. Contract version v1.4.0 invariant validation failure] +Scope: Reverting backend to v1.3.9 and contract WASM to hash [hash_prefix] +Authorized By: IC @[Name] +Expected Downtime: 2-5 minutes +Next update: Upon rollback verification. +``` -### Incident Roles -- **Incident Commander (IC):** Leads response, authorizes pauses or rollbacks. -- **Lead Developer:** Drafts hotfix or executes rollback commands. -- **Verifier:** Runs post-deployment health checks before resolving incident. +#### Template: Incident Resolved +```text +✅ INCIDENT RESOLVED: [Summary of resolution] +Final Severity: [Sev0 / Sev1 / Sev2] +Impact Window: YYYY-MM-DD HH:MM to HH:MM UTC (Total: XX mins) +Actions Taken: [e.g. Rolled back backend to commit sha, verified health checks, unpaused vault] +Postmortem: Target date YYYY-MM-DD (within 48h) in docs/incidents/ +``` --- -## 2. Deployment Rollback Procedure +## 6. Post-Rollback Verification & Sign-Off -### Phase A: Immediate Emergency Pause (Circuit Breaker) -If the issue affects money movement or contract state, pause the contracts immediately. +Before resolving the incident and reopening public services, verify: -1. **Pause YieldVault Contracts:** - ```bash - # Run pause script or call pause method via CLI - cargo run -p scripts -- bin/pause_vault --network mainnet +- [ ] `GET /health` returns HTTP 200 `healthy`. +- [ ] `GET /ready` returns HTTP 200 `ready: true`. +- [ ] Vault contract `paused()` is `false` (if deliberately unpaused) and share pricing math is verified. +- [ ] Test deposit and withdrawal executed successfully on staging/mainnet canary account. +- [ ] Sentry error rates returned to pre-incident baseline for > 10 minutes. +- [ ] Public status page marked as **Resolved**. +- [ ] Postmortem issue scheduled (within 48 hours for Sev0/Sev1) per [`docs/postmortem-playbook.md`](../postmortem-playbook.md). diff --git a/frontend/src/components/AccountStatementExport.test.tsx b/frontend/src/components/AccountStatementExport.test.tsx index 26a2f191..188b0fa2 100644 --- a/frontend/src/components/AccountStatementExport.test.tsx +++ b/frontend/src/components/AccountStatementExport.test.tsx @@ -106,6 +106,63 @@ describe("AccountStatementExport", () => { expect(mockGetTransactions).not.toHaveBeenCalled(); }); + it("filters statement by custom date range", async () => { + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: /export statement/i })); + + const startInput = screen.getByLabelText(/start date/i); + const endInput = screen.getByLabelText(/end date/i); + + fireEvent.change(startInput, { target: { value: "2026-03-01" } }); + fireEvent.change(endInput, { target: { value: "2026-03-31" } }); + + fireEvent.click(screen.getByRole("button", { name: /download statement/i })); + + await waitFor(() => { + expect(mockDownloadTextFile).toHaveBeenCalledTimes(1); + }); + + const arg = mockDownloadTextFile.mock.calls[0][0]; + expect(arg.content).toContain("tx-2"); + expect(arg.content).not.toContain("tx-1"); + }); + + it("closes modal on cancel click", async () => { + render(); + + fireEvent.click(screen.getByRole("button", { name: /export statement/i })); + expect(await screen.findByRole("dialog")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /cancel/i })); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + it("surfaces an error when transaction fetch fails", async () => { mockGetTransactions.mockRejectedValue(new Error("Horizon unavailable")); diff --git a/frontend/src/components/NetworkWarningBanner.test.tsx b/frontend/src/components/NetworkWarningBanner.test.tsx new file mode 100644 index 00000000..996674b1 --- /dev/null +++ b/frontend/src/components/NetworkWarningBanner.test.tsx @@ -0,0 +1,86 @@ +import React from "react"; +import { render, screen, fireEvent } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import NetworkWarningBanner from "./NetworkWarningBanner"; +import * as useWalletNetworkModule from "../hooks/useWalletNetwork"; + +vi.mock("../hooks/useWalletNetwork"); + +const mockUseWalletNetwork = vi.mocked(useWalletNetworkModule.useWalletNetwork); + +describe("NetworkWarningBanner", () => { + const checkNowMock = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("renders nothing when there is no network mismatch", () => { + mockUseWalletNetwork.mockReturnValue({ + walletNetwork: "Testnet", + isMismatch: false, + expectedNetwork: "Testnet", + isChecking: false, + checkNow: checkNowMock, + }); + + const { container } = render(); + expect(container.firstChild).toBeNull(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("renders warning banner when wallet network is mismatched", () => { + mockUseWalletNetwork.mockReturnValue({ + walletNetwork: "Mainnet", + isMismatch: true, + expectedNetwork: "Testnet", + isChecking: false, + checkNow: checkNowMock, + }); + + render(); + + const alert = screen.getByRole("alert"); + expect(alert).toBeInTheDocument(); + expect(screen.getByText("Mainnet")).toBeInTheDocument(); + expect(screen.getByText("Testnet")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /show me how to fix this/i })).toBeInTheDocument(); + }); + + it("opens the guided fix modal when clicking the fix button", () => { + mockUseWalletNetwork.mockReturnValue({ + walletNetwork: "Mainnet", + isMismatch: true, + expectedNetwork: "Testnet", + isChecking: false, + checkNow: checkNowMock, + }); + + render(); + + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + + const fixButton = screen.getByRole("button", { name: /show me how to fix this/i }); + fireEvent.click(fixButton); + + expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /check again/i })).toBeInTheDocument(); + }); + + it("triggers checkNow callback when requested in guide modal", () => { + mockUseWalletNetwork.mockReturnValue({ + walletNetwork: "Mainnet", + isMismatch: true, + expectedNetwork: "Testnet", + isChecking: false, + checkNow: checkNowMock, + }); + + render(); + + fireEvent.click(screen.getByRole("button", { name: /show me how to fix this/i })); + fireEvent.click(screen.getByRole("button", { name: /check again/i })); + + expect(checkNowMock).toHaveBeenCalledTimes(1); + }); +});