Merge pull request #1107 from maccoder374-sudo/feat/devops-monitoring-pitr-csp-stress-issues-1098-1101

feat(devops): prometheus alerting rules, db PITR backup/DR runbook, strict CSP headers, and stress benchmark suite

Author: ayomideadeniran

.github/workflows/stress-benchmark.yml

@@ -0,0 +1,32 @@
+name: End-to-End Stress & Chaos Engineering Benchmark Suite
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  stress-benchmark:
+    name: Stress & Chaos Load Benchmark Suite
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Run End-to-End Stress & Chaos Benchmark Suite
+        run: |
+          node tests/stress/load_stress_suite.js
+
+      - name: Upload Benchmark Report Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: stress-benchmark-report
+          path: tests/stress/stress-benchmark-report.json

docs/DR_RUNBOOK.md

@@ -0,0 +1,66 @@
+# SoroTask Disaster Recovery & Point-in-Time Recovery (PITR) Runbook
+
+## Overview
+This runbook provides step-by-step instructions for performing automated database backups, continuous Write-Ahead Log (WAL) archiving, Point-in-Time Recovery (PITR), and disaster recovery drills for SoroTask production databases.
+
+## Service Level Objectives (SLOs)
+- **Recovery Point Objective (RPO)**: < 5 minutes (Maximum acceptable data loss window).
+- **Recovery Time Objective (RTO)**: < 15 minutes (Maximum acceptable downtime during restoration).
+
+---
+
+## 1. Automated Backup Architecture
+
+### Daily Base Snapshots
+- Base snapshots are generated daily at `02:00 UTC` using `scripts/backup/db_pitr_backup.sh`.
+- Snapshots are compressed with `gzip` and uploaded to `s3://sorotask-db-backups/snapshots/`.
+- Retention Policy: 30 days automatic retention.
+
+### Continuous WAL Archiving
+- PostgreSQL / DB Write-Ahead Logs (WAL) are shipped continuously as segments are filled to `s3://sorotask-db-backups/wal/`.
+- Archiving frequency: Continuous (or maximum 60-second window).
+
+---
+
+## 2. Emergency Disaster Recovery Procedure (PITR)
+
+When database corruption or accidental data deletion occurs, follow these steps to perform Point-in-Time Recovery:
+
+### Step 1: Declare Incident & Stop Writing Services
+```bash
+# Scale down API services to prevent new incoming writes
+docker-compose stop indexer keeper
+```
+
+### Step 2: Determine Recovery Target Timestamp
+Identify the exact UTC timestamp immediately prior to the incident (e.g., `2026-08-26T14:00:00Z`).
+
+### Step 3: Run Automated Recovery Script
+Execute the disaster recovery drill script with your target timestamp:
+```bash
+./scripts/backup/dr_restore_drill.sh "2026-08-26T14:00:00Z"
+```
+
+### Step 4: Verify Restored Data Integrity
+Validate state consistency using indexer and keeper verification checks:
+```bash
+# Verify restored indexer database schema and latest synced ledger
+sqlite3 /tmp/sorotask_dr_restore/restored_sorotask.db "SELECT count(*) FROM tasks;"
+```
+
+### Step 5: Promote Restored Database & Restart Services
+```bash
+# Replace target database with restored file
+cp /tmp/sorotask_dr_restore/restored_sorotask.db indexer/indexer.db
+docker-compose start indexer keeper
+```
+
+---
+
+## 3. Automated DR Restoration Drill Verification
+
+To execute an automated restoration drill and verify RTO SLA compliance:
+```bash
+./scripts/backup/dr_restore_drill.sh
+```
+Expected output: `✓ RTO SLA PASSED: Restoration completed in < 15 minutes`.

frontend/__tests__/securityHeaders.test.ts

@@ -0,0 +1,79 @@
+import nextConfig from "../next.config";
+
+describe("Frontend Security Headers & Content Security Policy Configuration", () => {
+  it("should export an async headers function", () => {
+    expect(typeof nextConfig.headers).toBe("function");
+  });
+
+  it("should apply security headers to all routes (/:path*)", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    expect(headerConfigs.length).toBeGreaterThan(0);
+    expect(headerConfigs[0].source).toBe("/:path*");
+  });
+
+  it("should configure strict Content-Security-Policy (CSP)", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    const headers = headerConfigs[0].headers;
+    const csp = headers.find(
+      (h: { key: string }) => h.key === "Content-Security-Policy",
+    );
+
+    expect(csp).toBeDefined();
+    expect(csp?.value).toContain("default-src 'self'");
+    expect(csp?.value).toContain("script-src 'self'");
+    expect(csp?.value).toContain("connect-src 'self'");
+    expect(csp?.value).toContain("frame-ancestors 'none'");
+    expect(csp?.value).toContain("object-src 'none'");
+  });
+
+  it("should configure X-Frame-Options DENY", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    const headers = headerConfigs[0].headers;
+    const xfo = headers.find(
+      (h: { key: string }) => h.key === "X-Frame-Options",
+    );
+
+    expect(xfo).toBeDefined();
+    expect(xfo?.value).toBe("DENY");
+  });
+
+  it("should configure X-Content-Type-Options nosniff", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    const headers = headerConfigs[0].headers;
+    const xcto = headers.find(
+      (h: { key: string }) => h.key === "X-Content-Type-Options",
+    );
+
+    expect(xcto).toBeDefined();
+    expect(xcto?.value).toBe("nosniff");
+  });
+
+  it("should configure HSTS Strict-Transport-Security", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    const headers = headerConfigs[0].headers;
+    const hsts = headers.find(
+      (h: { key: string }) => h.key === "Strict-Transport-Security",
+    );
+
+    expect(hsts).toBeDefined();
+    expect(hsts?.value).toContain("max-age=63072000");
+    expect(hsts?.value).toContain("includeSubDomains");
+  });
+
+  it("should configure Referrer-Policy strict-origin-when-cross-origin", async () => {
+    if (!nextConfig.headers) return;
+    const headerConfigs = await nextConfig.headers();
+    const headers = headerConfigs[0].headers;
+    const rp = headers.find(
+      (h: { key: string }) => h.key === "Referrer-Policy",
+    );
+
+    expect(rp).toBeDefined();
+    expect(rp?.value).toBe("strict-origin-when-cross-origin");
+  });
+});

frontend/next.config.ts

@@ -1,13 +1,63 @@
 import type { NextConfig } from "next";
 import { withSentryConfig } from "@sentry/nextjs";
 
+const cspHeader = `
+  default-src 'self';
+  script-src 'self' 'unsafe-inline' 'unsafe-eval';
+  style-src 'self' 'unsafe-inline';
+  img-src 'self' blob: data: https:;
+  font-src 'self' data:;
+  connect-src 'self' https://*.stellar.org https://*.soroban.org https://soroban-testnet.stellar.org https://horizon-testnet.stellar.org wss://*.stellar.org http://localhost:* ws://localhost:*;
+  frame-ancestors 'none';
+  object-src 'none';
+  base-uri 'self';
+  form-action 'self';
+`
+  .replace(/\s{2,}/g, " ")
+  .trim();
+
+const securityHeaders = [
+  {
+    key: "Content-Security-Policy",
+    value: cspHeader,
+  },
+  {
+    key: "X-Frame-Options",
+    value: "DENY",
+  },
+  {
+    key: "X-Content-Type-Options",
+    value: "nosniff",
+  },
+  {
+    key: "Strict-Transport-Security",
+    value: "max-age=63072000; includeSubDomains; preload",
+  },
+  {
+    key: "Referrer-Policy",
+    value: "strict-origin-when-cross-origin",
+  },
+  {
+    key: "Permissions-Policy",
+    value: "camera=(), microphone=(), geolocation=()",
+  },
+];
+
 const nextConfig: NextConfig = {
   experimental: {
     useTypeScriptCli: true,
   },
   typescript: {
     ignoreBuildErrors: true,
   },
+  async headers() {
+    return [
+      {
+        source: "/:path*",
+        headers: securityHeaders,
+      },
+    ];
+  },
 };
 
 // Issue #813: wrap the default config so Sentry's webpack plugin instruments

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,36 @@
+# Alertmanager Configuration for SoroTask (Issue #1098)
+global:
+  resolve_timeout: 5m
+  pagerduty_url: "https://events.pagerduty.com/v2/enqueue"
+
+route:
+  group_by: ['alertname', 'service', 'severity']
+  group_wait: 10s
+  group_interval: 5m
+  repeat_interval: 4h
+  receiver: 'telegram-default'
+  routes:
+    - match:
+        severity: critical
+      receiver: 'pagerduty-oncall'
+      continue: true
+
+    - match:
+        severity: warning
+      receiver: 'telegram-default'
+
+receivers:
+  - name: 'pagerduty-oncall'
+    pagerduty_configs:
+      - service_key: '${PAGERDUTY_ROUTING_KEY:-sorotask-oncall-key}'
+        send_resolved: true
+        severity: 'critical'
+        description: '{{ .CommonAnnotations.summary }} - {{ .CommonAnnotations.description }}'
+
+  - name: 'telegram-default'
+    telegram_configs:
+      - bot_token: '${TELEGRAM_BOT_TOKEN:-disabled}'
+        chat_id: ${TELEGRAM_CHAT_ID:-0}
+        send_resolved: true
+        parse_mode: 'HTML'
+        message: '<b>[ALERT] {{ .Status | toUpper }}</b>\n<b>Summary:</b> {{ .CommonAnnotations.summary }}\n<b>Description:</b> {{ .CommonAnnotations.description }}\n<b>Service:</b> {{ .CommonLabels.service }}'

monitoring/prometheus/alerts.yml

@@ -0,0 +1,47 @@
+# Prometheus Alerting Rules for SoroTask Services (Issue #1098)
+groups:
+  - name: sorotask-sla-alerts
+    rules:
+      - alert: IndexerHighLedgerLag
+        expr: (sorotask_indexer_current_ledger - sorotask_indexer_synced_ledger) > 5 or indexer_ledger_lag_ledgers > 5
+        for: 2m
+        labels:
+          severity: warning
+          service: indexer
+          component: sync
+        annotations:
+          summary: "Indexer ledger lag exceeds threshold"
+          description: "Indexer is lagging behind stellar network by {{ $value }} ledgers for more than 2 minutes."
+
+      - alert: KeeperBalanceLow
+        expr: sorotask_keeper_balance_xlm < 50 or keeper_wallet_balance_xlm < 50
+        for: 5m
+        labels:
+          severity: critical
+          service: keeper
+          component: wallet
+        annotations:
+          summary: "Keeper wallet balance critically low"
+          description: "Keeper bot wallet balance is {{ $value }} XLM, which is below the minimum operational threshold of 50 XLM."
+
+      - alert: KeeperHighFailureRate
+        expr: (rate(keeper_task_execution_failures_total[5m]) / (rate(keeper_task_executions_total[5m]) + 0.001)) * 100 > 5
+        for: 2m
+        labels:
+          severity: critical
+          service: keeper
+          component: execution
+        annotations:
+          summary: "Keeper task execution failure rate high"
+          description: "Keeper task execution failure rate is {{ $value }}% over the last 5 minutes (exceeds 5% SLA limit)."
+
+      - alert: ZKProverQueueHighLatency
+        expr: zk_queue_wait_ms > 60000 or (histogram_quantile(0.95, sum(rate(zk_queue_wait_ms_bucket[5m])) by (le)) > 60000)
+        for: 2m
+        labels:
+          severity: warning
+          service: zk-proof-service
+          component: prover-queue
+        annotations:
+          summary: "ZK prover queue latency high"
+          description: "ZK prover queue wait time is {{ $value }}ms, exceeding the 60 second SLA limit."

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,26 @@
+# Prometheus Configuration for SoroTask
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - "alerts.yml"
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - "alertmanager:9093"
+
+scrape_configs:
+  - job_name: "sorotask-keeper"
+    static_configs:
+      - targets: ["keeper:3000"]
+
+  - job_name: "sorotask-indexer"
+    static_configs:
+      - targets: ["indexer:4000"]
+
+  - job_name: "zk-proof-service"
+    static_configs:
+      - targets: ["zk-proof-service:5000"]