Add-disaster-recovery-automation-with-runbooks

Author: Chidubemkingsley

backend/dr/DisasterRecoveryService.ts

@@ -71,7 +71,23 @@ export type {
 export { BatchChargeService } from './batchChargeService';
 export type { BatchChargeCandidate, BatchChargeOptions, BatchChargeResult } from './batchChargeService';
 
-// ── Feature Flags (Issue #TBD) ──────────────────────────────────────────────
+// ── Disaster Recovery ───────────────────────────────────────────────────────
+export { DisasterRecoveryService, disasterRecoveryService } from '../dr/DisasterRecoveryService';
+export type {
+  BackupManifest,
+  BackupEntry,
+  VerificationResult,
+  RecoveryResult,
+  DrDrillResult,
+  DrDrillSchedule,
+  RtoMonitorEntry,
+  RpoMonitorEntry,
+  DrIncident,
+  GeoRegionStatus,
+  ConsistencyProof,
+} from '../dr/DisasterRecoveryService';
+
+// ── Feature Flags ───────────────────────────────────────────────────────────
 export { BackendFeatureFlagsService, backendFeatureFlagsService } from './featureFlags';
 export type {
   FeatureFlag,

backend/dr/__tests__/DisasterRecoveryService.test.ts

@@ -0,0 +1,50 @@
+import {
+  simulateCrossServiceBackup,
+  injectBackupInconsistency,
+  runBackupConsistencyExperiment,
+} from '../experiments/backup-consistency';
+
+describe('Backup Consistency Experiment', () => {
+  it('detects consistent cross-service backup', () => {
+    const result = simulateCrossServiceBackup(
+      { shared_count: '100', app_key: 'val' },
+      { shared_count: '100', contract_key: 'val' }
+    );
+    expect(result.consistent).toBe(true);
+    expect(result.mismatches).toHaveLength(0);
+  });
+
+  it('detects inconsistent shared keys', () => {
+    const result = simulateCrossServiceBackup(
+      { shared_count: '100', app_key: 'val' },
+      { shared_count: '200', contract_key: 'val' }
+    );
+    expect(result.consistent).toBe(false);
+    expect(result.mismatches).toHaveLength(1);
+    expect(result.mismatches[0].key).toBe('shared_count');
+  });
+
+  it('ignores non-shared keys', () => {
+    const result = simulateCrossServiceBackup({ app_only: 'a' }, { contract_only: 'b' });
+    expect(result.consistent).toBe(true);
+  });
+
+  it('injects inconsistency for testing', () => {
+    const result = injectBackupInconsistency(
+      { shared_x: '1' },
+      { shared_x: '1' },
+      'shared_x',
+      '10',
+      '20'
+    );
+    expect(result.appData.shared_x).toBe('10');
+    expect(result.contractData.shared_x).toBe('20');
+  });
+
+  it('runBackupConsistencyExperiment passes', async () => {
+    const result = await runBackupConsistencyExperiment();
+    expect(result.experiment).toBe('backup-consistency');
+    expect(result.passed).toBe(true);
+    expect(result.recovery).toBe('inconsistency-detected');
+  });
+});

backend/services/index.ts

@@ -0,0 +1,46 @@
+import {
+  simulateGeoRequest,
+  simulateRegionFailover,
+  runGeoPartitionExperiment,
+} from '../experiments/geo-partition';
+
+describe('Geo Partition Experiment', () => {
+  it('simulateGeoRequest succeeds for available region', async () => {
+    const result = await simulateGeoRequest('us-east-1', () => Promise.resolve('ok'));
+    expect(result).toBe('ok');
+  });
+
+  it('simulateGeoRequest fails for unavailable region', async () => {
+    await expect(
+      simulateGeoRequest('us-east-1', () => Promise.resolve('ok'), {
+        primary: { region: 'us-east-1', available: false, latencyMs: 0 },
+        replicas: [],
+      })
+    ).rejects.toThrow('Region unavailable');
+  });
+
+  it('simulateRegionFailover fails over to replica', async () => {
+    const result = await simulateRegionFailover(() => Promise.resolve({ data: 'ok' }), {
+      primary: { region: 'us-east-1', available: false, latencyMs: 0 },
+      replicas: [{ region: 'eu-west-1', available: true, latencyMs: 10 }],
+    });
+    expect(result.failoverRegion).toBe('eu-west-1');
+    expect(result.result).toEqual({ data: 'ok' });
+  });
+
+  it('simulateRegionFailover throws when all regions down', async () => {
+    await expect(
+      simulateRegionFailover(() => Promise.resolve('ok'), {
+        primary: { region: 'us-east-1', available: false, latencyMs: 0 },
+        replicas: [{ region: 'eu-west-1', available: false, latencyMs: 0 }],
+      })
+    ).rejects.toThrow('All regions unavailable');
+  });
+
+  it('runGeoPartitionExperiment passes', async () => {
+    const result = await runGeoPartitionExperiment();
+    expect(result.experiment).toBe('geo-partition');
+    expect(result.passed).toBe(true);
+    expect(result.recovery).toBe('failover-to-eu-west-1');
+  });
+});

chaos/__tests__/backup-consistency.test.ts

@@ -3,7 +3,7 @@ import { runAllExperiments, summarize } from '../runner';
 describe('Chaos Runner', () => {
   it('runs all experiments and all pass', async () => {
     const results = await runAllExperiments();
-    expect(results).toHaveLength(3);
+    expect(results).toHaveLength(5);
     const failed = results.filter((r) => !r.passed);
     expect(failed).toHaveLength(0);
   });

chaos/__tests__/geo-partition.test.ts

@@ -0,0 +1,95 @@
+import type { ChaosResult } from './network-partition';
+
+export interface ConsistencyCheckResult {
+  serviceA: string[];
+  serviceB: string[];
+  mismatches: { key: string; aValue: string | null; bValue: string | null }[];
+  consistent: boolean;
+}
+
+export function simulateCrossServiceBackup(
+  appData: Record<string, string>,
+  contractData: Record<string, string>
+): ConsistencyCheckResult {
+  const mismatches: { key: string; aValue: string | null; bValue: string | null }[] = [];
+  const serviceA = Object.keys(appData);
+  const serviceB = Object.keys(contractData);
+
+  const allKeys = new Set([...serviceA, ...serviceB]);
+  for (const key of allKeys) {
+    const aVal = appData[key] ?? null;
+    const bVal = contractData[key] ?? null;
+
+    if (key.startsWith('shared_') && aVal !== bVal) {
+      mismatches.push({ key, aValue: aVal, bValue: bVal });
+    }
+  }
+
+  return {
+    serviceA,
+    serviceB,
+    mismatches,
+    consistent: mismatches.length === 0,
+  };
+}
+
+export function injectBackupInconsistency(
+  appData: Record<string, string>,
+  contractData: Record<string, string>,
+  inconsistencyKey: string,
+  appValue: string,
+  contractValue: string
+): { appData: Record<string, string>; contractData: Record<string, string> } {
+  return {
+    appData: { ...appData, [inconsistencyKey]: appValue },
+    contractData: { ...contractData, [inconsistencyKey]: contractValue },
+  };
+}
+
+export async function runBackupConsistencyExperiment(): Promise<ChaosResult> {
+  const start = Date.now();
+
+  const appData: Record<string, string> = {
+    shared_user_count: '150',
+    shared_subscription_count: '300',
+    app_config: 'enabled',
+  };
+
+  const contractData: Record<string, string> = {
+    shared_user_count: '150',
+    shared_subscription_count: '300',
+    contract_state: 'active',
+  };
+
+  const cleanCheck = simulateCrossServiceBackup(appData, contractData);
+  if (!cleanCheck.consistent) {
+    return {
+      experiment: 'backup-consistency',
+      passed: false,
+      duration: Date.now() - start,
+      error: 'Clean data reported as inconsistent',
+    };
+  }
+
+  const corrupted = injectBackupInconsistency(
+    appData,
+    contractData,
+    'shared_user_count',
+    '150',
+    '200'
+  );
+
+  const corruptedCheck = simulateCrossServiceBackup(corrupted.appData, corrupted.contractData);
+
+  const passed = !corruptedCheck.consistent && corruptedCheck.mismatches.length === 1;
+
+  return {
+    experiment: 'backup-consistency',
+    passed,
+    duration: Date.now() - start,
+    recovery: passed ? 'inconsistency-detected' : undefined,
+    error: passed
+      ? undefined
+      : `Expected 1 mismatch, got ${corruptedCheck.mismatches.length}, consistent=${corruptedCheck.consistent}`,
+  };
+}

chaos/__tests__/runner.test.ts

@@ -0,0 +1,106 @@
+import type { ChaosResult } from './network-partition';
+
+export interface GeoRegionState {
+  region: string;
+  available: boolean;
+  latencyMs: number;
+}
+
+export interface GeoPartitionScenario {
+  primary: GeoRegionState;
+  replicas: GeoRegionState[];
+}
+
+const DEFAULT_SCENARIO: GeoPartitionScenario = {
+  primary: { region: 'us-east-1', available: true, latencyMs: 5 },
+  replicas: [
+    { region: 'eu-west-1', available: true, latencyMs: 80 },
+    { region: 'ap-southeast-1', available: true, latencyMs: 150 },
+  ],
+};
+
+export async function simulateGeoRequest<T>(
+  region: string,
+  fn: () => Promise<T>,
+  scenario: GeoPartitionScenario = DEFAULT_SCENARIO
+): Promise<T> {
+  const allRegions = [scenario.primary, ...scenario.replicas];
+  const regionState = allRegions.find((r) => r.region === region);
+
+  if (!regionState) throw new Error(`Unknown region: ${region}`);
+  if (!regionState.available) throw new Error(`Region unavailable: ${region}`);
+
+  if (regionState.latencyMs > 0) {
+    await new Promise((r) => setTimeout(r, regionState.latencyMs));
+  }
+
+  return fn();
+}
+
+export async function simulateRegionFailover<T>(
+  fn: () => Promise<T>,
+  scenario: GeoPartitionScenario
+): Promise<{ result: T | null; failoverRegion: string; failoverDurationMs: number }> {
+  const start = Date.now();
+
+  if (!scenario.primary.available) {
+    for (const replica of scenario.replicas) {
+      if (!replica.available) continue;
+      const replicaStart = Date.now();
+      try {
+        const result = await simulateGeoRequest(replica.region, fn, scenario);
+        return {
+          result,
+          failoverRegion: replica.region,
+          failoverDurationMs: Date.now() - replicaStart,
+        };
+      } catch {
+        continue;
+      }
+    }
+    throw new Error('All regions unavailable');
+  }
+
+  const result = await fn();
+  return {
+    result,
+    failoverRegion: scenario.primary.region,
+    failoverDurationMs: Date.now() - start,
+  };
+}
+
+export async function runGeoPartitionExperiment(): Promise<ChaosResult> {
+  const start = Date.now();
+
+  const scenario: GeoPartitionScenario = {
+    primary: { region: 'us-east-1', available: false, latencyMs: 0 },
+    replicas: [
+      { region: 'eu-west-1', available: true, latencyMs: 80 },
+      { region: 'ap-southeast-1', available: false, latencyMs: 0 },
+    ],
+  };
+
+  try {
+    const { failoverRegion, failoverDurationMs } = await simulateRegionFailover(
+      async () => ({ data: 'recovered' }),
+      scenario
+    );
+
+    const passed = failoverRegion === 'eu-west-1' && failoverDurationMs < 500;
+
+    return {
+      experiment: 'geo-partition',
+      passed,
+      duration: Date.now() - start,
+      recovery: `failover-to-${failoverRegion}`,
+      error: passed ? undefined : `Failover took ${failoverDurationMs}ms or went to wrong region`,
+    };
+  } catch (err) {
+    return {
+      experiment: 'geo-partition',
+      passed: false,
+      duration: Date.now() - start,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}

chaos/experiments/backup-consistency.ts

@@ -1,17 +1,17 @@
-/**
- * Chaos Runner — executes all experiments and reports results.
- */
-
 import { runNetworkPartitionExperiment } from './experiments/network-partition';
 import { runServiceDegradationExperiment } from './experiments/service-degradation';
 import { runFailureInjectionExperiment } from './experiments/failure-injection';
+import { runGeoPartitionExperiment } from './experiments/geo-partition';
+import { runBackupConsistencyExperiment } from './experiments/backup-consistency';
 import type { ChaosResult } from './experiments/network-partition';
 
 export async function runAllExperiments(): Promise<ChaosResult[]> {
   const results = await Promise.all([
     runNetworkPartitionExperiment(),
     runServiceDegradationExperiment(),
     runFailureInjectionExperiment(),
+    runGeoPartitionExperiment(),
+    runBackupConsistencyExperiment(),
   ]);
   return results;
 }
@@ -20,9 +20,9 @@ export function summarize(results: ChaosResult[]): void {
   const passed = results.filter((r) => r.passed).length;
   console.log(`\nChaos Engineering Results: ${passed}/${results.length} passed\n`);
   for (const r of results) {
-    const status = r.passed ? '✅' : '❌';
-    console.log(`${status} ${r.experiment} (${r.duration}ms)`);
-    if (r.recovery) console.log(`   recovery: ${r.recovery}`);
-    if (r.error) console.log(`   error: ${r.error}`);
+    const status = r.passed ? 'PASS' : 'FAIL';
+    console.log(`${status}  ${r.experiment} (${r.duration}ms)`);
+    if (r.recovery) console.log(`     recovery: ${r.recovery}`);
+    if (r.error) console.log(`     error: ${r.error}`);
   }
 }