Merge pull request #61 from robertocarlous/robbert

ritik4ever · web-flow · commit 17aa6f4b9d3d · 2026-02-21T03:16:32.000+05:30
feat #53 Build portfolio migration tool for schema/version upgrades
diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml
@@ -4,13 +4,15 @@ on:
   pull_request:
     paths:
       - 'backend/**'
+      - 'docs/MIGRATION.md'
       - '.github/workflows/backend-tests.yml'
   push:
     branches:
       - main
       - develop
     paths:
       - 'backend/**'
+      - 'docs/MIGRATION.md'
 
 jobs:
   test:
@@ -35,6 +37,10 @@ jobs:
         working-directory: backend
         run: npm ci
       
+      - name: Verify migrations (dry-run)
+        working-directory: backend
+        run: npm run db:migrate:dry-run
+      
       - name: Run tests
         working-directory: backend
         run: npm run test
diff --git a/README.md b/README.md
@@ -67,6 +67,8 @@ cp backend/.env.example backend/.env
 cp frontend/.env.example frontend/.env
 # Edit with contract addresses
 
+**Database migrations** (when using PostgreSQL with `DATABASE_URL`): see [docs/MIGRATION.md](docs/MIGRATION.md). Apply with `cd backend && npm run db:migrate`; use `--dry-run` to preview.
+
 Configure SMTP for Email Notifications (Optional)
 
 To enable email notifications for rebalancing events:
diff --git a/backend/package.json b/backend/package.json
@@ -7,6 +7,9 @@
         "build": "tsc",
         "start": "node dist/index.js",
         "db:migrate": "tsx src/db/migrate.ts",
+        "db:migrate:dry-run": "tsx src/db/migrate.ts -- --dry-run",
+        "db:migrate:rollback": "tsx src/db/migrate.ts -- --rollback",
+        "db:migrate:status": "tsx src/db/migrate.ts -- --status",
         "db:setup": "tsx src/db/migrate.ts",
         "test": "vitest run",
         "test:watch": "vitest"
diff --git a/backend/src/db/migrate.ts b/backend/src/db/migrate.ts
@@ -1,19 +1,200 @@
+#!/usr/bin/env node
+/**
+ * Versioned migration runner for PostgreSQL.
+ * Usage:
+ *   npm run db:migrate           - Apply all pending migrations
+ *   npm run db:migrate -- --dry-run   - Show what would run without applying
+ *   npm run db:migrate -- --rollback [n] - Roll back last n migrations (default 1)
+ *   npm run db:migrate -- --status     - List applied and pending migrations
+ */
 import 'dotenv/config'
-import { readFileSync } from 'fs'
+import { readFileSync, readdirSync } from 'fs'
 import { fileURLToPath } from 'url'
 import { dirname, join } from 'path'
 import { getPool, closePool, isDbConfigured } from './client.js'
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
+const MIGRATIONS_DIR = join(__dirname, 'migrations')
+
+const MIGRATION_TABLE = `CREATE TABLE IF NOT EXISTS schema_migrations (
+    version VARCHAR(64) PRIMARY KEY,
+    name VARCHAR(256) NOT NULL,
+    applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);`
+
+interface MigrationFile {
+    version: string
+    name: string
+    upPath: string
+    downPath: string
+}
+
+function parseArgs(): { dryRun: boolean; rollback: number | null; status: boolean } {
+    const args = process.argv.slice(2)
+    let dryRun = false
+    let rollback: number | null = null
+    let status = false
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--dry-run') dryRun = true
+        else if (args[i] === '--status') status = true
+        else if (args[i] === '--rollback') {
+            const next = args[i + 1]
+            rollback = next !== undefined && /^\d+$/.test(next) ? parseInt(next, 10) : 1
+            if (next !== undefined && /^\d+$/.test(next)) i++
+        }
+    }
+    return { dryRun, rollback, status }
+}
+
+function discoverMigrations(): MigrationFile[] {
+    const files = readdirSync(MIGRATIONS_DIR)
+    const upFiles = files.filter((f) => f.endsWith('.up.sql'))
+    const migrations: MigrationFile[] = []
+    for (const up of upFiles) {
+        const match = up.match(/^(\d+)_(.+)\.up\.sql$/)
+        if (!match) continue
+        const down = `${match[1]}_${match[2]}.down.sql`
+        if (!files.includes(down)) {
+            console.warn(`Warning: missing down migration ${down}`)
+        }
+        migrations.push({
+            version: match[1],
+            name: match[2],
+            upPath: join(MIGRATIONS_DIR, up),
+            downPath: join(MIGRATIONS_DIR, down)
+        })
+    }
+    migrations.sort((a, b) => a.version.localeCompare(b.version, 'en'))
+    return migrations
+}
+
+async function ensureMigrationsTable(): Promise<void> {
+    await getPool().query(MIGRATION_TABLE)
+}
+
+async function getAppliedVersions(): Promise<string[]> {
+    const result = await getPool().query<{ version: string }>(
+        'SELECT version FROM schema_migrations ORDER BY version ASC'
+    )
+    return result.rows.map((r) => r.version)
+}
 
 async function run() {
+    const { dryRun, rollback, status } = parseArgs()
+    const migrations = discoverMigrations()
+    if (migrations.length === 0) {
+        console.log('No migration files found in', MIGRATIONS_DIR)
+        process.exit(0)
+    }
+
     if (!isDbConfigured()) {
+        if (dryRun && rollback === null && !status) {
+            console.log('[DRY RUN] DATABASE_URL not set. Showing migration files that would be applied:')
+            for (const m of migrations) {
+                console.log('  ', m.version, m.name)
+            }
+            console.log('Total:', migrations.length, 'migration(s).')
+            process.exit(0)
+        }
         console.error('DATABASE_URL is not set. Set it to run migrations.')
         process.exit(1)
     }
+
     try {
-        const schema = readFileSync(join(__dirname, 'schema.sql'), 'utf8')
-        await getPool().query(schema)
+        await ensureMigrationsTable()
+        const applied = await getAppliedVersions()
+
+        if (status) {
+            console.log('Applied migrations:')
+            for (const v of applied) {
+                const m = migrations.find((x) => x.version === v)
+                console.log('  ', v, m ? m.name : '(unknown)')
+            }
+            console.log('\nPending migrations:')
+            const pending = migrations.filter((m) => !applied.includes(m.version))
+            for (const m of pending) {
+                console.log('  ', m.version, m.name)
+            }
+            await closePool()
+            process.exit(0)
+            return
+        }
+
+        if (rollback !== null) {
+            const toRollback = applied.slice(-rollback).reverse()
+            if (toRollback.length === 0) {
+                console.log('No migrations to roll back.')
+                await closePool()
+                process.exit(0)
+                return
+            }
+            if (dryRun) {
+                console.log('[DRY RUN] Would roll back:', toRollback.join(', '))
+                for (const v of toRollback) {
+                    const m = migrations.find((x) => x.version === v)
+                    if (m) {
+                        try {
+                            const sql = readFileSync(m.downPath, 'utf8')
+                            console.log('---', m.version, m.name, '(down) ---\n', sql)
+                        } catch {
+                            console.log('---', m.version, m.name, '(down file missing) ---')
+                        }
+                    }
+                }
+                await closePool()
+                process.exit(0)
+            }
+            for (const v of toRollback) {
+                const m = migrations.find((x) => x.version === v)
+                if (!m) {
+                    console.error('Migration version not found:', v)
+                    process.exit(1)
+                }
+                let downSql: string
+                try {
+                    downSql = readFileSync(m.downPath, 'utf8')
+                } catch (err) {
+                    console.error('Cannot read down migration', m.downPath, err)
+                    process.exit(1)
+                }
+                console.log('Rolling back', m.version, m.name, '...')
+                await getPool().query(downSql)
+                await getPool().query('DELETE FROM schema_migrations WHERE version = $1', [m.version])
+                console.log('Rolled back', m.version, m.name)
+            }
+            console.log('Rollback completed.')
+            await closePool()
+            process.exit(0)
+        }
+
+        const pending = migrations.filter((m) => !applied.includes(m.version))
+        if (pending.length === 0) {
+            console.log('No pending migrations.')
+            await closePool()
+            process.exit(0)
+        }
+
+        if (dryRun) {
+            console.log('[DRY RUN] Pending migrations (not applied):')
+            for (const m of pending) {
+                const sql = readFileSync(m.upPath, 'utf8')
+                console.log('---', m.version, m.name, '---\n', sql.substring(0, 500) + (sql.length > 500 ? '...' : ''), '\n')
+            }
+            console.log('Total:', pending.length, 'migration(s). Run without --dry-run to apply.')
+            await closePool()
+            process.exit(0)
+        }
+
+        for (const m of pending) {
+            const sql = readFileSync(m.upPath, 'utf8')
+            console.log('Applying', m.version, m.name, '...')
+            await getPool().query(sql)
+            await getPool().query(
+                'INSERT INTO schema_migrations (version, name) VALUES ($1, $2)',
+                [m.version, m.name]
+            )
+            console.log('Applied', m.version, m.name)
+        }
         console.log('Migrations completed successfully.')
     } catch (err) {
         console.error('Migration failed:', err)
diff --git a/backend/src/db/migrations/001_initial_schema.down.sql b/backend/src/db/migrations/001_initial_schema.down.sql
@@ -0,0 +1,15 @@
+-- Migration: 001_initial_schema (down)
+-- Rollback: Drop tables in reverse dependency order to respect foreign keys.
+
+DROP INDEX IF EXISTS idx_notification_preferences_user;
+DROP TABLE IF EXISTS notification_preferences;
+
+DROP INDEX IF EXISTS idx_analytics_portfolio_time;
+DROP TABLE IF EXISTS analytics_snapshots;
+
+DROP INDEX IF EXISTS idx_rebalance_events_timestamp;
+DROP INDEX IF EXISTS idx_rebalance_events_portfolio;
+DROP TABLE IF EXISTS rebalance_events;
+
+DROP INDEX IF EXISTS idx_portfolios_user;
+DROP TABLE IF EXISTS portfolios;
diff --git a/backend/src/db/migrations/001_initial_schema.up.sql b/backend/src/db/migrations/001_initial_schema.up.sql
@@ -0,0 +1,60 @@
+-- Migration: 001_initial_schema (up)
+-- Description: Initial schema for portfolios, rebalance_events, analytics_snapshots, notification_preferences.
+-- Rollback: See 001_initial_schema.down.sql
+
+CREATE TABLE IF NOT EXISTS portfolios (
+    id VARCHAR(64) PRIMARY KEY,
+    user_address VARCHAR(256) NOT NULL,
+    allocations JSONB NOT NULL DEFAULT '{}',
+    threshold INTEGER NOT NULL,
+    balances JSONB NOT NULL DEFAULT '{}',
+    total_value NUMERIC NOT NULL DEFAULT 0,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    last_rebalance TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_portfolios_user ON portfolios(user_address);
+
+CREATE TABLE IF NOT EXISTS rebalance_events (
+    id VARCHAR(64) PRIMARY KEY,
+    portfolio_id VARCHAR(64) NOT NULL REFERENCES portfolios(id) ON DELETE CASCADE,
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    trigger VARCHAR(512) NOT NULL,
+    trades INTEGER NOT NULL DEFAULT 0,
+    gas_used VARCHAR(64) NOT NULL DEFAULT '',
+    status VARCHAR(32) NOT NULL CHECK (status IN ('completed', 'failed', 'pending')),
+    is_automatic BOOLEAN NOT NULL DEFAULT FALSE,
+    risk_alerts JSONB,
+    error TEXT,
+    details JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_rebalance_events_portfolio ON rebalance_events(portfolio_id);
+CREATE INDEX IF NOT EXISTS idx_rebalance_events_timestamp ON rebalance_events(timestamp DESC);
+
+CREATE TABLE IF NOT EXISTS analytics_snapshots (
+    id SERIAL PRIMARY KEY,
+    portfolio_id VARCHAR(64) NOT NULL,
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    total_value NUMERIC NOT NULL,
+    allocations JSONB NOT NULL DEFAULT '{}',
+    balances JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_analytics_portfolio_time ON analytics_snapshots(portfolio_id, timestamp DESC);
+
+CREATE TABLE IF NOT EXISTS notification_preferences (
+    user_id VARCHAR(256) PRIMARY KEY,
+    email_enabled BOOLEAN NOT NULL DEFAULT FALSE,
+    email_address VARCHAR(512),
+    webhook_enabled BOOLEAN NOT NULL DEFAULT FALSE,
+    webhook_url VARCHAR(1024),
+    event_rebalance BOOLEAN NOT NULL DEFAULT TRUE,
+    event_circuit_breaker BOOLEAN NOT NULL DEFAULT TRUE,
+    event_price_movement BOOLEAN NOT NULL DEFAULT TRUE,
+    event_risk_change BOOLEAN NOT NULL DEFAULT TRUE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_notification_preferences_user ON notification_preferences(user_id);
diff --git a/backend/src/db/migrations/002_seed_demo_data.down.sql b/backend/src/db/migrations/002_seed_demo_data.down.sql
@@ -0,0 +1,5 @@
+-- Migration: 002_seed_demo_data (down)
+-- Rollback: Remove demo portfolio and its rebalance events.
+
+DELETE FROM rebalance_events WHERE portfolio_id = 'demo-portfolio-1';
+DELETE FROM portfolios WHERE id = 'demo-portfolio-1';
diff --git a/backend/src/db/migrations/002_seed_demo_data.up.sql b/backend/src/db/migrations/002_seed_demo_data.up.sql
@@ -0,0 +1,60 @@
+-- Migration: 002_seed_demo_data (up)
+-- Description: Optional demo/seed data for development and staging. Safe to skip in production.
+-- Rollback: See 002_seed_demo_data.down.sql
+-- Idempotent: Uses ON CONFLICT DO NOTHING so re-run is safe.
+
+INSERT INTO portfolios (id, user_address, allocations, threshold, balances, total_value, created_at, last_rebalance)
+VALUES (
+    'demo-portfolio-1',
+    'DEMO-USER',
+    '{"XLM": 40, "BTC": 30, "ETH": 20, "USDC": 10}',
+    5,
+    '{"XLM": 11173.18, "BTC": 0.02697, "ETH": 0.68257, "USDC": 1000}',
+    10000,
+    NOW(),
+    NOW()
+)
+ON CONFLICT (id) DO NOTHING;
+
+INSERT INTO rebalance_events (id, portfolio_id, timestamp, trigger, trades, gas_used, status, is_automatic, risk_alerts, error, details)
+VALUES
+    (
+        'demo-evt-1',
+        'demo-portfolio-1',
+        NOW() - INTERVAL '2 hours',
+        'Threshold exceeded (8.2%)',
+        3,
+        '0.0234 XLM',
+        'completed',
+        FALSE,
+        NULL,
+        NULL,
+        '{"fromAsset": "XLM", "toAsset": "ETH", "amount": 1200, "reason": "Portfolio allocation drift exceeded rebalancing threshold", "riskLevel": "medium", "priceDirection": "down", "performanceImpact": "neutral"}'::jsonb
+    ),
+    (
+        'demo-evt-2',
+        'demo-portfolio-1',
+        NOW() - INTERVAL '12 hours',
+        'Automatic Rebalancing',
+        2,
+        '0.0156 XLM',
+        'completed',
+        TRUE,
+        NULL,
+        NULL,
+        '{"reason": "Automated scheduled rebalancing executed", "riskLevel": "low", "priceDirection": "up", "performanceImpact": "positive"}'::jsonb
+    ),
+    (
+        'demo-evt-3',
+        'demo-portfolio-1',
+        NOW() - INTERVAL '3 days',
+        'Volatility circuit breaker',
+        1,
+        '0.0089 XLM',
+        'completed',
+        TRUE,
+        NULL,
+        NULL,
+        '{"reason": "High market volatility detected, protective rebalance executed", "volatilityDetected": true, "riskLevel": "high", "priceDirection": "down", "performanceImpact": "negative"}'::jsonb
+    )
+ON CONFLICT (id) DO NOTHING;
diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md
