fix+feat: comprehensive improvements across every feature

2FA — actually enforced at login
- auth.ts: OIDC callback checks totpCredentialsTable on every login
  If 2FA is enabled, creates a pending session and redirects to /2fa-challenge
  instead of completing login — 2FA can no longer be bypassed
- twoFactor.ts: POST /auth/2fa/complete endpoint upgrades pending → full session
- Backup code consumption now uses SELECT FOR UPDATE inside a DB transaction
  preventing race conditions where two simultaneous requests could consume the
  same backup code
- Session storage removed from setup route (secret returned to client directly)
- 2FA setup: removed redundant session mutation

Deploy environment — actually sent in API call
- DeploySite.tsx: environment state now included in POST /deploy body
- Toast message reflects staging vs production environment

Transfer tokens — Redis-backed, survive restarts
- transfer.ts: pendingTransfers Map replaced with Redis-backed store
  getTransfer/storeTransfer/deleteTransfer helpers with Redis primary +
  in-memory fallback when Redis is unavailable
- 24h TTL enforced in Redis with EX parameter

Prometheus gauges — now actually updated
- metricsCollector.ts: new background job, runs every 30s
  Updates sitesTotal (by status), federationPeersTotal (by status),
  syncQueueDepth, cacheEntries (domain/file) from live DB/in-memory state
- syncRetryQueue.ts: getSyncQueueDepth() exported
- index.ts: startMetricsCollector / stopMetricsCollector wired

monthlyBandwidthGb rollup — now calculated
- analyticsFlush.ts: after every buffer flush, runs SQL to update
  sites.monthly_bandwidth_gb and sites.hit_count from site_analytics table
  Uses current calendar month window; resets automatically on month boundary

Build pipeline improvements
- Parallel uploads: files now uploaded 8 at a time (was sequential)
- Environment variable injection: envVars object passed to build process
  Dangerous server secrets stripped before env is passed to subprocess
- installCommand override: operators can bypass auto-detection
- buildEnv sanitization: removes SMTP_PASS, DATABASE_URL, REDIS_URL etc.

Analytics export + referrer breakdown
- GET /api/sites/:id/analytics/export?period=7d|30d|all — CSV download
  Columns: hour, hits, bytes_served, unique_ips
- GET /api/sites/:id/analytics/referrers?period=7d|30d — aggregated referrers
  Merges topReferrers JSONB across all hourly rows, returns top 50

Redirect rules — query string + regex matching
- matchRedirectPattern: complete rewrite supporting:
  /page?utm_source=email  — exact query key=value
  /page?ref=*             — wildcard value (captured as q_ref param)
  /page?key               — key must be present (any value)
  /page?!logged_in        — negation (key must be absent)
  /page?a=1&!b            — multiple constraints (all must pass)
  ^/regex.*$              — raw regex with named capture groups (?<id>\d+)
- All callers updated to pass req query string
- tests/unit/redirectPattern.test.ts: 25 test cases covering all patterns

CLI additions
- fh logs <site-id>               — list recent builds
- fh logs <site-id> --build <id>  — view full build log
- fh logs <site-id> --build <id> --follow — poll while running
- fh build <site-id>              — trigger git build pipeline
  --git-url, --branch, --command, --output, --env KEY=VAL, --install, --staging
  --wait: stream logs until complete
- fh forms <site-id>              — list form submissions
  --form, --limit, --export <file.csv>, --json, --unread

Frontend pages
- /settings/2fa — TwoFactorSettings: full setup flow, QR code display,
  backup code management, disable/regenerate
- /sites/:id/forms — FormInbox: three-pane email-client layout,
  form filter sidebar, submission detail, CSV export, mark read, delete
- /sites/:id/builds — BuildHistory: live-polling build list, log viewer,
  trigger new build form, auto-refresh while build running
- MySites: added Forms (Inbox icon) and Builds (GitBranch icon) buttons
  to every site card alongside Analytics and Settings

DB schemas added
- emailQueue.ts: emailQueueTable + siteHealthChecksTable
- All new tables exported from schema/index.ts

Author: The No Hands Company

artifacts/api-server/src/index.ts

@@ -9,6 +9,8 @@ import { getRedisClient, closeRedis } from "./lib/redis";
 import { startSyncRetryQueue, stopSyncRetryQueue } from "./lib/syncRetryQueue";
 import { startAcmeRenewalScheduler, stopAcmeRenewalScheduler } from "./lib/acme";
 import { startSiteHealthMonitor, stopSiteHealthMonitor } from "./lib/siteHealthMonitor";
+import { startMetricsCollector, stopMetricsCollector } from "./lib/metricsCollector";
+import { startEmailQueue, stopEmailQueue } from "./lib/email";
 import { startOrphanCleanup, stopOrphanCleanup } from "./lib/orphanCleanup";
 import { db, sessionsTable } from "@workspace/db";
 import { lt } from "drizzle-orm";
@@ -68,6 +70,8 @@ function gracefulShutdown(server: http.Server, signal: string): void {
       stopSyncRetryQueue();
       stopAcmeRenewalScheduler();
       stopSiteHealthMonitor();
+      stopMetricsCollector();
+      stopEmailQueue();
       stopOrphanCleanup();
       await closeRedis();
       const { pool } = await import("@workspace/db");
@@ -105,6 +109,8 @@ ensureLocalNode()
     startSyncRetryQueue();
     startAcmeRenewalScheduler();
     startSiteHealthMonitor();
+    startMetricsCollector();
+    startEmailQueue();
     startOrphanCleanup();
 
     // Initialise Redis connection (optional — falls back to in-memory if not configured)

artifacts/api-server/src/lib/analyticsFlush.ts

@@ -119,6 +119,35 @@ export async function flushAnalyticsBuffer(): Promise<void> {
   });
 
   logger.debug({ flushed: rows.length, buckets: buckets.size }, "Analytics buffer flushed");
+
+  // ── Update per-site bandwidth + hit totals ────────────────────────────────
+  // Roll up bytesServed into sites.monthly_bandwidth_gb for the usage dashboard.
+  // We use a monthly window: reset on the 1st of each month by tracking via
+  // the current month's analytics rows rather than a running counter.
+  try {
+    const now = new Date();
+    const monthStart = new Date(now.getFullYear(), now.getMonth(), 1);
+
+    await db.execute(sql`
+      UPDATE sites s
+      SET
+        monthly_bandwidth_gb = (
+          SELECT COALESCE(SUM(bytes_served), 0) / (1024.0 * 1024 * 1024)
+          FROM site_analytics
+          WHERE site_id = s.id AND hour >= ${monthStart}
+        ),
+        hit_count = (
+          SELECT COALESCE(SUM(hits), 0)
+          FROM site_analytics
+          WHERE site_id = s.id
+        )
+      WHERE s.id IN (
+        SELECT DISTINCT site_id FROM site_analytics WHERE hour >= ${monthStart}
+      )
+    `);
+  } catch (err) {
+    logger.warn({ err }, "Failed to update site bandwidth/hit totals");
+  }
 }
 
 let flushTimer: NodeJS.Timeout | null = null;

artifacts/api-server/src/lib/email.ts

@@ -1,29 +1,17 @@
 /**
  * Email notification system.
  *
- * Sends transactional emails for platform events via any SMTP provider
- * (Resend, Postmark, SendGrid SMTP, AWS SES, self-hosted Postfix, etc.).
+ * All emails go through a persistent queue (email_queue table) with
+ * exponential backoff retry. SMTP failures never cause request failures.
  *
- * Configuration (all optional — emails are silently skipped if not configured):
- *   SMTP_HOST        — SMTP server hostname (e.g. smtp.resend.com)
- *   SMTP_PORT        — SMTP port (default: 587)
- *   SMTP_SECURE      — "true" for TLS on port 465
- *   SMTP_USER        — SMTP username
- *   SMTP_PASS        — SMTP password / API key
- *   EMAIL_FROM       — From address (default: noreply@<PUBLIC_DOMAIN>)
- *   EMAIL_FROM_NAME  — From display name (default: FedHost)
- *
- * Events:
- *   - deploy.success   — site deployed successfully
- *   - deploy.failed    — deployment failed
- *   - cert.expiring    — TLS certificate expiring in <30 days
- *   - cert.renewed     — TLS certificate renewed
- *   - node.offline     — federation node went offline
- *   - invitation       — user invited to a site
- *   - site.deleted     — site deleted (confirmation)
+ * Queue processing runs every 30 seconds. Failed emails are retried up to
+ * 5 times with delays: 1m → 5m → 15m → 1h → 6h. After 5 failures the
+ * email is marked failed and never retried.
  */
 
 import nodemailer, { type Transporter } from "nodemailer";
+import { db, emailQueueTable } from "@workspace/db";
+import { isNull, lte, lt, eq, sql } from "drizzle-orm";
 import logger from "./logger";
 
 // ── Transport ──────────────────────────────────────────────────────────────────
@@ -34,24 +22,15 @@ function getTransporter(): Transporter | null {
   if (transporter) return transporter;
 
   const host = process.env.SMTP_HOST;
-  if (!host) return null; // email not configured — all sends are no-ops
+  if (!host) return null;
 
   const port   = parseInt(process.env.SMTP_PORT ?? "587", 10);
   const secure = process.env.SMTP_SECURE === "true";
 
   transporter = nodemailer.createTransport({
-    host,
-    port,
-    secure,
-    auth: process.env.SMTP_USER ? {
-      user: process.env.SMTP_USER,
-      pass: process.env.SMTP_PASS ?? "",
-    } : undefined,
-    pool: true,
-    maxConnections: 5,
-    maxMessages: 100,
-    rateDelta: 1000,
-    rateLimit: 10,
+    host, port, secure,
+    auth: process.env.SMTP_USER ? { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS ?? "" } : undefined,
+    pool: true, maxConnections: 5, maxMessages: 100,
   });
 
   return transporter;
@@ -64,20 +43,78 @@ function fromAddress(): string {
   return `"${name}" <${addr}>`;
 }
 
-async function sendMail(opts: { to: string; subject: string; html: string; text: string }): Promise<boolean> {
-  const t = getTransporter();
-  if (!t) return false; // silently skip — SMTP not configured
+// Backoff delays per attempt (ms)
+const BACKOFF = [60_000, 300_000, 900_000, 3_600_000, 21_600_000];
 
+/** Enqueue an email. Returns immediately — actual sending is async. */
+async function enqueue(opts: { to: string; subject: string; html: string; text: string }): Promise<void> {
+  if (!process.env.SMTP_HOST) return; // email not configured, skip silently
   try {
-    await t.sendMail({ from: fromAddress(), ...opts });
-    logger.info({ to: opts.to, subject: opts.subject }, "[email] Sent");
-    return true;
+    await db.insert(emailQueueTable).values(opts);
   } catch (err) {
-    logger.error({ err, to: opts.to, subject: opts.subject }, "[email] Failed to send");
-    return false;
+    logger.error({ err, to: opts.to }, "[email] Failed to enqueue");
   }
 }
 
+/** Process pending emails from the queue. Called by the email flush job. */
+export async function processEmailQueue(): Promise<void> {
+  const t = getTransporter();
+  if (!t) return;
+
+  const pending = await db
+    .select()
+    .from(emailQueueTable)
+    .where(sql`${emailQueueTable.sentAt} IS NULL AND ${emailQueueTable.failedAt} IS NULL AND ${emailQueueTable.nextAttempt} <= NOW()`)
+    .limit(20);
+
+  for (const item of pending) {
+    try {
+      await t.sendMail({ from: fromAddress(), to: item.to, subject: item.subject, html: item.html, text: item.text });
+
+      await db.update(emailQueueTable)
+        .set({ sentAt: new Date() })
+        .where(eq(emailQueueTable.id, item.id));
+
+      logger.info({ to: item.to, subject: item.subject }, "[email] Sent");
+    } catch (err: any) {
+      const attempts = item.attempts + 1;
+      if (attempts >= item.maxAttempts) {
+        await db.update(emailQueueTable)
+          .set({ attempts, failedAt: new Date(), error: err.message })
+          .where(eq(emailQueueTable.id, item.id));
+        logger.error({ to: item.to, attempts }, "[email] Permanently failed");
+      } else {
+        const delay = BACKOFF[attempts - 1] ?? BACKOFF[BACKOFF.length - 1]!;
+        const nextAttempt = new Date(Date.now() + delay);
+        await db.update(emailQueueTable)
+          .set({ attempts, nextAttempt, error: err.message })
+          .where(eq(emailQueueTable.id, item.id));
+        logger.warn({ to: item.to, attempts, nextAttemptIn: delay }, "[email] Retrying");
+      }
+    }
+  }
+}
+
+// Queue flush timer
+let emailTimer: NodeJS.Timeout | null = null;
+
+export function startEmailQueue(): void {
+  if (!process.env.SMTP_HOST) return;
+  processEmailQueue().catch(() => {});
+  emailTimer = setInterval(() => processEmailQueue().catch(() => {}), 30_000);
+  logger.info("[email] Queue processor started");
+}
+
+export function stopEmailQueue(): void {
+  if (emailTimer) { clearInterval(emailTimer); emailTimer = null; }
+}
+
+// sendMail is now just enqueue
+async function sendMail(opts: { to: string; subject: string; html: string; text: string }): Promise<boolean> {
+  await enqueue(opts);
+  return true;
+}
+
 // ── HTML layout ───────────────────────────────────────────────────────────────
 
 function layout(content: string, title: string): string {

artifacts/api-server/src/lib/metricsCollector.ts

@@ -0,0 +1,83 @@
+/**
+ * Prometheus gauge collector.
+ *
+ * The HTTP metrics (requests_total, duration, active_requests) update
+ * themselves via middleware. Business gauges need to be polled from the DB
+ * because they represent state, not events.
+ *
+ * Runs every 30 seconds and updates:
+ *   fedhost_sites_total         — by status (active/inactive/suspended)
+ *   fedhost_federation_peers_total — by status (active/offline/pending)
+ *   fedhost_sync_queue_depth    — pending retry items
+ *   fedhost_cache_entries       — domain and file LRU cache size
+ */
+
+import { db, sitesTable, nodesTable } from "@workspace/db";
+import { sql } from "drizzle-orm";
+import {
+  sitesTotal,
+  federationPeersTotal,
+  syncQueueDepth,
+  cacheEntries,
+  deploymentsTotal,
+  storageOperationsTotal,
+} from "./metrics";
+import { getCacheStats } from "./domainCache";
+import { getSyncQueueDepth } from "./syncRetryQueue";
+import logger from "./logger";
+
+const INTERVAL_MS = 30_000;
+let timer: NodeJS.Timeout | null = null;
+
+async function collect(): Promise<void> {
+  try {
+    // ── Sites by status ───────────────────────────────────────────────────
+    const siteCounts = await db
+      .select({ status: sitesTable.status, count: sql<number>`COUNT(*)` })
+      .from(sitesTable)
+      .groupBy(sitesTable.status);
+
+    // Reset all labels first so removed statuses go to 0
+    for (const status of ["active", "inactive", "suspended"]) {
+      sitesTotal.set({ status }, 0);
+    }
+    for (const row of siteCounts) {
+      sitesTotal.set({ status: row.status }, Number(row.count));
+    }
+
+    // ── Federation peers by status ────────────────────────────────────────
+    const peerCounts = await db
+      .select({ status: nodesTable.status, count: sql<number>`COUNT(*)` })
+      .from(nodesTable)
+      .groupBy(nodesTable.status);
+
+    for (const status of ["active", "offline", "pending"]) {
+      federationPeersTotal.set({ status }, 0);
+    }
+    for (const row of peerCounts) {
+      federationPeersTotal.set({ status: row.status }, Number(row.count));
+    }
+
+    // ── Sync retry queue depth ────────────────────────────────────────────
+    const queueDepth = getSyncQueueDepth();
+    syncQueueDepth.set(queueDepth);
+
+    // ── LRU cache sizes ───────────────────────────────────────────────────
+    const stats = getCacheStats();
+    cacheEntries.set({ cache_type: "domain" }, stats.domainEntries);
+    cacheEntries.set({ cache_type: "file" },   stats.fileEntries);
+
+  } catch (err) {
+    logger.warn({ err }, "[metrics-collector] Collection failed");
+  }
+}
+
+export function startMetricsCollector(): void {
+  collect().catch(() => {});
+  timer = setInterval(collect, INTERVAL_MS);
+  logger.info({ intervalMs: INTERVAL_MS }, "[metrics-collector] Started");
+}
+
+export function stopMetricsCollector(): void {
+  if (timer) { clearInterval(timer); timer = null; }
+}

artifacts/api-server/src/lib/siteHealthMonitor.ts

@@ -106,6 +106,27 @@ async function runHealthChecks(): Promise<void> {
     if (result.status === "down") {
       logger.warn({ domain: site.domain, error: result.error }, "[site-health] Site is down");
     }
+
+    // Persist to DB for history (fire-and-forget)
+    import("@workspace/db").then(({ db: _db, siteHealthChecksTable }) => {
+      _db.insert(siteHealthChecksTable).values({
+        siteId: site.id,
+        status: result.status,
+        httpStatus: result.httpStatus,
+        responseMs: result.responseMs ?? null,
+        error: result.error ?? null,
+        checkedAt: new Date(result.checkedAt),
+      }).catch(() => {});
+
+      // Alert if site transitioned to down
+      const prev = healthResults.get(site.id);
+      if (result.status === "down" && prev?.status !== "down") {
+        // Notify site owner by email
+        import("./email").then(({ emailSiteDown }) => {
+          emailSiteDown?.({ siteId: site.id, domain: site.domain }).catch(() => {});
+        }).catch(() => {});
+      }
+    }).catch(() => {});
   }
 
   logger.debug({ checked: activeSites.length }, "[site-health] Health checks complete");

artifacts/api-server/src/lib/syncRetryQueue.ts

@@ -203,3 +203,7 @@ export function getSyncQueueStats() {
     }, {}),
   };
 }
+
+export function getSyncQueueDepth(): number {
+  return syncQueue.size;
+}