Merge pull request #193 from alanshurafa/contrib/alanshurafa/brain-backup

[recipes] Brain backup and export

Author: justfinethanku

recipes/brain-backup/README.md

@@ -1,5 +1,13 @@
 # Brain Backup and Export
 
+<div align="center">
+
+![Community Contribution](https://img.shields.io/badge/OB1_COMMUNITY-Approved_Contribution-2ea44f?style=for-the-badge&logo=github)
+
+**Created by [@alanshurafa](https://github.com/alanshurafa)**
+
+</div>
+
 Export all Open Brain Supabase tables to local JSON files. The script paginates through PostgREST (1 000 rows per request), writes each table to a dated JSON file, and prints a summary.
 
 ## Prerequisites
@@ -31,10 +39,27 @@ Export all Open Brain Supabase tables to local JSON files. The script paginates
 
 ## Expected Result
 
-After running the script you will have a `backup/` directory containing dated JSON exports of every Open Brain table (thoughts, entities, edges, thought_entities, reflections, ingestion_jobs, ingestion_items). The console output shows row counts and file sizes for each table, making it easy to verify the backup is complete.
+After running the script you will have a `backup/` directory containing dated JSON exports of every Open Brain table present in your project.
+
+- `thoughts` is always backed up (required).
+- Optional companion tables — `entities`, `edges`, `thought_entities`, `ingestion_jobs`, `ingestion_items` — are backed up only if they exist. They ship with companion contributions (e.g. the entity-extraction and smart-ingest schemas). Stock Open Brain installs will see `skipped (table not present)` for those, which is expected.
+
+The console output shows row counts and file sizes for each table, making it easy to verify the backup is complete.
 
 ## Tips
 
 - Schedule the script with cron or Task Scheduler for automatic daily backups.
 - Commit the `backup/` directory to a private repo for versioned history.
 - The script streams rows to disk, so it handles large tables without running out of memory.
+
+## Troubleshooting
+
+- **`PostgREST error 404 on thoughts`** -- the script could reach the server but the `thoughts` table isn't visible to it. Only a PostgREST "schema cache" 404 (`code: "PGRST205"`) is treated as "table not present"; everything else is surfaced so you can diagnose it. Common causes:
+  - Typo in `SUPABASE_URL` (for example pointing at `/v1` instead of the project root -- the script appends `/rest/v1` itself).
+  - Supabase project is paused or deleted.
+  - The `thoughts` table lives in a non-`public` schema that PostgREST isn't exposing.
+  - You're using the `anon` key instead of the `service_role` key. The anon key can be restricted by RLS and return empty or 404 responses; service-role keys bypass RLS.
+- **`skipped (table not present)` for optional tables** -- expected on stock Open Brain installs. The optional tables ship with companion contributions (entity extraction, smart ingest).
+- **`PostgREST error 401`** -- `SUPABASE_SERVICE_ROLE_KEY` is wrong, revoked, or truncated.
+- **`PostgREST error 403`** -- unusual for service-role keys, which should bypass RLS. Double-check you're not using a custom-minted JWT with narrower claims.
+- **Script hangs or aborts after ~60s** -- set `FETCH_TIMEOUT_MS` to a larger value (milliseconds) if your project is on a slow tier or has very large tables.

recipes/brain-backup/backup-brain.mjs

@@ -23,14 +23,16 @@ const SCRIPT_DIR = process.cwd();
 
 const PAGE_SIZE = 1000;
 
+// Stock Open Brain only has `thoughts`. The other tables are from optional
+// companion contributions (entity extraction, smart ingest). Missing tables
+// are skipped at runtime so this recipe works against any Open Brain install.
 const TABLES = [
-  { name: "thoughts",         orderBy: "id" },
-  { name: "entities",         orderBy: "id" },
-  { name: "edges",            orderBy: "id" },
-  { name: "thought_entities", orderBy: "thought_id,entity_id" },
-  { name: "reflections",      orderBy: "id" },
-  { name: "ingestion_jobs",   orderBy: "id" },
-  { name: "ingestion_items",  orderBy: "id" },
+  { name: "thoughts",         orderBy: "id", required: true  },
+  { name: "entities",         orderBy: "id", required: false },
+  { name: "edges",            orderBy: "id", required: false },
+  { name: "thought_entities", orderBy: "thought_id,entity_id", required: false },
+  { name: "ingestion_jobs",   orderBy: "id", required: false },
+  { name: "ingestion_items",  orderBy: "id", required: false },
 ];
 
 // ---------------------------------------------------------------------------
@@ -41,7 +43,14 @@ function loadEnvFile() {
   const envPath = path.join(SCRIPT_DIR, ".env.local");
   const vars = {};
   if (fs.existsSync(envPath)) {
-    for (const line of fs.readFileSync(envPath, "utf8").split("\n")) {
+    let isFirstLine = true;
+    for (let line of fs.readFileSync(envPath, "utf8").split("\n")) {
+      // Strip UTF-8 BOM from the first line -- Notepad and some VS Code
+      // configurations on Windows write it, which would otherwise poison
+      // the first key name (e.g. "\uFEFFSUPABASE_URL") and cause a
+      // confusing "SUPABASE_URL not found" even though it's right there.
+      if (isFirstLine && line.charCodeAt(0) === 0xFEFF) line = line.slice(1);
+      isFirstLine = false;
       const trimmed = line.trim();
       if (!trimmed || trimmed.startsWith("#")) continue;
       const eqIdx = trimmed.indexOf("=");
@@ -90,6 +99,19 @@ const HEADERS = {
   Prefer: "count=exact",
 };
 
+// Bounded per-request timeout. Unattended backup jobs must either finish or
+// fail within a predictable window -- a hung connection should not keep a
+// cron job alive forever. 60s is generous for a 1000-row page; override with
+// FETCH_TIMEOUT_MS for slow tiers or very large tables.
+const FETCH_TIMEOUT_MS = (() => {
+  const raw =
+    process.env.FETCH_TIMEOUT_MS ||
+    envVars.FETCH_TIMEOUT_MS ||
+    "";
+  const parsed = parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 60_000;
+})();
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -101,19 +123,54 @@ function today() {
 function humanSize(bytes) {
   if (bytes < 1024) return `${bytes} B`;
   if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
 }
 
 /** Fetch a single page of rows from a table. */
 async function fetchPage(table, orderBy, offset, limit) {
   const url = `${REST_BASE}/${table}?order=${orderBy}&limit=${limit}&offset=${offset}`;
   const rangeEnd = offset + limit - 1;
-  const res = await fetch(url, {
-    headers: {
-      ...HEADERS,
-      Range: `${offset}-${rangeEnd}`,
-    },
-  });
+
+  // Node 18+ fetch() has no default timeout. Wire up AbortController so a
+  // hung Supabase connection can't hang the whole backup run.
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+  let res;
+  try {
+    res = await fetch(url, {
+      headers: {
+        ...HEADERS,
+        Range: `${offset}-${rangeEnd}`,
+      },
+      signal: controller.signal,
+    });
+  } catch (err) {
+    if (err && err.name === "AbortError") {
+      throw new Error(
+        `PostgREST request for ${table} timed out after ${FETCH_TIMEOUT_MS} ms ` +
+        `(raise FETCH_TIMEOUT_MS if this table is legitimately slow)`
+      );
+    }
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+
+  if (res.status === 404) {
+    // PostgREST returns 404 with `code: "PGRST205"` when the table is not in
+    // the schema cache. Any other 404 (typo in SUPABASE_URL, paused project,
+    // wrong schema, custom API gateway) should surface loudly, not be
+    // silently treated as "table missing" -- that's how backup tools lose
+    // data without anyone noticing.
+    const rawBody = await res.text();
+    let parsed = null;
+    try { parsed = JSON.parse(rawBody); } catch {}
+    if (parsed && parsed.code === "PGRST205") {
+      return { rows: [], total: null, missing: true };
+    }
+    throw new Error(`PostgREST error 404 on ${table}: ${rawBody}`);
+  }
 
   if (!res.ok && res.status !== 206) {
     const body = await res.text();
@@ -132,54 +189,89 @@ async function fetchPage(table, orderBy, offset, limit) {
 }
 
 /** Export one table, streaming rows to disk. */
-async function exportTable(tableName, orderBy, backupDir, dateStr) {
+async function exportTable(tableName, orderBy, backupDir, dateStr, required) {
   const filePath = path.join(backupDir, `${tableName}-${dateStr}.json`);
+  // Write to a sibling .tmp file and atomically rename on success. Any crash
+  // (network error, process kill) leaves only the .tmp behind, so yesterday's
+  // valid backup is never overwritten by today's partial one.
+  const tmpPath = `${filePath}.tmp`;
   let offset = 0;
   let total = null;
   let rowCount = 0;
 
   const first = await fetchPage(tableName, orderBy, 0, PAGE_SIZE);
-  total = first.total;
 
   const label = `  ${tableName}`;
-  if (first.rows.length === 0) {
-    process.stdout.write(`${label}: 0 rows (empty table)\n`);
-    fs.writeFileSync(filePath, "[]");
-    return { rowCount: 0, filePath, fileSize: 2 };
+  if (first.missing) {
+    if (required) {
+      throw new Error(`Required table "${tableName}" not found in Supabase project`);
+    }
+    process.stdout.write(`${label}: skipped (table not present)\n`);
+    return { rowCount: 0, filePath: null, fileSize: 0, skipped: true };
   }
 
-  const fd = fs.openSync(filePath, "w");
-  fs.writeSync(fd, "[\n");
-  let firstRow = true;
+  total = first.total;
 
-  function writeRows(rows) {
-    for (const row of rows) {
-      if (!firstRow) fs.writeSync(fd, ",\n");
-      fs.writeSync(fd, JSON.stringify(row));
-      firstRow = false;
-      rowCount++;
+  if (first.rows.length === 0) {
+    process.stdout.write(`${label}: 0 rows (empty table)\n`);
+    // Even the two-byte "[]" path writes via tmp+rename so we never leave a
+    // half-written file in the final location.
+    try {
+      fs.writeFileSync(tmpPath, "[]");
+      fs.renameSync(tmpPath, filePath);
+    } catch (err) {
+      try { fs.unlinkSync(tmpPath); } catch {}
+      throw err;
     }
+    return { rowCount: 0, filePath, fileSize: 2 };
   }
 
-  writeRows(first.rows);
-  process.stdout.write(
-    `${label}: ${rowCount}${total != null ? "/" + total : ""} rows\r`
-  );
-
-  offset = PAGE_SIZE;
-  while (first.rows.length === PAGE_SIZE && (total == null || offset < total)) {
-    const page = await fetchPage(tableName, orderBy, offset, PAGE_SIZE);
-    if (page.rows.length === 0) break;
-    writeRows(page.rows);
-    offset += page.rows.length;
+  const fd = fs.openSync(tmpPath, "w");
+  let closed = false;
+  try {
+    fs.writeSync(fd, "[\n");
+    let firstRow = true;
+
+    function writeRows(rows) {
+      for (const row of rows) {
+        if (!firstRow) fs.writeSync(fd, ",\n");
+        fs.writeSync(fd, JSON.stringify(row));
+        firstRow = false;
+        rowCount++;
+      }
+    }
 
+    writeRows(first.rows);
     process.stdout.write(
       `${label}: ${rowCount}${total != null ? "/" + total : ""} rows\r`
     );
-  }
 
-  fs.writeSync(fd, "\n]");
-  fs.closeSync(fd);
+    let lastPageSize = first.rows.length;
+    offset = PAGE_SIZE;
+    while (lastPageSize === PAGE_SIZE && (total == null || offset < total)) {
+      const page = await fetchPage(tableName, orderBy, offset, PAGE_SIZE);
+      lastPageSize = page.rows.length;
+      if (lastPageSize === 0) break;
+      writeRows(page.rows);
+      offset += lastPageSize;
+
+      process.stdout.write(
+        `${label}: ${rowCount}${total != null ? "/" + total : ""} rows\r`
+      );
+    }
+
+    fs.writeSync(fd, "\n]");
+    fs.closeSync(fd);
+    closed = true;
+
+    fs.renameSync(tmpPath, filePath);
+  } catch (err) {
+    if (!closed) {
+      try { fs.closeSync(fd); } catch {}
+    }
+    try { fs.unlinkSync(tmpPath); } catch {}
+    throw err;
+  }
 
   const fileSize = fs.statSync(filePath).size;
 
@@ -209,7 +301,7 @@ async function main() {
   const results = [];
   for (const table of TABLES) {
     try {
-      const result = await exportTable(table.name, table.orderBy, backupDir, dateStr);
+      const result = await exportTable(table.name, table.orderBy, backupDir, dateStr, table.required);
       results.push({ table: table.name, ...result });
     } catch (err) {
       console.error(`\n  ERROR exporting ${table.name}: ${err.message}`);