Add a script to restore the production database from a backup.

Author: jyasskin

webserver/fly-release-command.mjs

@@ -1,5 +1,5 @@
 import { execSync } from 'node:child_process';
-import { replaceDbWithBackup } from './replaceDbWithBackup.mjs';
+import { replaceStagingDbWithBackup } from './replaceDbWithBackup.mjs';
 
 // Turn on swap to avoid out-of-memory errors during `prisma migrate deploy`
 console.log(execSync('fallocate -l 256M /swapfile', {encoding: 'utf-8'}));
@@ -11,7 +11,7 @@ console.log(execSync('swapon /swapfile', {encoding: 'utf-8'}));
 if (process.env.FLY_APP_NAME === 'vlach-cookbook-staging') {
   // Copy the latest backup over the staging database before running the Prisma
   // migration, to make sure the migration will work when the app is released.
-  await replaceDbWithBackup({
+  await replaceStagingDbWithBackup({
     DATABASE_URL: process.env.ADMIN_DATABASE_URL,
     dbName: "cookbook_staging",
     GOOGLE_CREDENTIALS: process.env.MIGRATION_GOOGLE_SERVICE_ACCOUNT_CREDENTIALS,

webserver/prisma/seed.mjs

@@ -1,12 +1,12 @@
-import { replaceDbWithBackup } from '../replaceDbWithBackup.mjs';
+import { replaceStagingDbWithBackup } from '../replaceDbWithBackup.mjs';
 
 // Load the production database into the local database.
 
 const DATABASE_URL = new URL(process.env.DATABASE_URL);
 // Remove a parameter that psql doesn't understand.
 DATABASE_URL.searchParams.delete('statement_cache_size');
 
-await replaceDbWithBackup({
+await replaceStagingDbWithBackup({
     DATABASE_URL: DATABASE_URL.href,
     dbName: "cookbook",
     GOOGLE_CREDENTIALS: process.env.COOKBOOK_GOOGLE_SERVICE_ACCOUNT_CREDENTIALS,

webserver/replaceDbWithBackup.mjs

@@ -1,7 +1,7 @@
-import { Storage } from '@google-cloud/storage';
+import { Storage } from "@google-cloud/storage";
 import * as age from "age-encryption";
-import { execSync } from 'node:child_process';
-import zlib from 'node:zlib';
+import { execSync } from "node:child_process";
+import zlib from "node:zlib";
 
 /**
  * Fetches the newest backup of the production database from gs://vlach-cookbook-backup, and
@@ -16,18 +16,47 @@ import zlib from 'node:zlib';
  * service account key.
  * @param {string} options.AGE_IDENTITY The https://age-encryption.org/ identity file to decrypt the
  * backup.
+ */
+export async function replaceStagingDbWithBackup({
+  DATABASE_URL,
+  dbName,
+  GOOGLE_CREDENTIALS,
+  AGE_IDENTITY,
+}) {
+  const dump = await fetchLatestBackup({ GOOGLE_CREDENTIALS, AGE_IDENTITY });
+  replaceDbWithBackupContents({
+    dump,
+    dbName,
+    DATABASE_URL,
+    webserverRole: "cookbook_staging_webserver",
+  });
+}
+
+/**
+ * Fetches, decrypts, and returns the newest backup of the production database from
+ * gs://vlach-cookbook-backup.
  *
+ * @param {Object} options
+ * @param {string} options.GOOGLE_CREDENTIALS The JSON-serialized contents of the file expected by
+ * https://cloud.google.com/docs/authentication/application-default-credentials#GAC. Usually a
+ * service account key.
+ * @param {string} options.AGE_IDENTITY The https://age-encryption.org/ identity file to decrypt the
+ * backup.
+ *
+ * @returns {Promise<string>}
  */
-export async function replaceDbWithBackup({DATABASE_URL, dbName, GOOGLE_CREDENTIALS, AGE_IDENTITY}) {
+async function fetchLatestBackup({ GOOGLE_CREDENTIALS, AGE_IDENTITY }) {
   const storage = new Storage({
-    projectId: 'vlach-cookbook',
-    credentials: JSON.parse(GOOGLE_CREDENTIALS)
+    projectId: "vlach-cookbook",
+    credentials: JSON.parse(GOOGLE_CREDENTIALS),
   });
 
-  const bucket = storage.bucket('vlach-cookbook-backup');
+  const bucket = storage.bucket("vlach-cookbook-backup");
   // Look for backups starting a month ago.
   let earliestBackupTime = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
-  const [recentBackups] = await bucket.getFiles({ startOffset: `backup-${earliestBackupTime.toISOString()}` });
+  const [recentBackups] = await bucket.getFiles({
+    startOffset: `backup-${earliestBackupTime.toISOString()}`,
+  });
   if (recentBackups.length === 0) {
     throw new Error(`Couldn't find a backup since ${earliestBackupTime}.`);
   }
@@ -37,18 +66,41 @@ export async function replaceDbWithBackup({DATABASE_URL, dbName, GOOGLE_CREDENTI
   d.addIdentity(AGE_IDENTITY);
   const decrypted = await d.decrypt(contents);
 
-  const dump = zlib.brotliDecompressSync(decrypted).toString('utf-8');
+  const dump = zlib.brotliDecompressSync(decrypted).toString("utf-8");
+  return dump;
+}
 
+/**
+ * Replaces the database at {@link DATABASE_URL} with the SQL {@link dump}.
+ *
+ * @param {Object} options
+ * @param {string} options.dump Decrypted contents of a backup.
+ * @param {string} options.DATABASE_URL A postgres:// URL with credentials.
+ * @param {string} options.dbName The name of the database to write into. This should match the end
+ * of DATABASE_URL.
+ * @param {string} options.webserverRole The database role name that the webserver uses.
+ */
+export function replaceDbWithBackupContents({
+  dump,
+  dbName,
+  DATABASE_URL,
+  webserverRole,
+}) {
   // Clear the whole database without actually deleting it. Otherwise, when we
   // add new tables that aren't in Prod yet, the *second* deployment to Staging
   // will fail because the above `pg_dump` won't know to remove those tables.
   const restore = `DROP SCHEMA IF EXISTS public CASCADE;
   CREATE SCHEMA public;
-  GRANT USAGE ON SCHEMA public TO cookbook_staging_webserver;
+  GRANT USAGE ON SCHEMA public TO ${webserverRole};
   ${dump.replaceAll(/cookbook_prod/g, dbName)}`;
 
   try {
-    console.log(execSync(`psql -d "${DATABASE_URL}"`, { input: restore, encoding: 'utf-8' }));
+    console.log(
+      execSync(`psql -d "${DATABASE_URL}"`, {
+        input: restore,
+        encoding: "utf-8",
+      })
+    );
   } catch (e) {
     console.error(`Restoring failed with\n>>>>>\n${restore}\n<<<<<`);
     throw e;

webserver/restore-prod-from-backup.mjs

@@ -0,0 +1,36 @@
+import assert from "node:assert";
+import { execSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { replaceDbWithBackupContents } from "./replaceDbWithBackup.mjs";
+
+assert.equal(process.env.FLY_APP_NAME, "vlach-cookbook");
+
+// Turn on swap to avoid out-of-memory errors during `prisma migrate deploy`
+console.log(execSync("fallocate -l 256M /swapfile", { encoding: "utf-8" }));
+console.log(execSync("chmod 0600 /swapfile", { encoding: "utf-8" }));
+console.log(execSync("mkswap /swapfile", { encoding: "utf-8" }));
+console.log(
+  execSync("echo 10 > /proc/sys/vm/swappiness", { encoding: "utf-8" })
+);
+console.log(execSync("swapon /swapfile", { encoding: "utf-8" }));
+
+// Expect the admin to have manually downloaded and decrypted a backup.
+const dump = readFileSync(process.env.BACKUP_FILE, { encoding: "utf8" });
+
+// Replace the production database with the loaded backup.
+replaceDbWithBackupContents({
+  DATABASE_URL: process.env.ADMIN_DATABASE_URL,
+  dbName: "cookbook_prod",
+  dump,
+  webserverRole: "cookbook_prod_webserver",
+});
+
+// Run the Prisma migration, in case the backup is older than the latest release.
+process.env.DATABASE_URL = process.env.ADMIN_DATABASE_URL;
+console.log(execSync("prisma migrate deploy", { encoding: "utf-8" }));
+console.log(
+  execSync(
+    "node prisma/migrations/20240916002444_cooking_history/migrate-history-categories.mjs"
+  ),
+  { encoding: "utf-8" }
+);