backup fix

Author: terion-labs

scripts/ctl/backup.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
-import { assertNewRestoreTargetIsUnused, rewriteRecoveredBackupMetadata } from "./backup";
-import { chooseLatestExportSnapshot, isRetriableS3ExportError } from "../terrarium-s3-export";
+import { assertNewRestoreTargetIsUnused, rewriteRecoveredBackupMetadata, rollbackSnapshotsForDatasetTree } from "./backup";
+import { chooseLatestExportSnapshot, isRetriableS3ExportError, zfsReplicationSendCommand } from "../terrarium-s3-export";
 
 describe("backup restore metadata", () => {
   test("classifies transient S3 export errors for retry", () => {
@@ -27,6 +27,13 @@ describe("backup restore metadata", () => {
     ).toBe("terrarium/containers/app@manual-keep");
   });
 
+  test("exports recursive ZFS replication streams so child rootfs datasets are included", () => {
+    expect(zfsReplicationSendCommand("terrarium/containers/app@manual-keep")).toBe("zfs send -R 'terrarium/containers/app@manual-keep'");
+    expect(zfsReplicationSendCommand("terrarium/containers/app@manual-keep", "terrarium/containers/app@manual-base")).toBe(
+      "zfs send -R -I 'terrarium/containers/app@manual-base' 'terrarium/containers/app@manual-keep'"
+    );
+  });
+
   test("renames restored LXD metadata and removes generated identity", () => {
     const rewritten = rewriteRecoveredBackupMetadata(
       {
@@ -112,6 +119,22 @@ describe("backup restore metadata", () => {
 });
 
 describe("backup restore target safety", () => {
+  test("rolls back descendant snapshots before the parent container dataset", () => {
+    expect(
+      rollbackSnapshotsForDatasetTree("terrarium/containers/app@manual-keep", [
+        "terrarium/containers/app@manual-keep",
+        "terrarium/containers/app/rootfs@manual-keep",
+        "terrarium/containers/app/rootfs/nested@manual-keep",
+        "terrarium/containers/app/rootfs@other",
+        "terrarium/containers/other/rootfs@manual-keep"
+      ])
+    ).toEqual([
+      "terrarium/containers/app/rootfs/nested@manual-keep",
+      "terrarium/containers/app/rootfs@manual-keep",
+      "terrarium/containers/app@manual-keep"
+    ]);
+  });
+
   test("rejects restore-as-new when the target LXD instance already exists", async () => {
     await expect(
       assertNewRestoreTargetIsUnused("victim", "terrarium/containers/victim", async (cmd) => {

scripts/ctl/backup.ts

@@ -339,6 +339,35 @@ async function zfsSnapshotExists(snapshot: string): Promise<boolean> {
   return (await runAllowFailure(["zfs", "list", "-H", "-t", "snapshot", snapshot])).exitCode === 0;
 }
 
+export function rollbackSnapshotsForDatasetTree(snapshot: string, availableSnapshots: string[]): string[] {
+  const snapshotMarker = snapshot.indexOf("@");
+  if (snapshotMarker === -1) {
+    throw new Error(`invalid ZFS snapshot name: ${snapshot}`);
+  }
+
+  const dataset = snapshot.slice(0, snapshotMarker);
+  const snapshotSuffix = snapshot.slice(snapshotMarker);
+  const descendants = availableSnapshots
+    .map((item) => item.trim())
+    .filter((item) => item.startsWith(`${dataset}/`) && item.endsWith(snapshotSuffix))
+    .sort((left, right) => right.split("/").length - left.split("/").length || right.localeCompare(left));
+
+  return [...descendants, snapshot];
+}
+
+async function rollbackDatasetTreeToSnapshot(snapshot: string): Promise<void> {
+  const snapshotMarker = snapshot.indexOf("@");
+  if (snapshotMarker === -1) {
+    throw new Error(`invalid ZFS snapshot name: ${snapshot}`);
+  }
+
+  const dataset = snapshot.slice(0, snapshotMarker);
+  const snapshots = await runText(["zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-r", dataset], PREFIX);
+  for (const item of rollbackSnapshotsForDatasetTree(snapshot, snapshots.split("\n"))) {
+    await runText(["zfs", "rollback", "-r", item], PREFIX);
+  }
+}
+
 function restoreTempDatasetName(targetDataset: string, label: string): string {
   const suffix = `${label}-${process.pid}-${Date.now()}`.replace(/[^A-Za-z0-9_.:-]/g, "-");
   return `${targetDataset}-${suffix}`;
@@ -452,7 +481,7 @@ async function restoreLocal(
   if (mode === "in-place") {
     await confirmDestructive(`Rollback ${instance} in place to ${snapshot}?`);
     await stopInstanceForRestore(instance);
-    await runText(["zfs", "rollback", "-r", snapshot], PREFIX);
+    await rollbackDatasetTreeToSnapshot(snapshot);
     console.log(success(`Rolled back ${instance} to ${snapshot}`));
     console.log(`${label("Next:")} ${value(`lxc start ${instance}`)}`);
     return;

scripts/terrarium-s3-export.ts

@@ -58,6 +58,13 @@ async function latestSnapshot(dataset: string): Promise<string> {
   return chooseLatestExportSnapshot(stdout.split("\n"), dataset);
 }
 
+export function zfsReplicationSendCommand(latestSnapshot: string, parentSnapshot = ""): string {
+  if (parentSnapshot) {
+    return `zfs send -R -I ${shellEscape(parentSnapshot)} ${shellEscape(latestSnapshot)}`;
+  }
+  return `zfs send -R ${shellEscape(latestSnapshot)}`;
+}
+
 export function isRetriableS3ExportError(message: string): boolean {
   const lowered = message.toLowerCase();
   return [
@@ -161,8 +168,8 @@ export async function backupExportCmd(configPath = DEFAULT_CONFIG_PATH): Promise
 
     const streamSource =
       last && (await runAllowFailure(["zfs", "list", "-H", "-t", "snapshot", last])).exitCode === 0
-        ? `zfs send -I ${shellEscape(last)} ${shellEscape(latest)}`
-        : `zfs send ${shellEscape(latest)}`;
+        ? zfsReplicationSendCommand(latest, last)
+        : zfsReplicationSendCommand(latest);
 
     await runRetriableS3Command(
       `${streamSource} | zstd -T0 | ${awsBase.map(shellEscape).join(" ")} s3 cp - ${shellEscape(`s3://${bucket}/${objectKey}`)}`,