Add stacked disk integration coverage and documentation

benchmark/README.md

@@ -22,6 +22,32 @@ You can also enable the debugging output to diagnose issues:
 go run cmd/main.go fio --debug
 ```
 
+To compare an empty Tart home with the same pull after its immutable base has
+been prewarmed, provide a standalone remote base image and a stacked image built
+from it. For example, create and push a stacked child of the public Tahoe base:
+
+```shell
+BASE_IMAGE=ghcr.io/cirruslabs/macos-tahoe-base:latest
+STACKED_IMAGE=ghcr.io/your-org/macos-tahoe-stacked:latest
+
+tart clone --stacked "$BASE_IMAGE" macos-tahoe-stacked
+tart push macos-tahoe-stacked "$STACKED_IMAGE"
+```
+
+Then benchmark that pair:
+
+```shell
+go run cmd/main.go stacked-oci \
+  --base-image "$BASE_IMAGE" \
+  --image "$STACKED_IMAGE"
+```
+
+The command first performs an unmeasured pull to warm registry, CDN, and
+filesystem caches. It then uses disposable `TART_HOME` directories for both
+measured scenarios. The prewarmed scenario keeps the VM created by
+`tart clone --stacked` alive while pulling the child, so the shared immutable
+base layer remains referenced and available for reuse.
+
 ## Results
 
 ### Mar 27, 2024

benchmark/internal/command/root.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/stackedoci"
 	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )
@@ -15,6 +16,7 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		fio.NewCommand(),
+		stackedoci.NewCommand(),
 		xcode.NewCommand(),
 	)
 

benchmark/internal/command/stackedoci/stackedoci.go

@@ -0,0 +1,161 @@
+package stackedoci
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+)
+
+var (
+	debug        bool
+	baseImage    string
+	stackedImage string
+	insecure     bool
+	concurrency  uint
+)
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "stacked-oci",
+		Short: "benchmark empty and prewarmed Tart homes for stacked OCI pulls",
+		Long: "Warm the registry once, then compare an empty Tart home with one whose " +
+			"immutable base has already been materialized by tart clone --stacked. " +
+			"Every scenario uses a disposable TART_HOME and leaves the user's Tart home untouched.",
+		RunE: run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&baseImage, "base-image", "", "remote flat OCI image used as the stacked image's base")
+	cmd.Flags().StringVar(&stackedImage, "image", "", "remote stacked OCI image to pull and clone")
+	cmd.Flags().BoolVar(&insecure, "insecure", false, "connect to the OCI registry via insecure HTTP")
+	cmd.Flags().UintVar(&concurrency, "concurrency", 4, "network concurrency passed to tart pull and clone")
+	_ = cmd.MarkFlagRequired("base-image")
+	_ = cmd.MarkFlagRequired("image")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, _ []string) error {
+	if concurrency < 1 {
+		return fmt.Errorf("concurrency cannot be less than 1")
+	}
+
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = logger.Sync() }()
+
+	warmupHome, err := os.MkdirTemp("", "tart-stacked-oci-warmup-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(warmupHome)
+
+	emptyHome, err := os.MkdirTemp("", "tart-stacked-oci-empty-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(emptyHome)
+
+	warmHome, err := os.MkdirTemp("", "tart-stacked-oci-warm-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(warmHome)
+
+	table := uitable.New()
+	table.AddRow("Scenario", "Operation", "Time")
+
+	// Warm registry, CDN, and filesystem caches before either measured
+	// scenario so their difference reflects Tart's local base reuse.
+	if _, err := timedTart(cmd.Context(), logger, warmupHome, pullArguments(stackedImage)...); err != nil {
+		return fmt.Errorf("registry warmup failed: %w", err)
+	}
+	if err := os.RemoveAll(warmupHome); err != nil {
+		return fmt.Errorf("removing registry warmup home: %w", err)
+	}
+
+	duration, err := timedTart(cmd.Context(), logger, emptyHome, pullArguments(stackedImage)...)
+	if err != nil {
+		return fmt.Errorf("empty-home stacked pull failed: %w", err)
+	}
+	table.AddRow("empty", "pull stacked image", duration)
+
+	duration, err = timedTart(cmd.Context(), logger, emptyHome, "clone", stackedImage, "empty-clone")
+	if err != nil {
+		return fmt.Errorf("empty-home stacked clone failed: %w", err)
+	}
+	table.AddRow("empty", "clone stacked image", duration)
+	if err := os.RemoveAll(emptyHome); err != nil {
+		return fmt.Errorf("removing empty home: %w", err)
+	}
+
+	duration, err = timedTart(cmd.Context(), logger, warmHome, cloneBaseArguments(baseImage)...)
+	if err != nil {
+		return fmt.Errorf("base prewarm failed: %w", err)
+	}
+	table.AddRow("prewarmed", "clone --stacked base image", duration)
+
+	duration, err = timedTart(cmd.Context(), logger, warmHome, pullArguments(stackedImage)...)
+	if err != nil {
+		return fmt.Errorf("prewarmed stacked pull failed: %w", err)
+	}
+	table.AddRow("prewarmed", "pull stacked image", duration)
+
+	duration, err = timedTart(cmd.Context(), logger, warmHome, "clone", stackedImage, "warm-clone")
+	if err != nil {
+		return fmt.Errorf("prewarmed stacked clone failed: %w", err)
+	}
+	table.AddRow("prewarmed", "clone stacked image", duration)
+
+	fmt.Println(table.String())
+	return nil
+}
+
+func pullArguments(image string) []string {
+	args := []string{"pull", "--concurrency", fmt.Sprint(concurrency)}
+	if insecure {
+		args = append(args, "--insecure")
+	}
+	return append(args, image)
+}
+
+func cloneBaseArguments(image string) []string {
+	args := []string{"clone", "--stacked", "--concurrency", fmt.Sprint(concurrency)}
+	if insecure {
+		args = append(args, "--insecure")
+	}
+	return append(args, image, "prewarmed-base")
+}
+
+func timedTart(
+	ctx context.Context,
+	logger *zap.Logger,
+	tartHome string,
+	args ...string,
+) (time.Duration, error) {
+	logger.Sugar().Debugf("TART_HOME=%s tart %s", tartHome, strings.Join(args, " "))
+	start := time.Now()
+
+	command := exec.CommandContext(ctx, "tart", args...)
+	command.Env = append(os.Environ(), "TART_HOME="+tartHome)
+	loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+	command.Stdout = loggerWriter
+	command.Stderr = loggerWriter
+
+	err := command.Run()
+	return time.Since(start).Round(time.Millisecond), err
+}

docs/faq.md

@@ -231,6 +231,28 @@ export TART_NO_AUTO_PRUNE=
 TART_NO_AUTO_PRUNE= tart pull ...
 ```
 
+## Stacked disk images
+
+On macOS 27 or newer, `tart clone --stacked` can create a VM from a remote,
+standalone macOS OCI image whose writes are stored in a private ASIF overlay while
+its source disk remains a shared read-only base:
+
+```shell
+tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base:latest macos-build
+```
+
+Running and pushing `macos-build` preserves that disk relationship. Pulling
+another image from the same lineage only downloads immutable disk files that
+are not already present in Tart's cache. For a stopped stacked VM,
+`tart set --disk-size` grows its private writable overlay without changing the
+base; a subsequent push records the new guest-visible disk size.
+
+`tart pull` can cache a stacked image without assembling its disk. Clone, run,
+import, and export require a Tart build with DiskImageKit support and macOS 27
+or newer. Existing standalone raw and ASIF images continue to work on older
+hosts. Keep published lineages shallow when possible: every additional parent
+overlay adds another ASIF file to validate and assemble at run time.
+
 ## Disk resizing
 
 Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images have the `cloud-initramfs-growroot` package installed that runs on boot) and on the rest of the distributions by running the `growpart` or `resize2fs` commands.

docs/quick-start.md

@@ -265,3 +265,15 @@ tart clone acme.io/remoteorg/name:latest my-local-vm-name
 ```
 
 If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning.
+
+### Creating a Stacked Disk
+
+On macOS 27 or newer, use `--stacked` to create a VM that keeps a remote standalone
+macOS OCI image as an immutable base and stores only its own writes separately:
+
+```bash
+tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base my-local-vm-name
+```
+
+Pushing this VM preserves the disk relationship. Pulling another image from the
+same lineage reuses immutable disk files that are already in Tart's cache.

integration-tests/test_stacked_oci.py

@@ -0,0 +1,94 @@
+import os
+import platform
+import subprocess
+import uuid
+from typing import Optional
+
+import pytest
+from paramiko.client import AutoAddPolicy, SSHClient
+
+
+def _macos_major_version() -> int:
+    version = platform.mac_ver()[0]
+    return int(version.split(".", maxsplit=1)[0]) if version else 0
+
+
+def _shutdown_vm(tart, vm_name: str, guest_command: Optional[str] = None) -> None:
+    tart_run_process = tart.run_async(["run", "--no-graphics", vm_name])
+    client = None
+
+    try:
+        stdout, _ = tart.run(["ip", vm_name, "--wait", "180"])
+        client = SSHClient()
+        client.set_missing_host_key_policy(AutoAddPolicy)
+        client.connect(stdout.strip(), username="admin", password="admin")
+        if guest_command:
+            _, command_stdout, command_stderr = client.exec_command(guest_command)
+            assert command_stdout.channel.recv_exit_status() == 0, command_stderr.read().decode()
+        client.exec_command("sudo shutdown -h now")
+
+        tart_run_process.wait(timeout=180)
+        assert tart_run_process.returncode == 0
+    finally:
+        if client is not None:
+            client.close()
+        if tart_run_process.poll() is None:
+            tart_run_process.terminate()
+            try:
+                tart_run_process.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                tart_run_process.kill()
+                tart_run_process.wait(timeout=30)
+
+
+@pytest.mark.skipif(
+    _macos_major_version() < 27,
+    reason="stacked disk images require DiskImageKit on macOS 27 or newer",
+)
+def test_stacked_oci_round_trip_and_boot(tart, docker_registry):
+    suffix = str(uuid.uuid4())
+    source_image = os.environ.get(
+        "TART_STACKED_INTEGRATION_BASE_IMAGE",
+        "ghcr.io/cirruslabs/macos-tahoe-base:latest",
+    )
+    source_clone_args = ["clone"]
+    if os.environ.get("TART_STACKED_INTEGRATION_BASE_INSECURE") == "1":
+        source_clone_args.append("--insecure")
+    standalone_vm = f"stacked-base-{suffix}"
+    stacked_vm = f"stacked-child-{suffix}"
+    restored_vm = f"stacked-restored-{suffix}"
+    base_remote = docker_registry.remote_name(f"stacked-base-{suffix}")
+    child_remote = docker_registry.remote_name(f"stacked-child-{suffix}")
+
+    try:
+        # Publish a normal Tart image, then start a stacked lineage from that
+        # remote image. The base remains a normal flat OCI image.
+        tart.run(source_clone_args + [source_image, standalone_vm])
+        tart.run(["push", "--insecure", standalone_vm, base_remote])
+        tart.run(["clone", "--insecure", "--stacked", base_remote, stacked_vm])
+
+        stacked_path = os.path.join(tart.home(), "vms", stacked_vm)
+        assert os.path.isfile(os.path.join(stacked_path, "overlay.asif"))
+        assert os.path.isfile(os.path.join(stacked_path, "manifest.json"))
+        assert not os.path.exists(os.path.join(stacked_path, "disk.img"))
+
+        # Boot once so the top ASIF overlay contains real guest writes, then
+        # exercise stacked push, pull, clone, assembly, and a second boot.
+        _shutdown_vm(tart, stacked_vm, "touch ~/stacked-round-trip-marker")
+        tart.run(["push", "--insecure", stacked_vm, child_remote])
+        tart.run(["delete", stacked_vm])
+        tart.run(["pull", "--insecure", child_remote])
+        tart.run(["clone", child_remote, restored_vm])
+
+        restored_path = os.path.join(tart.home(), "vms", restored_vm)
+        assert os.path.isfile(os.path.join(restored_path, "overlay.asif"))
+        assert os.path.isfile(os.path.join(restored_path, "manifest.json"))
+        assert not os.path.exists(os.path.join(restored_path, "disk.img"))
+
+        _shutdown_vm(tart, restored_vm, "test -f ~/stacked-round-trip-marker")
+    finally:
+        for vm_name in (restored_vm, stacked_vm, standalone_vm, child_remote, base_remote):
+            try:
+                tart.run(["delete", vm_name])
+            except Exception:
+                pass