Skip to content

Commit 84c37ea

Browse files
committed
Add stacked disk integration coverage and documentation
1 parent 5f8795b commit 84c37ea

6 files changed

Lines changed: 317 additions & 0 deletions

File tree

benchmark/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,32 @@ You can also enable the debugging output to diagnose issues:
2222
go run cmd/main.go fio --debug
2323
```
2424

25+
To compare an empty Tart home with the same pull after its immutable base has
26+
been prewarmed, provide a standalone remote base image and a stacked image built
27+
from it. For example, create and push a stacked child of the public Tahoe base:
28+
29+
```shell
30+
BASE_IMAGE=ghcr.io/cirruslabs/macos-tahoe-base:latest
31+
STACKED_IMAGE=ghcr.io/your-org/macos-tahoe-stacked:latest
32+
33+
tart clone --stacked "$BASE_IMAGE" macos-tahoe-stacked
34+
tart push macos-tahoe-stacked "$STACKED_IMAGE"
35+
```
36+
37+
Then benchmark that pair:
38+
39+
```shell
40+
go run cmd/main.go stacked-oci \
41+
--base-image "$BASE_IMAGE" \
42+
--image "$STACKED_IMAGE"
43+
```
44+
45+
The command first performs an unmeasured pull to warm registry, CDN, and
46+
filesystem caches. It then uses disposable `TART_HOME` directories for both
47+
measured scenarios. The prewarmed scenario keeps the VM created by
48+
`tart clone --stacked` alive while pulling the child, so the shared immutable
49+
base layer remains referenced and available for reuse.
50+
2551
## Results
2652

2753
### Mar 27, 2024

benchmark/internal/command/root.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package command
22

33
import (
44
"github.com/cirruslabs/tart/benchmark/internal/command/fio"
5+
"github.com/cirruslabs/tart/benchmark/internal/command/stackedoci"
56
"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
67
"github.com/spf13/cobra"
78
)
@@ -15,6 +16,7 @@ func NewCommand() *cobra.Command {
1516

1617
cmd.AddCommand(
1718
fio.NewCommand(),
19+
stackedoci.NewCommand(),
1820
xcode.NewCommand(),
1921
)
2022

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
package stackedoci
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/exec"
8+
"strings"
9+
"time"
10+
11+
"github.com/gosuri/uitable"
12+
"github.com/spf13/cobra"
13+
"go.uber.org/zap"
14+
"go.uber.org/zap/zapio"
15+
)
16+
17+
var (
18+
debug bool
19+
baseImage string
20+
stackedImage string
21+
insecure bool
22+
concurrency uint
23+
)
24+
25+
func NewCommand() *cobra.Command {
26+
cmd := &cobra.Command{
27+
Use: "stacked-oci",
28+
Short: "benchmark empty and prewarmed Tart homes for stacked OCI pulls",
29+
Long: "Warm the registry once, then compare an empty Tart home with one whose " +
30+
"immutable base has already been materialized by tart clone --stacked. " +
31+
"Every scenario uses a disposable TART_HOME and leaves the user's Tart home untouched.",
32+
RunE: run,
33+
}
34+
35+
cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
36+
cmd.Flags().StringVar(&baseImage, "base-image", "", "remote flat OCI image used as the stacked image's base")
37+
cmd.Flags().StringVar(&stackedImage, "image", "", "remote stacked OCI image to pull and clone")
38+
cmd.Flags().BoolVar(&insecure, "insecure", false, "connect to the OCI registry via insecure HTTP")
39+
cmd.Flags().UintVar(&concurrency, "concurrency", 4, "network concurrency passed to tart pull and clone")
40+
_ = cmd.MarkFlagRequired("base-image")
41+
_ = cmd.MarkFlagRequired("image")
42+
43+
return cmd
44+
}
45+
46+
func run(cmd *cobra.Command, _ []string) error {
47+
if concurrency < 1 {
48+
return fmt.Errorf("concurrency cannot be less than 1")
49+
}
50+
51+
config := zap.NewProductionConfig()
52+
if debug {
53+
config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
54+
}
55+
logger, err := config.Build()
56+
if err != nil {
57+
return err
58+
}
59+
defer func() { _ = logger.Sync() }()
60+
61+
warmupHome, err := os.MkdirTemp("", "tart-stacked-oci-warmup-*")
62+
if err != nil {
63+
return err
64+
}
65+
defer os.RemoveAll(warmupHome)
66+
67+
emptyHome, err := os.MkdirTemp("", "tart-stacked-oci-empty-*")
68+
if err != nil {
69+
return err
70+
}
71+
defer os.RemoveAll(emptyHome)
72+
73+
warmHome, err := os.MkdirTemp("", "tart-stacked-oci-warm-*")
74+
if err != nil {
75+
return err
76+
}
77+
defer os.RemoveAll(warmHome)
78+
79+
table := uitable.New()
80+
table.AddRow("Scenario", "Operation", "Time")
81+
82+
// Warm registry, CDN, and filesystem caches before either measured
83+
// scenario so their difference reflects Tart's local base reuse.
84+
if _, err := timedTart(cmd.Context(), logger, warmupHome, pullArguments(stackedImage)...); err != nil {
85+
return fmt.Errorf("registry warmup failed: %w", err)
86+
}
87+
if err := os.RemoveAll(warmupHome); err != nil {
88+
return fmt.Errorf("removing registry warmup home: %w", err)
89+
}
90+
91+
duration, err := timedTart(cmd.Context(), logger, emptyHome, pullArguments(stackedImage)...)
92+
if err != nil {
93+
return fmt.Errorf("empty-home stacked pull failed: %w", err)
94+
}
95+
table.AddRow("empty", "pull stacked image", duration)
96+
97+
duration, err = timedTart(cmd.Context(), logger, emptyHome, "clone", stackedImage, "empty-clone")
98+
if err != nil {
99+
return fmt.Errorf("empty-home stacked clone failed: %w", err)
100+
}
101+
table.AddRow("empty", "clone stacked image", duration)
102+
if err := os.RemoveAll(emptyHome); err != nil {
103+
return fmt.Errorf("removing empty home: %w", err)
104+
}
105+
106+
duration, err = timedTart(cmd.Context(), logger, warmHome, cloneBaseArguments(baseImage)...)
107+
if err != nil {
108+
return fmt.Errorf("base prewarm failed: %w", err)
109+
}
110+
table.AddRow("prewarmed", "clone --stacked base image", duration)
111+
112+
duration, err = timedTart(cmd.Context(), logger, warmHome, pullArguments(stackedImage)...)
113+
if err != nil {
114+
return fmt.Errorf("prewarmed stacked pull failed: %w", err)
115+
}
116+
table.AddRow("prewarmed", "pull stacked image", duration)
117+
118+
duration, err = timedTart(cmd.Context(), logger, warmHome, "clone", stackedImage, "warm-clone")
119+
if err != nil {
120+
return fmt.Errorf("prewarmed stacked clone failed: %w", err)
121+
}
122+
table.AddRow("prewarmed", "clone stacked image", duration)
123+
124+
fmt.Println(table.String())
125+
return nil
126+
}
127+
128+
func pullArguments(image string) []string {
129+
args := []string{"pull", "--concurrency", fmt.Sprint(concurrency)}
130+
if insecure {
131+
args = append(args, "--insecure")
132+
}
133+
return append(args, image)
134+
}
135+
136+
func cloneBaseArguments(image string) []string {
137+
args := []string{"clone", "--stacked", "--concurrency", fmt.Sprint(concurrency)}
138+
if insecure {
139+
args = append(args, "--insecure")
140+
}
141+
return append(args, image, "prewarmed-base")
142+
}
143+
144+
func timedTart(
145+
ctx context.Context,
146+
logger *zap.Logger,
147+
tartHome string,
148+
args ...string,
149+
) (time.Duration, error) {
150+
logger.Sugar().Debugf("TART_HOME=%s tart %s", tartHome, strings.Join(args, " "))
151+
start := time.Now()
152+
153+
command := exec.CommandContext(ctx, "tart", args...)
154+
command.Env = append(os.Environ(), "TART_HOME="+tartHome)
155+
loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
156+
command.Stdout = loggerWriter
157+
command.Stderr = loggerWriter
158+
159+
err := command.Run()
160+
return time.Since(start).Round(time.Millisecond), err
161+
}

docs/faq.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,28 @@ export TART_NO_AUTO_PRUNE=
231231
TART_NO_AUTO_PRUNE= tart pull ...
232232
```
233233

234+
## Stacked disk images
235+
236+
On macOS 27 or newer, `tart clone --stacked` can create a VM from a remote,
237+
standalone macOS OCI image whose writes are stored in a private ASIF overlay while
238+
its source disk remains a shared read-only base:
239+
240+
```shell
241+
tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base:latest macos-build
242+
```
243+
244+
Running and pushing `macos-build` preserves that disk relationship. Pulling
245+
another image from the same lineage only downloads immutable disk files that
246+
are not already present in Tart's cache. For a stopped stacked VM,
247+
`tart set --disk-size` grows its private writable overlay without changing the
248+
base; a subsequent push records the new guest-visible disk size.
249+
250+
`tart pull` can cache a stacked image without assembling its disk. Clone, run,
251+
import, and export require a Tart build with DiskImageKit support and macOS 27
252+
or newer. Existing standalone raw and ASIF images continue to work on older
253+
hosts. Keep published lineages shallow when possible: every additional parent
254+
overlay adds another ASIF file to validate and assemble at run time.
255+
234256
## Disk resizing
235257

236258
Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images have the `cloud-initramfs-growroot` package installed that runs on boot) and on the rest of the distributions by running the `growpart` or `resize2fs` commands.

docs/quick-start.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,3 +265,15 @@ tart clone acme.io/remoteorg/name:latest my-local-vm-name
265265
```
266266

267267
If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning.
268+
269+
### Creating a Stacked Disk
270+
271+
On macOS 27 or newer, use `--stacked` to create a VM that keeps a remote standalone
272+
macOS OCI image as an immutable base and stores only its own writes separately:
273+
274+
```bash
275+
tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base my-local-vm-name
276+
```
277+
278+
Pushing this VM preserves the disk relationship. Pulling another image from the
279+
same lineage reuses immutable disk files that are already in Tart's cache.
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import os
2+
import platform
3+
import subprocess
4+
import uuid
5+
from typing import Optional
6+
7+
import pytest
8+
from paramiko.client import AutoAddPolicy, SSHClient
9+
10+
11+
def _macos_major_version() -> int:
12+
version = platform.mac_ver()[0]
13+
return int(version.split(".", maxsplit=1)[0]) if version else 0
14+
15+
16+
def _shutdown_vm(tart, vm_name: str, guest_command: Optional[str] = None) -> None:
17+
tart_run_process = tart.run_async(["run", "--no-graphics", vm_name])
18+
client = None
19+
20+
try:
21+
stdout, _ = tart.run(["ip", vm_name, "--wait", "180"])
22+
client = SSHClient()
23+
client.set_missing_host_key_policy(AutoAddPolicy)
24+
client.connect(stdout.strip(), username="admin", password="admin")
25+
if guest_command:
26+
_, command_stdout, command_stderr = client.exec_command(guest_command)
27+
assert command_stdout.channel.recv_exit_status() == 0, command_stderr.read().decode()
28+
client.exec_command("sudo shutdown -h now")
29+
30+
tart_run_process.wait(timeout=180)
31+
assert tart_run_process.returncode == 0
32+
finally:
33+
if client is not None:
34+
client.close()
35+
if tart_run_process.poll() is None:
36+
tart_run_process.terminate()
37+
try:
38+
tart_run_process.wait(timeout=30)
39+
except subprocess.TimeoutExpired:
40+
tart_run_process.kill()
41+
tart_run_process.wait(timeout=30)
42+
43+
44+
@pytest.mark.skipif(
45+
_macos_major_version() < 27,
46+
reason="stacked disk images require DiskImageKit on macOS 27 or newer",
47+
)
48+
def test_stacked_oci_round_trip_and_boot(tart, docker_registry):
49+
suffix = str(uuid.uuid4())
50+
source_image = os.environ.get(
51+
"TART_STACKED_INTEGRATION_BASE_IMAGE",
52+
"ghcr.io/cirruslabs/macos-tahoe-base:latest",
53+
)
54+
source_clone_args = ["clone"]
55+
if os.environ.get("TART_STACKED_INTEGRATION_BASE_INSECURE") == "1":
56+
source_clone_args.append("--insecure")
57+
standalone_vm = f"stacked-base-{suffix}"
58+
stacked_vm = f"stacked-child-{suffix}"
59+
restored_vm = f"stacked-restored-{suffix}"
60+
base_remote = docker_registry.remote_name(f"stacked-base-{suffix}")
61+
child_remote = docker_registry.remote_name(f"stacked-child-{suffix}")
62+
63+
try:
64+
# Publish a normal Tart image, then start a stacked lineage from that
65+
# remote image. The base remains a normal flat OCI image.
66+
tart.run(source_clone_args + [source_image, standalone_vm])
67+
tart.run(["push", "--insecure", standalone_vm, base_remote])
68+
tart.run(["clone", "--insecure", "--stacked", base_remote, stacked_vm])
69+
70+
stacked_path = os.path.join(tart.home(), "vms", stacked_vm)
71+
assert os.path.isfile(os.path.join(stacked_path, "overlay.asif"))
72+
assert os.path.isfile(os.path.join(stacked_path, "manifest.json"))
73+
assert not os.path.exists(os.path.join(stacked_path, "disk.img"))
74+
75+
# Boot once so the top ASIF overlay contains real guest writes, then
76+
# exercise stacked push, pull, clone, assembly, and a second boot.
77+
_shutdown_vm(tart, stacked_vm, "touch /Users/admin/stacked-round-trip-marker")
78+
tart.run(["push", "--insecure", stacked_vm, child_remote])
79+
tart.run(["delete", stacked_vm])
80+
tart.run(["pull", "--insecure", child_remote])
81+
tart.run(["clone", child_remote, restored_vm])
82+
83+
restored_path = os.path.join(tart.home(), "vms", restored_vm)
84+
assert os.path.isfile(os.path.join(restored_path, "overlay.asif"))
85+
assert os.path.isfile(os.path.join(restored_path, "manifest.json"))
86+
assert not os.path.exists(os.path.join(restored_path, "disk.img"))
87+
88+
_shutdown_vm(tart, restored_vm, "test -f /Users/admin/stacked-round-trip-marker")
89+
finally:
90+
for vm_name in (restored_vm, stacked_vm, standalone_vm, child_remote, base_remote):
91+
try:
92+
tart.run(["delete", vm_name])
93+
except Exception:
94+
pass

0 commit comments

Comments
 (0)