diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index c7f7738910..f22bc4d85a 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -58,6 +58,9 @@ Self-backed rootfs overlay (`--overlay2=root:self`) is enabled by default in runsc for performance. If you need to propagate rootfs changes to the host filesystem, then disable it with `--overlay2=none`. +Overlay has `size=` option which is passed as `size=` tmpfs mount option. +For example, `--overlay2=root:memory,size=2g`. + ## Directfs Directfs is a feature that allows the sandbox process to directly access the diff --git a/runsc/boot/gofer_conf.go b/runsc/boot/gofer_conf.go index 9d94cb8f40..fdd98214a3 100644 --- a/runsc/boot/gofer_conf.go +++ b/runsc/boot/gofer_conf.go @@ -124,19 +124,24 @@ func (l *GoferMountConfLowerType) Set(v string) error { // GoferMountConf describes how a gofer mount is configured in the sandbox. type GoferMountConf struct { - Upper GoferMountConfUpperType `json:"upper"` Lower GoferMountConfLowerType `json:"lower"` + Upper GoferMountConfUpperType `json:"upper"` + Size string `json:"size,omitempty"` } // String returns a human-readable string representing the gofer mount config. func (g GoferMountConf) String() string { - return fmt.Sprintf("%s:%s", g.Lower, g.Upper) + res := fmt.Sprintf("%s:%s", g.Lower, g.Upper) + if g.Size != "" { + res += ":size=" + g.Size + } + return res } // Set sets the value. Set(String()) should be idempotent. func (g *GoferMountConf) Set(v string) error { parts := strings.Split(v, ":") - if len(parts) != 2 { + if len(parts) < 2 || len(parts) > 3 { return fmt.Errorf("invalid gofer mount config format: %q", v) } if err := g.Lower.Set(parts[0]); err != nil { @@ -145,8 +150,17 @@ func (g *GoferMountConf) Set(v string) error { if err := g.Upper.Set(parts[1]); err != nil { return err } + g.Size = "" + if len(parts) >= 3 { + sizeArg := parts[2] + size, cut := strings.CutPrefix(sizeArg, "size=") + if !cut { + return fmt.Errorf("invalid gofer mount config format: %q", v) + } + g.Size = size + } if !g.valid() { - return fmt.Errorf("invalid gofer mount config: %+v", g) + return fmt.Errorf("invalid gofer mount config: %q", v) } return nil } diff --git a/runsc/boot/gofer_conf_test.go b/runsc/boot/gofer_conf_test.go index 49f1f7ab2b..7d028d3ce5 100644 --- a/runsc/boot/gofer_conf_test.go +++ b/runsc/boot/gofer_conf_test.go @@ -175,3 +175,28 @@ func TestGoferConfFlags(t *testing.T) { } } } + +func TestGoferMountConfSetGet(t *testing.T) { + t.Run("Without size", func(t *testing.T) { + conf := GoferMountConf{} + err := conf.Set("lisafs:anon") + if err != nil { + t.Fatalf("Expect success: %v", err) + } + s := conf.String() + if s != "lisafs:anon" { + t.Fatalf("Expected lisafs:anon, got %s", s) + } + }) + t.Run("With size", func(t *testing.T) { + conf := GoferMountConf{} + err := conf.Set("lisafs:anon:size=1719") + if err != nil { + t.Fatalf("Expect success: %v", err) + } + s := conf.String() + if s != "lisafs:anon:size=1719" { + t.Fatalf("Expected lisafs:anon:size=1719, got %s", s) + } + }) +} diff --git a/runsc/boot/mount_hints.go b/runsc/boot/mount_hints.go index 3b1090e871..ce84f81fd3 100644 --- a/runsc/boot/mount_hints.go +++ b/runsc/boot/mount_hints.go @@ -238,6 +238,9 @@ func (p *PodMountHints) FindMount(mountSrc string) *MountHint { type RootfsHint struct { Mount specs.Mount Overlay config.OverlayMedium + // Size of overlay tmpfs. Passed as `size={Size}` to tmpfs mount. + // Use default if unspecified. + Size string } func (r *RootfsHint) setSource(val string) error { @@ -258,6 +261,30 @@ func (r *RootfsHint) setType(val string) error { return nil } +func (r *RootfsHint) setOption(key, val string) error { + switch key { + case "size": + r.Size = val + default: + return fmt.Errorf("invalid rootfs option: %s=%s", key, val) + } + return nil +} + +func (r *RootfsHint) setOptions(val string) error { + for _, option := range strings.Split(val, ",") { + parts := strings.SplitN(option, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("rootfs options must be key=value: %s", option) + } + key, val := parts[0], parts[1] + if err := r.setOption(key, val); err != nil { + return err + } + } + return nil +} + func (r *RootfsHint) setField(key, val string) error { switch key { case "source": @@ -266,6 +293,8 @@ func (r *RootfsHint) setField(key, val string) error { return r.setType(val) case "overlay": return r.Overlay.Set(val) + case "options": + return r.setOptions(val) default: return fmt.Errorf("invalid rootfs annotation: %s=%s", key, val) } diff --git a/runsc/boot/mount_hints_test.go b/runsc/boot/mount_hints_test.go index da8028c550..df41658618 100644 --- a/runsc/boot/mount_hints_test.go +++ b/runsc/boot/mount_hints_test.go @@ -258,6 +258,7 @@ func TestRootfsHintHappy(t *testing.T) { RootfsPrefix + "source": imagePath, RootfsPrefix + "type": erofs.Name, RootfsPrefix + "overlay": config.MemoryOverlay.String(), + RootfsPrefix + "options": "size=100m", }, } hint, err := NewRootfsHint(spec) @@ -275,6 +276,9 @@ func TestRootfsHintHappy(t *testing.T) { if hint.Overlay != config.MemoryOverlay { t.Errorf("rootfs overlay, want: %q, got: %q", config.MemoryOverlay, hint.Overlay) } + if hint.Size != "100m" { + t.Errorf("rootfs size, want: 100m, got: %q", hint.Size) + } } // TestRootfsHintErrors tests that proper errors will be returned when parsing diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index cf7aa06bee..cf32430b22 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -585,6 +585,12 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co // filesystem specific options. upperOpts := *lowerOpts upperOpts.GetFilesystemOptions = vfs.GetFilesystemOptions{InternalMount: true} + if mountConf.Size != "" { + if upperOpts.GetFilesystemOptions.Data != "" { + upperOpts.GetFilesystemOptions.Data += "," + } + upperOpts.GetFilesystemOptions.Data += "size=" + mountConf.Size + } overlayOpts := *lowerOpts overlayOpts.GetFilesystemOptions = vfs.GetFilesystemOptions{InternalMount: true} diff --git a/runsc/config/config.go b/runsc/config/config.go index 7910db473b..a0953cd90b 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -906,6 +906,11 @@ type Overlay2 struct { rootMount bool subMounts bool medium OverlayMedium + // Size of overlay upper layer. + // Passed as is to tmpfs mount, as `size={size}`. + // Empty means use default. + // Size is applied to each overlay independently and not shared by overlays. + size string } func defaultOverlay2() *Overlay2 { @@ -913,17 +918,30 @@ func defaultOverlay2() *Overlay2 { return &Overlay2{rootMount: true, subMounts: false, medium: SelfOverlay} } +func setOverlay2Err(v string) error { + return fmt.Errorf("expected format is --overlay2={mount}:{medium}[,size={size}], got %q", v) +} + +// `--overlay2=...` param `size=`. +const overlay2SizeEq = "size=" + // Set implements flag.Value. Set(String()) should be idempotent. func (o *Overlay2) Set(v string) error { if v == "none" { o.rootMount = false o.subMounts = false o.medium = NoOverlay + o.size = "" return nil } - vs := strings.Split(v, ":") + parts := strings.Split(v, ",") + if len(parts) < 1 { + return setOverlay2Err(v) + } + + vs := strings.Split(parts[0], ":") if len(vs) != 2 { - return fmt.Errorf("expected format is --overlay2={mount}:{medium}, got %q", v) + return setOverlay2Err(v) } switch mount := vs[0]; mount { @@ -936,7 +954,25 @@ func (o *Overlay2) Set(v string) error { return fmt.Errorf("unexpected mount specifier for --overlay2: %q", mount) } - return o.medium.Set(vs[1]) + err := o.medium.Set(vs[1]) + if err != nil { + return err + } + + if len(parts) == 1 { + o.size = "" + } else if len(parts) == 2 { + sizeArg := parts[1] + size, cut := strings.CutPrefix(sizeArg, overlay2SizeEq) + if !cut { + return setOverlay2Err(v) + } + o.size = size + } else { + return setOverlay2Err(v) + } + + return nil } // Get implements flag.Value. @@ -958,7 +994,13 @@ func (o Overlay2) String() string { default: panic("invalid state of subMounts = true and rootMount = false") } - return res + ":" + o.medium.String() + + var sizeSuffix string + if o.size != "" { + sizeSuffix = fmt.Sprintf(",%s%s", overlay2SizeEq, o.size) + } + + return res + ":" + o.medium.String() + sizeSuffix } // Enabled returns true if the overlay option is enabled for any mounts. @@ -974,6 +1016,13 @@ func (o *Overlay2) RootOverlayMedium() OverlayMedium { return o.medium } +func (o *Overlay2) RootOverlaySize() string { + if !o.rootMount { + return "" + } + return o.size +} + // SubMountOverlayMedium returns the overlay medium config of submounts. func (o *Overlay2) SubMountOverlayMedium() OverlayMedium { if !o.subMounts { @@ -982,6 +1031,13 @@ func (o *Overlay2) SubMountOverlayMedium() OverlayMedium { return o.medium } +func (o *Overlay2) SubMountOverlaySize() string { + if !o.subMounts { + return "" + } + return o.size +} + // Medium returns the overlay medium config. func (o Overlay2) Medium() OverlayMedium { return o.medium diff --git a/runsc/config/config_test.go b/runsc/config/config_test.go index 6ecfff3b1d..20a88a576e 100644 --- a/runsc/config/config_test.go +++ b/runsc/config/config_test.go @@ -231,6 +231,11 @@ func TestInvalidFlags(t *testing.T) { value: "root:dir=tmp", error: "overlay host file directory should be an absolute path, got \"tmp\"", }, + { + name: "overlay2", + value: "root:memory,sz=sdg", + error: "expected format is --overlay2", + }, } { t.Run(tc.name, func(t *testing.T) { testFlags := flag.NewFlagSet("test", flag.ContinueOnError) @@ -781,3 +786,32 @@ root = "%s" } } + +func TestParseSerializeOverlay2(t *testing.T) { + t.Run("Without size", func(t *testing.T) { + o := Overlay2{} + err := o.Set("all:memory") + if err != nil { + t.Fatalf("Set failed: %v", err) + } + if o.RootOverlaySize() != "" || o.RootOverlayMedium() != "" { + t.Fatalf("Size mismatch, expecting empty, got %q, %q", o.RootOverlaySize(), o.SubMountOverlaySize()) + } + if o.String() != "all:memory" { + t.Fatalf("String mismatch, expecting all:memory, got %q", o.String()) + } + }) + t.Run("With size", func(t *testing.T) { + o := Overlay2{} + err := o.Set("root:memory,size=1g") + if err != nil { + t.Fatalf("Set failed: %v", err) + } + if o.RootOverlaySize() != "1g" || o.SubMountOverlaySize() != "" { + t.Fatalf("Size mismatch, expecting 1g, empty, got %q, %q", o.RootOverlaySize(), o.SubMountOverlaySize()) + } + if o.String() != "all:memory,size=1g" { + t.Fatalf("String mismatch, expecting all:memory, got %q", o.String()) + } + }) +} diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 379da85cd8..b0e1901e96 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -119,7 +119,12 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Var(fileAccessTypePtr(FileAccessExclusive), "file-access", "specifies which filesystem validation to use for the root mount: exclusive (default), shared.") flagSet.Var(fileAccessTypePtr(FileAccessShared), "file-access-mounts", "specifies which filesystem validation to use for volumes other than the root mount: shared (default), exclusive.") flagSet.Bool("overlay", false, "DEPRECATED: use --overlay2=all:memory to achieve the same effect") - flagSet.Var(defaultOverlay2(), flagOverlay2, "wrap mounts with overlayfs. Format is {mount}:{medium}, where 'mount' can be 'root' or 'all' and medium can be 'memory', 'self' or 'dir=/abs/dir/path' in which filestore will be created. 'none' will turn overlay mode off.") + flagSet.Var(defaultOverlay2(), flagOverlay2, "wrap mounts with overlayfs. Format is\n"+ + "* 'none' to turn overlay mode off\n"+ + "* {mount}:{medium}[,size={size}], where\n"+ + " 'mount' can be 'root' or 'all'\n"+ + " 'medium' can be 'memory', 'self' or 'dir=/abs/dir/path' in which filestore will be created\n"+ + " 'size' optional parameter overrides default overlay upper layer size\n") flagSet.Bool("fsgofer-host-uds", false, "DEPRECATED: use host-uds=all") flagSet.Var(hostUDSPtr(HostUDSNone), flagHostUDS, "controls permission to access host Unix-domain sockets. Values: none|open|create|all, default: none") flagSet.Var(hostFifoPtr(HostFifoNone), "host-fifo", "controls permission to access host FIFOs (or named pipes). Values: none|open, default: none") diff --git a/runsc/container/container.go b/runsc/container/container.go index c6a85ddfda..5ee32e6c5e 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -899,7 +899,7 @@ func (c *Container) forEachSelfMount(fn func(mountSrc string)) { } } -func createGoferConf(overlayMedium config.OverlayMedium, mountType string, mountSrc string) (boot.GoferMountConf, error) { +func createGoferConf(overlayMedium config.OverlayMedium, overlaySize string, mountType string, mountSrc string) (boot.GoferMountConf, error) { var lower boot.GoferMountConfLowerType switch mountType { case boot.Bind: @@ -915,7 +915,7 @@ func createGoferConf(overlayMedium config.OverlayMedium, mountType string, mount case config.NoOverlay: return boot.GoferMountConf{Lower: lower, Upper: boot.NoOverlay}, nil case config.MemoryOverlay: - return boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay}, nil + return boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay, Size: overlaySize}, nil case config.SelfOverlay: mountSrcInfo, err := os.Stat(mountSrc) if err != nil { @@ -923,12 +923,12 @@ func createGoferConf(overlayMedium config.OverlayMedium, mountType string, mount } if !mountSrcInfo.IsDir() { log.Warningf("self filestore is only supported for directory mounts, but mount %q is not a directory, falling back to memory", mountSrc) - return boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay}, nil + return boot.GoferMountConf{Lower: lower, Upper: boot.MemoryOverlay, Size: overlaySize}, nil } - return boot.GoferMountConf{Lower: lower, Upper: boot.SelfOverlay}, nil + return boot.GoferMountConf{Lower: lower, Upper: boot.SelfOverlay, Size: overlaySize}, nil default: if overlayMedium.IsBackedByAnon() { - return boot.GoferMountConf{Lower: lower, Upper: boot.AnonOverlay}, nil + return boot.GoferMountConf{Lower: lower, Upper: boot.AnonOverlay, Size: overlaySize}, nil } return boot.GoferMountConf{}, fmt.Errorf("unexpected overlay medium %q", overlayMedium) } @@ -939,17 +939,19 @@ func createGoferConf(overlayMedium config.OverlayMedium, mountType string, mount func (c *Container) initGoferConfs(ovlConf config.Overlay2, mountHints *boot.PodMountHints, rootfsHint *boot.RootfsHint) error { // Handle root mount first. overlayMedium := ovlConf.RootOverlayMedium() + overlaySize := ovlConf.RootOverlaySize() mountType := boot.Bind if rootfsHint != nil { overlayMedium = rootfsHint.Overlay if !specutils.IsGoferMount(rootfsHint.Mount) { mountType = rootfsHint.Mount.Type } + overlaySize = rootfsHint.Size } if c.Spec.Root.Readonly { overlayMedium = config.NoOverlay } - goferConf, err := createGoferConf(overlayMedium, mountType, c.Spec.Root.Path) + goferConf, err := createGoferConf(overlayMedium, overlaySize, mountType, c.Spec.Root.Path) if err != nil { return err } @@ -960,7 +962,8 @@ func (c *Container) initGoferConfs(ovlConf config.Overlay2, mountHints *boot.Pod if !specutils.IsGoferMount(c.Spec.Mounts[i]) { continue } - overlayMedium = ovlConf.SubMountOverlayMedium() + overlayMedium := ovlConf.SubMountOverlayMedium() + overlaySize := ovlConf.SubMountOverlaySize() mountType = boot.Bind if specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) { overlayMedium = config.NoOverlay @@ -972,8 +975,9 @@ func (c *Container) initGoferConfs(ovlConf config.Overlay2, mountHints *boot.Pod if !specutils.IsGoferMount(hint.Mount) { mountType = hint.Mount.Type } + overlaySize = "" } - goferConf, err := createGoferConf(overlayMedium, mountType, c.Spec.Mounts[i].Source) + goferConf, err := createGoferConf(overlayMedium, overlaySize, mountType, c.Spec.Mounts[i].Source) if err != nil { return err }