Skip to content

feat: add erofs snapshotter configuration for new kata-preview runtime class #9085

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Camelron wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: add erofs snapshotter configuration for new kata-preview runtime class #9085

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4664f7b
containerd: Configure erofs snapshotter for kata images
Camelron Jul 10, 2026
222decb
containerd: add erofs kata blocks to the aks-node-controller path
Camelron Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions aks-node-controller/parser/templates/containerd.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,27 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
enable_cdi = true
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if .GetIsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Comment thread
Camelron marked this conversation as resolved.
Expand Down Expand Up @@ -62,6 +77,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
Expand All @@ -74,6 +90,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]

@Bickor Bickor Jul 31, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the final name of the runtimeclass? CC @allyford

Maybe let's leave a TODO to define the exact name when we're ready to onboard the rc.

runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
22 changes: 22 additions & 0 deletions aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,26 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if .GetIsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Comment thread
Camelron marked this conversation as resolved.
Expand Down Expand Up @@ -46,6 +61,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
Expand All @@ -58,6 +74,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
21 changes: 21 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -56,8 +70,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
21 changes: 21 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -43,8 +57,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
86 changes: 86 additions & 0 deletions pkg/agent/baker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1821,12 +1821,27 @@ const (
containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = -999{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
{{- if IsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
Comment thread
Camelron marked this conversation as resolved.
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
enable_cdi = true
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if IsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
Comment thread
Camelron marked this conversation as resolved.
Expand Down Expand Up @@ -1883,8 +1898,15 @@ root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
Comment on lines +1904 to +1908

@Camelron Camelron Jul 30, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm pushing a second commit containing the same template changes mirrored to these files. @cameronmeissner @Bickor can someone double check here and explain the difference?

@Bickor Bickor Jul 31, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll let @cameronmeissner explain the difference here, I don't know the answer.

@Bickor Bickor Jul 31, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the past I've made changes only to baker.go (ex: https://github.com/Azure/AgentBaker/pull/6816/changes) I think with that we should be good, but I'll let Cameron chime in in case it has changed now and we need changes to both.

ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand All @@ -1901,6 +1923,20 @@ root = "{{GetDataDir}}"{{- end}}
containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = -999{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
{{- if IsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -1956,8 +1992,15 @@ root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand All @@ -1974,6 +2017,20 @@ root = "{{GetDataDir}}"{{- end}}
containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = -999{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
{{- if IsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -2016,8 +2073,15 @@ root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand All @@ -2027,11 +2091,26 @@ root = "{{GetDataDir}}"{{- end}}
containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2
oom_score = -999{{if HasDataDir }}
root = "{{GetDataDir}}"{{- end}}
{{- if IsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{GetPodInfraContainerSpec}}"
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if IsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
{{- if IsArtifactStreamingEnabled }}
snapshotter = "overlaybd"
Comment thread
Camelron marked this conversation as resolved.
Expand Down Expand Up @@ -2073,8 +2152,15 @@ root = "{{GetDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
Loading