diff --git a/aks-node-controller/parser/templates/containerd.toml.gtpl b/aks-node-controller/parser/templates/containerd.toml.gtpl index 6b9d7442fdb..ecd492bee80 100644 --- a/aks-node-controller/parser/templates/containerd.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd.toml.gtpl @@ -1,12 +1,27 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false + snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -62,6 +77,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -74,6 +90,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl index 2eb27cab54c..90c52b01120 100644 --- a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl @@ -1,11 +1,26 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} disable_snapshot_annotations = false + snapshotter = "overlayfs" {{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -46,6 +61,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli] runtime_type = "io.containerd.runc.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options] @@ -58,6 +74,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} Root = "" CriuPath = "" SystemdCgroup = false +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index fd11c508145..42442b8c57f 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,6 +1,20 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -56,8 +70,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index 45a2129f743..c28dd4f560c 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,6 +1,20 @@ version = 2 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" @@ -43,8 +57,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 9451b6d13b5..c8e9ac90745 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -1821,12 +1821,27 @@ const ( containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} +{{- if IsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false + snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -1883,8 +1898,15 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -1901,6 +1923,20 @@ root = "{{GetDataDir}}"{{- end}} containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} +{{- if IsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -1956,8 +1992,15 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -1974,6 +2017,20 @@ root = "{{GetDataDir}}"{{- end}} containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} +{{- if IsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2016,8 +2073,15 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" @@ -2027,11 +2091,26 @@ root = "{{GetDataDir}}"{{- end}} containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} +{{- if IsKata }} +[plugins."io.containerd.snapshotter.v1.erofs"] + default_size = "10G" + enable_fsverity = false + ovl_mount_options = [] + max_unmerged_layers = 1 + +[plugins."io.containerd.service.v1.diff-service"] + default = ["erofs", "walking"] + +[plugins."io.containerd.differ.v1.erofs"] + mkfs_options = ["-T0", "--mkfs-time", "--sort=none"] + enable_tar_index = false +{{- end}} [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{GetPodInfraContainerSpec}}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if IsKata }} disable_snapshot_annotations = false + snapshotter = "overlayfs" {{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" @@ -2073,8 +2152,15 @@ root = "{{GetDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true + snapshotter = "overlayfs" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "erofs" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-templating.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot"