Support for Kubernetes v1.23 (#386)

* Add Kubernetes 1.23 support to README.md

* Upgrade github.com/kubernetes/cloud-provider-openstack

* Replace deprecated kubelet flags

Author: rfranzke

README.md

@@ -23,6 +23,7 @@ This extension controller supports the following Kubernetes versions:
 
 | Version         | Support     | Conformance test results |
 | --------------- | ----------- | ------------------------ |
+| Kubernetes 1.23 | 1.23.0+     | N/A |
 | Kubernetes 1.22 | 1.22.0+     | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20OpenStack) |
 | Kubernetes 1.21 | 1.21.0+     | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20OpenStack) |
 | Kubernetes 1.20 | 1.20.0+     | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20OpenStack) |

charts/images.yaml

@@ -33,7 +33,12 @@ images:
   sourceRepository: github.com/kubernetes/cloud-provider-openstack
   repository: k8scloudprovider/openstack-cloud-controller-manager
   tag: "v1.22.0"
-  targetVersion: ">= 1.22"
+  targetVersion: "1.22.x"
+- name: cloud-controller-manager
+  sourceRepository: github.com/kubernetes/cloud-provider-openstack
+  repository: k8scloudprovider/openstack-cloud-controller-manager
+  tag: "v1.23.0"
+  targetVersion: ">= 1.23"
 
 - name: machine-controller-manager
   sourceRepository: github.com/gardener/machine-controller-manager

pkg/webhook/controlplane/ensurer.go

@@ -39,6 +39,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -367,7 +368,7 @@ func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, gctx gcon
 
 	if opt := extensionswebhook.UnitOptionWithSectionAndName(new, "Service", "ExecStart"); opt != nil {
 		command := extensionswebhook.DeserializeCommandLine(opt.Value)
-		command = ensureKubeletCommandLineArgs(command, csiEnabled)
+		command = ensureKubeletCommandLineArgs(command, csiEnabled, kubeletVersion)
 		opt.Value = extensionswebhook.SerializeCommandLine(command, 1, " \\\n    ")
 	}
 
@@ -379,10 +380,12 @@ func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, gctx gcon
 	return new, nil
 }
 
-func ensureKubeletCommandLineArgs(command []string, csiEnabled bool) []string {
+func ensureKubeletCommandLineArgs(command []string, csiEnabled bool, kubeletVersion *semver.Version) []string {
 	if csiEnabled {
-		command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-provider=", "external")
-		command = extensionswebhook.EnsureStringWithPrefix(command, "--enable-controller-attach-detach=", "true")
+		if !versionutils.ConstraintK8sGreaterEqual123.Check(kubeletVersion) {
+			command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-provider=", "external")
+			command = extensionswebhook.EnsureStringWithPrefix(command, "--enable-controller-attach-detach=", "true")
+		}
 	} else {
 		command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-provider=", "openstack")
 		command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-config=", "/var/lib/kubelet/cloudprovider.conf")
@@ -415,6 +418,10 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, gctx gcontext.
 		new.FeatureGates["CSIMigrationOpenStack"] = true
 		// kubelets of new worker nodes can directly be started with the the <csiMigrationCompleteFeatureGate> feature gate
 		new.FeatureGates[csiMigrationCompleteFeatureGate] = true
+
+		if versionutils.ConstraintK8sGreaterEqual123.Check(kubeletVersion) {
+			new.EnableControllerAttachDetach = pointer.Bool(true)
+		}
 	}
 
 	// resolv-for-kubelet.conf is created by update-resolv-conf.service

pkg/webhook/controlplane/ensurer_test.go

@@ -477,41 +477,42 @@ var _ = Describe("Ensurer", func() {
 			}
 		})
 
-		It("should modify existing elements of kubelet.service unit options (k8s < 1.19)", func() {
-			newUnitOptions := []*unit.UnitOption{
-				{
-					Section: "Service",
-					Name:    "ExecStart",
-					Value: `/opt/bin/hyperkube kubelet \
-    --config=/var/lib/kubelet/config/kubelet \
-    --cloud-provider=openstack \
-    --cloud-config=/var/lib/kubelet/cloudprovider.conf`,
-				},
-				hostnamectlUnitOption,
-			}
+		DescribeTable("should modify existing elements of kubelet.service unit options",
+			func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, cloudProvider string, withControllerAttachDetachFlag bool) {
+				newUnitOptions := []*unit.UnitOption{
+					{
+						Section: "Service",
+						Name:    "ExecStart",
+						Value: `/opt/bin/hyperkube kubelet \
+    --config=/var/lib/kubelet/config/kubelet`,
+					},
+					hostnamectlUnitOption,
+				}
 
-			opts, err := ensurer.EnsureKubeletServiceUnitOptions(ctx, eContextK8s116, semver.MustParse("1.16.0"), oldUnitOptions, nil)
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(opts).To(Equal(newUnitOptions))
-		})
+				if cloudProvider != "" {
+					newUnitOptions[0].Value += ` \
+    --cloud-provider=` + cloudProvider
 
-		It("should modify existing elements of kubelet.service unit options (k8s >= 1.19)", func() {
-			newUnitOptions := []*unit.UnitOption{
-				{
-					Section: "Service",
-					Name:    "ExecStart",
-					Value: `/opt/bin/hyperkube kubelet \
-    --config=/var/lib/kubelet/config/kubelet \
-    --cloud-provider=external \
-    --enable-controller-attach-detach=true`,
-				},
-				hostnamectlUnitOption,
-			}
+					if cloudProvider != "external" {
+						newUnitOptions[0].Value += ` \
+    --cloud-config=/var/lib/kubelet/cloudprovider.conf`
+					}
+				}
 
-			opts, err := ensurer.EnsureKubeletServiceUnitOptions(ctx, eContextK8s119, semver.MustParse("1.19.0"), oldUnitOptions, nil)
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(opts).To(Equal(newUnitOptions))
-		})
+				if withControllerAttachDetachFlag {
+					newUnitOptions[0].Value += ` \
+    --enable-controller-attach-detach=true`
+				}
+
+				opts, err := ensurer.EnsureKubeletServiceUnitOptions(ctx, gctx, kubeletVersion, oldUnitOptions, nil)
+				Expect(err).To(Not(HaveOccurred()))
+				Expect(opts).To(Equal(newUnitOptions))
+			},
+
+			Entry("kubelet version < 1.19", eContextK8s116, semver.MustParse("1.16.0"), "openstack", false),
+			Entry("1.19 <= kubelet version < 1.23", eContextK8s119, semver.MustParse("1.19.0"), "external", true),
+			Entry("kubelet version >= 1.23", eContextK8s119, semver.MustParse("1.23.0"), "", false),
+		)
 	})
 
 	Describe("#EnsureKubeletConfiguration", func() {
@@ -526,12 +527,13 @@ var _ = Describe("Ensurer", func() {
 		})
 
 		DescribeTable("should modify existing elements of kubelet configuration",
-			func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, unregisterFeatureGateName string) {
+			func(gctx gcontext.GardenContext, kubeletVersion *semver.Version, unregisterFeatureGateName string, enableControllerAttachDetach *bool) {
 				newKubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{
 					FeatureGates: map[string]bool{
 						"Foo": true,
 					},
-					ResolverConfig: "/etc/resolv-for-kubelet.conf",
+					ResolverConfig:               "/etc/resolv-for-kubelet.conf",
+					EnableControllerAttachDetach: enableControllerAttachDetach,
 				}
 
 				if unregisterFeatureGateName != "" {
@@ -547,10 +549,11 @@ var _ = Describe("Ensurer", func() {
 				Expect(&kubeletConfig).To(Equal(newKubeletConfig))
 			},
 
-			Entry("control plane, kubelet < 1.19", eContextK8s116, semver.MustParse("1.16.0"), ""),
-			Entry("1.19 <= control plane, kubelet <= 1.21", eContextK8s119, semver.MustParse("1.19.0"), "CSIMigrationOpenStackComplete"),
-			Entry("control plane >= 1.21, kubelet < 1.21", eContextK8s121, semver.MustParse("1.20.0"), "CSIMigrationOpenStackComplete"),
-			Entry("control plane, kubelet >= 1.21", eContextK8s121, semver.MustParse("1.21.0"), "InTreePluginOpenStackUnregister"),
+			Entry("control plane, kubelet < 1.19", eContextK8s116, semver.MustParse("1.16.0"), "", nil),
+			Entry("1.19 <= control plane, kubelet <= 1.21", eContextK8s119, semver.MustParse("1.19.0"), "CSIMigrationOpenStackComplete", nil),
+			Entry("control plane >= 1.21, kubelet < 1.21", eContextK8s121, semver.MustParse("1.20.0"), "CSIMigrationOpenStackComplete", nil),
+			Entry("1.21 <= kubelet < 1.23", eContextK8s121, semver.MustParse("1.22.0"), "InTreePluginOpenStackUnregister", nil),
+			Entry("kubelet >= 1.23", eContextK8s121, semver.MustParse("1.23.0"), "InTreePluginOpenStackUnregister", pointer.Bool(true)),
 		)
 	})