feat: AL2023 AMI family support (#5604)

Co-authored-by: Jonathan Innis <[email protected]>

Author: jmdeal

Makefile

@@ -119,7 +119,8 @@ vulncheck: ## Verify code vulnerabilities
 	@govulncheck ./pkg/...
 
 licenses: download ## Verifies dependency licenses
-	! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0'
+	# TODO: remove nodeadm check once license is updated
+	! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0' -e 'github.com/awslabs/amazon-eks-ami/nodeadm'
 
 image: ## Build the Karpenter controller images using ko build
 	$(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --bare github.com/aws/karpenter-provider-aws/cmd/controller))

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.9.0
 	github.com/aws/aws-sdk-go v1.50.25
 	github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623
+	github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240226194241-7da3d23779cf
 	github.com/go-logr/zapr v1.3.0
 	github.com/imdario/mergo v0.3.16
 	github.com/mitchellh/hashstructure/v2 v2.0.2
@@ -24,10 +25,11 @@ require (
 	k8s.io/apiextensions-apiserver v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
 	sigs.k8s.io/controller-runtime v0.17.2
 	sigs.k8s.io/karpenter v0.34.1-0.20240227070119-01b8127e7068
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -113,5 +115,4 @@ require (
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -58,6 +58,8 @@ github.com/aws/aws-sdk-go v1.50.25 h1:vhiHtLYybv1Nhx3Kv18BBC6L0aPJHaG9aeEsr92W99
 github.com/aws/aws-sdk-go v1.50.25/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623 h1:DQEFtmPyyMVHOyqva+DaWR6iAQG4h0KJbpSJAYlsnEo=
 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623/go.mod h1:fpKKbSoh7nKrbAw8V44Ov1sgosfUvR1ZtyN9k44zHfY=
+github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240226194241-7da3d23779cf h1:RH5DuK+nbRBD4S2c0kSXUhNiH4wjkDlOWosra3pBq3k=
+github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240226194241-7da3d23779cf/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -746,8 +748,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
+k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE=
 knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -50,6 +50,7 @@ spec:
                 description: AMIFamily is the AMI family that instances use.
                 enum:
                 - AL2
+                - AL2023
                 - Bottlerocket
                 - Ubuntu
                 - Custom

pkg/apis/v1beta1/ec2nodeclass.go

@@ -51,7 +51,7 @@ type EC2NodeClassSpec struct {
 	// +optional
 	AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"`
 	// AMIFamily is the AMI family that instances use.
-	// +kubebuilder:validation:Enum:={AL2,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
+	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
 	// +required
 	AMIFamily *string `json:"amiFamily"`
 	// UserData to be applied to the provisioned nodes.

pkg/apis/v1beta1/labels.go

@@ -75,6 +75,7 @@ var (
 	}
 	AMIFamilyBottlerocket                      = "Bottlerocket"
 	AMIFamilyAL2                               = "AL2"
+	AMIFamilyAL2023                            = "AL2023"
 	AMIFamilyUbuntu                            = "Ubuntu"
 	AMIFamilyWindows2019                       = "Windows2019"
 	AMIFamilyWindows2022                       = "Windows2022"

pkg/controllers/nodeclass/controller.go

@@ -87,6 +87,11 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl
 		c.resolveAMIs(ctx, nodeClass),
 		c.resolveInstanceProfile(ctx, nodeClass),
 	)
+	if lo.FromPtr(nodeClass.Spec.AMIFamily) == v1beta1.AMIFamilyAL2023 {
+		if cidrErr := c.launchTemplateProvider.ResolveClusterCIDR(ctx); err != nil {
+			err = multierr.Append(err, fmt.Errorf("resolving cluster CIDR, %w", cidrErr))
+		}
+	}
 	if !equality.Semantic.DeepEqual(stored, nodeClass) {
 		statusCopy := nodeClass.DeepCopy()
 		if patchErr := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil {

pkg/controllers/nodeclass/suite_test.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/aws/aws-sdk-go/service/iam"
 	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/events"
@@ -107,6 +108,46 @@ var _ = Describe("NodeClassController", func() {
 			},
 		})
 	})
+	Context("Cluster CIDR Resolution", func() {
+		BeforeEach(func() {
+			// Cluster CIDR will only be resolved once per lifetime of the launch template provider, reset to nil between tests
+			awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil)
+		})
+		It("shouldn't resolve cluster CIDR for non-AL2023 NodeClasses", func() {
+			for _, family := range []string{
+				v1beta1.AMIFamilyAL2,
+				v1beta1.AMIFamilyBottlerocket,
+				v1beta1.AMIFamilyUbuntu,
+				v1beta1.AMIFamilyWindows2019,
+				v1beta1.AMIFamilyWindows2022,
+				v1beta1.AMIFamilyCustom,
+			} {
+				nodeClass.Spec.AMIFamily = lo.ToPtr(family)
+				ExpectApplied(ctx, env.Client, nodeClass)
+				ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass))
+				Expect(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load()).To(BeNil())
+			}
+		})
+		It("should resolve cluster CIDR for IPv4 clusters", func() {
+			nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023)
+			ExpectApplied(ctx, env.Client, nodeClass)
+			ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass))
+			Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("10.100.0.0/16"))
+		})
+		It("should resolve cluster CIDR for IPv6 clusters", func() {
+			awsEnv.EKSAPI.DescribeClusterBehavior.Output.Set(&eks.DescribeClusterOutput{
+				Cluster: &eks.Cluster{
+					KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{
+						ServiceIpv6Cidr: lo.ToPtr("2001:db8::/64"),
+					},
+				},
+			})
+			nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023)
+			ExpectApplied(ctx, env.Client, nodeClass)
+			ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass))
+			Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("2001:db8::/64"))
+		})
+	})
 	Context("Subnet Status", func() {
 		It("Should update EC2NodeClass status for Subnets", func() {
 			ExpectApplied(ctx, env.Client, nodeClass)

pkg/fake/eksapi.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
+	"github.com/samber/lo"
 )
 
 const ()
@@ -35,6 +36,10 @@ type EKSAPI struct {
 	EKSAPIBehavior
 }
 
+func NewEKSAPI() *EKSAPI {
+	return &EKSAPI{}
+}
+
 // Reset must be called between tests otherwise tests will pollute
 // each other.
 func (s *EKSAPI) Reset() {
@@ -43,6 +48,12 @@ func (s *EKSAPI) Reset() {
 
 func (s *EKSAPI) DescribeClusterWithContext(_ context.Context, input *eks.DescribeClusterInput, _ ...request.Option) (*eks.DescribeClusterOutput, error) {
 	return s.DescribeClusterBehavior.Invoke(input, func(*eks.DescribeClusterInput) (*eks.DescribeClusterOutput, error) {
-		return nil, nil
+		return &eks.DescribeClusterOutput{
+			Cluster: &eks.Cluster{
+				KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{
+					ServiceIpv4Cidr: lo.ToPtr("10.100.0.0/16"),
+				},
+			},
+		}, nil
 	})
 }

pkg/operator/operator.go

@@ -149,6 +149,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		ctx,
 		cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval),
 		ec2api,
+		eks.New(sess),
 		amiResolver,
 		securityGroupProvider,
 		subnetProvider,

pkg/providers/amifamily/al2023.go

@@ -0,0 +1,79 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package amifamily
+
+import (
+	"fmt"
+
+	"github.com/samber/lo"
+	v1 "k8s.io/api/core/v1"
+	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+	"sigs.k8s.io/karpenter/pkg/cloudprovider"
+	"sigs.k8s.io/karpenter/pkg/scheduling"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap"
+)
+
+type AL2023 struct {
+	DefaultFamily
+	*Options
+}
+
+func (a AL2023) DefaultAMIs(version string) []DefaultAMIOutput {
+	return []DefaultAMIOutput{
+		{
+			Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version),
+			Requirements: scheduling.NewRequirements(
+				scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
+			),
+		},
+		{
+			Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version),
+			Requirements: scheduling.NewRequirements(
+				scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
+			),
+		},
+	}
+}
+
+func (a AL2023) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper {
+	return bootstrap.Nodeadm{
+		Options: bootstrap.Options{
+			ClusterName:             a.Options.ClusterName,
+			ClusterEndpoint:         a.Options.ClusterEndpoint,
+			ClusterCIDR:             a.Options.ClusterCIDR,
+			KubeletConfig:           kubeletConfig,
+			Taints:                  taints,
+			Labels:                  labels,
+			CABundle:                caBundle,
+			AWSENILimitedPodDensity: false,
+			CustomUserData:          customUserData,
+			InstanceStorePolicy:     instanceStorePolicy,
+		},
+	}
+}
+
+// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family
+func (a AL2023) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping {
+	return []*v1beta1.BlockDeviceMapping{{
+		DeviceName: a.EphemeralBlockDevice(),
+		EBS:        &DefaultEBS,
+	}}
+}
+
+func (a AL2023) EphemeralBlockDevice() *string {
+	return lo.ToPtr("/dev/xvda")
+}

pkg/providers/amifamily/bootstrap/bootstrap.go

@@ -34,6 +34,7 @@ import (
 type Options struct {
 	ClusterName             string
 	ClusterEndpoint         string
+	ClusterCIDR             *string
 	KubeletConfig           *corev1beta1.KubeletConfiguration
 	Taints                  []core.Taint      `hash:"set"`
 	Labels                  map[string]string `hash:"set"`

pkg/providers/amifamily/bootstrap/mime/mime.go

@@ -0,0 +1,121 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mime
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/mail"
+	"net/textproto"
+	"strings"
+
+	admapi "github.com/awslabs/amazon-eks-ami/nodeadm/api"
+)
+
+type ContentType string
+
+const (
+	boundary      = "//"
+	versionHeader = "MIME-Version: 1.0"
+
+	ContentTypeShellScript ContentType = `text/x-shellscript; charset="us-ascii"`
+	ContentTypeNodeConfig  ContentType = "application/" + admapi.GroupName
+	ContentTypeMultipart   ContentType = `multipart/mixed; boundary="` + boundary + `"`
+)
+
+type Entry struct {
+	ContentType ContentType
+	Content     string
+}
+
+type Archive []Entry
+
+func NewArchive(content string) (Archive, error) {
+	archive := Archive{}
+	if content == "" {
+		return archive, nil
+	}
+	reader, err := archive.getReader(content)
+	if err != nil {
+		return nil, err
+	}
+	for {
+		p, err := reader.NextPart()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, fmt.Errorf("parsing content, %w", err)
+		}
+		slurp, err := io.ReadAll(p)
+		if err != nil {
+			return nil, fmt.Errorf("parsing content, %s, %w", string(slurp), err)
+		}
+		archive = append(archive, Entry{
+			ContentType: ContentType(p.Header.Get("Content-Type")),
+			Content:     string(slurp),
+		})
+	}
+	return archive, nil
+}
+
+// Serialize returns a base64 encoded serialized MIME multi-part archive
+func (ma Archive) Serialize() (string, error) {
+	buffer := bytes.Buffer{}
+	writer := multipart.NewWriter(&buffer)
+	if err := writer.SetBoundary(boundary); err != nil {
+		return "", err
+	}
+	buffer.WriteString(versionHeader + "\n")
+	buffer.WriteString(fmt.Sprintf("Content-Type: %s\n\n", ContentTypeMultipart))
+	for _, entry := range ma {
+		partWriter, err := writer.CreatePart(textproto.MIMEHeader{
+			"Content-Type": []string{string(entry.ContentType)},
+		})
+		if err != nil {
+			return "", fmt.Errorf("creating multi-part section for entry, %w", err)
+		}
+		_, err = partWriter.Write([]byte(entry.Content))
+		if err != nil {
+			return "", fmt.Errorf("writing entry, %w", err)
+		}
+	}
+	if err := writer.Close(); err != nil {
+		return "", fmt.Errorf("terminating multi-part archive, %w", err)
+	}
+	// The mime/multipart package adds carriage returns, while the rest of our logic does not. Remove all
+	// carriage returns for consistency.
+	return base64.StdEncoding.EncodeToString([]byte(strings.ReplaceAll(buffer.String(), "\r", ""))), nil
+}
+
+func (Archive) getReader(content string) (*multipart.Reader, error) {
+	mailMsg, err := mail.ReadMessage(strings.NewReader(content))
+	if err != nil {
+		return nil, err
+	}
+	mediaType, params, err := mime.ParseMediaType(mailMsg.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, fmt.Errorf("archive doesn't have Content-Type header, %w", err)
+	}
+	if !strings.HasPrefix(mediaType, "multipart/") {
+		return nil, fmt.Errorf("archive is not in multipart format, %w", err)
+	}
+	return multipart.NewReader(mailMsg.Body, params["boundary"]), nil
+}