diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml index a8494041ff15..e673922e9b05 100644 --- a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -114,6 +114,9 @@ spec: Owner is the owner for the ami. You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string + ssmParameterName: + description: SSMParameterName is the name (or ARN) of the SSM parameter containing the Image ID. The parameter data type should be aws:ec2:image + type: string tags: additionalProperties: type: string @@ -130,8 +133,8 @@ spec: minItems: 1 type: array x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameterName'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameterName)) - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index a2444bfcd03c..fb8211a7db59 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -111,6 +111,9 @@ spec: Owner is the owner for the ami. You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string + ssmParameterName: + description: SSMParameterName is the name (or ARN) of the SSM parameter containing the Image ID. The parameter data type should be aws:ec2:image + type: string tags: additionalProperties: type: string @@ -127,8 +130,8 @@ spec: minItems: 1 type: array x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameterName'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameterName)) - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' diff --git a/pkg/apis/v1/ec2nodeclass.go b/pkg/apis/v1/ec2nodeclass.go index 8d61dc78d156..e11ada72ec76 100644 --- a/pkg/apis/v1/ec2nodeclass.go +++ b/pkg/apis/v1/ec2nodeclass.go @@ -47,7 +47,7 @@ type EC2NodeClassSpec struct { // +optional AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. - // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias))" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameterName']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameterName))" // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))" // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))" // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)" @@ -202,6 +202,9 @@ type AMISelectorTerm struct { // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" // +optional Owner string `json:"owner,omitempty"` + //SSMParameterName is the name (or ARN) of the SSM parameter containing the Image ID. The parameter data type should be aws:ec2:image + // +optional + SSMParameterName string `json:"ssmParameterName,omitempty"` } // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index ee6fdfc7257d..84f5224c3ffe 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -104,6 +104,8 @@ func (p *DefaultProvider) DescribeImageQueries(ctx context.Context, nodeClass *v switch { case term.ID != "": idFilter.Values = append(idFilter.Values, term.ID) + case term.SSMParameterName != "": + p.updateFilterFromCustomParameter(ctx, term.SSMParameterName, &idFilter) default: query := DescribeImageQuery{ Owners: lo.Ternary(term.Owner != "", []string{term.Owner}, []string{}), @@ -142,6 +144,17 @@ func (p *DefaultProvider) DescribeImageQueries(ctx context.Context, nodeClass *v return queries, nil } +func (p *DefaultProvider) updateFilterFromCustomParameter(ctx context.Context, ssmParameterName string, idFilter *ec2types.Filter) { + imageID, err := p.ssmProvider.GetCustomParameter(ctx, ssm.Parameter{ + Name: ssmParameterName, + }) + if err != nil { + log.FromContext(ctx).WithValues("ssmParameterName", ssmParameterName).V(1).Error(err, "parameter not found") + } else { + idFilter.Values = append(idFilter.Values, imageID) + } +} + //nolint:gocyclo func (p *DefaultProvider) amis(ctx context.Context, queries []DescribeImageQuery) (AMIs, error) { hash, err := hashstructure.Hash(queries, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) diff --git a/pkg/providers/ssm/provider.go b/pkg/providers/ssm/provider.go index 7d698e8d1ebd..f57cca784f0d 100644 --- a/pkg/providers/ssm/provider.go +++ b/pkg/providers/ssm/provider.go @@ -28,6 +28,7 @@ import ( type Provider interface { Get(context.Context, Parameter) (string, error) + GetCustomParameter(context.Context, Parameter) (string, error) } type DefaultProvider struct { @@ -60,3 +61,11 @@ func (p *DefaultProvider) Get(ctx context.Context, parameter Parameter) (string, log.FromContext(ctx).WithValues("parameter", parameter.Name, "value", result.Parameter.Value).Info("discovered ssm parameter") return lo.FromPtr(result.Parameter.Value), nil } + +func (p *DefaultProvider) GetCustomParameter(ctx context.Context, parameter Parameter) (string, error) { + result, err := p.ssmapi.GetParameter(ctx, parameter.GetParameterInput()) + if err != nil { + return "", fmt.Errorf("getting ssm parameter %q, %w", parameter.Name, err) + } + return lo.FromPtr(result.Parameter.Value), nil +} diff --git a/test/suites/ami/suite_test.go b/test/suites/ami/suite_test.go index e102bbe823ea..f20d47476c27 100644 --- a/test/suites/ami/suite_test.go +++ b/test/suites/ami/suite_test.go @@ -33,6 +33,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/ssm" + ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" "github.com/awslabs/operatorpkg/status" . "github.com/awslabs/operatorpkg/test/expectations" "github.com/samber/lo" @@ -71,11 +73,19 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("AMI", func() { + var ssmPath string var customAMI string var deprecatedAMI string + var customParameter string + BeforeEach(func() { - customAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) + ssmPath = fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion()) + customAMI = env.GetAMIBySSMPath(ssmPath) deprecatedAMI = env.GetDeprecatedAMI(customAMI, "AL2023") + customParameter = createCustomSSMParameter(&customAMI) + }) + AfterEach(func() { + deleteCustomSSMParameter(customParameter) }) It("should use the AMI defined by the AMI Selector Terms", func() { @@ -159,6 +169,36 @@ var _ = Describe("AMI", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) + It("should support custom ssm parameters", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + SSMParameterName: customParameter, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) + It("should support shared ssm parameters by ARN", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + SSMParameterName: ssmPath, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) It("should support launching nodes with a deprecated ami", func() { nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ @@ -253,6 +293,26 @@ var _ = Describe("AMI", func() { ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionTrue}) ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) }) + It("should have the EC2NodeClass status for AMIs using custom ssm parameters", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{SSMParameterName: customParameter}} + env.ExpectCreated(nodeClass) + nc := EventuallyExpectAMIsToExist(nodeClass) + Expect(len(nc.Status.AMIs)).To(BeNumerically("==", 1)) + Expect(nc.Status.AMIs[0].ID).To(Equal(customAMI)) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) + It("should have the EC2NodeClass status for AMIs using public ssm parameter ARN", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{SSMParameterName: ssmPath}} + env.ExpectCreated(nodeClass) + nc := EventuallyExpectAMIsToExist(nodeClass) + Expect(len(nc.Status.AMIs)).To(BeNumerically("==", 1)) + Expect(nc.Status.AMIs[0].ID).To(Equal(customAMI)) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionTrue}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionTrue}) + }) It("should have ec2nodeClass status as not ready since AMI was not resolved", func() { nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: "ami-123"}} @@ -260,6 +320,13 @@ var _ = Describe("AMI", func() { ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionFalse, Message: "AMISelector did not match any AMIs"}) ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionFalse, Message: "AMIsReady=False"}) }) + It("should have ec2nodeClass status as not ready since ssm parameter was not found", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{SSMParameterName: "parameter-123"}} + env.ExpectCreated(nodeClass) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: v1.ConditionTypeAMIsReady, Status: metav1.ConditionFalse, Message: "AMISelector did not match any AMIs"}) + ExpectStatusConditions(env, env.Client, 1*time.Minute, nodeClass, status.Condition{Type: status.ConditionReady, Status: metav1.ConditionFalse, Message: "AMIsReady=False"}) + }) }) Context("UserData", func() { @@ -388,6 +455,26 @@ func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstan return instanceAttribute } +func createCustomSSMParameter(imageID *string) string { + parameterName := coretest.RandomName() + dataType := "aws:ec2:image" + _, err := env.SSMAPI.PutParameter(env.Context, &ssm.PutParameterInput{ + Name: awssdk.String(parameterName), + Value: imageID, + DataType: &dataType, + Type: ssmtypes.ParameterTypeString, + }) + Expect(err).ToNot(HaveOccurred()) + return parameterName +} + +func deleteCustomSSMParameter(parameterName string) { + _, err := env.SSMAPI.DeleteParameter(env.Context, &ssm.DeleteParameterInput{ + Name: awssdk.String(parameterName), + }) + Expect(err).ToNot(HaveOccurred()) +} + func EventuallyExpectAMIsToExist(nodeClass *v1.EC2NodeClass) *v1.EC2NodeClass { nc := &v1.EC2NodeClass{} Eventually(func(g Gomega) { diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index 9388d487e265..b8ef20c696b8 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -100,12 +100,13 @@ spec: amiSelectorTerms: # Select on any AMI that has both the `karpenter.sh/discovery: ${CLUSTER_NAME}` # AND `environment: test` tags OR any AMI with the name `my-ami` OR an AMI with - # ID `ami-123` + # ID `ami-123` OR an AMI with ID matching the value of my-custom-parameter - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test - name: my-ami - id: ami-123 + - ssmParameterName: my-custom-parameter # ssm parameter name or ARN # Select EKS optimized AL2023 AMIs with version `v20240703`. This term is mutually # exclusive and can't be specified with other terms. # - alias: al2023@v20240703 @@ -705,12 +706,13 @@ The example below shows how this selection logic is fulfilled. amiSelectorTerms: # Select on any AMI that has both the `karpenter.sh/discovery: ${CLUSTER_NAME}` # AND `environment: test` tags OR any AMI with the name `my-ami` OR an AMI with - # ID `ami-123` + # ID `ami-123` OR an AMI with ID matching the value of my-custom-parameter - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test - name: my-ami - id: ami-123 + - ssmParameterName: my-custom-parameter # ssm parameter name or ARN # Select EKS optimized AL2023 AMIs with version `v20240807`. This term is mutually # exclusive and can't be specified with other terms. # - alias: al2023@v20240807 @@ -842,6 +844,12 @@ Specify using ids: - id: "ami-456" ``` +Specify using custom ssm parameter name or ARN: +```yaml + amiSelectorTerms: + - ssmParameterName: "my-custom-parameter" +``` + ## spec.tags Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of tags are listed below.