Support storing Ignition user data in S3 bucket for AWSMachinePool

Author: AndiDog

api/v1beta2/awsmachine_types.go

@@ -30,6 +30,16 @@ const (
 
 	// DefaultIgnitionVersion represents default Ignition version generated for machine userdata.
 	DefaultIgnitionVersion = "2.3"
+
+	// DefaultIgnitionStorageType represents the default storage type of Ignition userdata
+	DefaultIgnitionStorageType = IgnitionStorageTypeOptionClusterObjectStore
+
+	// DefaultMachinePoolIgnitionStorageType represents the default storage type of Ignition userdata for machine pools.
+	//
+	// This is only different from DefaultIgnitionStorageType because of backward compatibility. Machine pools used to
+	// default to store Ignition user data directly on the EC2 instance. Since the choice between remote storage (S3)
+	// and direct storage was introduced, the default was kept, but might change in newer API versions.
+	DefaultMachinePoolIgnitionStorageType = IgnitionStorageTypeOptionUnencryptedUserData
 )
 
 // SecretBackend defines variants for backend secret storage.

api/v1beta2/awsmachine_webhook.go

@@ -414,12 +414,11 @@ func (r *AWSMachine) Default() {
 	}
 
 	if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
-		if r.Spec.Ignition == nil {
-			r.Spec.Ignition = &Ignition{}
-		}
-
 		r.Spec.Ignition.Version = DefaultIgnitionVersion
 	}
+	if r.ignitionEnabled() && r.Spec.Ignition.StorageType == "" {
+		r.Spec.Ignition.StorageType = DefaultIgnitionStorageType
+	}
 }
 
 func (r *AWSMachine) validateAdditionalSecurityGroups() field.ErrorList {

api/v1beta2/tags.go

@@ -195,6 +195,12 @@ const (
 	// of the bootstrap secret that was used to create the user data for the latest launch
 	// template version.
 	LaunchTemplateBootstrapDataSecret = NameAWSProviderPrefix + "bootstrap-data-secret"
+
+	// LaunchTemplateBootstrapDataHash is the tag we use to store the hash of the raw bootstrap data.
+	// If bootstrap data is stored in S3, this hash relates to that data, not to the EC2 instance
+	// user data which only references the S3 object. We store this tag on launch template versions
+	// so that S3 bootstrap data objects can be deleted when they get outdated.
+	LaunchTemplateBootstrapDataHash = NameAWSProviderPrefix + "bootstrap-data-hash"
 )
 
 // ClusterTagKey generates the key for resources associated with a cluster.

cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go

@@ -291,11 +291,13 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 			Action: iamv1.Actions{
 				"s3:CreateBucket",
 				"s3:DeleteBucket",
-				"s3:GetObject",
-				"s3:PutObject",
 				"s3:DeleteObject",
+				"s3:GetObject",
+				"s3:ListBucket",
 				"s3:PutBucketPolicy",
 				"s3:PutBucketTagging",
+				"s3:PutLifecycleConfiguration",
+				"s3:PutObject",
 			},
 		})
 	}

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml

@@ -297,11 +297,13 @@ Resources:
         - Action:
           - s3:CreateBucket
           - s3:DeleteBucket
-          - s3:GetObject
-          - s3:PutObject
           - s3:DeleteObject
+          - s3:GetObject
+          - s3:ListBucket
           - s3:PutBucketPolicy
           - s3:PutBucketTagging
+          - s3:PutLifecycleConfiguration
+          - s3:PutObject
           Effect: Allow
           Resource:
           - arn:*:s3:::cluster-api-provider-aws-*

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

@@ -889,6 +889,101 @@ spec:
                   after it enters the InService state.
                   If no value is supplied by user a default value of 300 seconds is set
                 type: string
+              ignition:
+                description: Ignition defined options related to the bootstrapping
+                  systems where Ignition is used.
+                properties:
+                  proxy:
+                    description: |-
+                      Proxy defines proxy settings for Ignition.
+                      Only valid for Ignition versions 3.1 and above.
+                    properties:
+                      httpProxy:
+                        description: |-
+                          HTTPProxy is the HTTP proxy to use for Ignition.
+                          A single URL that specifies the proxy server to use for HTTP and HTTPS requests,
+                          unless overridden by the HTTPSProxy or NoProxy options.
+                        type: string
+                      httpsProxy:
+                        description: |-
+                          HTTPSProxy is the HTTPS proxy to use for Ignition.
+                          A single URL that specifies the proxy server to use for HTTPS requests,
+                          unless overridden by the NoProxy option.
+                        type: string
+                      noProxy:
+                        description: |-
+                          NoProxy is the list of domains to not proxy for Ignition.
+                          Specifies a list of strings to hosts that should be excluded from proxying.
+
+                          Each value is represented by:
+                          - An IP address prefix (1.2.3.4)
+                          - An IP address prefix in CIDR notation (1.2.3.4/8)
+                          - A domain name
+                            - A domain name matches that name and all subdomains
+                            - A domain name with a leading . matches subdomains only
+                          - A special DNS label (*), indicates that no proxying should be done
+
+                          An IP address prefix and domain name can also include a literal port number (1.2.3.4:80).
+                        items:
+                          description: IgnitionNoProxy defines the list of domains
+                            to not proxy for Ignition.
+                          maxLength: 2048
+                          type: string
+                        maxItems: 64
+                        type: array
+                    type: object
+                  storageType:
+                    default: ClusterObjectStore
+                    description: |-
+                      StorageType defines how to store the boostrap user data for Ignition.
+                      This can be used to instruct Ignition from where to fetch the user data to bootstrap an instance.
+
+                      When omitted, the storage option will default to ClusterObjectStore.
+
+                      When set to "ClusterObjectStore", if the capability is available and a Cluster ObjectStore configuration
+                      is correctly provided in the Cluster object (under .spec.s3Bucket),
+                      an object store will be used to store bootstrap user data.
+
+                      When set to "UnencryptedUserData", EC2 Instance User Data will be used to store the machine bootstrap user data, unencrypted.
+                      This option is considered less secure than others as user data may contain sensitive informations (keys, certificates, etc.)
+                      and users with ec2:DescribeInstances permission or users running pods
+                      that can access the ec2 metadata service have access to this sensitive information.
+                      So this is only to be used at ones own risk, and only when other more secure options are not viable.
+                    enum:
+                    - ClusterObjectStore
+                    - UnencryptedUserData
+                    type: string
+                  tls:
+                    description: |-
+                      TLS defines TLS settings for Ignition.
+                      Only valid for Ignition versions 3.1 and above.
+                    properties:
+                      certificateAuthorities:
+                        description: |-
+                          CASources defines the list of certificate authorities to use for Ignition.
+                          The value is the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates.
+                          Supported schemes are http, https, tftp, s3, arn, gs, and `data` (RFC 2397) URL scheme.
+                        items:
+                          description: IgnitionCASource defines the source of the
+                            certificate authority to use for Ignition.
+                          maxLength: 65536
+                          type: string
+                        maxItems: 64
+                        type: array
+                    type: object
+                  version:
+                    default: "2.3"
+                    description: Version defines which version of Ignition will be
+                      used to generate bootstrap data.
+                    enum:
+                    - "2.3"
+                    - "3.0"
+                    - "3.1"
+                    - "3.2"
+                    - "3.3"
+                    - "3.4"
+                    type: string
+                type: object
               maxSize:
                 default: 1
                 description: MaxSize defines the maximum size of the group.

controllers/awsmachine_controller.go

@@ -739,7 +739,7 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
 	if machineScope.UseIgnition(userDataFormat) {
 		var ignitionStorageType infrav1.IgnitionStorageTypeOption
 		if machineScope.AWSMachine.Spec.Ignition == nil {
-			ignitionStorageType = infrav1.IgnitionStorageTypeOptionClusterObjectStore
+			ignitionStorageType = infrav1.DefaultIgnitionStorageType
 		} else {
 			ignitionStorageType = machineScope.AWSMachine.Spec.Ignition.StorageType
 		}
@@ -795,8 +795,8 @@ func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScop
 // then returns the config to instruct ignition on how to pull the user data from the bucket.
 func (r *AWSMachineReconciler) generateIgnitionWithRemoteStorage(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
 	if objectStoreSvc == nil {
-		return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.Spec.Ignition.S3Bucket`. " +
-			"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.Spec.Ignition.StorageType` to `UnencryptedUserData`")
+		return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.spec.s3Bucket`. " +
+			"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.spec.ignition.storageType` to `UnencryptedUserData`")
 	}
 
 	objectURL, err := objectStoreSvc.Create(scope, userData)

exp/api/v1beta1/conversion.go

@@ -52,6 +52,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Spec.AvailabilityZoneSubnetType != nil {
 		dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
 	}
+	if restored.Spec.Ignition != nil {
+		dst.Spec.Ignition = restored.Spec.Ignition
+	}
 
 	if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
 		dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName

exp/api/v1beta1/zz_generated.conversion.go

@@ -101,6 +101,10 @@ type AWSMachinePoolSpec struct {
 	// SuspendProcesses defines a list of processes to suspend for the given ASG. This is constantly reconciled.
 	// If a process is removed from this list it will automatically be resumed.
 	SuspendProcesses *SuspendProcessesTypes `json:"suspendProcesses,omitempty"`
+
+	// Ignition defined options related to the bootstrapping systems where Ignition is used.
+	// +optional
+	Ignition *infrav1.Ignition `json:"ignition,omitempty"`
 }
 
 // SuspendProcessesTypes contains user friendly auto-completable values for suspended process names.

exp/api/v1beta2/awsmachinepool_types.go

@@ -28,6 +28,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	"sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
 )
 
 var log = ctrl.Log.WithName("awsmachinepool-resource")
@@ -62,12 +64,12 @@ func (r *AWSMachinePool) validateRootVolume() field.ErrorList {
 		return allErrs
 	}
 
-	if v1beta2.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
+	if infrav1.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
 		allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.iops"), "iops required if type is 'io1' or 'io2'"))
 	}
 
 	if r.Spec.AWSLaunchTemplate.RootVolume.Throughput != nil {
-		if r.Spec.AWSLaunchTemplate.RootVolume.Type != v1beta2.VolumeTypeGP3 {
+		if r.Spec.AWSLaunchTemplate.RootVolume.Type != infrav1.VolumeTypeGP3 {
 			allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.throughput"), "throughput is valid only for type 'gp3'"))
 		}
 		if *r.Spec.AWSLaunchTemplate.RootVolume.Throughput < 0 {
@@ -86,12 +88,12 @@ func (r *AWSMachinePool) validateNonRootVolumes() field.ErrorList {
 	var allErrs field.ErrorList
 
 	for _, volume := range r.Spec.AWSLaunchTemplate.NonRootVolumes {
-		if v1beta2.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
+		if infrav1.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
 			allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.iops"), "iops required if type is 'io1' or 'io2'"))
 		}
 
 		if volume.Throughput != nil {
-			if volume.Type != v1beta2.VolumeTypeGP3 {
+			if volume.Type != infrav1.VolumeTypeGP3 {
 				allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.throughput"), "throughput is valid only for type 'gp3'"))
 			}
 			if *volume.Throughput < 0 {
@@ -162,6 +164,22 @@ func (r *AWSMachinePool) validateRefreshPreferences() field.ErrorList {
 	return allErrs
 }
 
+func (r *AWSMachinePool) ignitionEnabled() bool {
+	return r.Spec.Ignition != nil
+}
+
+func (r *AWSMachinePool) validateIgnition() field.ErrorList {
+	var allErrs field.ErrorList
+
+	// Feature gate is not enabled but ignition is enabled then send a forbidden error.
+	if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) && r.ignitionEnabled() {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "ignition"),
+			"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
+	}
+
+	return allErrs
+}
+
 // ValidateCreate will do any extra validation when creating a AWSMachinePool.
 func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
@@ -177,6 +195,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	allErrs = append(allErrs, r.validateSpotInstances()...)
 	allErrs = append(allErrs, r.validateRefreshPreferences()...)
 	allErrs = append(allErrs, r.validateInstanceMarketType()...)
+	allErrs = append(allErrs, r.validateIgnition()...)
 
 	if len(allErrs) == 0 {
 		return nil, nil
@@ -243,4 +262,11 @@ func (r *AWSMachinePool) Default() {
 		log.Info("DefaultInstanceWarmup is zero, setting 300 seconds as default")
 		r.Spec.DefaultInstanceWarmup.Duration = 300 * time.Second
 	}
+
+	if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
+		r.Spec.Ignition.Version = infrav1.DefaultIgnitionVersion
+	}
+	if r.ignitionEnabled() && r.Spec.Ignition.StorageType == "" {
+		r.Spec.Ignition.StorageType = infrav1.DefaultMachinePoolIgnitionStorageType
+	}
 }