Skip to content

Commit 0972f68

Browse files
AndiDogAndiDog
committed
Support storing Ignition user data in S3 bucket for AWSMachinePool 
1 parent fef3eea commit 0972f68

27 files changed

+950
-196
lines changed

‎api/v1beta2/awsmachine_types.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@ const (
3030

3131
// DefaultIgnitionVersion represents default Ignition version generated for machine userdata.
3232
DefaultIgnitionVersion = "2.3"
33+
34+
// DefaultIgnitionStorageType represents the default storage type of Ignition userdata
35+
DefaultIgnitionStorageType = IgnitionStorageTypeOptionClusterObjectStore
36+
37+
// DefaultMachinePoolIgnitionStorageType represents the default storage type of Ignition userdata for machine pools.
38+
//
39+
// This is only different from DefaultIgnitionStorageType because of backward compatibility. Machine pools used to
40+
// default to store Ignition user data directly on the EC2 instance. Since the choice between remote storage (S3)
41+
// and direct storage was introduced, the default was kept, but might change in newer API versions.
42+
DefaultMachinePoolIgnitionStorageType = IgnitionStorageTypeOptionUnencryptedUserData
3343
)
3444

3545
// SecretBackend defines variants for backend secret storage.

‎api/v1beta2/awsmachine_webhook.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -414,12 +414,11 @@ func (r *AWSMachine) Default() {
414414
}
415415

416416
if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
417-
if r.Spec.Ignition == nil {
418-
r.Spec.Ignition = &Ignition{}
419-
}
420-
421417
r.Spec.Ignition.Version = DefaultIgnitionVersion
422418
}
419+
if r.ignitionEnabled() && r.Spec.Ignition.StorageType == "" {
420+
r.Spec.Ignition.StorageType = DefaultIgnitionStorageType
421+
}
423422
}
424423

425424
func (r *AWSMachine) validateAdditionalSecurityGroups() field.ErrorList {

‎api/v1beta2/tags.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,12 @@ const (
195195
// of the bootstrap secret that was used to create the user data for the latest launch
196196
// template version.
197197
LaunchTemplateBootstrapDataSecret = NameAWSProviderPrefix + "bootstrap-data-secret"
198+
199+
// LaunchTemplateBootstrapDataHash is the tag we use to store the hash of the raw bootstrap data.
200+
// If bootstrap data is stored in S3, this hash relates to that data, not to the EC2 instance
201+
// user data which only references the S3 object. We store this tag on launch template versions
202+
// so that S3 bootstrap data objects can be deleted when they get outdated.
203+
LaunchTemplateBootstrapDataHash = NameAWSProviderPrefix + "bootstrap-data-hash"
198204
)
199205

200206
// ClusterTagKey generates the key for resources associated with a cluster.

‎cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,11 +291,13 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
291291
Action: iamv1.Actions{
292292
"s3:CreateBucket",
293293
"s3:DeleteBucket",
294-
"s3:GetObject",
295-
"s3:PutObject",
296294
"s3:DeleteObject",
295+
"s3:GetObject",
296+
"s3:ListBucket",
297297
"s3:PutBucketPolicy",
298298
"s3:PutBucketTagging",
299+
"s3:PutLifecycleConfiguration",
300+
"s3:PutObject",
299301
},
300302
})
301303
}

‎cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,11 +297,13 @@ Resources:
297297
- Action:
298298
- s3:CreateBucket
299299
- s3:DeleteBucket
300-
- s3:GetObject
301-
- s3:PutObject
302300
- s3:DeleteObject
301+
- s3:GetObject
302+
- s3:ListBucket
303303
- s3:PutBucketPolicy
304304
- s3:PutBucketTagging
305+
- s3:PutLifecycleConfiguration
306+
- s3:PutObject
305307
Effect: Allow
306308
Resource:
307309
- arn:*:s3:::cluster-api-provider-aws-*

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -889,6 +889,101 @@ spec:
889889
after it enters the InService state.
890890
If no value is supplied by user a default value of 300 seconds is set
891891
type: string
892+
ignition:
893+
description: Ignition defined options related to the bootstrapping
894+
systems where Ignition is used.
895+
properties:
896+
proxy:
897+
description: |-
898+
Proxy defines proxy settings for Ignition.
899+
Only valid for Ignition versions 3.1 and above.
900+
properties:
901+
httpProxy:
902+
description: |-
903+
HTTPProxy is the HTTP proxy to use for Ignition.
904+
A single URL that specifies the proxy server to use for HTTP and HTTPS requests,
905+
unless overridden by the HTTPSProxy or NoProxy options.
906+
type: string
907+
httpsProxy:
908+
description: |-
909+
HTTPSProxy is the HTTPS proxy to use for Ignition.
910+
A single URL that specifies the proxy server to use for HTTPS requests,
911+
unless overridden by the NoProxy option.
912+
type: string
913+
noProxy:
914+
description: |-
915+
NoProxy is the list of domains to not proxy for Ignition.
916+
Specifies a list of strings to hosts that should be excluded from proxying.
917+
918+
Each value is represented by:
919+
- An IP address prefix (1.2.3.4)
920+
- An IP address prefix in CIDR notation (1.2.3.4/8)
921+
- A domain name
922+
- A domain name matches that name and all subdomains
923+
- A domain name with a leading . matches subdomains only
924+
- A special DNS label (*), indicates that no proxying should be done
925+
926+
An IP address prefix and domain name can also include a literal port number (1.2.3.4:80).
927+
items:
928+
description: IgnitionNoProxy defines the list of domains
929+
to not proxy for Ignition.
930+
maxLength: 2048
931+
type: string
932+
maxItems: 64
933+
type: array
934+
type: object
935+
storageType:
936+
default: ClusterObjectStore
937+
description: |-
938+
StorageType defines how to store the boostrap user data for Ignition.
939+
This can be used to instruct Ignition from where to fetch the user data to bootstrap an instance.
940+
941+
When omitted, the storage option will default to ClusterObjectStore.
942+
943+
When set to "ClusterObjectStore", if the capability is available and a Cluster ObjectStore configuration
944+
is correctly provided in the Cluster object (under .spec.s3Bucket),
945+
an object store will be used to store bootstrap user data.
946+
947+
When set to "UnencryptedUserData", EC2 Instance User Data will be used to store the machine bootstrap user data, unencrypted.
948+
This option is considered less secure than others as user data may contain sensitive informations (keys, certificates, etc.)
949+
and users with ec2:DescribeInstances permission or users running pods
950+
that can access the ec2 metadata service have access to this sensitive information.
951+
So this is only to be used at ones own risk, and only when other more secure options are not viable.
952+
enum:
953+
- ClusterObjectStore
954+
- UnencryptedUserData
955+
type: string
956+
tls:
957+
description: |-
958+
TLS defines TLS settings for Ignition.
959+
Only valid for Ignition versions 3.1 and above.
960+
properties:
961+
certificateAuthorities:
962+
description: |-
963+
CASources defines the list of certificate authorities to use for Ignition.
964+
The value is the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates.
965+
Supported schemes are http, https, tftp, s3, arn, gs, and `data` (RFC 2397) URL scheme.
966+
items:
967+
description: IgnitionCASource defines the source of the
968+
certificate authority to use for Ignition.
969+
maxLength: 65536
970+
type: string
971+
maxItems: 64
972+
type: array
973+
type: object
974+
version:
975+
default: "2.3"
976+
description: Version defines which version of Ignition will be
977+
used to generate bootstrap data.
978+
enum:
979+
- "2.3"
980+
- "3.0"
981+
- "3.1"
982+
- "3.2"
983+
- "3.3"
984+
- "3.4"
985+
type: string
986+
type: object
892987
maxSize:
893988
default: 1
894989
description: MaxSize defines the maximum size of the group.

‎controllers/awsmachine_controller.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -739,7 +739,7 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
739739
if machineScope.UseIgnition(userDataFormat) {
740740
var ignitionStorageType infrav1.IgnitionStorageTypeOption
741741
if machineScope.AWSMachine.Spec.Ignition == nil {
742-
ignitionStorageType = infrav1.IgnitionStorageTypeOptionClusterObjectStore
742+
ignitionStorageType = infrav1.DefaultIgnitionStorageType
743743
} else {
744744
ignitionStorageType = machineScope.AWSMachine.Spec.Ignition.StorageType
745745
}
@@ -795,8 +795,8 @@ func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScop
795795
// then returns the config to instruct ignition on how to pull the user data from the bucket.
796796
func (r *AWSMachineReconciler) generateIgnitionWithRemoteStorage(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
797797
if objectStoreSvc == nil {
798-
return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.Spec.Ignition.S3Bucket`. " +
799-
"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.Spec.Ignition.StorageType` to `UnencryptedUserData`")
798+
return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.spec.s3Bucket`. " +
799+
"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.spec.ignition.storageType` to `UnencryptedUserData`")
800800
}
801801

802802
objectURL, err := objectStoreSvc.Create(scope, userData)

‎exp/api/v1beta1/conversion.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
5252
if restored.Spec.AvailabilityZoneSubnetType != nil {
5353
dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
5454
}
55+
if restored.Spec.Ignition != nil {
56+
dst.Spec.Ignition = restored.Spec.Ignition
57+
}
5558

5659
if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
5760
dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName

‎exp/api/v1beta1/zz_generated.conversion.go

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎exp/api/v1beta2/awsmachinepool_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ type AWSMachinePoolSpec struct {
101101
// SuspendProcesses defines a list of processes to suspend for the given ASG. This is constantly reconciled.
102102
// If a process is removed from this list it will automatically be resumed.
103103
SuspendProcesses *SuspendProcessesTypes `json:"suspendProcesses,omitempty"`
104+
105+
// Ignition defined options related to the bootstrapping systems where Ignition is used.
106+
// +optional
107+
Ignition *infrav1.Ignition `json:"ignition,omitempty"`
104108
}
105109

106110
// SuspendProcessesTypes contains user friendly auto-completable values for suspended process names.

‎exp/api/v1beta2/awsmachinepool_webhook.go

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import (
2828
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
2929

3030
"sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
31+
infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
32+
"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
3133
)
3234

3335
var log = ctrl.Log.WithName("awsmachinepool-resource")
@@ -62,12 +64,12 @@ func (r *AWSMachinePool) validateRootVolume() field.ErrorList {
6264
return allErrs
6365
}
6466

65-
if v1beta2.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
67+
if infrav1.VolumeTypesProvisioned.Has(string(r.Spec.AWSLaunchTemplate.RootVolume.Type)) && r.Spec.AWSLaunchTemplate.RootVolume.IOPS == 0 {
6668
allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.iops"), "iops required if type is 'io1' or 'io2'"))
6769
}
6870

6971
if r.Spec.AWSLaunchTemplate.RootVolume.Throughput != nil {
70-
if r.Spec.AWSLaunchTemplate.RootVolume.Type != v1beta2.VolumeTypeGP3 {
72+
if r.Spec.AWSLaunchTemplate.RootVolume.Type != infrav1.VolumeTypeGP3 {
7173
allErrs = append(allErrs, field.Required(field.NewPath("spec.awsLaunchTemplate.rootVolume.throughput"), "throughput is valid only for type 'gp3'"))
7274
}
7375
if *r.Spec.AWSLaunchTemplate.RootVolume.Throughput < 0 {
@@ -86,12 +88,12 @@ func (r *AWSMachinePool) validateNonRootVolumes() field.ErrorList {
8688
var allErrs field.ErrorList
8789

8890
for _, volume := range r.Spec.AWSLaunchTemplate.NonRootVolumes {
89-
if v1beta2.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
91+
if infrav1.VolumeTypesProvisioned.Has(string(volume.Type)) && volume.IOPS == 0 {
9092
allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.iops"), "iops required if type is 'io1' or 'io2'"))
9193
}
9294

9395
if volume.Throughput != nil {
94-
if volume.Type != v1beta2.VolumeTypeGP3 {
96+
if volume.Type != infrav1.VolumeTypeGP3 {
9597
allErrs = append(allErrs, field.Required(field.NewPath("spec.template.spec.nonRootVolumes.throughput"), "throughput is valid only for type 'gp3'"))
9698
}
9799
if *volume.Throughput < 0 {
@@ -162,6 +164,22 @@ func (r *AWSMachinePool) validateRefreshPreferences() field.ErrorList {
162164
return allErrs
163165
}
164166

167+
func (r *AWSMachinePool) ignitionEnabled() bool {
168+
return r.Spec.Ignition != nil
169+
}
170+
171+
func (r *AWSMachinePool) validateIgnition() field.ErrorList {
172+
var allErrs field.ErrorList
173+
174+
// Feature gate is not enabled but ignition is enabled then send a forbidden error.
175+
if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) && r.ignitionEnabled() {
176+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "ignition"),
177+
"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
178+
}
179+
180+
return allErrs
181+
}
182+
165183
// ValidateCreate will do any extra validation when creating a AWSMachinePool.
166184
func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
167185
log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
@@ -177,6 +195,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
177195
allErrs = append(allErrs, r.validateSpotInstances()...)
178196
allErrs = append(allErrs, r.validateRefreshPreferences()...)
179197
allErrs = append(allErrs, r.validateInstanceMarketType()...)
198+
allErrs = append(allErrs, r.validateIgnition()...)
180199

181200
if len(allErrs) == 0 {
182201
return nil, nil
@@ -243,4 +262,11 @@ func (r *AWSMachinePool) Default() {
243262
log.Info("DefaultInstanceWarmup is zero, setting 300 seconds as default")
244263
r.Spec.DefaultInstanceWarmup.Duration = 300 * time.Second
245264
}
265+
266+
if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
267+
r.Spec.Ignition.Version = infrav1.DefaultIgnitionVersion
268+
}
269+
if r.ignitionEnabled() && r.Spec.Ignition.StorageType == "" {
270+
r.Spec.Ignition.StorageType = infrav1.DefaultMachinePoolIgnitionStorageType
271+
}
246272
}

‎exp/api/v1beta2/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)