Skip to content

Commit b424f32

Browse files
AndiDogAndiDog
committed
Support storing Ignition user data in S3 bucket for AWSMachinePool 
1 parent 71928f2 commit b424f32

27 files changed

+949
-196
lines changed

‎api/v1beta2/awsmachine_types.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@ const (
3030

3131
// DefaultIgnitionVersion represents default Ignition version generated for machine userdata.
3232
DefaultIgnitionVersion = "2.3"
33+
34+
// DefaultIgnitionStorageType represents the default storage type of Ignition userdata
35+
DefaultIgnitionStorageType = IgnitionStorageTypeOptionClusterObjectStore
36+
37+
// DefaultMachinePoolIgnitionStorageType represents the default storage type of Ignition userdata for machine pools.
38+
//
39+
// This is only different from DefaultIgnitionStorageType because of backward compatibility. Machine pools used to
40+
// default to store Ignition user data directly on the EC2 instance. Since the choice between remote storage (S3)
41+
// and direct storage was introduced, the default was kept, but might change in newer API versions.
42+
DefaultMachinePoolIgnitionStorageType = IgnitionStorageTypeOptionUnencryptedUserData
3343
)
3444

3545
// SecretBackend defines variants for backend secret storage.

‎api/v1beta2/awsmachine_webhook.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -399,12 +399,11 @@ func (r *AWSMachine) Default() {
399399
}
400400

401401
if r.ignitionEnabled() && r.Spec.Ignition.Version == "" {
402-
if r.Spec.Ignition == nil {
403-
r.Spec.Ignition = &Ignition{}
404-
}
405-
406402
r.Spec.Ignition.Version = DefaultIgnitionVersion
407403
}
404+
if r.ignitionEnabled() && r.Spec.Ignition.StorageType == "" {
405+
r.Spec.Ignition.StorageType = DefaultIgnitionStorageType
406+
}
408407
}
409408

410409
func (r *AWSMachine) validateAdditionalSecurityGroups() field.ErrorList {

‎api/v1beta2/tags.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,12 @@ const (
195195
// of the bootstrap secret that was used to create the user data for the latest launch
196196
// template version.
197197
LaunchTemplateBootstrapDataSecret = NameAWSProviderPrefix + "bootstrap-data-secret"
198+
199+
// LaunchTemplateBootstrapDataHash is the tag we use to store the hash of the raw bootstrap data.
200+
// If bootstrap data is stored in S3, this hash relates to that data, not to the EC2 instance
201+
// user data which only references the S3 object. We store this tag on launch template versions
202+
// so that S3 bootstrap data objects can be deleted when they get outdated.
203+
LaunchTemplateBootstrapDataHash = NameAWSProviderPrefix + "bootstrap-data-hash"
198204
)
199205

200206
// ClusterTagKey generates the key for resources associated with a cluster.

‎cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,11 +291,13 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
291291
Action: iamv1.Actions{
292292
"s3:CreateBucket",
293293
"s3:DeleteBucket",
294-
"s3:GetObject",
295-
"s3:PutObject",
296294
"s3:DeleteObject",
295+
"s3:GetObject",
296+
"s3:ListBucket",
297297
"s3:PutBucketPolicy",
298298
"s3:PutBucketTagging",
299+
"s3:PutLifecycleConfiguration",
300+
"s3:PutObject",
299301
},
300302
})
301303
}

‎cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,11 +297,13 @@ Resources:
297297
- Action:
298298
- s3:CreateBucket
299299
- s3:DeleteBucket
300-
- s3:GetObject
301-
- s3:PutObject
302300
- s3:DeleteObject
301+
- s3:GetObject
302+
- s3:ListBucket
303303
- s3:PutBucketPolicy
304304
- s3:PutBucketTagging
305+
- s3:PutLifecycleConfiguration
306+
- s3:PutObject
305307
Effect: Allow
306308
Resource:
307309
- arn:*:s3:::cluster-api-provider-aws-*

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,101 @@ spec:
872872
after it enters the InService state.
873873
If no value is supplied by user a default value of 300 seconds is set
874874
type: string
875+
ignition:
876+
description: Ignition defined options related to the bootstrapping
877+
systems where Ignition is used.
878+
properties:
879+
proxy:
880+
description: |-
881+
Proxy defines proxy settings for Ignition.
882+
Only valid for Ignition versions 3.1 and above.
883+
properties:
884+
httpProxy:
885+
description: |-
886+
HTTPProxy is the HTTP proxy to use for Ignition.
887+
A single URL that specifies the proxy server to use for HTTP and HTTPS requests,
888+
unless overridden by the HTTPSProxy or NoProxy options.
889+
type: string
890+
httpsProxy:
891+
description: |-
892+
HTTPSProxy is the HTTPS proxy to use for Ignition.
893+
A single URL that specifies the proxy server to use for HTTPS requests,
894+
unless overridden by the NoProxy option.
895+
type: string
896+
noProxy:
897+
description: |-
898+
NoProxy is the list of domains to not proxy for Ignition.
899+
Specifies a list of strings to hosts that should be excluded from proxying.
900+
901+
Each value is represented by:
902+
- An IP address prefix (1.2.3.4)
903+
- An IP address prefix in CIDR notation (1.2.3.4/8)
904+
- A domain name
905+
- A domain name matches that name and all subdomains
906+
- A domain name with a leading . matches subdomains only
907+
- A special DNS label (*), indicates that no proxying should be done
908+
909+
An IP address prefix and domain name can also include a literal port number (1.2.3.4:80).
910+
items:
911+
description: IgnitionNoProxy defines the list of domains
912+
to not proxy for Ignition.
913+
maxLength: 2048
914+
type: string
915+
maxItems: 64
916+
type: array
917+
type: object
918+
storageType:
919+
default: ClusterObjectStore
920+
description: |-
921+
StorageType defines how to store the boostrap user data for Ignition.
922+
This can be used to instruct Ignition from where to fetch the user data to bootstrap an instance.
923+
924+
When omitted, the storage option will default to ClusterObjectStore.
925+
926+
When set to "ClusterObjectStore", if the capability is available and a Cluster ObjectStore configuration
927+
is correctly provided in the Cluster object (under .spec.s3Bucket),
928+
an object store will be used to store bootstrap user data.
929+
930+
When set to "UnencryptedUserData", EC2 Instance User Data will be used to store the machine bootstrap user data, unencrypted.
931+
This option is considered less secure than others as user data may contain sensitive informations (keys, certificates, etc.)
932+
and users with ec2:DescribeInstances permission or users running pods
933+
that can access the ec2 metadata service have access to this sensitive information.
934+
So this is only to be used at ones own risk, and only when other more secure options are not viable.
935+
enum:
936+
- ClusterObjectStore
937+
- UnencryptedUserData
938+
type: string
939+
tls:
940+
description: |-
941+
TLS defines TLS settings for Ignition.
942+
Only valid for Ignition versions 3.1 and above.
943+
properties:
944+
certificateAuthorities:
945+
description: |-
946+
CASources defines the list of certificate authorities to use for Ignition.
947+
The value is the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates.
948+
Supported schemes are http, https, tftp, s3, arn, gs, and `data` (RFC 2397) URL scheme.
949+
items:
950+
description: IgnitionCASource defines the source of the
951+
certificate authority to use for Ignition.
952+
maxLength: 65536
953+
type: string
954+
maxItems: 64
955+
type: array
956+
type: object
957+
version:
958+
default: "2.3"
959+
description: Version defines which version of Ignition will be
960+
used to generate bootstrap data.
961+
enum:
962+
- "2.3"
963+
- "3.0"
964+
- "3.1"
965+
- "3.2"
966+
- "3.3"
967+
- "3.4"
968+
type: string
969+
type: object
875970
maxSize:
876971
default: 1
877972
description: MaxSize defines the maximum size of the group.

‎controllers/awsmachine_controller.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -739,7 +739,7 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
739739
if machineScope.UseIgnition(userDataFormat) {
740740
var ignitionStorageType infrav1.IgnitionStorageTypeOption
741741
if machineScope.AWSMachine.Spec.Ignition == nil {
742-
ignitionStorageType = infrav1.IgnitionStorageTypeOptionClusterObjectStore
742+
ignitionStorageType = infrav1.DefaultIgnitionStorageType
743743
} else {
744744
ignitionStorageType = machineScope.AWSMachine.Spec.Ignition.StorageType
745745
}
@@ -795,8 +795,8 @@ func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScop
795795
// then returns the config to instruct ignition on how to pull the user data from the bucket.
796796
func (r *AWSMachineReconciler) generateIgnitionWithRemoteStorage(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
797797
if objectStoreSvc == nil {
798-
return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.Spec.Ignition.S3Bucket`. " +
799-
"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.Spec.Ignition.StorageType` to `UnencryptedUserData`")
798+
return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.spec.s3Bucket`. " +
799+
"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.spec.ignition.storageType` to `UnencryptedUserData`")
800800
}
801801

802802
objectURL, err := objectStoreSvc.Create(scope, userData)

‎exp/api/v1beta1/conversion.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
5252
if restored.Spec.AvailabilityZoneSubnetType != nil {
5353
dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
5454
}
55+
if restored.Spec.Ignition != nil {
56+
dst.Spec.Ignition = restored.Spec.Ignition
57+
}
5558

5659
if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
5760
dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName

‎exp/api/v1beta1/zz_generated.conversion.go

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎exp/api/v1beta2/awsmachinepool_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ type AWSMachinePoolSpec struct {
101101
// SuspendProcesses defines a list of processes to suspend for the given ASG. This is constantly reconciled.
102102
// If a process is removed from this list it will automatically be resumed.
103103
SuspendProcesses *SuspendProcessesTypes `json:"suspendProcesses,omitempty"`
104+
105+
// Ignition defined options related to the bootstrapping systems where Ignition is used.
106+
// +optional
107+
Ignition *infrav1.Ignition `json:"ignition,omitempty"`
104108
}
105109

106110
// SuspendProcessesTypes contains user friendly auto-completable values for suspended process names.

0 commit comments

Comments
 (0)