From 6ddfa8900626d3f1fa19c9ee66a345e42f419563 Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Wed, 16 Apr 2025 10:33:12 +0200 Subject: [PATCH] feat: add capa+cabpk clusterclass example and test Signed-off-by: Carlos Salas --- .../applications/ccm/aws/fleet-bundle.yaml | 190 ++++++++++++++++++ .../aws/clusterclass-kubeadm-example.yaml | 104 +--------- test/e2e/const.go | 6 +- .../aws-kubeadm-topology.yaml | 62 ++++++ .../import-gitops-v3/import_gitops_v3_test.go | 42 ++-- 5 files changed, 293 insertions(+), 111 deletions(-) create mode 100644 examples/applications/ccm/aws/fleet-bundle.yaml rename test/e2e/data/cluster-templates/aws-ec2-kubeadm.yaml => examples/clusterclasses/aws/clusterclass-kubeadm-example.yaml (70%) create mode 100644 test/e2e/data/cluster-templates/aws-kubeadm-topology.yaml diff --git a/examples/applications/ccm/aws/fleet-bundle.yaml b/examples/applications/ccm/aws/fleet-bundle.yaml new file mode 100644 index 000000000..e51b91860 --- /dev/null +++ b/examples/applications/ccm/aws/fleet-bundle.yaml @@ -0,0 +1,190 @@ +kind: Bundle +apiVersion: fleet.cattle.io/v1alpha1 +metadata: + name: aws-cloud-controller-manager +spec: + resources: + - content: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + name: aws-ccm.yaml + targets: + - clusterSelector: + matchLabels: + cloud-provider: aws + matchExpressions: + - key: clusterclass-name.fleet.addons.cluster.x-k8s.io + operator: In + values: [aws-kubeadm-example] diff --git a/test/e2e/data/cluster-templates/aws-ec2-kubeadm.yaml b/examples/clusterclasses/aws/clusterclass-kubeadm-example.yaml similarity index 70% rename from test/e2e/data/cluster-templates/aws-ec2-kubeadm.yaml rename to examples/clusterclasses/aws/clusterclass-kubeadm-example.yaml index d79826f95..af38e49c5 100644 --- a/test/e2e/data/cluster-templates/aws-ec2-kubeadm.yaml +++ b/examples/clusterclasses/aws/clusterclass-kubeadm-example.yaml @@ -1,58 +1,23 @@ apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - ccm: external - csi: external - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - topology: - class: ${CLUSTER_NAME} - controlPlane: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - variables: - - name: region - value: ${AWS_REGION} - - name: sshKeyName - value: ${AWS_SSH_KEY_NAME} - - name: controlPlaneMachineType - value: ${AWS_CONTROL_PLANE_MACHINE_TYPE} - - name: workerMachineType - value: ${AWS_NODE_MACHINE_TYPE} - version: ${KUBERNETES_VERSION} - workers: - machineDeployments: - - class: default-worker - name: md-0 - replicas: ${WORKER_MACHINE_COUNT} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: ClusterClass metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} + name: aws-kubeadm-example spec: controlPlane: ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate - name: ${CLUSTER_NAME}-control-plane + name: aws-kubeadm-example-control-plane machineInfrastructure: ref: kind: AWSMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: ${CLUSTER_NAME}-control-plane + name: aws-kubeadm-example-control-plane infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSClusterTemplate - name: ${CLUSTER_NAME} + name: aws-kubeadm-example workers: machineDeployments: - class: default-worker @@ -61,12 +26,12 @@ spec: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-worker-bootstraptemplate + name: aws-kubeadm-example-worker-bootstraptemplate infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-worker-machinetemplate + name: aws-kubeadm-example-worker-machinetemplate variables: - name: region required: true @@ -160,8 +125,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSClusterTemplate metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} + name: aws-kubeadm-example spec: template: spec: @@ -172,8 +136,7 @@ spec: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} + name: aws-kubeadm-example-control-plane spec: template: spec: @@ -199,8 +162,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: ${NAMESPACE} + name: aws-kubeadm-example-control-plane spec: template: spec: @@ -213,8 +175,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: ${CLUSTER_NAME}-worker-machinetemplate - namespace: ${NAMESPACE} + name: aws-kubeadm-example-worker-machinetemplate spec: template: spec: @@ -227,8 +188,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_NAME}-worker-bootstraptemplate" - namespace: ${NAMESPACE} + name: "aws-kubeadm-example-worker-bootstraptemplate" spec: template: spec: @@ -237,45 +197,3 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-cni - namespace: ${NAMESPACE} -spec: - clusterSelector: - matchLabels: - cni: calico - resources: - - kind: ConfigMap - name: calico-cni - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-ccm - namespace: ${NAMESPACE} -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-csi - namespace: ${NAMESPACE} -spec: - clusterSelector: - matchLabels: - csi: external - resources: - - kind: ConfigMap - name: aws-ebs-csi-driver-addon - strategy: ApplyOnce diff --git a/test/e2e/const.go b/test/e2e/const.go index d9c45b3c1..5bdba5bf0 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -96,12 +96,12 @@ var ( //go:embed data/cluster-templates/aws-eks-mmp.yaml CAPIAwsEKSMMP []byte - //go:embed data/cluster-templates/aws-ec2-kubeadm.yaml - CAPIAwsEC2Kubeadm []byte - //go:embed data/cluster-templates/aws-ec2-rke2-topology.yaml CAPIAwsEC2RKE2Topology []byte + //go:embed data/cluster-templates/aws-kubeadm-topology.yaml + CAPIAwsKubeadmTopology []byte + //go:embed data/cluster-templates/gcp-gke.yaml CAPIGCPGKE []byte diff --git a/test/e2e/data/cluster-templates/aws-kubeadm-topology.yaml b/test/e2e/data/cluster-templates/aws-kubeadm-topology.yaml new file mode 100644 index 000000000..7f6750de8 --- /dev/null +++ b/test/e2e/data/cluster-templates/aws-kubeadm-topology.yaml @@ -0,0 +1,62 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: calico + cloud-provider: aws + csi: aws-ebs-csi-driver + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: aws-kubeadm-example + classNamespace: ${TOPOLOGY_NAMESPACE} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: region + value: ${AWS_REGION} + - name: sshKeyName + value: ${AWS_SSH_KEY_NAME} + - name: controlPlaneMachineType + value: ${AWS_CONTROL_PLANE_MACHINE_TYPE} + - name: workerMachineType + value: ${AWS_NODE_MACHINE_TYPE} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cni-aws-kubeadm + namespace: ${NAMESPACE} +spec: + clusterSelector: + matchLabels: + cni: calico + resources: + - kind: ConfigMap + name: calico-cni + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi-aws-kubeadm + namespace: ${NAMESPACE} +spec: + clusterSelector: + matchLabels: + csi: aws-ebs-csi-driver + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go index 65ba83aeb..ccfcd11dd 100644 --- a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go +++ b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go @@ -142,9 +142,7 @@ var _ = Describe("[Docker] [RKE2] Create and delete CAPI cluster functionality s }) var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster from cluster class", Label(e2e.FullTestLabel), func() { - var ( - topologyNamespace string - ) + var topologyNamespace string BeforeEach(func() { komega.SetClient(bootstrapClusterProxy.GetClient()) @@ -199,9 +197,7 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster from cluster clas }) var _ = Describe("[Azure] [RKE2] - [management.cattle.io/v3] Create and delete CAPI cluster from cluster class", Label(e2e.FullTestLabel, e2e.Rke2TestLabel), func() { - var ( - topologyNamespace string - ) + var topologyNamespace string BeforeEach(func() { komega.SetClient(bootstrapClusterProxy.GetClient()) @@ -311,9 +307,13 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul }) var _ = Describe("[AWS] [EC2 Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel, e2e.KubeadmTestLabel), func() { + var topologyNamespace string + BeforeEach(func() { komega.SetClient(bootstrapClusterProxy.GetClient()) komega.SetContext(ctx) + + topologyNamespace = "creategitops-aws-kubeadm" }) specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { @@ -337,12 +337,13 @@ var _ = Describe("[AWS] [EC2 Kubeadm] Create and delete CAPI cluster functionali return specs.CreateMgmtV3UsingGitOpsSpecInput{ E2EConfig: e2e.LoadE2EConfig(), BootstrapClusterProxy: bootstrapClusterProxy, - ClusterTemplate: e2e.CAPIAwsEC2Kubeadm, - AdditionalTemplates: [][]byte{e2e.CAPICalico, e2e.CAPIAWSCPICSI}, - ClusterName: "cluster-ec2", + ClusterTemplate: e2e.CAPIAwsKubeadmTopology, + AdditionalTemplates: [][]byte{e2e.CAPICalico}, + ClusterName: "cluster-aws-kubeadm", ControlPlaneMachineCount: ptr.To(1), WorkerMachineCount: ptr.To(1), GitAddr: gitAddress, + SkipDeletionTest: false, LabelNamespace: true, RancherServerURL: hostName, CAPIClusterCreateWaitName: "wait-capa-create-cluster", @@ -353,14 +354,27 @@ var _ = Describe("[AWS] [EC2 Kubeadm] Create and delete CAPI cluster functionali AdditionalTemplateVariables: map[string]string{ e2e.KubernetesVersionVar: e2e.LoadE2EConfig().GetVariable(e2e.AWSKubernetesVersionVar), // override the default k8s version }, + TopologyNamespace: topologyNamespace, + AdditionalFleetGitRepos: []turtlesframework.FleetCreateGitRepoInput{ + { + Name: "aws-cluster-classes-regular", + Paths: []string{"examples/clusterclasses/aws"}, + ClusterProxy: bootstrapClusterProxy, + TargetNamespace: topologyNamespace, + }, + { + Name: "aws-ccm", + Paths: []string{"examples/applications/ccm/aws"}, + ClusterProxy: bootstrapClusterProxy, + TargetNamespace: topologyNamespace, + }, + }, } }) }) var _ = Describe("[AWS] [EC2 RKE2] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel, e2e.Rke2TestLabel), func() { - var ( - topologyNamespace string - ) + var topologyNamespace string BeforeEach(func() { komega.SetClient(bootstrapClusterProxy.GetClient()) @@ -459,9 +473,7 @@ var _ = Describe("[GCP] [GKE] Create and delete CAPI cluster functionality shoul }) var _ = Describe("[vSphere] [Kubeadm] Create and delete CAPI cluster from cluster class", Label(e2e.VsphereTestLabel, e2e.KubeadmTestLabel), func() { - var ( - topologyNamespace string - ) + var topologyNamespace string BeforeEach(func() { komega.SetClient(bootstrapClusterProxy.GetClient())