diff --git a/examples/applications/ccm/azure/helm-chart.yaml b/examples/applications/ccm/azure/helm-chart.yaml index 331ea191c..455e608cc 100644 --- a/examples/applications/ccm/azure/helm-chart.yaml +++ b/examples/applications/ccm/azure/helm-chart.yaml @@ -15,7 +15,11 @@ spec: caCertDir: /etc/ssl hostNetworking: true nodeSelector: + ${- if contains "KubeadmControlPlane" ( .ClusterValues | quote ) } + node-role.kubernetes.io/control-plane: "" + ${- else } node-role.kubernetes.io/control-plane: "true" + ${- end } insecureSkipTLSVerify: true targets: - clusterSelector: @@ -24,4 +28,4 @@ spec: matchExpressions: - key: clusterclass-name.fleet.addons.cluster.x-k8s.io operator: In - values: [azure-rke2-example] + values: [azure-rke2-example, azure-kubeadm-example] diff --git a/examples/applications/cni/calico/helm-chart.yaml b/examples/applications/cni/calico/helm-chart.yaml index 2da2e527f..a727862c4 100644 --- a/examples/applications/cni/calico/helm-chart.yaml +++ b/examples/applications/cni/calico/helm-chart.yaml @@ -36,6 +36,7 @@ spec: operator: In values: - azure-rke2-example + - azure-kubeadm-example - docker-kubeadm-example - docker-rke2-example - etcd-snapshot-restore diff --git a/examples/clusterclasses/azure/kubeadm/clusterclass-kubeadm-example.yaml b/examples/clusterclasses/azure/kubeadm/clusterclass-kubeadm-example.yaml new file mode 100644 index 000000000..41b71764e --- /dev/null +++ b/examples/clusterclasses/azure/kubeadm/clusterclass-kubeadm-example.yaml @@ -0,0 +1,335 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: azure-kubeadm-example +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: azure-machine-control-plane + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: kubeadm-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterTemplate + name: azure-kubeadm-cluster + workers: + machineDeployments: + - class: kubeadm-default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: kubeadm-default-worker-bootstrap + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: kubeadm-default-worker + variables: + - name: subscriptionID + required: true + schema: + openAPIV3Schema: + description: "The Azure Subscription ID where the Cluster will be created." + type: string + - name: location + required: true + schema: + openAPIV3Schema: + description: "The Azure location where the Cluster will be created." + type: string + enum: + - australiaeast + - francecentral + - germanywestcentral + - northcentralus + - northeurope + - switzerlandnorth + - uksouth + - westeurope + - westus2 + - name: resourceGroup + required: true + schema: + openAPIV3Schema: + description: "The Azure Resource Group where the Cluster will be created." + type: string + - name: azureClusterIdentityName + required: true + schema: + openAPIV3Schema: + description: "The AzureClusterIdentity resource name referencing the credentials to create the Cluster." + type: string + default: "cluster-identity" + - name: imageGallery + required: true + schema: + openAPIV3Schema: + description: "The image Public gallery name." + type: string + default: "ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019" + - name: imageName + required: true + schema: + openAPIV3Schema: + description: "The image name" + type: string + default: "capi-ubun2-2404" + - name: vmSize + required: true + schema: + openAPIV3Schema: + description: "The VM size used by machines." + type: string + default: "Standard_D2s_v3" + patches: + - name: azureClusterTemplate + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: "/spec/template/spec/subscriptionID" + valueFrom: + variable: subscriptionID + - op: add + path: "/spec/template/spec/location" + valueFrom: + variable: location + - op: add + path: "/spec/template/spec/resourceGroup" + valueFrom: + variable: resourceGroup + - op: add + path: "/spec/template/spec/identityRef/name" + valueFrom: + variable: azureClusterIdentityName + - op: add + path: "/spec/template/spec/networkSpec/vnet/name" + valueFrom: + variable: builtin.cluster.name + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/controllerManager/extraArgs/cluster-name + valueFrom: + variable: builtin.cluster.name + - op: replace + path: /spec/template/spec/kubeadmConfigSpec/files + valueFrom: + template: | + - contentFrom: + secret: + key: control-plane-azure.json + name: "{{ .builtin.controlPlane.machineTemplate.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + matchResources: + controlPlane: true + machineDeploymentClass: + names: + - kubeadm-default-worker + jsonPatches: + - op: add + path: "/spec/template/spec/image/computeGallery/gallery" + valueFrom: + variable: imageGallery + - op: add + path: "/spec/template/spec/image/computeGallery/name" + valueFrom: + variable: imageName + - op: replace + path: /spec/template/spec/image/computeGallery/version + valueFrom: + template: '{{ trimPrefix "v" (trimSuffix "+rke2r1" .builtin.cluster.topology.version) }}' + - op: add + path: "/spec/template/spec/vmSize" + valueFrom: + variable: vmSize + name: controlPlaneAzureJsonSecretName + - definitions: + - jsonPatches: + - op: replace + path: /spec/template/spec/files + valueFrom: + template: | + - contentFrom: + secret: + key: worker-node-azure.json + name: "{{ .builtin.machineDeployment.infrastructureRef.name }}-azure-json" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - kubeadm-default-worker + name: workerAzureJsonSecretName +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterTemplate +metadata: + name: azure-kubeadm-cluster +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: cluster-identity + location: "TO_BE_REPLACED_BY_PATCH" + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + subscriptionID: "TO_BE_REPLACED_BY_PATCH" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: azure-machine-control-plane +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + vmSize: "TO_BE_REPLACED_BY_PATCH" + image: + computeGallery: + gallery: "TO_BE_REPLACED_BY_PATCH" + name: "TO_BE_REPLACED_BY_PATCH" + version: "TO_BE_REPLACED_BY_PATCH" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: kubeadm-default-worker +spec: + template: + spec: + image: + computeGallery: + gallery: "TO_BE_REPLACED_BY_PATCH" + name: "TO_BE_REPLACED_BY_PATCH" + version: "TO_BE_REPLACED_BY_PATCH" + osDisk: + diskSizeGB: 128 + osType: Linux + vmSize: "TO_BE_REPLACED_BY_PATCH" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: kubeadm-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: {} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: "TO_BE_REPLACED_BY_PATCH" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + verbosity: 10 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: kubeadm-default-worker-bootstrap +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] diff --git a/test/e2e/const.go b/test/e2e/const.go index bb0cc9733..36c902514 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -108,6 +108,9 @@ var ( //go:embed data/cluster-templates/azure-rke2-topology.yaml CAPIAzureRKE2Topology []byte + //go:embed data/cluster-templates/azure-kubeadm-topology.yaml + CAPIAzureKubeadmTopology []byte + //go:embed data/cluster-templates/vsphere-kubeadm-topology.yaml CAPIvSphereKubeadmTopology []byte diff --git a/test/e2e/data/cluster-templates/azure-kubeadm-topology.yaml b/test/e2e/data/cluster-templates/azure-kubeadm-topology.yaml new file mode 100644 index 000000000..b38937217 --- /dev/null +++ b/test/e2e/data/cluster-templates/azure-kubeadm-topology.yaml @@ -0,0 +1,53 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: cluster-identity + namespace: ${NAMESPACE} +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: cluster-identity-secret + namespace: capz-system + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster-api.cattle.io/rancher-auto-import: "true" + cni: calico + cloud-provider: azure + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: azure-kubeadm-example + classNamespace: ${TOPOLOGY_NAMESPACE} + controlPlane: + replicas: 1 + variables: + - name: subscriptionID + value: ${AZURE_SUBSCRIPTION_ID} + - name: location + value: germanywestcentral + - name: resourceGroup + value: highlander-e2e-azure-kubeadm + - name: azureClusterIdentityName + value: cluster-identity + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: kubeadm-default-worker + name: md-0 + replicas: 1 + diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go index 4d9bb64df..c43c31607 100644 --- a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go +++ b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go @@ -192,6 +192,73 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster from cluster clas }) }) +var _ = FDescribe("[Azure] [Kubeadm] - [management.cattle.io/v3] Create and delete CAPI cluster from cluster class", Label(e2e.FullTestLabel, e2e.KubeadmTestLabel), func() { + var topologyNamespace string + + BeforeEach(func() { + komega.SetClient(bootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + + topologyNamespace = "creategitops-azure-kubeadm" + }) + specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { + testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + CAPIProvidersSecretsYAML: [][]byte{ + e2e.AzureIdentitySecret, + }, + CAPIProvidersYAML: [][]byte{ + e2e.AzureProvider, + }, + WaitForDeployments: []testenv.NamespaceName{ + { + Name: "capz-controller-manager", + Namespace: "capz-system", + }, + }, + }) + + return specs.CreateMgmtV3UsingGitOpsSpecInput{ + E2EConfig: e2e.LoadE2EConfig(), + BootstrapClusterProxy: bootstrapClusterProxy, + ClusterTemplate: e2e.CAPIAzureKubeadmTopology, + ClusterName: "cluster-azure-kubeadm", + ControlPlaneMachineCount: ptr.To(1), + WorkerMachineCount: ptr.To(1), + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-capz-create-cluster", + DeleteClusterWaitName: "wait-aks-delete", + CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, + CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, + OwnedLabelName: e2e.OwnedLabelName, + TopologyNamespace: topologyNamespace, + AdditionalFleetGitRepos: []turtlesframework.FleetCreateGitRepoInput{ + { + Name: "azure-cluster-classes-regular", + Paths: []string{"examples/clusterclasses/azure/kubeadm"}, + ClusterProxy: bootstrapClusterProxy, + TargetNamespace: topologyNamespace, + }, + { + Name: "azure-ccm-regular", + Paths: []string{"examples/applications/ccm/azure"}, + ClusterProxy: bootstrapClusterProxy, + TargetNamespace: topologyNamespace, + }, + { + Name: "azure-cni", + Paths: []string{"examples/applications/cni/calico"}, + ClusterProxy: bootstrapClusterProxy, + TargetNamespace: topologyNamespace, + }, + }, + } + }) + +}) + var _ = Describe("[Azure] [RKE2] - [management.cattle.io/v3] Create and delete CAPI cluster from cluster class", Label(e2e.FullTestLabel, e2e.Rke2TestLabel), func() { var topologyNamespace string