Merge pull request #268 from Danil-Grigorev/fix-e2e-tests-cherry-pick

[release-0.2] Fix: e2e tests

Author: Danil-Grigorev

.github/workflows/e2e.yml

@@ -1,5 +1,6 @@
 name: E2E tests
 on:
+  pull_request:
   schedule:
     - cron: '0 1 * * *'
   workflow_dispatch:
@@ -8,6 +9,11 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     steps:
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          docker-images: false
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -38,4 +44,4 @@ jobs:
         with:
           name: e2e-artifacts
           path: _artifacts
-          if-no-files-found: ignore
+          if-no-files-found: ignore

Makefile

@@ -99,7 +99,7 @@ GOLANGCI_LINT_VER := v1.55.1
 GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN))
 
-GINKGO_VER := v2.9.4
+GINKGO_VER := v2.14.0
 GINKGO_BIN := ginkgo
 GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER))
 GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo

metadata.yaml

@@ -11,4 +11,4 @@ releaseSeries:
     contract: v1beta1
   - major: 0
     minor: 2
-    contract: v1beta1
+    contract: v1beta1

test/e2e/config/e2e_conf.yaml

@@ -5,68 +5,97 @@ images:
     loadBehavior: mustLoad
   - name: ghcr.io/rancher-sandbox/cluster-api-provider-rke2-controlplane:dev
     loadBehavior: mustLoad
+  - name: quay.io/jetstack/cert-manager-cainjector:v1.13.2
+    loadBehavior: tryLoad
+  - name: quay.io/jetstack/cert-manager-webhook:v1.13.2
+    loadBehavior: tryLoad
+  - name: quay.io/jetstack/cert-manager-controller:v1.13.2
+    loadBehavior: tryLoad
 
 providers:
   - name: cluster-api
     type: CoreProvider
     versions:
-      - name: v1.3.0
-        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0/core-components.yaml
-        type: url
-        contract: v1beta1
-        files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
+    - name: "v1.6.0"
+      value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/core-components.yaml"
+      type: "url"
+      contract: v1beta1
+      files:
+        - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
   - name: docker
     type: InfrastructureProvider
     versions:
-      - name: v1.3.3
-        value: "https://github.com/belgaied2/cluster-api/releases/download/v1.3.3-cabpr-fix/infrastructure-components.yaml" # NOTE: this uses the docker form for the time being
-        type: url
-        files:
-          - sourcePath: "../data/infrastructure/cluster-template-docker.yaml"
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
-          - old: --metrics-addr=127.0.0.1:8080
-            new: --metrics-addr=:8080
-        contract: v1beta1
-
+    - name: "v1.6.0"
+      value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/infrastructure-components-development.yaml"
+      type: "url"
+      contract: v1beta1
+      files:
+        - sourcePath: "../data/infrastructure/cluster-template-docker.yaml"
+        - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
+        - old: --metrics-addr=127.0.0.1:8080
+          new: --metrics-addr=:8080
   - name: rke2-control-plane
     type: ControlPlaneProvider
     versions:
-      - name: v0.1.99
-        value: "../../../controlplane/config/default"
-        contract: v1beta1
-        files:
-          - sourcePath: "../../../metadata.yaml"
-            targetName: "metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
+    - name: "v0.2.3"
+      value: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/download/v0.2.3/control-plane-components.yaml"
+      type: "url"
+      contract: v1beta1
+      files:
+        - sourcePath: "../../../metadata.yaml"
+          targetName: "metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
+    - name: v0.2.99
+      value: "../../../controlplane/config/default"
+      contract: v1beta1
+      files:
+        - sourcePath: "../../../metadata.yaml"
+          targetName: "metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
   - name: rke2-bootstrap
     type: BootstrapProvider
     versions:
-      - name: v0.1.99
-        value: ../../../bootstrap/config/default
-        contract: v1beta1
-        files:
-          - sourcePath: "../../../metadata.yaml"
-            targetName: "metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
-          - old: "--leader-elect"
-            new: "--leader-elect=false"
+    - name: "v0.2.3"
+      value: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/download/v0.2.3/bootstrap-components.yaml"
+      type: "url"
+      contract: v1beta1
+      files:
+        - sourcePath: "../../../metadata.yaml"
+          targetName: "metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
+    - name: v0.2.99
+      value: ../../../bootstrap/config/default
+      contract: v1beta1
+      files:
+        - sourcePath: "../../../metadata.yaml"
+          targetName: "metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
+        - old: "--leader-elect"
+          new: "--leader-elect=false"
 
 variables:
   KUBERNETES_VERSION_MANAGEMENT: "v1.28.0"

test/e2e/data/infrastructure/cluster-template-docker.yaml

@@ -1,3 +1,66 @@
+apiVersion: v1
+data:
+  value: |-
+    # generated by kind
+    global
+      log /dev/log local0
+      log /dev/log local1 notice
+      daemon
+      # limit memory usage to approximately 18 MB
+      # (see https://github.com/kubernetes-sigs/kind/pull/3115)
+      maxconn 100000
+
+    resolvers docker
+      nameserver dns 127.0.0.11:53
+
+    defaults
+      log global
+      mode tcp
+      option dontlognull
+      # TODO: tune these
+      timeout connect 5000
+      timeout client 50000
+      timeout server 50000
+      # allow to boot despite dns don't resolve backends
+      default-server init-addr none
+
+    frontend stats
+      bind *:8404
+      stats enable
+      stats uri /
+      stats refresh 10s
+
+    frontend control-plane
+      bind *:{{ .FrontendControlPlanePort }}
+      {{ if .IPv6 -}}
+      bind :::{{ .FrontendControlPlanePort }};
+      {{- end }}
+      default_backend kube-apiservers
+
+    backend kube-apiservers
+      option httpchk GET /healthz
+      http-check expect status 401
+      # TODO: we should be verifying (!)
+      {{range $server, $address := .BackendServers}}
+      server {{ $server }} {{ JoinHostPort $address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
+      {{- end}}
+
+    frontend rke2-join
+      bind *:9345
+      {{ if .IPv6 -}}
+      bind :::9345;
+      {{- end }}
+      default_backend rke2-servers
+
+    backend rke2-servers
+      option httpchk GET /v1-rke2/readyz
+      http-check expect status 403
+      {{range $server, $address := .BackendServers}}
+      server {{ $server }} {{ $address }}:9345 check check-ssl verify none
+      {{- end}}
+kind: ConfigMap
+metadata:
+  name: ${CLUSTER_NAME}-lb-config
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster 
@@ -25,6 +88,10 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerCluster
 metadata:
   name: ${CLUSTER_NAME}
+spec:
+  loadBalancer:
+    customHAProxyConfigTemplateRef:
+      name: ${CLUSTER_NAME}-lb-config
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
 kind: RKE2ControlPlane
@@ -36,6 +103,10 @@ spec:
     version: ${KUBERNETES_VERSION}+rke2r1
     nodeAnnotations:
       test: "true"
+  serverConfig:
+    disableComponents:
+      kubernetesComponents:
+      - cloudController
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: DockerMachineTemplate

test/e2e/data/shared/v1beta1/metadata.yaml

@@ -2,5 +2,5 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
 kind: Metadata
 releaseSeries:
   - major: 1
-    minor: 3
+    minor: 6
     contract: v1beta1

test/e2e/e2e_test.go

@@ -221,7 +221,7 @@ var _ = Describe("Workload cluster creation", func() {
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
-					ControlPlaneMachineCount: pointer.Int64Ptr(1),
+					ControlPlaneMachineCount: pointer.Int64Ptr(3), //TODO: change this back to 1 when scaling is supported
 					WorkerMachineCount:       pointer.Int64Ptr(1),
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),