Skip to content

Commit a615527

Browse files
alexander-demicevalexander-demicev
committed
Add e2e suite for in-place updates
Signed-off-by: Alex Demicev <alex.demicev@lambdal.com>
1 parent 042a012 commit a615527

5 files changed

Lines changed: 671 additions & 2 deletions

File tree

‎test/e2e/config/e2e_conf.yaml‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ images:
55
loadBehavior: mustLoad
66
- name: ghcr.io/rancher/cluster-api-provider-rke2-controlplane:dev
77
loadBehavior: mustLoad
8+
- name: ghcr.io/rancher/cluster-api-provider-rke2-test-extension:dev
9+
loadBehavior: mustLoad
810
- name: quay.io/jetstack/cert-manager-cainjector:v1.16.3
911
loadBehavior: tryLoad
1012
- name: quay.io/jetstack/cert-manager-webhook:v1.16.3
@@ -125,6 +127,20 @@ providers:
125127
new: "imagePullPolicy: IfNotPresent"
126128
- old: "--leader-elect"
127129
new: "--leader-elect=false"
130+
- name: rke2-test-extension
131+
type: RuntimeExtensionProvider
132+
versions:
133+
- name: v0.25.99
134+
value: ../../extension/config/default
135+
contract: v1beta2
136+
files:
137+
- sourcePath: "../../../metadata.yaml"
138+
targetName: "metadata.yaml"
139+
replacements:
140+
- old: "imagePullPolicy: Always"
141+
new: "imagePullPolicy: IfNotPresent"
142+
- old: "--leader-elect"
143+
new: "--leader-elect=false"
128144

129145
variables:
130146
KUBERNETES_VERSION_MANAGEMENT: "v1.34.0"
@@ -146,8 +162,8 @@ variables:
146162
intervals:
147163
default/wait-controllers: ["3m", "10s"]
148164
default/wait-cluster: ["20m", "10s"]
149-
default/wait-control-plane: ["45m", "10s"]
150-
default/wait-worker-nodes: ["45m", "10s"]
165+
default/wait-control-plane: ["120m", "10s"]
166+
default/wait-worker-nodes: ["120m", "10s"]
151167
default/wait-delete-cluster: ["20m", "10s"]
152168
default/wait-machine-upgrade: ["50m", "10s"]
153169
default/wait-machine-remediation: ["30m", "10s"]

‎test/e2e/const.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ var (
4545
ClusterClassDocker []byte
4646
//go:embed data/infrastructure/cluster-from-clusterclass-template-docker.yaml
4747
ClusterFromClusterClassDocker []byte
48+
//go:embed data/infrastructure/clusterclass-template-docker-in-place.yaml
49+
ClusterClassDockerInPlace []byte
50+
//go:embed data/infrastructure/cluster-from-clusterclass-template-docker-in-place.yaml
51+
ClusterFromClusterClassDockerInPlace []byte
4852
//go:embed data/infrastructure/cluster-template-docker-external-datastore.yaml
4953
ClusterTemplateDockerExternalDatastore []byte
5054
//go:embed data/infrastructure/postgres.yaml

‎test/e2e/data/infrastructure/cluster-from-clusterclass-template-docker-in-place.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta2
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
namespace: "${NAMESPACE}"
7+
spec:
8+
clusterNetwork:
9+
pods:
10+
cidrBlocks:
11+
- 10.45.0.0/16
12+
serviceDomain: cluster.local
13+
services:
14+
cidrBlocks:
15+
- 10.46.0.0/16
16+
topology:
17+
classRef:
18+
name: "${CLASS_NAME}"
19+
version: ${KUBERNETES_VERSION}+rke2r1
20+
controlPlane:
21+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
22+
workers:
23+
machineDeployments:
24+
- class: default-worker
25+
name: md-0
26+
replicas: ${WORKER_MACHINE_COUNT}
27+
rollout:
28+
strategy:
29+
type: RollingUpdate
30+
rollingUpdate:
31+
maxSurge: 0
32+
maxUnavailable: 1
33+
variables:
34+
- name: dockerKindImage
35+
value: kindest/node:${KIND_IMAGE_VERSION}
36+
- name: files
37+
value: []
38+
- name: preRKE2Commands
39+
value: []

‎test/e2e/data/infrastructure/clusterclass-template-docker-in-place.yaml‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
apiVersion: cluster.x-k8s.io/v1beta2
2+
kind: ClusterClass
3+
metadata:
4+
name: "${CLASS_NAME}"
5+
namespace: "${NAMESPACE}"
6+
spec:
7+
controlPlane:
8+
templateRef:
9+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
10+
kind: RKE2ControlPlaneTemplate
11+
name: ${CLASS_NAME}-control-plane
12+
machineInfrastructure:
13+
templateRef:
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
15+
kind: DockerMachineTemplate
16+
name: ${CLASS_NAME}-control-plane
17+
infrastructure:
18+
templateRef:
19+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
20+
kind: DockerClusterTemplate
21+
name: ${CLASS_NAME}-cluster
22+
workers:
23+
machineDeployments:
24+
- class: default-worker
25+
bootstrap:
26+
templateRef:
27+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
28+
kind: RKE2ConfigTemplate
29+
name: ${CLASS_NAME}-default-worker-bootstraptemplate
30+
infrastructure:
31+
templateRef:
32+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
33+
kind: DockerMachineTemplate
34+
name: ${CLASS_NAME}-default-worker-machinetemplate
35+
variables:
36+
- name: dockerKindImage
37+
required: true
38+
schema:
39+
openAPIV3Schema:
40+
type: string
41+
default: kindest/node:v1.28.12
42+
- name: files
43+
required: false
44+
schema:
45+
openAPIV3Schema:
46+
type: array
47+
default: []
48+
items:
49+
type: object
50+
properties:
51+
path:
52+
type: string
53+
content:
54+
type: string
55+
owner:
56+
type: string
57+
permissions:
58+
type: string
59+
- name: preRKE2Commands
60+
required: false
61+
schema:
62+
openAPIV3Schema:
63+
type: array
64+
default: []
65+
items:
66+
type: string
67+
patches:
68+
- name: controlPlaneDockerKindImage
69+
definitions:
70+
- selector:
71+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
72+
kind: DockerMachineTemplate
73+
matchResources:
74+
controlPlane: true
75+
jsonPatches:
76+
- op: add
77+
path: /spec/template/spec/customImage
78+
valueFrom:
79+
variable: dockerKindImage
80+
- name: workerDockerKindImage
81+
definitions:
82+
- selector:
83+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
84+
kind: DockerMachineTemplate
85+
matchResources:
86+
machineDeploymentClass:
87+
names:
88+
- default-worker
89+
jsonPatches:
90+
- op: add
91+
path: "/spec/template/spec/customImage"
92+
valueFrom:
93+
variable: dockerKindImage
94+
- name: controlPlaneFiles
95+
definitions:
96+
- selector:
97+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
98+
kind: RKE2ControlPlaneTemplate
99+
matchResources:
100+
controlPlane: true
101+
jsonPatches:
102+
- op: add
103+
path: /spec/template/spec/files
104+
valueFrom:
105+
variable: files
106+
- name: workerFiles
107+
definitions:
108+
- selector:
109+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
110+
kind: RKE2ConfigTemplate
111+
matchResources:
112+
machineDeploymentClass:
113+
names:
114+
- default-worker
115+
jsonPatches:
116+
- op: add
117+
path: /spec/template/spec/files
118+
valueFrom:
119+
variable: files
120+
- name: controlPlanePreRKE2Commands
121+
definitions:
122+
- selector:
123+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
124+
kind: RKE2ControlPlaneTemplate
125+
matchResources:
126+
controlPlane: true
127+
jsonPatches:
128+
- op: add
129+
path: /spec/template/spec/preRKE2Commands
130+
valueFrom:
131+
variable: preRKE2Commands
132+
- name: workerPreRKE2Commands
133+
definitions:
134+
- selector:
135+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
136+
kind: RKE2ConfigTemplate
137+
matchResources:
138+
machineDeploymentClass:
139+
names:
140+
- default-worker
141+
jsonPatches:
142+
- op: add
143+
path: /spec/template/spec/preRKE2Commands
144+
valueFrom:
145+
variable: preRKE2Commands
146+
---
147+
apiVersion: v1
148+
data:
149+
value: |-
150+
# generated by kind
151+
global
152+
log /dev/log local0
153+
log /dev/log local1 notice
154+
daemon
155+
# limit memory usage to approximately 18 MB
156+
# (see https://github.com/kubernetes-sigs/kind/pull/3115)
157+
maxconn 100000
158+
159+
resolvers docker
160+
nameserver dns 127.0.0.11:53
161+
162+
defaults
163+
log global
164+
mode tcp
165+
option dontlognull
166+
# TODO: tune these
167+
timeout connect 5000
168+
timeout client 50000
169+
timeout server 50000
170+
# allow to boot despite dns don't resolve backends
171+
default-server init-addr none
172+
173+
frontend stats
174+
mode http
175+
bind *:8404
176+
stats enable
177+
stats uri /stats
178+
stats refresh 1s
179+
stats admin if TRUE
180+
181+
frontend control-plane
182+
bind *:{{ .FrontendControlPlanePort }}
183+
{{ if .IPv6 -}}
184+
bind :::{{ .FrontendControlPlanePort }};
185+
{{- end }}
186+
default_backend kube-apiservers
187+
188+
backend kube-apiservers
189+
option httpchk GET /healthz
190+
191+
{{range $server, $backend := .BackendServers}}
192+
server {{ $server }} {{ JoinHostPort $backend.Address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
193+
{{- end}}
194+
195+
frontend rke2-join
196+
bind *:9345
197+
{{ if .IPv6 -}}
198+
bind :::9345;
199+
{{- end }}
200+
default_backend rke2-servers
201+
202+
backend rke2-servers
203+
option httpchk GET /v1-rke2/readyz
204+
http-check expect status 403
205+
{{range $server, $backend := .BackendServers}}
206+
server {{ $server }} {{ $backend.Address }}:9345 check check-ssl verify none
207+
{{- end}}
208+
kind: ConfigMap
209+
metadata:
210+
name: ${CLASS_NAME}-lb-config
211+
namespace: "${NAMESPACE}"
212+
---
213+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
214+
kind: DockerClusterTemplate
215+
metadata:
216+
name: ${CLASS_NAME}-cluster
217+
namespace: "${NAMESPACE}"
218+
spec:
219+
template:
220+
spec:
221+
loadBalancer:
222+
customHAProxyConfigTemplateRef:
223+
name: ${CLASS_NAME}-lb-config
224+
---
225+
kind: RKE2ControlPlaneTemplate
226+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
227+
metadata:
228+
name: ${CLASS_NAME}-control-plane
229+
namespace: "${NAMESPACE}"
230+
spec:
231+
template:
232+
spec:
233+
gzipUserData: false
234+
serverConfig:
235+
cni: calico
236+
disableComponents:
237+
kubernetesComponents: [ "cloudController"]
238+
kubeAPIServer:
239+
extraArgs:
240+
- --anonymous-auth=true
241+
rolloutStrategy:
242+
type: "RollingUpdate"
243+
rollingUpdate:
244+
maxSurge: 0
245+
---
246+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
247+
kind: DockerMachineTemplate
248+
metadata:
249+
name: ${CLASS_NAME}-control-plane
250+
namespace: "${NAMESPACE}"
251+
spec:
252+
template:
253+
spec:
254+
customImage: kindest/node:v1.28.0
255+
extraMounts:
256+
- containerPath: "/var/run/docker.sock"
257+
hostPath: "/var/run/docker.sock"
258+
bootstrapTimeout: 15m
259+
---
260+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
261+
kind: DockerMachineTemplate
262+
metadata:
263+
name: ${CLASS_NAME}-default-worker-machinetemplate
264+
namespace: "${NAMESPACE}"
265+
spec:
266+
template:
267+
spec:
268+
customImage: kindest/node:v1.28.0
269+
extraMounts:
270+
- containerPath: "/var/run/docker.sock"
271+
hostPath: "/var/run/docker.sock"
272+
bootstrapTimeout: 15m
273+
---
274+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
275+
kind: RKE2ConfigTemplate
276+
metadata:
277+
name: ${CLASS_NAME}-default-worker-bootstraptemplate
278+
namespace: "${NAMESPACE}"
279+
spec:
280+
template:
281+
spec: {}

0 commit comments

Comments
 (0)