Skip to content

Commit 611558f

Browse files
dtantsurdtantsur
committed
Add an e2e test for forced detachment during provisioning
Exercise forced detachment followed by deletion, new enrollment and finished provisioning. Signed-off-by: Dmitry Tantsur <dtantsur@protonmail.com>
1 parent 52b2698 commit 611558f

File tree

7 files changed

+357
-1
lines changed

7 files changed

+357
-1
lines changed

‎config/overlays/e2e/force-detach/kustomization.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
resources:
4+
- ../roles-rolebindings
5+
- namespace.yaml
6+
7+
namespace: force-detach

‎config/overlays/e2e/force-detach/namespace.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
apiVersion: v1
3+
kind: Namespace
4+
metadata:
5+
name: force-detach

‎config/overlays/e2e/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ resources:
66
- ./basic-ops
77
- ./external-inspection
88
- ./externally-provisioned
9+
- ./force-detach
910
- ./inspection
1011
- ./live-iso-ops
1112
- ./provisioning-ops

‎config/overlays/e2e/namespaced-manager-patch.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,5 @@ spec:
99
containers:
1010
- env:
1111
- name: WATCH_NAMESPACE
12-
value: basic-ops,external-inspection,externally-provisioned,inspection,live-iso-ops,provisioning-ops,re-inspection,automated-cleaning,upgrade-bmo,upgrade-ironic
12+
value: basic-ops,external-inspection,externally-provisioned,inspection,live-iso-ops,provisioning-ops,re-inspection,automated-cleaning,upgrade-bmo,upgrade-ironic,force-detach
1313
name: manager

‎test/e2e/config/fixture.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ variables:
3434
IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
3535
CERT_MANAGER_VERSION: "v1.19.2"
3636
SSH_CHECK_PROVISIONED: "false"
37+
FORCE_DETACH_WAIT_PROVISIONING: "false"
3738
FETCH_IRONIC_NODES: "false"
3839
IRONIC_PROVISIONING_IP: "192.168.222.2"
3940
# Boot mode can be legacy, UEFI or UEFISecureBoot

‎test/e2e/config/ironic.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ variables:
5252
SSH_PORT: "22"
5353
SSH_PRIV_KEY: "./images/ssh_testkey"
5454
SSH_PUB_KEY: "./images/ssh_testkey.pub"
55+
FORCE_DETACH_WAIT_PROVISIONING: "true"
5556
FETCH_IRONIC_NODES: "true"
5657
IRONIC_USERNAME: "changeme"
5758
IRONIC_PASSWORD: "changeme"

‎test/e2e/forced_detachment_test.go‎

Lines changed: 341 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,341 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
package e2e
5+
6+
import (
7+
"context"
8+
"path"
9+
"time"
10+
11+
metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
12+
. "github.com/onsi/ginkgo/v2"
13+
. "github.com/onsi/gomega"
14+
corev1 "k8s.io/api/core/v1"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/types"
17+
"sigs.k8s.io/cluster-api/test/framework"
18+
"sigs.k8s.io/cluster-api/util/deprecated/v1beta1/patch"
19+
)
20+
21+
var _ = Describe("Start provisioning, force detachment, delete and recreate, provision, detach again and delete", Label("required", "provision", "detach", "force-detach"),
22+
func() {
23+
var (
24+
specName = "force-detach"
25+
namespace *corev1.Namespace
26+
cancelWatches context.CancelFunc
27+
)
28+
29+
BeforeEach(func() {
30+
namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
31+
Creator: clusterProxy.GetClient(),
32+
ClientSet: clusterProxy.GetClientSet(),
33+
Name: specName,
34+
LogFolder: artifactFolder,
35+
IgnoreAlreadyExists: true,
36+
})
37+
})
38+
39+
It("starts provisioning, forces detachment, removes the host", func() {
40+
bmhName := specName + "-remove"
41+
secretName := bmhName + "-bmc-creds"
42+
43+
By("Creating a secret with BMH credentials")
44+
bmcCredentialsData := map[string]string{
45+
"username": bmc.User,
46+
"password": bmc.Password,
47+
}
48+
CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
49+
50+
By("Creating a BMH with inspection and cleaning disabled")
51+
bmh := metal3api.BareMetalHost{
52+
ObjectMeta: metav1.ObjectMeta{
53+
Name: bmhName,
54+
Namespace: namespace.Name,
55+
},
56+
Spec: metal3api.BareMetalHostSpec{
57+
Online: true,
58+
BMC: metal3api.BMCDetails{
59+
Address: bmc.Address,
60+
CredentialsName: secretName,
61+
DisableCertificateVerification: bmc.DisableCertificateVerification,
62+
},
63+
BootMode: metal3api.BootMode(e2eConfig.GetVariable("BOOT_MODE")),
64+
BootMACAddress: bmc.BootMacAddress,
65+
AutomatedCleaningMode: metal3api.CleaningModeDisabled,
66+
InspectionMode: metal3api.InspectionModeDisabled,
67+
},
68+
}
69+
err := clusterProxy.GetClient().Create(ctx, &bmh)
70+
Expect(err).NotTo(HaveOccurred())
71+
72+
By("Waiting for the BMH to become available")
73+
WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
74+
Client: clusterProxy.GetClient(),
75+
Bmh: bmh,
76+
State: metal3api.StateAvailable,
77+
}, e2eConfig.GetIntervals(specName, "wait-available")...)
78+
79+
By("Patching the BMH to test provisioning")
80+
err = PatchBMHForProvisioning(ctx, PatchBMHForProvisioningInput{
81+
client: clusterProxy.GetClient(),
82+
bmh: &bmh,
83+
bmc: bmc,
84+
e2eConfig: e2eConfig,
85+
namespace: namespace.Name,
86+
})
87+
Expect(err).NotTo(HaveOccurred())
88+
89+
By("Waiting for the BMH to be in provisioning state")
90+
WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
91+
Client: clusterProxy.GetClient(),
92+
Bmh: bmh,
93+
State: metal3api.StateProvisioning,
94+
}, e2eConfig.GetIntervals(specName, "wait-provisioning")...)
95+
96+
if e2eConfig.GetVariable("FORCE_DETACH_WAIT_PROVISIONING") == "true" {
97+
By("Waiting for the status in Ironic to catch up")
98+
// NOTE(dtantsur): tests have no access to the state of the Ironic Node object.
99+
// Wait a bit more to make sure it has actually transitioned away from "available".
100+
// FIXME: this won't be needed when substates are added to the BMH API.
101+
time.Sleep(2 * time.Second)
102+
}
103+
104+
By("Retrieving the latest BMH object")
105+
err = clusterProxy.GetClient().Get(ctx, types.NamespacedName{
106+
Name: bmh.Name,
107+
Namespace: bmh.Namespace,
108+
}, &bmh)
109+
Expect(err).NotTo(HaveOccurred())
110+
111+
By("Adding the detached annotation")
112+
helper, err := patch.NewHelper(&bmh, clusterProxy.GetClient())
113+
Expect(err).NotTo(HaveOccurred())
114+
115+
if bmh.Annotations == nil {
116+
bmh.Annotations = make(map[string]string, 1)
117+
}
118+
bmh.Annotations["baremetalhost.metal3.io/detached"] = "{\"force\": true}"
119+
120+
Expect(helper.Patch(ctx, &bmh)).To(Succeed())
121+
122+
By("Waiting for the BMH to be detached")
123+
WaitForBmhInOperationalStatus(ctx, WaitForBmhInOperationalStatusInput{
124+
Client: clusterProxy.GetClient(),
125+
Bmh: bmh,
126+
State: metal3api.OperationalStatusDetached,
127+
UndesiredStates: []metal3api.OperationalStatus{
128+
metal3api.OperationalStatusError,
129+
},
130+
}, e2eConfig.GetIntervals(specName, "wait-detached")...)
131+
132+
By("Retrieving and checking the latest BMH object")
133+
err = clusterProxy.GetClient().Get(ctx, types.NamespacedName{
134+
Name: bmh.Name,
135+
Namespace: bmh.Namespace,
136+
}, &bmh)
137+
Expect(err).NotTo(HaveOccurred())
138+
Expect(bmh.Status.Provisioning.State).To(Equal(metal3api.StateProvisioning))
139+
140+
By("Deleting the BMH")
141+
err = clusterProxy.GetClient().Delete(ctx, &bmh)
142+
Expect(err).NotTo(HaveOccurred())
143+
144+
By("Waiting for the BMH to be deleted")
145+
WaitForBmhDeleted(ctx, WaitForBmhDeletedInput{
146+
Client: clusterProxy.GetClient(),
147+
BmhName: bmh.Name,
148+
Namespace: bmh.Namespace,
149+
UndesiredStates: []metal3api.ProvisioningState{
150+
metal3api.StateDeprovisioning,
151+
metal3api.StatePoweringOffBeforeDelete,
152+
},
153+
}, e2eConfig.GetIntervals(specName, "wait-bmh-deleted")...)
154+
})
155+
156+
It("starts provisioning, forces detachment, re-attaches and finishes provisioning", func() {
157+
if e2eConfig.GetVariable("FORCE_DETACH_WAIT_PROVISIONING") == "false" {
158+
Skip("Test on provisioning after detachment relies on provisioning taking non-zero time")
159+
}
160+
161+
bmhName := specName + "-finish"
162+
secretName := specName + "-bmc-creds"
163+
164+
By("Creating a secret with BMH credentials")
165+
bmcCredentialsData := map[string]string{
166+
"username": bmc.User,
167+
"password": bmc.Password,
168+
}
169+
CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
170+
171+
By("Creating a BMH with inspection and cleaning disabled")
172+
bmh := metal3api.BareMetalHost{
173+
ObjectMeta: metav1.ObjectMeta{
174+
Name: bmhName,
175+
Namespace: namespace.Name,
176+
},
177+
Spec: metal3api.BareMetalHostSpec{
178+
Online: true,
179+
BMC: metal3api.BMCDetails{
180+
Address: bmc.Address,
181+
CredentialsName: secretName,
182+
DisableCertificateVerification: bmc.DisableCertificateVerification,
183+
},
184+
BootMode: metal3api.BootMode(e2eConfig.GetVariable("BOOT_MODE")),
185+
BootMACAddress: bmc.BootMacAddress,
186+
AutomatedCleaningMode: metal3api.CleaningModeDisabled,
187+
InspectionMode: metal3api.InspectionModeDisabled,
188+
},
189+
}
190+
err := clusterProxy.GetClient().Create(ctx, &bmh)
191+
Expect(err).NotTo(HaveOccurred())
192+
193+
By("Waiting for the BMH to become available")
194+
WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
195+
Client: clusterProxy.GetClient(),
196+
Bmh: bmh,
197+
State: metal3api.StateAvailable,
198+
}, e2eConfig.GetIntervals(specName, "wait-available")...)
199+
200+
By("Patching the BMH to test provisioning")
201+
var userDataSecret *corev1.SecretReference
202+
if e2eConfig.GetVariable("SSH_CHECK_PROVISIONED") == "true" {
203+
userDataSecretName := "user-data"
204+
sshPubKeyPath := e2eConfig.GetVariable("SSH_PUB_KEY")
205+
createSSHSetupUserdata(ctx, clusterProxy.GetClient(), namespace.Name, userDataSecretName, sshPubKeyPath, bmc.IPAddress)
206+
userDataSecret = &corev1.SecretReference{
207+
Name: userDataSecretName,
208+
Namespace: namespace.Name,
209+
}
210+
}
211+
err = PatchBMHForProvisioning(ctx, PatchBMHForProvisioningInput{
212+
client: clusterProxy.GetClient(),
213+
bmh: &bmh,
214+
bmc: bmc,
215+
e2eConfig: e2eConfig,
216+
namespace: namespace.Name,
217+
userDataSecret: userDataSecret,
218+
})
219+
Expect(err).NotTo(HaveOccurred())
220+
221+
By("Waiting for the BMH to be in provisioning state")
222+
WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
223+
Client: clusterProxy.GetClient(),
224+
Bmh: bmh,
225+
State: metal3api.StateProvisioning,
226+
}, e2eConfig.GetIntervals(specName, "wait-provisioning")...)
227+
228+
By("Waiting for the status in Ironic to catch up")
229+
// NOTE(dtantsur): tests have no access to the state of the Ironic Node object.
230+
// Wait a bit more to make sure it has actually transitioned away from "available".
231+
// FIXME: this won't be needed when substates are added to the BMH API.
232+
time.Sleep(2 * time.Second)
233+
234+
By("Retrieving the latest BMH object")
235+
err = clusterProxy.GetClient().Get(ctx, types.NamespacedName{
236+
Name: bmh.Name,
237+
Namespace: bmh.Namespace,
238+
}, &bmh)
239+
Expect(err).NotTo(HaveOccurred())
240+
241+
By("Adding the detached annotation")
242+
helper, err := patch.NewHelper(&bmh, clusterProxy.GetClient())
243+
Expect(err).NotTo(HaveOccurred())
244+
245+
if bmh.Annotations == nil {
246+
bmh.Annotations = make(map[string]string, 1)
247+
}
248+
bmh.Annotations["baremetalhost.metal3.io/detached"] = "{\"force\": true}"
249+
250+
Expect(helper.Patch(ctx, &bmh)).To(Succeed())
251+
252+
By("Waiting for the BMH to be detached")
253+
WaitForBmhInOperationalStatus(ctx, WaitForBmhInOperationalStatusInput{
254+
Client: clusterProxy.GetClient(),
255+
Bmh: bmh,
256+
State: metal3api.OperationalStatusDetached,
257+
UndesiredStates: []metal3api.OperationalStatus{
258+
metal3api.OperationalStatusError,
259+
},
260+
}, e2eConfig.GetIntervals(specName, "wait-detached")...)
261+
262+
By("Retrieving and checking the latest BMH object")
263+
err = clusterProxy.GetClient().Get(ctx, types.NamespacedName{
264+
Name: bmh.Name,
265+
Namespace: bmh.Namespace,
266+
}, &bmh)
267+
Expect(err).NotTo(HaveOccurred())
268+
Expect(bmh.Status.Provisioning.State).To(Equal(metal3api.StateProvisioning))
269+
270+
By("Removing the detached annotation")
271+
helper, err = patch.NewHelper(&bmh, clusterProxy.GetClient())
272+
Expect(err).NotTo(HaveOccurred())
273+
274+
delete(bmh.Annotations, "baremetalhost.metal3.io/detached")
275+
Expect(helper.Patch(ctx, &bmh)).To(Succeed())
276+
277+
By("Waiting for the BMH to become provisioned")
278+
WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
279+
Client: clusterProxy.GetClient(),
280+
Bmh: bmh,
281+
State: metal3api.StateProvisioned,
282+
}, e2eConfig.GetIntervals(specName, "wait-provisioned")...)
283+
284+
// The ssh check is not possible in all situations (e.g. fixture) so it can be skipped
285+
if e2eConfig.GetVariable("SSH_CHECK_PROVISIONED") == "true" {
286+
By("Verifying the node booting from disk")
287+
PerformSSHBootCheck(e2eConfig, "disk", bmc.IPAddress)
288+
} else {
289+
Logf("WARNING: Skipping SSH check since SSH_CHECK_PROVISIONED != true")
290+
}
291+
292+
By("Retrieving the latest BMH object")
293+
err = clusterProxy.GetClient().Get(ctx, types.NamespacedName{
294+
Name: bmh.Name,
295+
Namespace: bmh.Namespace,
296+
}, &bmh)
297+
Expect(err).NotTo(HaveOccurred())
298+
299+
By("Adding the detached annotation")
300+
helper, err = patch.NewHelper(&bmh, clusterProxy.GetClient())
301+
Expect(err).NotTo(HaveOccurred())
302+
303+
if bmh.Annotations == nil {
304+
bmh.Annotations = make(map[string]string, 1)
305+
}
306+
// Making sure that forced detachment works in the normal case too
307+
bmh.Annotations["baremetalhost.metal3.io/detached"] = "{\"force\": true}"
308+
309+
Expect(helper.Patch(ctx, &bmh)).To(Succeed())
310+
311+
By("Waiting for the BMH to be detached")
312+
WaitForBmhInOperationalStatus(ctx, WaitForBmhInOperationalStatusInput{
313+
Client: clusterProxy.GetClient(),
314+
Bmh: bmh,
315+
State: metal3api.OperationalStatusDetached,
316+
UndesiredStates: []metal3api.OperationalStatus{
317+
metal3api.OperationalStatusError,
318+
},
319+
}, e2eConfig.GetIntervals(specName, "wait-detached")...)
320+
321+
By("Delete BMH")
322+
err = clusterProxy.GetClient().Delete(ctx, &bmh)
323+
Expect(err).NotTo(HaveOccurred())
324+
325+
By("Waiting for the BMH to be deleted")
326+
WaitForBmhDeleted(ctx, WaitForBmhDeletedInput{
327+
Client: clusterProxy.GetClient(),
328+
BmhName: bmh.Name,
329+
Namespace: bmh.Namespace,
330+
}, e2eConfig.GetIntervals(specName, "wait-bmh-deleted")...)
331+
})
332+
333+
AfterEach(func() {
334+
CollectSerialLogs(bmc.Name, path.Join(artifactFolder, specName))
335+
DumpResources(ctx, e2eConfig, clusterProxy, path.Join(artifactFolder, specName))
336+
if !skipCleanup {
337+
isNamespaced := e2eConfig.GetBoolVariable("NAMESPACE_SCOPED")
338+
Cleanup(ctx, clusterProxy, namespace, cancelWatches, isNamespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...)
339+
}
340+
})
341+
})

0 commit comments

Comments
 (0)