Merge pull request #2827 from Nordix/tuomo/fix-bmo-optional-flakyness

🌱 E2E: Fix race condition in upgrade tests

Author: metal3-io-bot

test/e2e/common.go

@@ -142,6 +142,40 @@ func PatchBMHForProvisioning(ctx context.Context, input PatchBMHForProvisioningI
 	return helper.Patch(ctx, input.bmh)
 }
 
+// WaitForBmhReconciled waits for the BMO controller to process a BMH update.
+// This is used after BMO deployment rollout to ensure the controller is actually
+// processing events before making further changes. It works by adding/updating
+// an annotation and waiting for the status.lastUpdated timestamp to change,
+// which proves the controller reconciled the BMH.
+func WaitForBmhReconciled(ctx context.Context, c client.Client, bmh metal3api.BareMetalHost, intervals ...interface{}) {
+	// Get the current BMH state
+	key := types.NamespacedName{Namespace: bmh.Namespace, Name: bmh.Name}
+	currentBmh := &metal3api.BareMetalHost{}
+	Expect(c.Get(ctx, key, currentBmh)).To(Succeed())
+	initialLastUpdated := currentBmh.Status.LastUpdated
+
+	// Touch the BMH with an annotation update to trigger a reconcile
+	helper, err := patch.NewHelper(currentBmh, c)
+	Expect(err).NotTo(HaveOccurred())
+	if currentBmh.Annotations == nil {
+		currentBmh.Annotations = make(map[string]string)
+	}
+	currentBmh.Annotations["e2e.metal3.io/reconcile-check"] = metav1.Now().Format("2006-01-02T15:04:05Z")
+	Expect(helper.Patch(ctx, currentBmh)).To(Succeed())
+
+	// Wait for lastUpdated to change, proving the controller processed our update
+	Eventually(func(g Gomega) {
+		updatedBmh := &metal3api.BareMetalHost{}
+		g.Expect(c.Get(ctx, key, updatedBmh)).To(Succeed())
+		// Check that lastUpdated has changed (controller reconciled)
+		g.Expect(updatedBmh.Status.LastUpdated).NotTo(BeNil(), "BMH status.lastUpdated should not be nil")
+		if initialLastUpdated != nil {
+			g.Expect(updatedBmh.Status.LastUpdated.Before(initialLastUpdated)).To(BeFalse(),
+				"BMH status.lastUpdated should have been updated by controller")
+		}
+	}, intervals...).Should(Succeed())
+}
+
 // DeleteBmhsInNamespace deletes all BMHs in the given namespace.
 func DeleteBmhsInNamespace(ctx context.Context, deleter client.Client, namespace string) {
 	bmh := metal3api.BareMetalHost{}

test/e2e/upgrade_test.go

@@ -319,6 +319,14 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl
 		})
 	}
 
+	// After deployment rollout, wait for the BMO controller to actually be processing events.
+	// There's a race condition where the deployment is "available" but the new pod hasn't
+	// acquired the leader lease yet or hasn't started its watches. If we patch the BMH
+	// during this window, the controller may miss the update event.
+	By("Waiting for BMO controller to be ready to process events")
+	WaitForBmhReconciled(ctx, upgradeClusterProxy.GetClient(), bmh,
+		e2eConfig.GetIntervals("default", "wait-deployment")...)
+
 	By("Patching the BMH to test provisioning")
 	// Using Eventually here since the webhook can take some time after the deployment is ready
 	Eventually(func() error {