feat(CollaSet): Add pod labels to indicate PodOpsLifecycle phase (#297)

* add pod label to indicate PodOpsLifecycle phase

Author: wu8685

go.mod

@@ -94,7 +94,7 @@ require (
 	k8s.io/apiserver v0.22.6 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/kubectl v0.29.0
-	kusionstack.io/kube-api v0.6.2
+	kusionstack.io/kube-api v0.6.3
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -1139,8 +1139,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-kusionstack.io/kube-api v0.6.2 h1:OCfDPXoXRvM8wB6muGcEBBYjfV6ivlOf1R3NSXU+QqA=
-kusionstack.io/kube-api v0.6.2/go.mod h1:J0+EHiroG/88X904Y9TV9iMRcoEuD5tXMTLMBDSwM+Y=
+kusionstack.io/kube-api v0.6.3 h1:HKgB6E95DVRGkZ2OjXkuV4YE327gGt24sBFJRB+tc6A=
+kusionstack.io/kube-api v0.6.3/go.mod h1:J0+EHiroG/88X904Y9TV9iMRcoEuD5tXMTLMBDSwM+Y=
 kusionstack.io/resourceconsist v0.0.1 h1:+k/jriq5Ld7fQUYfWSMGynz/FesHtl3Rk2fmQPjBe0g=
 kusionstack.io/resourceconsist v0.0.1/go.mod h1:816xS/fY6EOUbPFjXIWW/TGs8/YE46qP4ElKeIiwFdU=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=

pkg/controllers/collaset/collaset_controller_test.go

@@ -272,6 +272,9 @@ var _ = Describe("collaset controller", func() {
 		}, 5*time.Second, 1*time.Second).Should(BeTrue())
 		Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil())
 		Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil())
+		for _, pod := range podList.Items {
+			Expect(pod.Labels[appsv1alpha1.PodCreatingLabel]).ShouldNot(BeEquivalentTo(""))
+		}
 
 		// pod replica will be kept, after deleted
 		podToDelete := &podList.Items[0]
@@ -987,6 +990,8 @@ var _ = Describe("collaset controller", func() {
 		// manual changes should have lower priority than PodTemplate changes
 		Expect(pod.Labels["test1"]).Should(BeEquivalentTo("v4"))
 		Expect(pod.Labels["test2"]).Should(BeEquivalentTo("v1"))
+		Expect(pod.Labels[appsv1alpha1.PodCompletingLabel]).ShouldNot(BeEquivalentTo(""))
+		Expect(pod.Labels[appsv1alpha1.PodCreatingLabel]).Should(BeEquivalentTo(""))
 
 		_, exist = pod.Labels["manual-added"]
 		Expect(exist).Should(BeFalse())

pkg/controllers/collaset/synccontrol/sync_control.go

@@ -343,6 +343,11 @@ func (r *RealSyncControl) Scale(
 					revision,
 					func(in *corev1.Pod) (localErr error) {
 						in.Labels[appsv1alpha1.PodInstanceIDLabelKey] = fmt.Sprintf("%d", availableIDContext.ID)
+						if availableIDContext.Data[podcontext.JustCreateContextDataKey] == "true" {
+							in.Labels[appsv1alpha1.PodCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10)
+						} else {
+							in.Labels[appsv1alpha1.PodCompletingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10)
+						}
 						revisionsInfo, ok := availableIDContext.Get(podcontext.PodDecorationRevisionKey)
 						var pds map[string]*appsv1alpha1.PodDecoration
 						if !ok {

pkg/controllers/podopslifecycle/podopslifecycle_controller.go

@@ -185,20 +185,19 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc
 		v1alpha1.PodPreparingLabelPrefix:  false, // Set readiness gate to false
 		v1alpha1.PodCompletingLabelPrefix: true,  // Set readiness gate to true
 	}
-	for _, k := range []string{v1alpha1.PodPreparingLabelPrefix, v1alpha1.PodCompletingLabelPrefix} {
-		v := expected[k]
+	for phaseLabel, expectedReadinessGateStatus := range expected {
 		keeped := false
 		for _, labels := range idToLabelsMap {
-			if _, ok := labels[k]; !ok {
+			if _, ok := labels[phaseLabel]; !ok {
 				continue
 			}
 
-			updated, err := r.updateServiceReadiness(ctx, pod, v)
+			updated, err := r.updateServiceReadiness(ctx, pod, expectedReadinessGateStatus)
 			if err != nil {
 				return reconcile.Result{}, err // Only need set once
 			}
 			if updated {
-				logger.Info("update readiness gate", "status", v)
+				logger.Info("update readiness gate", "status", expectedReadinessGateStatus)
 			}
 			keeped = true
 		}
@@ -221,7 +220,7 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc
 // addServiceAvailable try to add service available label to pod
 func (r *ReconcilePodOpsLifecycle) addServiceAvailable(pod *corev1.Pod) (bool, error) {
 	if pod.Labels == nil {
-		return false, nil
+		pod.Labels = map[string]string{}
 	}
 	if _, ok := pod.Labels[v1alpha1.PodServiceAvailableLabel]; ok {
 		return false, nil
@@ -248,10 +247,29 @@ func (r *ReconcilePodOpsLifecycle) addServiceAvailable(pod *corev1.Pod) (bool, e
 		return false, nil
 	}
 
-	podLabelsToAdd := map[string]string{
-		v1alpha1.PodServiceAvailableLabel: strconv.FormatInt(time.Now().UnixNano(), 10),
+	key := controllerKey(pod)
+	r.expectation.ExpectUpdate(key, pod.ResourceVersion)
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		newPod := &corev1.Pod{}
+		err := r.Client.Get(context.Background(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, newPod)
+		if err != nil {
+			return err
+		}
+		if newPod.Labels == nil {
+			newPod.Labels = map[string]string{}
+		}
+		newPod.Labels[v1alpha1.PodServiceAvailableLabel] = strconv.FormatInt(time.Now().UnixNano(), 10)
+		delete(newPod.Labels, v1alpha1.PodCreatingLabel)
+		delete(newPod.Labels, v1alpha1.PodCompletingLabel)
+
+		return r.Client.Update(context.Background(), newPod)
+	})
+	if err != nil {
+		r.Logger.Error(err, "failed to set Pod as service available", "pod", utils.ObjectKeyString(pod))
+		r.expectation.DeleteExpectations(key)
 	}
-	return true, r.addLabels(context.Background(), pod, podLabelsToAdd)
+
+	return true, err
 }
 
 func (r *ReconcilePodOpsLifecycle) removeDirtyExpectedFinalizer(pod *corev1.Pod, notSatisfiedFinalizers map[string]string) (bool, error) {