ciliumendpoint-gc: skip ciliumendpoints which are too new

I have see real production occurrences when a ciliumendpoint was just
created, but then immediately deleted by cilium operator. The timing is
a strong indicator that we have hit this [known flaw][1] where it is
possible to delete a ciliumendpoint because the operator haven't
received the pod notification. To reduce the chance of this happenning
further, I propose that we wait for at least one interval before
starting to delete a just-created ciliumendpoint.

[1]: https://github.com/cilium/cilium/blob/2e216d1d735073932074497e1687c73bc2f56d1f/operator/endpointgc/gc.go#L134-L137

Signed-off-by: Haibing Zhou <zhouhaibing089@gmail.com>

Author: zhouhaibing089

operator/endpointgc/gc.go

@@ -156,6 +156,16 @@ func (g *GC) checkIfCEPShouldBeDeleted(ctx context.Context, cep *cilium_api_v2.C
 		return true
 	}
 
+	// In rare cases when ciliumendpoint is too new, and we haven't received pod
+	// notifications, we might delete this ciliumendpoint unexpectedly. Whie we
+	// are still prone to the cache delay if g.interval is too small or when the
+	// watch cache for pods has an arbitrary delay, the chance is much lower than
+	// it is today.
+	if time.Since(cep.CreationTimestamp.Time) < g.interval {
+		scopedLog.DebugContext(ctx, "CiliumEndpoint is too new", logfields.Name, cep.Namespace+"/"+cep.Name)
+		return false
+	}
+
 	podChecked := false
 	podStore, err := g.pods.Store(ctx)
 	if err != nil {

operator/endpointgc/gc_test.go

@@ -58,7 +58,7 @@ func TestRegisterController(t *testing.T) {
 	cepStore, _ := ciliumEndpoint.Store(t.Context())
 	// wait for all CEPs to be deleted except for those with running pods or
 	// cilium node owner reference
-	waitForCEPs(t, cepStore, 2)
+	waitForCEPs(t, cepStore, 3)
 	if err := hive.Stop(tlog, t.Context()); err != nil {
 		t.Fatalf("failed to stop: %s", err)
 	}
@@ -134,7 +134,7 @@ func TestRegisterControllerWithCRDDisabled(t *testing.T) {
 	// wait for potential GC
 	time.Sleep(500 * time.Millisecond)
 	// gc is disabled so no CEPs should be deleted
-	waitForCEPs(t, cepStore, 6)
+	waitForCEPs(t, cepStore, 7)
 	if err := hive.Stop(tlog, t.Context()); err != nil {
 		t.Fatalf("failed to stop: %s", err)
 	}
@@ -170,6 +170,10 @@ func prepareCiliumEndpoints(t *testing.T, fakeClient *k8sClient.FakeClientset) {
 	cepWithOwnerPodDoesntExist := createCiliumEndpoint("cep6", "ns")
 	cepWithOwnerPodDoesntExist.OwnerReferences = []meta_v1.OwnerReference{createOwnerReference("Pod", "pod6")}
 	fakeClient.CiliumV2().CiliumEndpoints("ns").Create(t.Context(), cepWithOwnerPodDoesntExist, meta_v1.CreateOptions{})
+	// - CEP that is just created
+	cepWithRecentCreationTime := createCiliumEndpoint("cep7", "ns")
+	cepWithRecentCreationTime.CreationTimestamp = meta_v1.Now()
+	fakeClient.CiliumV2().CiliumEndpoints("ns").Create(t.Context(), cepWithRecentCreationTime, meta_v1.CreateOptions{})
 
 	// Create Pods
 	// - pod that is running for cep2