vspherecsi_controller.go incorrectly attempts to reconcile TKG-system template pods

[jayunit100]

Problem

Any time that we have a csi config setup with template-config = true......

apiVersion: csi.tanzu.vmware.com/v1alpha1
kind: VSphereCSIConfig
metadata:
  annotations:
    tkg.tanzu.vmware.com/template-config: "true"

We expect that there there will never be a log message saying "i cant reconcile this......" but we are seeing this......

E0228 19:23:16.934701       1 controller.go:326]  "msg"="Reconciler error" "error"="no owner cluster could be determined for tkg-system/v1.24.9---vmware.1-tkg.1" "controller"="vspherecsiconfig" "controllerGroup"="csi.tanzu.vmware.com" "controllerKind"="VSphereCSIConfig" "name"="v1.24.9---vmware.1-tkg.1" "namespace"="tkg-system" "reconcileID"="fac88650-bbb4-47e1-b526-f0074e99dad6" "vSphereCSIConfig"={"name":"v1.24.9---vmware.1-tkg.1","namespace":"tkg-system"}

Talking w/ @adduarte , evidently Reconciliation fails, so our unit tests pass :) , but the end result is very confusing from an addons perspective. Lets make sure to not ever try to reconcile things in tkg-system and audit other controlelrs for same correctness

Details

in vsphereCSI_controller

func (r *VSphereCSIConfigReconciler) ConfigMapToVSphereCSIConfig(o client.Object) []ctrl.Request {
    configs := &csiv1alpha1.VSphereCSIConfigList{}
    _ = r.List(context.Background(), configs)
    requests := []ctrl.Request{}
    for i := 0; i < len(configs.Items); i++ {
        // avoid enqueuing reconcile requests for template vSphereCSIConfig CRs in event handler of ConfigMap CR
        if _, ok := configs.Items[i].Annotations[constants.TKGAnnotationTemplateConfig]; ok && configs.Items[i].Namespace == r.Config.SystemNamespace {
            continue
        }
        if configs.Items[i].Spec.VSphereCSI.Mode == VSphereCSIParavirtualMode {
            requests = append(requests,
                ctrl.Request{NamespacedName: client.ObjectKey{Namespace: configs.Items[i].Namespace,
                    Name: configs.Items[i].Name}})
        }
    }
    return requests
}

in other controllers

func (r *AntreaConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	log := r.Log.WithValues("antreaconfig", req.NamespacedName)

	r.Log.Info("Start reconciliation")

	// fetch AntreaConfig resource, ignore not-found errors
	antreaConfig := &cniv1alpha1.AntreaConfig{}
	if err := r.Client.Get(ctx, req.NamespacedName, antreaConfig); err != nil {
		if apierrors.IsNotFound(err) {
			r.Log.Info(fmt.Sprintf("AntreaConfig resource '%v' not found", req.NamespacedName))
			return ctrl.Result{}, nil
		}
		return ctrl.Result{}, err
	}

	annotations := antreaConfig.GetAnnotations()
	if _, ok := annotations[constants.TKGAnnotationTemplateConfig]; ok {
		log.Info(fmt.Sprintf("resource '%v' is a config template. Skipping reconciling", req.NamespacedName))