fix(fluentd): propagate sidecarContainers to the configcheck pod

docs/configuration/crds/v1beta1/fluentd_types.md

@@ -45,6 +45,11 @@ Overrides the default logging level configCheck setup. This field is not used di
 ### configCheckAnnotations (map[string]string, optional) {#fluentdspec-configcheckannotations}
 
 
+### configCheckPod (*ConfigCheckPodOverrides, optional) {#fluentdspec-configcheckpod}
+
+ConfigCheckPod adds helper init containers and volumes to the transient configcheck pod; contents must exit on their own, or set restartPolicy: Always for a long-running helper. Takes effect on the next config change. 
+
+
 ### configCheckResources (corev1.ResourceRequirements, optional) {#fluentdspec-configcheckresources}
 
 
@@ -202,6 +207,27 @@ Duration in seconds for graceful pod termination. Set higher than expected clean
 
 
 
+## ConfigCheckPodOverrides
+
+ConfigCheckPodOverrides adds helper containers, volumes and a deadline to the
+transient fluentd configcheck pod, merged onto the generated pod spec last.
+
+### activeDeadlineSeconds (*int64, optional) {#configcheckpodoverrides-activedeadlineseconds}
+
+ActiveDeadlineSeconds bounds how long the configcheck pod may run before it is treated as failed and deleted so a new one can be created and retried. Without it, a helper container that never terminates leaves the pod running indefinitely and blocks config rollout, with no timeout. 
+
+
+### initContainers ([]corev1.Container, optional) {#configcheckpodoverrides-initcontainers}
+
+InitContainers to add to the configcheck pod. Plain init containers run to completion, in order, before the dry-run container starts - the merge prepends these ahead of the operator's own init containers, so they run first. A long-running helper must instead be a native sidecar (restartPolicy: Always, k8s 1.29+): the kubelet ends it once the dry-run container exits, letting the pod reach Succeeded/Failed. 
+
+
+### volumes ([]corev1.Volume, optional) {#configcheckpodoverrides-volumes}
+
+Volumes available to the configcheck pod only. FluentdSpec.ExtraVolumes is already mounted here too, but a PersistentVolumeClaim-backed entry is downgraded to an emptyDir on the check pod; give a same-named entry here to override that. An entry whose name matches an operator-managed volume field-merges into it (volumes use retainKeys, so the whole VolumeSource is replaced, not deep-merged). 
+
+
+
 ## FluentOutLogrotate
 
 ### age (string, optional) {#fluentoutlogrotate-age}

pkg/resources/fluentd/appconfigmap.go

@@ -21,8 +21,10 @@ import (
 	"maps"
 
 	"emperror.dev/errors"
+	"github.com/cisco-open/operator-tools/pkg/merge"
 	"github.com/cisco-open/operator-tools/pkg/reconciler"
 	"github.com/cisco-open/operator-tools/pkg/utils"
+	"github.com/cisco-open/operator-tools/pkg/volume"
 	"github.com/spf13/cast"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -70,8 +72,10 @@ func (r *Reconciler) configHash() (string, error) {
 }
 
 func (r *Reconciler) hasConfigCheckPod(ctx context.Context, hashKey string, fluentdSpec v1beta1.FluentdSpec) (bool, error) {
-	var err error
-	pod := r.newCheckPod(hashKey, fluentdSpec)
+	pod, err := r.newCheckPod(hashKey, fluentdSpec)
+	if err != nil {
+		return false, err
+	}
 
 	p := &corev1.Pod{}
 	err = r.Client.Get(ctx, client.ObjectKeyFromObject(pod), p)
@@ -143,7 +147,10 @@ func (r *Reconciler) configCheck(ctx context.Context) (*ConfigCheckResult, error
 		return res, errors.WrapIf(err, "failed to find output secret for fluentd configcheck")
 	}
 
-	pod := r.newCheckPod(hashKey, *r.fluentdSpec)
+	pod, err := r.newCheckPod(hashKey, *r.fluentdSpec)
+	if err != nil {
+		return nil, err
+	}
 	configcheck.WithHashLabel(pod, hashKey)
 
 	existingPods := &corev1.PodList{}
@@ -184,6 +191,25 @@ func (r *Reconciler) configCheck(ctx context.Context) (*ConfigCheckResult, error
 		case corev1.PodRunning:
 			return &ConfigCheckResult{}, nil
 		case corev1.PodFailed:
+			if pod.Status.Reason != "" {
+				// A pod that fails with a Reason (DeadlineExceeded, Evicted, Shutdown,
+				// NodeAffinity, ...) didn't fail because of an invalid config - a real
+				// config validation failure leaves Status.Reason empty. Delete it so a
+				// fresh one is created and retried instead of latching a false
+				// "config is invalid" verdict.
+				r.Log.Info("configcheck pod did not complete normally, deleting it to retry",
+					"pod", pod.Name,
+					"reason", pod.Status.Reason,
+					"activeDeadlineSeconds", pod.Spec.ActiveDeadlineSeconds,
+					"runningContainer", stillRunningContainer(pod))
+				if err := client.IgnoreNotFound(r.Client.Delete(ctx, pod)); err != nil {
+					return nil, errors.WrapIf(err, "failed to delete configcheck pod that did not complete normally")
+				}
+				return &ConfigCheckResult{
+					Ready:   false,
+					Message: fmt.Sprintf("configcheck pod %s did not complete normally (reason: %s), deleted for retry", pod.Name, pod.Status.Reason),
+				}, nil
+			}
 			return &ConfigCheckResult{
 				Ready: true,
 				Valid: false,
@@ -207,6 +233,24 @@ func (r *Reconciler) configCheck(ctx context.Context) (*ConfigCheckResult, error
 	return &ConfigCheckResult{}, nil
 }
 
+// stillRunningContainer returns the name of the container that was still
+// running when its pod stopped, if any - useful for diagnosing why a
+// configcheck pod with a helper container (e.g. a native sidecar) didn't
+// complete in time.
+func stillRunningContainer(pod *corev1.Pod) string {
+	for _, cs := range pod.Status.InitContainerStatuses {
+		if cs.State.Running != nil {
+			return cs.Name
+		}
+	}
+	for _, cs := range pod.Status.ContainerStatuses {
+		if cs.State.Running != nil {
+			return cs.Name
+		}
+	}
+	return ""
+}
+
 func (r *Reconciler) newCheckSecret(hashKey string, fluentdSpec v1beta1.FluentdSpec) (*corev1.Secret, error) {
 	data, err := r.generateConfigSecret(fluentdSpec)
 	if err != nil {
@@ -262,7 +306,7 @@ func (r *Reconciler) newCheckOutputSecret(hashKey string) (*corev1.Secret, error
 	return nil, errors.New("output secret is invalid, unable to create output secret for config check")
 }
 
-func (r *Reconciler) newCheckPod(hashKey string, fluentdSpec v1beta1.FluentdSpec) *corev1.Pod {
+func (r *Reconciler) newCheckPod(hashKey string, fluentdSpec v1beta1.FluentdSpec) (*corev1.Pod, error) {
 	volumes := r.volumesCheckPod(hashKey, fluentdSpec)
 	container := r.containerCheckPod(fluentdSpec)
 	initContainer := r.initContainerCheckPod(fluentdSpec)
@@ -287,7 +331,11 @@ func (r *Reconciler) newCheckPod(hashKey string, fluentdSpec v1beta1.FluentdSpec
 	}
 	if fluentdSpec.Affinity != nil && fluentdSpec.Affinity.PodAntiAffinity != nil {
 		r.Log.Info("pod anti-affinity has been disabled to avoid blocking configcheck pods accidentally")
-		pod.Spec.Affinity.PodAntiAffinity = nil
+		// deep-copy before clearing PodAntiAffinity: pod.Spec.Affinity is the same
+		// pointer as fluentdSpec.Affinity, shared with the StatefulSet's pod spec
+		affinity := fluentdSpec.Affinity.DeepCopy()
+		affinity.PodAntiAffinity = nil
+		pod.Spec.Affinity = affinity
 	}
 	if fluentdSpec.ConfigCheckAnnotations != nil {
 		pod.Annotations = fluentdSpec.ConfigCheckAnnotations
@@ -309,12 +357,29 @@ func (r *Reconciler) newCheckPod(hashKey string, fluentdSpec v1beta1.FluentdSpec
 		pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMount)
 	}
 	for _, n := range fluentdSpec.ExtraVolumes {
+		// The check pod is a plain, one-shot Pod, not part of the StatefulSet, so a
+		// PVC-backed extraVolume meant to be provisioned via volumeClaimTemplates
+		// (statefulset.go) never has a matching PVC here; mount an emptyDir instead
+		// - the dry-run only needs the mount path to resolve, not real data.
+		if n.Volume != nil && n.Volume.PersistentVolumeClaim != nil && !isPersistentVolumeClaimSpecEmpty(n.Volume.PersistentVolumeClaim.PersistentVolumeClaimSpec) {
+			emptyDirVolume := volume.KubernetesVolume{EmptyDir: &corev1.EmptyDirVolumeSource{}}
+			if err := emptyDirVolume.ApplyVolumeForPodSpec(n.VolumeName, n.ContainerName, n.Path, &pod.Spec); err != nil {
+				r.Log.Error(err, "Fluentd Config check pod extraVolume attachment failed.")
+			}
+			continue
+		}
 		if err := n.ApplyVolumeForPodSpec(&pod.Spec); err != nil {
 			r.Log.Error(err, "Fluentd Config check pod extraVolume attachment failed.")
 		}
 	}
 
-	return pod
+	if fluentdSpec.ConfigCheckPod != nil {
+		if err := merge.Merge(&pod.Spec, fluentdSpec.ConfigCheckPod); err != nil {
+			return nil, errors.WrapIf(err, "failed to merge configCheckPod overrides into the configcheck pod")
+		}
+	}
+
+	return pod, nil
 }
 
 func (r *Reconciler) volumesCheckPod(hashKey string, fluentdSpec v1beta1.FluentdSpec) (v []corev1.Volume) {