🐛 fix: collect operator pod logs across restarts and rolling updates

Makefile

@@ -383,6 +383,18 @@ wait-%: lint-deployed-%
 	KUBECONFIG=$(E2E_KUBECONFIG) kubectl wait --for=condition=Available --namespace=$(CATD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
 	KUBECONFIG=$(E2E_KUBECONFIG) kubectl wait --for=condition=Ready --namespace=$(CATD_NAMESPACE) certificate/catalogd-service-cert
 
+.PHONY: fluentbit-%
+fluentbit-%: wait-% $(HELM)
+ifeq ($(strip $(ARTIFACT_PATH)),)
+	@echo "ARTIFACT_PATH unset; skipping fluent-bit deployment"
+else
+	KUBECONFIG=$(E2E_KUBECONFIG) $(HELM) upgrade --install fluent-bit oci://ghcr.io/fluent/helm-charts/fluent-bit \
+	  --namespace $(FLUENTBIT_NAMESPACE) --create-namespace \
+	  --version $(FLUENTBIT_CHART_VERSION) \
+	  -f testdata/fluentbit/values.yaml \
+	  --wait --timeout 2m
+endif
+
 .PHONY: prometheus-%
 prometheus-%: wait-% $(HELM)
 ifeq ($(strip $(E2E_SUMMARY_OUTPUT)),)
@@ -401,7 +413,7 @@ endif
 
 .PHONY: e2e-run-%
 e2e-run-%: GODOG_ARGS ?=
-e2e-run-%: prometheus-%
+e2e-run-%: prometheus-% fluentbit-%
 ifeq ($(strip $(GODOG_ARGS)),)
 	E2E_PROMETHEUS_PORT=$$(grep -A1 'containerPort: 30900' $(KIND_CONFIG) | grep hostPort | awk '{print $$2}'); \
 	if [[ -z "$$E2E_PROMETHEUS_PORT" ]]; then echo "error: failed to extract prometheus hostPort from $(KIND_CONFIG)" >&2; exit 1; fi; \
@@ -628,6 +640,8 @@ run-experimental: wait-$(KIND_CLUSTER_NAME) #HELP Build the operator-controller
 CATD_NAMESPACE := olmv1-system
 PROMETHEUS_NAMESPACE := olmv1-system
 PROMETHEUS_CHART_VERSION := 86.2.2
+FLUENTBIT_NAMESPACE := fluent-bit
+FLUENTBIT_CHART_VERSION := 0.57.9
 
 .PHONY: docker-build
 docker-build: build-linux #EXHELP Build docker image for operator-controller and catalog with GOOS=linux and local GOARCH.

test/e2e/support-bundle.yaml

@@ -7,3 +7,10 @@ spec:
     - clusterResources: {}
     - clusterInfo: {}
     - logs: {}
+    - copyFromHost:
+        collectorName: "persisted-operator-logs"
+        image: "busybox:1.36"
+        hostPath: "/var/log/fluentbit/output/"
+        name: "operator-logs"
+        extractArchive: true
+        timeout: "120s"

testdata/fluentbit/values.yaml

@@ -0,0 +1,55 @@
+# Fluent Bit values for e2e log persistence.
+# Tails operator-controller and catalogd container logs and writes them
+# to the node filesystem so they survive pod deletion during serial tests.
+resources:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 50m
+    memory: 64Mi
+config:
+  service: |
+    [SERVICE]
+        Flush        5
+        Log_Level    warn
+        Daemon       off
+        HTTP_Server  On
+  inputs: |
+    [INPUT]
+        Name              tail
+        Path              /var/log/containers/operator-controller-*_olmv1-system_*.log, /var/log/containers/catalogd-*_olmv1-system_*.log
+        multiline.parser  cri
+        Tag               <namespace_name>/<pod_name>.<container_name>
+        Tag_Regex         (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-[a-z0-9]{64}\.log$
+        Read_from_Head    true
+        Skip_Long_Lines   On
+        Skip_Empty_Lines  On
+        Mem_Buf_Limit     10MB
+        DB                /var/log/fluentbit/tail.db
+  filters: ""
+  outputs: |
+    [OUTPUT]
+        Name              file
+        Match             *
+        Path              /var/log/fluentbit/output
+        Mkdir             true
+  customParsers: ""
+extraVolumes:
+  - name: fluentbit-output
+    hostPath:
+      path: /var/log/fluentbit
+      type: DirectoryOrCreate
+extraVolumeMounts:
+  - name: fluentbit-output
+    mountPath: /var/log/fluentbit
+testFramework:
+  enabled: false
+serviceMonitor:
+  enabled: false
+prometheusRule:
+  enabled: false
+dashboards:
+  enabled: false
+networkPolicy:
+  enabled: false