K8SPG-851 introduce persistent logs

build/postgres-operator/Dockerfile

@@ -70,6 +70,7 @@ COPY build/postgres-operator/postgres-entrypoint.sh /usr/local/bin
 COPY build/postgres-operator/postgres-liveness-check.sh /usr/local/bin
 COPY build/postgres-operator/postgres-readiness-check.sh /usr/local/bin
 COPY build/postgres-operator/restore-command-wrapper.sh /usr/local/bin
+COPY build/postgres-operator/logcollector /logcollector
 COPY hack/tools/queries /opt/crunchy/conf
 
 RUN chgrp -R 0 /opt/crunchy/conf && chmod -R g=u opt/crunchy/conf

build/postgres-operator/init-entrypoint.sh

@@ -11,3 +11,14 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-liveness
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-readiness-check.sh" "${CRUNCHY_BINDIR}/bin/postgres-readiness-check.sh"
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/relocate-extensions.sh" "${CRUNCHY_BINDIR}/bin/relocate-extensions.sh"
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/restore-command-wrapper.sh" "${CRUNCHY_BINDIR}/bin/restore-command-wrapper.sh"
+
+# Distribute the log collector assets (entrypoint + fluent-bit/logrotate config)
+# into the shared bin volume so the log collector sidecars can run them without
+# baking them into the fluent-bit image. The assets are only present when the
+# operator image ships them, so guard the copy to avoid failing the init
+# container (and thus the whole pod) on images without them.
+if [ -d /logcollector ]; then
+	cp -a "/logcollector" "${CRUNCHY_BINDIR}/"
+	chown -R "$(id -u)":"$(id -g)" "${CRUNCHY_BINDIR}/logcollector"
+	chmod -R 0755 "${CRUNCHY_BINDIR}/logcollector"
+fi

build/postgres-operator/logcollector/entrypoint.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+set -e
+
+export PATH="$PATH:/opt/fluent-bit/bin"
+
+run_cron() {
+	local schedule="$1"
+	local cmd="$2"
+
+	if [ -f /usr/bin/supercronic ]; then
+        printf '%s %s\n' "$schedule" "$cmd" > /tmp/crontab
+        exec supercronic /tmp/crontab
+    else
+        exec go-cron "$schedule" sh -c "$cmd"
+    fi
+}
+
+is_logrotate_config_invalid() {
+	local config_file="$1"
+	if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then
+		return 1
+	fi
+	# Specifying -d runs in debug mode, so even in case of errors, it will exit with 0.
+	# We need to check the output for "error" but skip those lines that are related to the missing logrotate.status file.
+	# Filter out logrotate.status lines first, then check for remaining errors
+	(
+		set +e
+		logrotate -d "$config_file" 2>&1 | grep -v "logrotate.status" | grep -qi "error"
+	)
+	return $?
+}
+
+run_logrotate() {
+	local logrotate_status_file="/pgdata/logrotate.status"
+	local logrotate_conf_file="/opt/crunchy/logcollector/logrotate/logrotate.conf"
+	local logrotate_additional_conf_files=()
+	local conf_d_dir="/opt/crunchy/logcollector/logrotate/conf.d"
+
+    # Operator-managed postgres.conf overrides the default when present.
+	if [ -f "$conf_d_dir/postgres.conf" ]; then
+		logrotate_conf_file="$conf_d_dir/postgres.conf"
+		if is_logrotate_config_invalid "$logrotate_conf_file"; then
+			echo "ERROR: Logrotate configuration is invalid, fallback to default configuration"
+			logrotate_conf_file="/opt/crunchy/logcollector/logrotate/logrotate.conf"
+		fi
+	fi
+
+	# Process all other .conf files under conf.d (postgres.conf handled above).
+	if [ -d "$conf_d_dir" ]; then
+		for conf_file in "$conf_d_dir"/*.conf; do
+			[ -f "$conf_file" ] || continue
+			[ "$(basename "$conf_file")" = "postgres.conf" ] && continue
+			if is_logrotate_config_invalid "$conf_file"; then
+				echo "ERROR: Logrotate configuration file $conf_file is invalid, it will be ignored"
+			else
+				logrotate_additional_conf_files+=("$conf_file")
+			fi
+		done
+	fi
+
+	local logrotate_cmd="logrotate -s \"$logrotate_status_file\" \"$logrotate_conf_file\""
+	for additional_conf in "${logrotate_additional_conf_files[@]}"; do
+		logrotate_cmd="$logrotate_cmd \"$additional_conf\""
+	done
+
+	set -o xtrace
+	run_cron "$LOGROTATE_SCHEDULE" "$logrotate_cmd"
+}
+
+run_fluentbit() {
+	local fluentbit_opt=(-c /opt/crunchy/logcollector/fluentbit/fluentbit.conf)
+	mkdir -p /tmp/fluentbit/custom
+	mkdir -p /pgdata/fluentbit
+	set +e
+	local fluentbit_conf_dir="/opt/crunchy/logcollector/fluentbit/custom"
+	for conf_file in $fluentbit_conf_dir/*.conf; do
+		[ -f "$conf_file" ] || continue
+		if ! fluent-bit --dry-run -c "$conf_file" >/dev/null 2>&1; then
+			echo "ERROR: Fluentbit configuration file $conf_file is invalid, it will be ignored"
+		else
+			cp "$conf_file" /tmp/fluentbit/custom/
+		fi
+	done
+	touch /tmp/fluentbit/custom/default.conf || true
+
+	set -e
+	set -o xtrace
+	exec "$@" "${fluentbit_opt[@]}"
+}
+
+case "$1" in
+	logrotate)
+		run_logrotate
+		;;
+	fluent-bit)
+		run_fluentbit "$@"
+		;;
+	*)
+		echo "Invalid argument: $1"
+		exit 1
+		;;
+esac

build/postgres-operator/logcollector/fluentbit/fluentbit.conf

@@ -0,0 +1,2 @@
+@INCLUDE fluentbit_*.conf
+@INCLUDE /tmp/fluentbit/custom/*.conf

build/postgres-operator/logcollector/fluentbit/fluentbit_pg.conf

@@ -0,0 +1,31 @@
+[SERVICE]
+    Flush        1
+    Log_Level    error
+    Daemon       off
+    Parsers_File /opt/crunchy/logcollector/fluentbit/parsers_multiline.conf
+
+[INPUT]
+    Name             tail
+    Path             ${PG_LOG_DIR}/*.log,${PG_LOG_DIR}/*.csv
+    Tag              ${POD_NAMESPACE}.${POD_NAME}.postgres
+    Refresh_Interval 5
+    DB               /pgdata/fluentbit/flb_pg.db
+    read_from_head   true
+    Path_Key         file
+    Skip_Long_Lines  on
+
+[INPUT]
+    Name             tail
+    Path             ${PGBACKREST_LOG_DIR}/*.log
+    Tag              ${POD_NAMESPACE}.${POD_NAME}.pgbackrest
+    Refresh_Interval 5
+    DB               /pgdata/fluentbit/flb_pgbackrest.db
+    read_from_head   true
+    Path_Key         file
+    Skip_Long_Lines  on
+
+[OUTPUT]
+    Name          stdout
+    Match         *
+    Format        json_lines
+    json_date_key false

build/postgres-operator/logcollector/fluentbit/parsers_multiline.conf

@@ -0,0 +1,17 @@
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    #
+    # Regex rules for multiline parsing
+    # ---------------------------------
+    #
+    # configuration hints:
+    #
+    #  - first state always has the name: start_state
+    #  - every field in the rule must be inside double quotes
+    #
+    # rules |   state name  | regex pattern                  | next state
+    # ------|---------------|--------------------------------------------
+    rule      "start_state"   "/\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\.\d{1,6}Z(.*)|\d{2,6} \d{2,4}\:\d{2,4}\:\d{2,4}(.*)/" "cont"
+    rule      "cont"          "/^\D/"                                                     "cont"

build/postgres-operator/logcollector/logrotate/logrotate.conf

@@ -0,0 +1,41 @@
+# PostgreSQL server logs, written to the default log directory inside the
+# versioned data directory (/pgdata/pgNN/log).
+# PostgreSQL's logging_collector already rotates these by age (log_rotation_age)
+# and reuses weekday-named files, so this block is only a size-based safety net:
+# it stays purely size-triggered to avoid competing with that daily rotation and
+# fires only if a file grows past the cap (e.g. when logging_collector is off).
+/pgdata/pg*/log/*.log
+/pgdata/pg*/log/*.csv {
+    size 100M
+    rotate 7
+    missingok
+    nocompress
+    notifempty
+    copytruncate
+    sharedscripts
+}
+
+# pgBackRest client logs on instance pods.
+/pgdata/pgbackrest/log/*.log {
+    daily
+    maxsize 100M
+    rotate 7
+    missingok
+    nocompress
+    notifempty
+    copytruncate
+    sharedscripts
+}
+
+# pgBackRest server logs on the dedicated repo host. Each configured repo has
+# its own subdirectory under /pgbackrest.
+/pgbackrest/*/log/*.log {
+    daily
+    maxsize 100M
+    rotate 7
+    missingok
+    nocompress
+    notifempty
+    copytruncate
+    sharedscripts
+}