Skip to content

Commit 7006ec6

Browse files
gkechgkech
committed
K8SPG-851 introduce persistent logs
1 parent 6f687cc commit 7006ec6

38 files changed

Lines changed: 14081 additions & 3 deletions

‎build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml‎

Lines changed: 2471 additions & 0 deletions
Large diffs are not rendered by default.

‎build/postgres-operator/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ COPY build/postgres-operator/postgres-entrypoint.sh /usr/local/bin
7070
COPY build/postgres-operator/postgres-liveness-check.sh /usr/local/bin
7171
COPY build/postgres-operator/postgres-readiness-check.sh /usr/local/bin
7272
COPY build/postgres-operator/restore-command-wrapper.sh /usr/local/bin
73+
COPY build/postgres-operator/logcollector /logcollector
7374
COPY hack/tools/queries /opt/crunchy/conf
7475

7576
RUN chgrp -R 0 /opt/crunchy/conf && chmod -R g=u opt/crunchy/conf

‎build/postgres-operator/init-entrypoint.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,14 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-liveness
1111
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/postgres-readiness-check.sh" "${CRUNCHY_BINDIR}/bin/postgres-readiness-check.sh"
1212
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/relocate-extensions.sh" "${CRUNCHY_BINDIR}/bin/relocate-extensions.sh"
1313
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "/usr/local/bin/restore-command-wrapper.sh" "${CRUNCHY_BINDIR}/bin/restore-command-wrapper.sh"
14+
15+
# Distribute the log collector assets (entrypoint + fluent-bit/logrotate config)
16+
# into the shared bin volume so the log collector sidecars can run them without
17+
# baking them into the fluent-bit image. The assets are only present when the
18+
# operator image ships them, so guard the copy to avoid failing the init
19+
# container (and thus the whole pod) on images without them.
20+
if [ -d /logcollector ]; then
21+
cp -a "/logcollector" "${CRUNCHY_BINDIR}/"
22+
chown -R "$(id -u)":"$(id -g)" "${CRUNCHY_BINDIR}/logcollector"
23+
chmod -R 0755 "${CRUNCHY_BINDIR}/logcollector"
24+
fi

‎build/postgres-operator/logcollector/entrypoint.sh‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/bash
2+
set -e
3+
4+
export PATH="$PATH:/opt/fluent-bit/bin"
5+
6+
run_cron() {
7+
local schedule="$1"
8+
local cmd="$2"
9+
10+
if [ -f /usr/bin/supercronic ]; then
11+
printf '%s %s\n' "$schedule" "$cmd" > /tmp/crontab
12+
exec supercronic /tmp/crontab
13+
else
14+
exec go-cron "$schedule" sh -c "$cmd"
15+
fi
16+
}
17+
18+
is_logrotate_config_invalid() {
19+
local config_file="$1"
20+
if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then
21+
return 1
22+
fi
23+
# Specifying -d runs in debug mode, so even in case of errors, it will exit with 0.
24+
# We need to check the output for "error" but skip those lines that are related to the missing logrotate.status file.
25+
# Filter out logrotate.status lines first, then check for remaining errors
26+
(
27+
set +e
28+
logrotate -d "$config_file" 2>&1 | grep -v "logrotate.status" | grep -qi "error"
29+
)
30+
return $?
31+
}
32+
33+
run_logrotate() {
34+
local logrotate_status_file="/pgdata/logrotate.status"
35+
local logrotate_conf_file="/opt/crunchy/logcollector/logrotate/logrotate.conf"
36+
local logrotate_additional_conf_files=()
37+
local conf_d_dir="/opt/crunchy/logcollector/logrotate/conf.d"
38+
39+
# Operator-managed postgres.conf overrides the default when present.
40+
if [ -f "$conf_d_dir/postgres.conf" ]; then
41+
logrotate_conf_file="$conf_d_dir/postgres.conf"
42+
if is_logrotate_config_invalid "$logrotate_conf_file"; then
43+
echo "ERROR: Logrotate configuration is invalid, fallback to default configuration"
44+
logrotate_conf_file="/opt/crunchy/logcollector/logrotate/logrotate.conf"
45+
fi
46+
fi
47+
48+
# Process all other .conf files under conf.d (postgres.conf handled above).
49+
if [ -d "$conf_d_dir" ]; then
50+
for conf_file in "$conf_d_dir"/*.conf; do
51+
[ -f "$conf_file" ] || continue
52+
[ "$(basename "$conf_file")" = "postgres.conf" ] && continue
53+
if is_logrotate_config_invalid "$conf_file"; then
54+
echo "ERROR: Logrotate configuration file $conf_file is invalid, it will be ignored"
55+
else
56+
logrotate_additional_conf_files+=("$conf_file")
57+
fi
58+
done
59+
fi
60+
61+
local logrotate_cmd="logrotate -s \"$logrotate_status_file\" \"$logrotate_conf_file\""
62+
for additional_conf in "${logrotate_additional_conf_files[@]}"; do
63+
logrotate_cmd="$logrotate_cmd \"$additional_conf\""
64+
done
65+
66+
set -o xtrace
67+
run_cron "$LOGROTATE_SCHEDULE" "$logrotate_cmd"
68+
}
69+
70+
run_fluentbit() {
71+
local fluentbit_opt=(-c /opt/crunchy/logcollector/fluentbit/fluentbit.conf)
72+
mkdir -p /tmp/fluentbit/custom
73+
set +e
74+
local fluentbit_conf_dir="/opt/crunchy/logcollector/fluentbit/custom"
75+
for conf_file in $fluentbit_conf_dir/*.conf; do
76+
[ -f "$conf_file" ] || continue
77+
if ! fluent-bit --dry-run -c "$conf_file" >/dev/null 2>&1; then
78+
echo "ERROR: Fluentbit configuration file $conf_file is invalid, it will be ignored"
79+
else
80+
cp "$conf_file" /tmp/fluentbit/custom/
81+
fi
82+
done
83+
touch /tmp/fluentbit/custom/default.conf || true
84+
85+
set -e
86+
set -o xtrace
87+
exec "$@" "${fluentbit_opt[@]}"
88+
}
89+
90+
case "$1" in
91+
logrotate)
92+
run_logrotate
93+
;;
94+
fluent-bit)
95+
run_fluentbit "$@"
96+
;;
97+
*)
98+
echo "Invalid argument: $1"
99+
exit 1
100+
;;
101+
esac

‎build/postgres-operator/logcollector/fluentbit/custom/default.conf‎

Whitespace-only changes.

‎build/postgres-operator/logcollector/fluentbit/fluentbit.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@INCLUDE fluentbit_*.conf
2+
@INCLUDE /tmp/fluentbit/custom/*.conf

‎build/postgres-operator/logcollector/fluentbit/fluentbit_pg.conf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
[SERVICE]
2+
Flush 1
3+
Log_Level error
4+
Daemon off
5+
Parsers_File /opt/crunchy/logcollector/fluentbit/parsers_multiline.conf
6+
7+
[INPUT]
8+
Name tail
9+
Path ${PG_LOG_DIR}/*.log,${PG_LOG_DIR}/*.csv
10+
Tag ${POD_NAMESPACE}.${POD_NAME}.postgres
11+
Refresh_Interval 5
12+
DB /tmp/flb_pg.db
13+
read_from_head true
14+
Path_Key file
15+
Skip_Long_Lines on
16+
17+
[INPUT]
18+
Name tail
19+
Path ${PGBACKREST_LOG_DIR}/*.log
20+
Tag ${POD_NAMESPACE}.${POD_NAME}.pgbackrest
21+
Refresh_Interval 5
22+
DB /tmp/flb_pgbackrest.db
23+
read_from_head true
24+
Path_Key file
25+
Skip_Long_Lines on
26+
27+
[OUTPUT]
28+
Name stdout
29+
Match *
30+
Format json_lines
31+
json_date_key false

‎build/postgres-operator/logcollector/fluentbit/parsers_multiline.conf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[MULTILINE_PARSER]
2+
name multiline-regex-test
3+
type regex
4+
flush_timeout 1000
5+
#
6+
# Regex rules for multiline parsing
7+
# ---------------------------------
8+
#
9+
# configuration hints:
10+
#
11+
# - first state always has the name: start_state
12+
# - every field in the rule must be inside double quotes
13+
#
14+
# rules | state name | regex pattern | next state
15+
# ------|---------------|--------------------------------------------
16+
rule "start_state" "/\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\.\d{1,6}Z(.*)|\d{2,6} \d{2,4}\:\d{2,4}\:\d{2,4}(.*)/" "cont"
17+
rule "cont" "/^\D/" "cont"

‎build/postgres-operator/logcollector/logrotate/logrotate.conf‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# PostgreSQL server logs, written to the default log directory inside the
2+
# versioned data directory (/pgdata/pgNN/log).
3+
# PostgreSQL's logging_collector rotates by size/age on its own; this block is
4+
# a safety net for deployments that disable logging_collector or that want an
5+
# extra time-based cap.
6+
/pgdata/pg*/log/*.log
7+
/pgdata/pg*/log/*.csv {
8+
daily
9+
maxsize 100M
10+
rotate 7
11+
missingok
12+
nocompress
13+
notifempty
14+
copytruncate
15+
sharedscripts
16+
}
17+
18+
# pgBackRest client logs on instance pods.
19+
/pgdata/pgbackrest/log/*.log {
20+
daily
21+
maxsize 100M
22+
rotate 7
23+
missingok
24+
nocompress
25+
notifempty
26+
copytruncate
27+
sharedscripts
28+
}
29+
30+
# pgBackRest server logs on the dedicated repo host. Each configured repo has
31+
# its own subdirectory under /pgbackrest.
32+
/pgbackrest/*/log/*.log {
33+
daily
34+
maxsize 100M
35+
rotate 7
36+
missingok
37+
nocompress
38+
notifempty
39+
copytruncate
40+
sharedscripts
41+
}

0 commit comments

Comments
 (0)