Skip to content

K8SPS-856: Don't reload haproxy on DNS changes #1489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
egegunes wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

K8SPS-856: Don't reload haproxy on DNS changes #1489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
04a77b3
K8SPS-856: Don't reload haproxy on DNS changes
egegunes Aug 13, 2026
5d4cbe9
Merge branch 'main' into K8SPS-856
egegunes Aug 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions build/haproxy-entrypoint.sh
View file
Open in desktop

@github-actions github-actions Bot Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[shellcheck (suggestion)] reported by reviewdog 🐶

percona-server-mysql-operator/build/haproxy-entrypoint.sh

Line 112 in 04a77b3

exec "$@" ${haproxy_opt}

@github-actions github-actions Bot Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[shfmt] reported by reviewdog 🐶

percona-server-mysql-operator/build/haproxy-entrypoint.sh

Lines 56 to 58 in 04a77b3

if [ ! -f '/etc/haproxy/mysql/haproxy.cfg' ]; then
cp /opt/percona/haproxy.cfg /etc/haproxy/mysql
fi

@github-actions github-actions Bot Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[shfmt] reported by reviewdog 🐶

percona-server-mysql-operator/build/haproxy-entrypoint.sh

Lines 74 to 94 in 04a77b3

custom_conf='/etc/haproxy-custom/haproxy.cfg'
if [ -f "$custom_conf" ]; then
log "haproxy -c -f $custom_conf -f /etc/haproxy/mysql/haproxy.cfg"
haproxy -c -f $custom_conf -f /etc/haproxy/mysql/haproxy.cfg || EC=$?
if [ -n "$EC" ]; then
log "The custom config $custom_conf is not valid and will be ignored."
fi
fi
haproxy_opt='-W -db '
if [ -f "$custom_conf" -a -z "$EC" ]; then
haproxy_opt+="-f $custom_conf "
else
haproxy_opt+='-f /opt/percona/haproxy-global.cfg '
fi
haproxy_opt+='-f /etc/haproxy/mysql/haproxy.cfg -p /etc/haproxy/mysql/haproxy.pid -S /etc/haproxy/mysql/haproxy-main.sock'
if [ -f '/etc/haproxy/config/haproxy.cfg' ]; then
haproxy_opt="${haproxy_opt} -f /etc/haproxy/config/haproxy.cfg"
fi

Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,74 @@
set -e
set -o xtrace

MYSQL_PORT=3306
MYSQLX_PORT=33060
MYSQL_ADMIN_PORT=33062

# must be >= max cluster size
MAX_SLOTS="${HA_MAX_SLOTS:-9}"

path_to_haproxy_cfg='/etc/haproxy/mysql'

SERVER_OPTIONS=${HA_SERVER_OPTIONS:-'resolvers kubernetes inter 10000 rise 1 fall 2 check weight 1'}

send_proxy=''
if [[ ${IS_PROXY_PROTOCOL} == "yes" ]]; then
send_proxy='send-proxy-v2'
fi

log() {
local message=$1
local date=$(/usr/bin/date +"%d/%b/%Y:%H:%M:%S.%3N")

echo "{\"time\":\"${date}\", \"message\": \"${message}\"}"
}

emit_backend() {
local name=$1
local port=$2
local check_script=$3
local extra=$4

cat <<-EOF
backend ${name}
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/${check_script}
default-server ${SERVER_OPTIONS} init-addr none on-marked-down shutdown-sessions ${extra}
EOF

# pre-provisioned slots; disabled == start in MAINT, no checks until
# peer-list script sets a real fqdn and flips them to ready
local i
for i in $(seq 0 $((MAX_SLOTS - 1))); do
echo " server ${CLUSTER_NAME}-mysql-${i} ${CLUSTER_NAME}-mysql-${i}.${CLUSTER_NAME}-mysql:${port} disabled"
done
}

echo "${CLUSTER_TYPE}" >/tmp/cluster_type

if [ "$1" = 'haproxy' ]; then
if [ ! -f '/etc/haproxy/mysql/haproxy.cfg' ]; then
cp /opt/percona/haproxy.cfg /etc/haproxy/mysql
fi

{
emit_backend mysql-primary "$MYSQL_PORT" haproxy_check_primary.sh "$send_proxy"
emit_backend mysql-replicas "$MYSQL_PORT" haproxy_check_replicas.sh "$send_proxy"
emit_backend mysql-x "$MYSQLX_PORT" haproxy_check_replicas.sh "$send_proxy"
emit_backend mysql-admin "$MYSQL_ADMIN_PORT" haproxy_check_replicas.sh ''
} >"$path_to_haproxy_cfg/haproxy.cfg"

path_to_custom_global_cnf='/etc/haproxy-custom'
if [ -f "$path_to_custom_global_cnf/haproxy-global.cfg" ]; then
haproxy -c -f "$path_to_custom_global_cnf/haproxy-global.cfg" -f "$path_to_haproxy_cfg/haproxy.cfg"
fi

haproxy -c -f /opt/percona/haproxy-global.cfg -f "$path_to_haproxy_cfg/haproxy.cfg"

custom_conf='/etc/haproxy-custom/haproxy.cfg'
if [ -f "$custom_conf" ]; then
log "haproxy -c -f $custom_conf -f /etc/haproxy/mysql/haproxy.cfg"
Expand All @@ -27,7 +81,7 @@
fi

haproxy_opt='-W -db '
if [ -f "$custom_conf" -a -z "$EC" ]; then

Check warning on line 84 in build/haproxy-entrypoint.sh

View workflow job for this annotation

GitHub Actions / shellcheck

[shellcheck] build/haproxy-entrypoint.sh#L84 <ShellCheck.SC2166> 

Prefer [ p ] && [ q ] as [ p -a q ] is not well defined.
Raw output 
./build/haproxy-entrypoint.sh:84:26: warning: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined. (ShellCheck.SC2166)
haproxy_opt+="-f $custom_conf "
else
haproxy_opt+='-f /opt/percona/haproxy-global.cfg '
Expand Down Expand Up @@ -55,4 +109,4 @@
log "Failed to set open file limit to ${RLIMIT_NOFILE}, continuing with $(ulimit -n)."
fi

exec "$@" ${haproxy_opt}

Check notice on line 112 in build/haproxy-entrypoint.sh

View workflow job for this annotation

GitHub Actions / shellcheck

[shellcheck] build/haproxy-entrypoint.sh#L112 <ShellCheck.SC2086> 

Double quote to prevent globbing and word splitting.
Raw output 
./build/haproxy-entrypoint.sh:112:11: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)
191 changes: 55 additions & 136 deletions build/haproxy_add_mysql_nodes.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,155 +9,74 @@ log() {
echo "{\"time\":\"${date}\", \"message\": \"${message}\"}"
}

MYSQL_PORT=3306
MYSQLX_PORT=33060
MYSQL_ADMIN_PORT=33062
primary_mysql_node=''
primary_mysql_host=''

function main() {
log "Running $0"

NODE_LIST=()
NODE_LIST_REPL=()
NODE_LIST_MYSQLX=()
NODE_LIST_ADMIN=()

SERVER_OPTIONS=${HA_SERVER_OPTIONS:-'resolvers kubernetes inter 10000 rise 1 fall 2 check weight 1'}
send_proxy=''
path_to_haproxy_cfg='/etc/haproxy/mysql'
if [[ ${IS_PROXY_PROTOCOL} == "yes" ]]; then
send_proxy='send-proxy-v2'
SOCK="${HA_STATS_SOCKET:-/etc/haproxy/mysql/haproxy.sock}"
MAX_SLOTS="${HA_MAX_SLOTS:-9}"
path_to_haproxy_cfg='/etc/haproxy/mysql'
BACKENDS='mysql-primary mysql-replicas mysql-x mysql-admin'

haproxy_socket_cmd() {
local cmd=$1
local out

out=$(echo "$cmd" | socat -t 5 stdio "$SOCK" | tr -d '\n')
if [ -n "$out" ]; then
log "'${cmd}' -> ${out}"
fi
return 0
}

while read mysql_host; do
if [ -z "$mysql_host" ]; then
log 'Could not find PEERS ...'
exit 0
fi
main() {
log "Running $0"

node_name=$(echo "$mysql_host" | cut -d . -f -1)
if /opt/percona/haproxy_check_primary.sh '' '' "$mysql_host";then
primary_mysql_host="$mysql_host"
primary_mysql_node="server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS} on-marked-up shutdown-backup-sessions"
continue
local peers=()
local mysql_host
while read -r mysql_host; do
if [ -n "$mysql_host" ]; then
peers+=("$mysql_host")
fi

NODE_LIST_REPL+=("server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS}")
NODE_LIST+=("server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS} backup")
NODE_LIST_ADMIN+=("server ${node_name} ${mysql_host}:${MYSQL_ADMIN_PORT} ${SERVER_OPTIONS} backup")
NODE_LIST_MYSQLX+=("server ${node_name} ${mysql_host}:${MYSQLX_PORT} ${send_proxy} ${SERVER_OPTIONS} backup")
done

# mysql-x and mysql-admin replicas run the same external check against the
# same admin port as mysql-replicas, so mirror the state with 'track'
# instead of running duplicate checks. Only replicas can track: the primary
# node is not present in the mysql-replicas backend and keeps its own check.
if [ "${#NODE_LIST_REPL[@]}" -gt 1 ]; then
for i in "${!NODE_LIST_ADMIN[@]}"; do
node_name=$(echo "${NODE_LIST_ADMIN[$i]}" | cut -d ' ' -f 2)
NODE_LIST_ADMIN[$i]="$(echo "${NODE_LIST_ADMIN[$i]}" | sed 's/ check / /') track mysql-replicas/${node_name}"
done
for i in "${!NODE_LIST_MYSQLX[@]}"; do
node_name=$(echo "${NODE_LIST_MYSQLX[$i]}" | cut -d ' ' -f 2)
NODE_LIST_MYSQLX[$i]="$(echo "${NODE_LIST_MYSQLX[$i]}" | sed 's/ check / /') track mysql-replicas/${node_name}"
done
if [ "${#peers[@]}" -eq 0 ]; then
log 'Could not find PEERS ...'
exit 0
fi

if [ -n "$primary_mysql_host" ]; then
if [[ "${#NODE_LIST[@]}" -ne 0 ]]; then
NODE_LIST=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST[@]}" | sort --version-sort -r | uniq)")
NODE_LIST_ADMIN=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST_ADMIN[@]}" | sort --version-sort -r | uniq)")
NODE_LIST_MYSQLX=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST_MYSQLX[@]}" | sort --version-sort -r | uniq)")
else
NODE_LIST=("$primary_mysql_node")
NODE_LIST_ADMIN=("$primary_mysql_node")
NODE_LIST_MYSQLX=("$primary_mysql_node")
fi
else
if [[ "${#NODE_LIST[@]}" -ne 0 ]]; then
NODE_LIST=("$(printf '%s\n' "${NODE_LIST[@]}" | sort --version-sort -r | uniq)")
NODE_LIST_ADMIN=("$(printf '%s\n' "${NODE_LIST_ADMIN[@]}" | sort --version-sort -r | uniq)")
NODE_LIST_MYSQLX=("$(printf '%s\n' "${NODE_LIST_MYSQLX[@]}" | sort --version-sort -r | uniq)")
fi
fi


echo "${#NODE_LIST_REPL[@]}" >$path_to_haproxy_cfg/AVAILABLE_NODES
log "number of available nodes are ${#NODE_LIST_REPL[@]}"
# stable slot assignment: version-sort == pod ordinal order for statefulsets
mapfile -t peers < <(printf '%s\n' "${peers[@]}" | sort --version-sort | uniq)

haproxy_check_script='haproxy_check_primary.sh'
if [ "${#NODE_LIST_REPL[@]}" -gt 1 ]; then
haproxy_check_script='haproxy_check_replicas.sh'
if [ "${#peers[@]}" -gt "$MAX_SLOTS" ]; then
log "ERROR: ${#peers[@]} peers but only ${MAX_SLOTS} slots; raise HA_MAX_SLOTS"
exit 1
fi

cat <<-EOF >"$path_to_haproxy_cfg/haproxy.cfg"
backend mysql-primary
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_primary.sh
EOF

(
IFS=$'\n'
echo "${NODE_LIST[*]}"
) >>"$path_to_haproxy_cfg/haproxy.cfg"

cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
backend mysql-replicas
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_replicas.sh
EOF

(
IFS=$'\n'
echo "${NODE_LIST_REPL[*]}"
) >>"$path_to_haproxy_cfg/haproxy.cfg"

cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
backend mysql-x
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/$haproxy_check_script
EOF

(
IFS=$'\n'
echo "${NODE_LIST_MYSQLX[*]}"
) >>"$path_to_haproxy_cfg/haproxy.cfg"

cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
backend mysql-admin
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/$haproxy_check_script
EOF
(
IFS=$'\n'
echo "${NODE_LIST_ADMIN[*]}"
) >>"$path_to_haproxy_cfg/haproxy.cfg"

path_to_custom_global_cnf='/etc/haproxy-custom'
if [ -f "$path_to_custom_global_cnf/haproxy-global.cfg" ]; then
haproxy -c -f "$path_to_custom_global_cnf/haproxy-global.cfg" -f $path_to_haproxy_cfg/haproxy.cfg
fi
# replica count for the check scripts: peers minus primary (if one exists)
local n_replicas=${#peers[@]}
for mysql_host in "${peers[@]}"; do
if /opt/percona/haproxy_check_primary.sh '' '' "$mysql_host"; then
n_replicas=$((n_replicas - 1))
break
fi
done

haproxy -c -f /opt/percona/haproxy-global.cfg -f $path_to_haproxy_cfg/haproxy.cfg
echo "$n_replicas" >"$path_to_haproxy_cfg/AVAILABLE_NODES"
log "number of available nodes are ${n_replicas}"

# TODO: Can we do something for maintenance mode?
if [ -S "$path_to_haproxy_cfg/haproxy-main.sock" ]; then
log "reload | socat stdio $path_to_haproxy_cfg/haproxy-main.sock"
echo 'reload' | socat stdio "$path_to_haproxy_cfg/haproxy-main.sock"
fi
local be i
for be in $BACKENDS; do
i=0
for mysql_host in "${peers[@]}"; do
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} fqdn ${mysql_host}"
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} state ready"
i=$((i + 1))
done

# park unused slots and kill any sessions still pinned to them
while [ "$i" -lt "$MAX_SLOTS" ]; do
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} state maint"
haproxy_socket_cmd "shutdown sessions server ${be}/${CLUSTER_NAME}-mysql-${i}"
i=$((i + 1))
done
done
}

main
Expand Down
26 changes: 26 additions & 0 deletions pkg/haproxy/haproxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,19 @@ func haproxyContainer(cr *apiv1.PerconaServerMySQL) corev1.Container {
env = append(env, probsEnvs...)
}

if cr.CompareVersion("1.3.0") >= 0 {
env = append(env, []corev1.EnvVar{
{
Name: "CLUSTER_NAME",
Value: cr.Name,
},
{
Name: "HA_MAX_SLOTS",
Value: strconv.Itoa(int(max(9, cr.Spec.MySQL.Size))),
},
}...)
}

volumeMounts := []corev1.VolumeMount{
{
Name: "bin",
Expand Down Expand Up @@ -443,6 +456,19 @@ func mysqlMonitContainer(cr *apiv1.PerconaServerMySQL) corev1.Container {
env = append(env, cluserTypeEnv...)
}

if cr.CompareVersion("1.3.0") >= 0 {
env = append(env, []corev1.EnvVar{
{
Name: "CLUSTER_NAME",
Value: cr.Name,
},
{
Name: "HA_MAX_SLOTS",
Value: strconv.Itoa(int(max(9, cr.Spec.MySQL.Size))),
},
}...)
}

resources := spec.Resources
if cr.CompareVersion("1.2.0") >= 0 {
resources = spec.SidecarResources["mysql-monit"]
Expand Down
Loading