Skip to content

Commit 04a77b3

Browse files
egegunesegegunes
committed
K8SPS-856: Don't reload haproxy on DNS changes
1 parent 22ec299 commit 04a77b3

3 files changed

Lines changed: 135 additions & 136 deletions

File tree

‎build/haproxy-entrypoint.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,74 @@
33
set -e
44
set -o xtrace
55

6+
MYSQL_PORT=3306
7+
MYSQLX_PORT=33060
8+
MYSQL_ADMIN_PORT=33062
9+
10+
# must be >= max cluster size
11+
MAX_SLOTS="${HA_MAX_SLOTS:-9}"
12+
13+
path_to_haproxy_cfg='/etc/haproxy/mysql'
14+
15+
SERVER_OPTIONS=${HA_SERVER_OPTIONS:-'resolvers kubernetes inter 10000 rise 1 fall 2 check weight 1'}
16+
17+
send_proxy=''
18+
if [[ ${IS_PROXY_PROTOCOL} == "yes" ]]; then
19+
send_proxy='send-proxy-v2'
20+
fi
21+
622
log() {
723
local message=$1
824
local date=$(/usr/bin/date +"%d/%b/%Y:%H:%M:%S.%3N")
925

1026
echo "{\"time\":\"${date}\", \"message\": \"${message}\"}"
1127
}
1228

29+
emit_backend() {
30+
local name=$1
31+
local port=$2
32+
local check_script=$3
33+
local extra=$4
34+
35+
cat <<-EOF
36+
backend ${name}
37+
mode tcp
38+
option srvtcpka
39+
balance roundrobin
40+
option external-check
41+
external-check command /opt/percona/${check_script}
42+
default-server ${SERVER_OPTIONS} init-addr none on-marked-down shutdown-sessions ${extra}
43+
EOF
44+
45+
# pre-provisioned slots; disabled == start in MAINT, no checks until
46+
# peer-list script sets a real fqdn and flips them to ready
47+
local i
48+
for i in $(seq 0 $((MAX_SLOTS - 1))); do
49+
echo " server ${CLUSTER_NAME}-mysql-${i} ${CLUSTER_NAME}-mysql-${i}.${CLUSTER_NAME}-mysql:${port} disabled"
50+
done
51+
}
52+
1353
echo "${CLUSTER_TYPE}" >/tmp/cluster_type
1454

1555
if [ "$1" = 'haproxy' ]; then
1656
if [ ! -f '/etc/haproxy/mysql/haproxy.cfg' ]; then
1757
cp /opt/percona/haproxy.cfg /etc/haproxy/mysql
1858
fi
1959

60+
{
61+
emit_backend mysql-primary "$MYSQL_PORT" haproxy_check_primary.sh "$send_proxy"
62+
emit_backend mysql-replicas "$MYSQL_PORT" haproxy_check_replicas.sh "$send_proxy"
63+
emit_backend mysql-x "$MYSQLX_PORT" haproxy_check_replicas.sh "$send_proxy"
64+
emit_backend mysql-admin "$MYSQL_ADMIN_PORT" haproxy_check_replicas.sh ''
65+
} >"$path_to_haproxy_cfg/haproxy.cfg"
66+
67+
path_to_custom_global_cnf='/etc/haproxy-custom'
68+
if [ -f "$path_to_custom_global_cnf/haproxy-global.cfg" ]; then
69+
haproxy -c -f "$path_to_custom_global_cnf/haproxy-global.cfg" -f "$path_to_haproxy_cfg/haproxy.cfg"
70+
fi
71+
72+
haproxy -c -f /opt/percona/haproxy-global.cfg -f "$path_to_haproxy_cfg/haproxy.cfg"
73+
2074
custom_conf='/etc/haproxy-custom/haproxy.cfg'
2175
if [ -f "$custom_conf" ]; then
2276
log "haproxy -c -f $custom_conf -f /etc/haproxy/mysql/haproxy.cfg"

‎build/haproxy_add_mysql_nodes.sh‎

Lines changed: 55 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -9,155 +9,74 @@ log() {
99
echo "{\"time\":\"${date}\", \"message\": \"${message}\"}"
1010
}
1111

12-
MYSQL_PORT=3306
13-
MYSQLX_PORT=33060
14-
MYSQL_ADMIN_PORT=33062
15-
primary_mysql_node=''
16-
primary_mysql_host=''
17-
18-
function main() {
19-
log "Running $0"
20-
21-
NODE_LIST=()
22-
NODE_LIST_REPL=()
23-
NODE_LIST_MYSQLX=()
24-
NODE_LIST_ADMIN=()
25-
26-
SERVER_OPTIONS=${HA_SERVER_OPTIONS:-'resolvers kubernetes inter 10000 rise 1 fall 2 check weight 1'}
27-
send_proxy=''
28-
path_to_haproxy_cfg='/etc/haproxy/mysql'
29-
if [[ ${IS_PROXY_PROTOCOL} == "yes" ]]; then
30-
send_proxy='send-proxy-v2'
12+
SOCK="${HA_STATS_SOCKET:-/etc/haproxy/mysql/haproxy.sock}"
13+
MAX_SLOTS="${HA_MAX_SLOTS:-9}"
14+
path_to_haproxy_cfg='/etc/haproxy/mysql'
15+
BACKENDS='mysql-primary mysql-replicas mysql-x mysql-admin'
16+
17+
haproxy_socket_cmd() {
18+
local cmd=$1
19+
local out
20+
21+
out=$(echo "$cmd" | socat -t 5 stdio "$SOCK" | tr -d '\n')
22+
if [ -n "$out" ]; then
23+
log "'${cmd}' -> ${out}"
3124
fi
25+
return 0
26+
}
3227

33-
while read mysql_host; do
34-
if [ -z "$mysql_host" ]; then
35-
log 'Could not find PEERS ...'
36-
exit 0
37-
fi
28+
main() {
29+
log "Running $0"
3830

39-
node_name=$(echo "$mysql_host" | cut -d . -f -1)
40-
if /opt/percona/haproxy_check_primary.sh '' '' "$mysql_host";then
41-
primary_mysql_host="$mysql_host"
42-
primary_mysql_node="server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS} on-marked-up shutdown-backup-sessions"
43-
continue
31+
local peers=()
32+
local mysql_host
33+
while read -r mysql_host; do
34+
if [ -n "$mysql_host" ]; then
35+
peers+=("$mysql_host")
4436
fi
45-
46-
NODE_LIST_REPL+=("server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS}")
47-
NODE_LIST+=("server ${node_name} ${mysql_host}:${MYSQL_PORT} ${send_proxy} ${SERVER_OPTIONS} backup")
48-
NODE_LIST_ADMIN+=("server ${node_name} ${mysql_host}:${MYSQL_ADMIN_PORT} ${SERVER_OPTIONS} backup")
49-
NODE_LIST_MYSQLX+=("server ${node_name} ${mysql_host}:${MYSQLX_PORT} ${send_proxy} ${SERVER_OPTIONS} backup")
5037
done
5138

52-
# mysql-x and mysql-admin replicas run the same external check against the
53-
# same admin port as mysql-replicas, so mirror the state with 'track'
54-
# instead of running duplicate checks. Only replicas can track: the primary
55-
# node is not present in the mysql-replicas backend and keeps its own check.
56-
if [ "${#NODE_LIST_REPL[@]}" -gt 1 ]; then
57-
for i in "${!NODE_LIST_ADMIN[@]}"; do
58-
node_name=$(echo "${NODE_LIST_ADMIN[$i]}" | cut -d ' ' -f 2)
59-
NODE_LIST_ADMIN[$i]="$(echo "${NODE_LIST_ADMIN[$i]}" | sed 's/ check / /') track mysql-replicas/${node_name}"
60-
done
61-
for i in "${!NODE_LIST_MYSQLX[@]}"; do
62-
node_name=$(echo "${NODE_LIST_MYSQLX[$i]}" | cut -d ' ' -f 2)
63-
NODE_LIST_MYSQLX[$i]="$(echo "${NODE_LIST_MYSQLX[$i]}" | sed 's/ check / /') track mysql-replicas/${node_name}"
64-
done
39+
if [ "${#peers[@]}" -eq 0 ]; then
40+
log 'Could not find PEERS ...'
41+
exit 0
6542
fi
6643

67-
if [ -n "$primary_mysql_host" ]; then
68-
if [[ "${#NODE_LIST[@]}" -ne 0 ]]; then
69-
NODE_LIST=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST[@]}" | sort --version-sort -r | uniq)")
70-
NODE_LIST_ADMIN=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST_ADMIN[@]}" | sort --version-sort -r | uniq)")
71-
NODE_LIST_MYSQLX=("$primary_mysql_node" "$(printf '%s\n' "${NODE_LIST_MYSQLX[@]}" | sort --version-sort -r | uniq)")
72-
else
73-
NODE_LIST=("$primary_mysql_node")
74-
NODE_LIST_ADMIN=("$primary_mysql_node")
75-
NODE_LIST_MYSQLX=("$primary_mysql_node")
76-
fi
77-
else
78-
if [[ "${#NODE_LIST[@]}" -ne 0 ]]; then
79-
NODE_LIST=("$(printf '%s\n' "${NODE_LIST[@]}" | sort --version-sort -r | uniq)")
80-
NODE_LIST_ADMIN=("$(printf '%s\n' "${NODE_LIST_ADMIN[@]}" | sort --version-sort -r | uniq)")
81-
NODE_LIST_MYSQLX=("$(printf '%s\n' "${NODE_LIST_MYSQLX[@]}" | sort --version-sort -r | uniq)")
82-
fi
83-
fi
84-
85-
86-
echo "${#NODE_LIST_REPL[@]}" >$path_to_haproxy_cfg/AVAILABLE_NODES
87-
log "number of available nodes are ${#NODE_LIST_REPL[@]}"
44+
# stable slot assignment: version-sort == pod ordinal order for statefulsets
45+
mapfile -t peers < <(printf '%s\n' "${peers[@]}" | sort --version-sort | uniq)
8846

89-
haproxy_check_script='haproxy_check_primary.sh'
90-
if [ "${#NODE_LIST_REPL[@]}" -gt 1 ]; then
91-
haproxy_check_script='haproxy_check_replicas.sh'
47+
if [ "${#peers[@]}" -gt "$MAX_SLOTS" ]; then
48+
log "ERROR: ${#peers[@]} peers but only ${MAX_SLOTS} slots; raise HA_MAX_SLOTS"
49+
exit 1
9250
fi
9351

94-
cat <<-EOF >"$path_to_haproxy_cfg/haproxy.cfg"
95-
backend mysql-primary
96-
mode tcp
97-
option srvtcpka
98-
balance roundrobin
99-
option external-check
100-
external-check command /opt/percona/haproxy_check_primary.sh
101-
EOF
102-
103-
(
104-
IFS=$'\n'
105-
echo "${NODE_LIST[*]}"
106-
) >>"$path_to_haproxy_cfg/haproxy.cfg"
107-
108-
cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
109-
backend mysql-replicas
110-
mode tcp
111-
option srvtcpka
112-
balance roundrobin
113-
option external-check
114-
external-check command /opt/percona/haproxy_check_replicas.sh
115-
EOF
116-
117-
(
118-
IFS=$'\n'
119-
echo "${NODE_LIST_REPL[*]}"
120-
) >>"$path_to_haproxy_cfg/haproxy.cfg"
121-
122-
cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
123-
backend mysql-x
124-
mode tcp
125-
option srvtcpka
126-
balance roundrobin
127-
option external-check
128-
external-check command /opt/percona/$haproxy_check_script
129-
EOF
130-
131-
(
132-
IFS=$'\n'
133-
echo "${NODE_LIST_MYSQLX[*]}"
134-
) >>"$path_to_haproxy_cfg/haproxy.cfg"
135-
136-
cat <<-EOF >>"$path_to_haproxy_cfg/haproxy.cfg"
137-
backend mysql-admin
138-
mode tcp
139-
option srvtcpka
140-
balance roundrobin
141-
option external-check
142-
external-check command /opt/percona/$haproxy_check_script
143-
EOF
144-
(
145-
IFS=$'\n'
146-
echo "${NODE_LIST_ADMIN[*]}"
147-
) >>"$path_to_haproxy_cfg/haproxy.cfg"
148-
149-
path_to_custom_global_cnf='/etc/haproxy-custom'
150-
if [ -f "$path_to_custom_global_cnf/haproxy-global.cfg" ]; then
151-
haproxy -c -f "$path_to_custom_global_cnf/haproxy-global.cfg" -f $path_to_haproxy_cfg/haproxy.cfg
152-
fi
52+
# replica count for the check scripts: peers minus primary (if one exists)
53+
local n_replicas=${#peers[@]}
54+
for mysql_host in "${peers[@]}"; do
55+
if /opt/percona/haproxy_check_primary.sh '' '' "$mysql_host"; then
56+
n_replicas=$((n_replicas - 1))
57+
break
58+
fi
59+
done
15360

154-
haproxy -c -f /opt/percona/haproxy-global.cfg -f $path_to_haproxy_cfg/haproxy.cfg
61+
echo "$n_replicas" >"$path_to_haproxy_cfg/AVAILABLE_NODES"
62+
log "number of available nodes are ${n_replicas}"
15563

156-
# TODO: Can we do something for maintenance mode?
157-
if [ -S "$path_to_haproxy_cfg/haproxy-main.sock" ]; then
158-
log "reload | socat stdio $path_to_haproxy_cfg/haproxy-main.sock"
159-
echo 'reload' | socat stdio "$path_to_haproxy_cfg/haproxy-main.sock"
160-
fi
64+
local be i
65+
for be in $BACKENDS; do
66+
i=0
67+
for mysql_host in "${peers[@]}"; do
68+
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} fqdn ${mysql_host}"
69+
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} state ready"
70+
i=$((i + 1))
71+
done
72+
73+
# park unused slots and kill any sessions still pinned to them
74+
while [ "$i" -lt "$MAX_SLOTS" ]; do
75+
haproxy_socket_cmd "set server ${be}/${CLUSTER_NAME}-mysql-${i} state maint"
76+
haproxy_socket_cmd "shutdown sessions server ${be}/${CLUSTER_NAME}-mysql-${i}"
77+
i=$((i + 1))
78+
done
79+
done
16180
}
16281

16382
main

‎pkg/haproxy/haproxy.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,19 @@ func haproxyContainer(cr *apiv1.PerconaServerMySQL) corev1.Container {
351351
env = append(env, probsEnvs...)
352352
}
353353

354+
if cr.CompareVersion("1.3.0") >= 0 {
355+
env = append(env, []corev1.EnvVar{
356+
{
357+
Name: "CLUSTER_NAME",
358+
Value: cr.Name,
359+
},
360+
{
361+
Name: "HA_MAX_SLOTS",
362+
Value: strconv.Itoa(int(max(9, cr.Spec.MySQL.Size))),
363+
},
364+
}...)
365+
}
366+
354367
volumeMounts := []corev1.VolumeMount{
355368
{
356369
Name: "bin",
@@ -439,6 +452,19 @@ func mysqlMonitContainer(cr *apiv1.PerconaServerMySQL) corev1.Container {
439452
env = append(env, cluserTypeEnv...)
440453
}
441454

455+
if cr.CompareVersion("1.3.0") >= 0 {
456+
env = append(env, []corev1.EnvVar{
457+
{
458+
Name: "CLUSTER_NAME",
459+
Value: cr.Name,
460+
},
461+
{
462+
Name: "HA_MAX_SLOTS",
463+
Value: strconv.Itoa(int(max(9, cr.Spec.MySQL.Size))),
464+
},
465+
}...)
466+
}
467+
442468
resources := spec.Resources
443469
if cr.CompareVersion("1.2.0") >= 0 {
444470
resources = spec.SidecarResources["mysql-monit"]

0 commit comments

Comments
 (0)