forked from willfarrell/docker-autoheal
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdocker-entrypoint
More file actions
executable file
·190 lines (157 loc) · 5.92 KB
/
Copy pathdocker-entrypoint
File metadata and controls
executable file
·190 lines (157 loc) · 5.92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/usr/bin/env sh
set -e
# shellcheck disable=2039
set -o pipefail
DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock}
UNIX_SOCK=""
CURL_TIMEOUT=${CURL_TIMEOUT:-60}
WEBHOOK_URL=${WEBHOOK_URL:-""}
# only use unix domain socket if no TCP endpoint is defined
case "${DOCKER_SOCK}" in
"tcp://"*) HTTP_ENDPOINT="$(echo ${DOCKER_SOCK} | sed 's#tcp://#https://#')"
CA="--cacert /certs/ca.pem"
CLIENT_KEY="--key /certs/client-key.pem"
CLIENT_CERT="--cert /certs/client-cert.pem"
;;
*) HTTP_ENDPOINT="http://localhost"
UNIX_SOCK="--unix-socket ${DOCKER_SOCK}"
;;
esac
AUTOHEAL_CONTAINER_LABEL=${AUTOHEAL_CONTAINER_LABEL:-autoheal_vpn}
AUTOHEAL_START_PERIOD=${AUTOHEAL_START_PERIOD:-0}
AUTOHEAL_INTERVAL=${AUTOHEAL_INTERVAL:-20}
AUTOHEAL_DEFAULT_STOP_TIMEOUT=${AUTOHEAL_DEFAULT_STOP_TIMEOUT:-10}
log_notice() { local DATE=$(date +%d-%m-%Y" "%H:%M:%S); printf "\e[33m${DATE} [NOTICE] %s\e[0m\n" "$*"; } #yellow
log_info() { local DATE=$(date +%d-%m-%Y" "%H:%M:%S); printf "\e[32m${DATE} [INFO] %s\e[0m\n" "$*"; } #green
log_err() { local DATE=$(date +%d-%m-%Y" "%H:%M:%S); printf "\e[31m${DATE} [ERROR] %s\e[0m\n" "$*"; } #red
docker_curl() {
curl --max-time "${CURL_TIMEOUT}" --no-buffer --silent \
${CA} ${CLIENT_KEY} ${CLIENT_CERT} \
${UNIX_SOCK} \
"$@"
}
# shellcheck disable=2039
get_container_info() {
local label_filter
local url
# Set container selector
if [ "$AUTOHEAL_CONTAINER_LABEL" = "all" ]
then
label_filter=""
else
label_filter=",\"label\":\[\"${AUTOHEAL_CONTAINER_LABEL}=true\"\]"
fi
url="${HTTP_ENDPOINT}/containers/json?filters=\{\"health\":\[\"unhealthy\"\]${label_filter}\}"
docker_curl "$url"
}
docker_start() {
local container_id="$1"
local timeout=90
local api_url="${HTTP_ENDPOINT}/containers/${container_id}/start"
(curl --silent --max-time "${timeout}" --unix-socket /var/run/docker.sock -X POST "${api_url}" \
&& log_notice "started ${container_id}") \
|| log_err "starting ${container_id} failed"
}
docker_restart() {
local container_id="$1"
local timeout=90
local api_url="${HTTP_ENDPOINT}/containers/${container_id}/restart"
(curl --silent --max-time "${timeout}" --unix-socket /var/run/docker.sock -X POST "${api_url}" \
&& log_notice "restarted ${container_id}") \
|| log_err "restarting ${container_id} failed-"
}
docker_container_status() {
local container_id="$1"
local api_url="${HTTP_ENDPOINT}/containers/${container_id}/json"
curl --silent --max-time "${timeout}" --unix-socket /var/run/docker.sock -X GET "${api_url}" | jq '.State.Status'
}
docker_container_health() {
local container_id="$1"
local api_url="${HTTP_ENDPOINT}/containers/${container_id}/json"
curl --silent --max-time "${timeout}" --unix-socket /var/run/docker.sock -X GET "${api_url}" | jq '.State.Health.Status'
}
# shellcheck disable=2039
restart_stack() {
local container_id="$1"
local timeout=90
local DATE=$(date +%d-%m-%Y" "%H:%M:%S)
log_info "$Restarting $container_id stack ... "
# Check status of VPN container.
VPN_STATUS=$(docker_container_status "${container_id}_protonvpn")
if [ "${VPN_STATUS}" == "exited" ]
then
#Stopped container VPN container; Must be started first
log_err "${container_id}_protonvpn is Not Running, Starting ${container_id}_protonvpn"
docker_start "${container_id}_protonvpn"
else
#Silent failure
log_notice "${container_id}_protonvpn is ${VPN_STATUS}; Possible Silent Failure. Restarting ${container_id}_protonvpn"
docker_restart "${container_id}_protonvpn"
fi
docker_restart "${container_id}"
}
# shellcheck disable=2039
notify_webhook() {
local text="$@"
if [ -n "$WEBHOOK_URL" ]
then
# execute webhook requests as background process to prevent healer from blocking
curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL
fi
}
# https://towardsdatascience.com/proper-ways-to-pass-environment-variables-in-json-for-curl-post-f797d2698bf3
generate_webhook_payload() {
local text="$@"
cat <<EOF
{
"text":"$text"
}
EOF
}
# SIGTERM-handler
term_handler() {
exit 143 # 128 + 15 -- SIGTERM
}
# shellcheck disable=2039
trap 'kill $$; term_handler' SIGTERM
if [ "$1" = "autoheal" ] && [ -e "$DOCKER_SOCK" ];then
# Delayed startup
if [ "$AUTOHEAL_START_PERIOD" -gt 0 ]
then
log_notice "Monitoring containers for unhealthy status in $AUTOHEAL_START_PERIOD second(s)"
sleep "$AUTOHEAL_START_PERIOD"
fi
while true
do
STOP_TIMEOUT=".Labels[\"autoheal.stop.timeout\"] // $AUTOHEAL_DEFAULT_STOP_TIMEOUT"
get_container_info | \
jq -r "foreach .[] as \$CONTAINER([];[]; \$CONTAINER | .Id, .Names[0], .State, ${STOP_TIMEOUT})" | \
while read -r CONTAINER_ID && read -r CONTAINER_NAME && read -r CONTAINER_STATE && read -r TIMEOUT
do
# shellcheck disable=2039
CONTAINER_SHORT_ID=${CONTAINER_ID:0:12}
DATE=$(date +%d-%m-%Y" "%H:%M:%S)
if [ "$CONTAINER_NAME" = "null" ]
then
log_notice "Container name of (${CONTAINER_SHORT_ID}) is null, which implies container does not exist - don't restart"
elif [ "$CONTAINER_STATE" = "restarting" ]
then
log_notice "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be already restarting - don't restart"
else
#strip out leading "/"
CONTAINER_NAME=${CONTAINER_NAME:1}
log_err "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy"
if restart_stack "${CONTAINER_NAME}" 90
then
notify_webhook "Container ${CONTAINER_NAME} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" &
else
notify_webhook "Container ${CONTAINER_NAME} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" &
fi
fi
done
log_info "Work Done; Checking again in ${AUTOHEAL_INTERVAL} second(s)"
sleep "${AUTOHEAL_INTERVAL}"
done
else
exec "$@"
fi