@@ -27,11 +27,16 @@ TRACKING_FILE="$WORKDIR/uploaded_files.log"
2727python build_imposm3_config.py
2828
2929# Create config file for imposm
30+ # Add connection parameters to handle keepalives and prevent "bad connection" errors
31+ # keepalives_idle: send keepalive after 30s of inactivity
32+ # keepalives_interval: retransmit every 10s if unacknowledged
33+ # keepalives_count: consider dead after 5 unacknowledged keepalives
34+ # connect_timeout: connection timeout in seconds
3035cat << EOF >"$WORKDIR /config.json"
3136{
3237 "cachedir": "$CACHE_DIR ",
3338 "diffdir": "$DIFF_DIR ",
34- "connection": "postgis://$POSTGRES_USER :$POSTGRES_PASSWORD @$POSTGRES_HOST /$POSTGRES_DB ",
39+ "connection": "postgis://$POSTGRES_USER :$POSTGRES_PASSWORD @$POSTGRES_HOST /$POSTGRES_DB ?keepalives=1&keepalives_idle=30&keepalives_interval=10&keepalives_count=5&connect_timeout=10 ",
3540 "mapping": "/osm/config/imposm3.json",
3641 "replication_url": "$REPLICATION_URL "
3742}
@@ -137,6 +142,67 @@ function uploadLastState() {
137142 fi
138143}
139144
145+ # Function to monitor imposm process and handle connection errors
146+ # Parameters:
147+ # $1: IMPOSM_PID - Process ID of the imposm process
148+ # $2: UPLOADER_PID - Process ID of the uploader background process
149+ function monitorImposmErrors() {
150+ local IMPOSM_PID=$1
151+ local UPLOADER_PID=$2
152+ local ERROR_COUNT=0
153+ local MAX_ERRORS=3
154+ local LOG_FILE=" /tmp/imposm.log"
155+
156+ while true ; do
157+ log_message " Checking for errors during minute replication import into the database"
158+
159+ # Check for connection errors specifically
160+ if grep -q " driver: bad connection" " $LOG_FILE " || grep -q " \[error\] Importing.*bad connection" " $LOG_FILE " ; then
161+ ERROR_COUNT=$(( ERROR_COUNT + 1 ))
162+ log_message " Detected bad connection error (count: $ERROR_COUNT /$MAX_ERRORS ). Waiting before retry..."
163+
164+ # Check if imposm process is still running
165+ if ! kill -0 $IMPOSM_PID 2> /dev/null; then
166+ log_message " Imposm process has died. Restarting container..."
167+ kill $UPLOADER_PID 2> /dev/null
168+ exit 1
169+ fi
170+
171+ # If we've hit max errors, restart
172+ if [ $ERROR_COUNT -ge $MAX_ERRORS ]; then
173+ log_message " Max connection errors reached ($MAX_ERRORS ). Restarting container..."
174+ kill $UPLOADER_PID 2> /dev/null
175+ kill $IMPOSM_PID 2> /dev/null
176+ exit 1
177+ fi
178+
179+ # Wait a bit and check if connection recovers
180+ sleep 30
181+ # Clear the error from log to avoid immediate re-trigger
182+ sed -i ' /driver: bad connection/d' " $LOG_FILE " 2> /dev/null || true
183+ elif grep -q " \[error\] Importing" " $LOG_FILE " ; then
184+ # Other import errors - log but don't immediately restart
185+ log_message " Detected [error] Importing in Imposm log. Monitoring..."
186+ ERROR_COUNT=$(( ERROR_COUNT + 1 ))
187+
188+ if [ $ERROR_COUNT -ge $MAX_ERRORS ]; then
189+ log_message " Max errors reached ($MAX_ERRORS ). Restarting container..."
190+ kill $UPLOADER_PID 2> /dev/null
191+ kill $IMPOSM_PID 2> /dev/null
192+ exit 1
193+ fi
194+ else
195+ # Reset error count if no errors found
196+ if [ $ERROR_COUNT -gt 0 ]; then
197+ log_message " No errors detected. Resetting error count."
198+ ERROR_COUNT=0
199+ fi
200+ fi
201+
202+ sleep 10
203+ done
204+ }
205+
140206function updateData() {
141207 log_message " Starting database update process..."
142208
187253 -quiet 2>&1 | tee /tmp/imposm.log &
188254 IMPOSM_PID=$!
189255
190- # Step 5: Check update process and restart
191- while true ; do
192- log_message " Checking for errors during minute replication import into the database"
193- if grep -q " \[error\] Importing" /tmp/imposm.log; then
194- log_message " Detected [error] Importing in Imposm log. Restarting container..."
195- kill $UPLOADER_PID 2> /dev/null
196- kill $IMPOSM_PID 2> /dev/null
197- exit 1
198- fi
199- sleep 10
200- done
256+ # Step 5: Monitor imposm process and handle errors
257+ monitorImposmErrors $IMPOSM_PID $UPLOADER_PID
201258}
202259
203260function importData() {
0 commit comments