Add healthcheck to CI containers

To prevent geth containers from getting stuck

Signed-off-by: Takeshi Arabiki <takeshi.arabiki@datachain.jp>

Author: abicky

e2e/chains/bsc/Makefile

@@ -10,6 +10,7 @@ bootstrap:
 
 .PHONY:network
 network:
+	docker compose -f docker-compose.simple.yml up -d autoheal
 	docker compose -f docker-compose.simple.yml up -d bsc-rpc bsc-validator1-1 bsc-validator1-2 bsc-validator1-3  bsc-validator1-4 bsc-validator1-5
 	docker compose -f docker-compose.simple.yml up -d bsc-rpc2 bsc-validator2-1 bsc-validator2-2 bsc-validator2-3
 

e2e/chains/bsc/docker-compose.simple.yml

@@ -71,6 +71,10 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-rpc2: # This is the bootstrap node
     image: bsc-geth:docker-local
@@ -86,6 +90,10 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-1:
     image: bsc-geth:docker-local
@@ -217,6 +225,14 @@ services:
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
 
+  autoheal:
+    restart: always
+    image: willfarrell/autoheal
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
 networks:
   bsc:
     ipam:
@@ -227,4 +243,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: 99.98.0.0/16
+        - subnet: 99.98.0.0/16

e2e/chains/bsc/scripts/bsc-rpc.sh

@@ -13,6 +13,7 @@ while [ "$i" -lt ${account_cnt} ]; do
 done
 
 ETHSTATS=""
-geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \
+# Use exec to handle signals
+exec geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \
 	--state.scheme=hash --db.engine=leveldb --verbosity ${VERBOSE} --nousb ${ETHSTATS} \
 	--unlock ${unlock_sequences} --password /dev/null --ipcpath /gethipc --override.fixedturnlength 2

e2e/chains/bsc/scripts/bsc-validator.sh

@@ -10,7 +10,8 @@ HOST_IP=$(hostname -i)
 echo "validator id: ${HOST_IP}"
 
 ETHSTATS=""
-geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \
+# Use exec to handle signals
+exec geth --config ${DATA_DIR}/config.toml --datadir ${DATA_DIR} --netrestrict ${CLUSTER_CIDR} \
 	--verbosity ${VERBOSE} --nousb ${ETHSTATS} --state.scheme=hash --db.engine=leveldb \
 	--bootnodes enode://${BOOTSTRAP_PUB_KEY}@${BOOTSTRAP_IP}:${BOOTSTRAP_TCP_PORT} \
 	--mine --miner.etherbase=${VALIDATOR_ADDR} -unlock ${VALIDATOR_ADDR} --password /dev/null --blspassword /scripts/wallet_password.txt \

e2e/chains/bsc/scripts/healthcheck.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+timestamp=$(geth attach --exec 'parseInt(eth.getBlockByNumber("latest").timestamp)' /gethipc)
+[ $(($(date '+%s') - $timestamp)) -lt 5 ]