Add healthcheck to CI containers

To prevent geth containers from getting stuck

Author: abicky

e2e/chains/bsc/docker-compose.simple.yml

@@ -71,6 +71,11 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-rpc2: # This is the bootstrap node
     image: bsc-geth:docker-local
@@ -86,6 +91,10 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "5s"
 
   bsc-validator1-1:
     image: bsc-geth:docker-local
@@ -99,6 +108,11 @@ services:
       - bsc-validator1-1:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator1-2:
     image: bsc-geth:docker-local
@@ -112,6 +126,11 @@ services:
       - bsc-validator1-2:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator1-3:
     image: bsc-geth:docker-local
@@ -125,6 +144,11 @@ services:
       - bsc-validator1-3:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator1-4:
     image: bsc-geth:docker-local
@@ -138,6 +162,11 @@ services:
       - bsc-validator1-4:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator1-5:
     image: bsc-geth:docker-local
@@ -151,6 +180,11 @@ services:
       - bsc-validator1-5:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator2-1:
     image: bsc-geth:docker-local
@@ -164,6 +198,11 @@ services:
       - bsc-validator2-1:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator2-2:
     image: bsc-geth:docker-local
@@ -177,6 +216,11 @@ services:
       - bsc-validator2-2:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator2-3:
     image: bsc-geth:docker-local
@@ -190,6 +234,11 @@ services:
       - bsc-validator2-3:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator2-4:
     image: bsc-geth:docker-local
@@ -203,6 +252,11 @@ services:
       - bsc-validator2-4:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
 
   bsc-validator2-5:
     image: bsc-geth:docker-local
@@ -216,6 +270,19 @@ services:
       - bsc-validator2-5:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      retries: 2
+      start_period: "5s"
+
+  autoheal:
+    restart: always
+    image: willfarrell/autoheal
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 
 networks:
   bsc:

e2e/chains/bsc/scripts/healthcheck.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+[ "$(geth attach --exec eth.syncing /gethipc)" == "false" ]