Add healthcheck to CI containers

To prevent geth containers from getting stuck

Author: abicky

e2e/chains/bsc/docker-compose.simple.yml

@@ -71,6 +71,10 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-rpc2: # This is the bootstrap node
     image: bsc-geth:docker-local
@@ -86,6 +90,10 @@ services:
       - ./scripts:/scripts
       - ./config:/config
     command: ash /scripts/bsc-rpc.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-1:
     image: bsc-geth:docker-local
@@ -99,6 +107,10 @@ services:
       - bsc-validator1-1:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-2:
     image: bsc-geth:docker-local
@@ -112,6 +124,10 @@ services:
       - bsc-validator1-2:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-3:
     image: bsc-geth:docker-local
@@ -125,6 +141,10 @@ services:
       - bsc-validator1-3:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-4:
     image: bsc-geth:docker-local
@@ -138,6 +158,10 @@ services:
       - bsc-validator1-4:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator1-5:
     image: bsc-geth:docker-local
@@ -151,6 +175,10 @@ services:
       - bsc-validator1-5:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator2-1:
     image: bsc-geth:docker-local
@@ -164,6 +192,10 @@ services:
       - bsc-validator2-1:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator2-2:
     image: bsc-geth:docker-local
@@ -177,6 +209,10 @@ services:
       - bsc-validator2-2:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator2-3:
     image: bsc-geth:docker-local
@@ -190,6 +226,10 @@ services:
       - bsc-validator2-3:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator2-4:
     image: bsc-geth:docker-local
@@ -203,6 +243,10 @@ services:
       - bsc-validator2-4:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
 
   bsc-validator2-5:
     image: bsc-geth:docker-local
@@ -216,6 +260,18 @@ services:
       - bsc-validator2-5:/root/.ethereum
       - ./scripts:/scripts
     command: ash /scripts/bsc-validator.sh
+    healthcheck:
+      test: ["CMD", "ash", "/scripts/healthcheck.sh"]
+      interval: "5s"
+      start_period: "10s"
+
+  autoheal:
+    restart: always
+    image: willfarrell/autoheal
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 
 networks:
   bsc:
@@ -227,4 +283,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: 99.98.0.0/16
+        - subnet: 99.98.0.0/16

e2e/chains/bsc/scripts/healthcheck.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+[ "$(geth attach --exec eth.syncing /gethipc)" == "false" ]