fix: Verify all compose services survive deploy recreate cycle (#2135)

Docker compose can lose sidecar containers during image recreate.
When the MCP server receives SIGTERM during compose up, it exits
cleanly (code 0) but Docker marks it as "manually stopped" and
the unless-stopped restart policy won't bring it back.

Add a post-deploy verification step that checks for exited services
and retries them before proceeding.

Co-authored-by: Ben Coombs <bjcoombs@users.noreply.github.com>

Author: bjcoombs

.github/workflows/deploy-demo.yml

@@ -209,6 +209,25 @@ jobs:
             docker pull ghcr.io/meridianhub/meridian@${IMAGE_DIGEST}
             docker tag ghcr.io/meridianhub/meridian@${IMAGE_DIGEST} ghcr.io/meridianhub/meridian:demo
             docker compose up -d --remove-orphans
+
+            # Verify all services came up. Docker compose can lose containers
+            # during recreate due to depends_on race conditions (the MCP server
+            # receives SIGTERM, exits cleanly, and isn't restarted because Docker
+            # marks it as "manually stopped").
+            sleep 5
+            failed=$(docker compose ps --status exited --format '{{.Service}}')
+            if [ -n "$failed" ]; then
+              echo "Services failed to start: $failed - retrying"
+              docker compose up -d $failed
+              sleep 3
+              still_failed=$(docker compose ps --status exited --format '{{.Service}}')
+              if [ -n "$still_failed" ]; then
+                echo "Services still down after retry: $still_failed"
+                exit 1
+              fi
+            fi
+            echo "All services running"
+
             docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile
             docker image prune -f
 

.github/workflows/deploy-develop.yml

@@ -220,6 +220,22 @@ jobs:
             docker pull ghcr.io/meridianhub/meridian@${IMAGE_DIGEST}
             docker tag ghcr.io/meridianhub/meridian@${IMAGE_DIGEST} ghcr.io/meridianhub/meridian:develop
             docker compose -f docker-compose.develop.yml up -d --remove-orphans
+
+            # Verify all services came up (see deploy-demo.yml for rationale)
+            sleep 5
+            failed=$(docker compose -f docker-compose.develop.yml ps --status exited --format '{{.Service}}')
+            if [ -n "$failed" ]; then
+              echo "Services failed to start: $failed - retrying"
+              docker compose -f docker-compose.develop.yml up -d $failed
+              sleep 3
+              still_failed=$(docker compose -f docker-compose.develop.yml ps --status exited --format '{{.Service}}')
+              if [ -n "$still_failed" ]; then
+                echo "Services still down after retry: $still_failed"
+                exit 1
+              fi
+            fi
+            echo "All services running"
+
             # Reload Caddy from the demo stack (Caddy runs there, serves both environments)
             docker compose -f /opt/meridian/docker-compose.yml exec -T caddy caddy reload --config /etc/caddy/Caddyfile
             docker image prune -f