Skip to content

Commit 73db68f

Browse files
committed
fix: replace docker compose --wait with manual health polling in CI
Docker's --wait respects the compose healthcheck parameters, so once retries are exhausted the container is marked permanently unhealthy regardless of --wait-timeout. Switch to manual curl/exec polling loops with generous timeouts (60×2s for ClickHouse, 30×2s for Postgres/Redis) which are independent of compose healthcheck config.
1 parent d6df3aa commit 73db68f

1 file changed

Lines changed: 35 additions & 13 deletions

File tree

.github/workflows/ci.yml

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ jobs:
9595
${{ runner.os }}-cargo-integration-
9696
9797
- name: Start infrastructure services
98-
run: docker compose up -d --wait --wait-timeout 120 clickhouse postgres redis
98+
run: docker compose up -d clickhouse postgres redis
9999
env:
100100
CLICKHOUSE_PORT: "8123"
101101
CLICKHOUSE_NATIVE_PORT: "9000"
@@ -105,26 +105,48 @@ jobs:
105105
POSTGRES_DB: llmtrace
106106
REDIS_PORT: "6379"
107107

108-
- name: Verify service health
108+
- name: Wait for services to be ready
109109
run: |
110+
echo "Waiting for ClickHouse..."
111+
for i in $(seq 1 60); do
112+
if curl -sf http://localhost:8123/ping >/dev/null 2>&1; then
113+
echo "ClickHouse ready after ${i}s"
114+
break
115+
fi
116+
if [ "$i" -eq 60 ]; then
117+
echo "ClickHouse failed after 60 attempts"
118+
docker compose logs clickhouse
119+
exit 1
120+
fi
121+
sleep 2
122+
done
123+
echo "Waiting for PostgreSQL..."
110124
for i in $(seq 1 30); do
111-
echo "Attempt $i/30: Checking ClickHouse..."
112-
if docker compose exec -T clickhouse wget --spider -q http://localhost:8123/ping 2>/dev/null; then
113-
echo "ClickHouse is ready."
125+
if docker compose exec -T postgres pg_isready -U llmtrace >/dev/null 2>&1; then
126+
echo "PostgreSQL ready after ${i}s"
114127
break
115128
fi
116129
if [ "$i" -eq 30 ]; then
117-
echo "ClickHouse failed to become healthy after 30 attempts."
118-
docker compose logs clickhouse
130+
echo "PostgreSQL failed after 30 attempts"
131+
docker compose logs postgres
132+
exit 1
133+
fi
134+
sleep 2
135+
done
136+
echo "Waiting for Redis..."
137+
for i in $(seq 1 30); do
138+
if docker compose exec -T redis redis-cli ping >/dev/null 2>&1; then
139+
echo "Redis ready after ${i}s"
140+
break
141+
fi
142+
if [ "$i" -eq 30 ]; then
143+
echo "Redis failed after 30 attempts"
144+
docker compose logs redis
119145
exit 1
120146
fi
121-
sleep 3
147+
sleep 2
122148
done
123-
echo "Checking PostgreSQL..."
124-
docker compose exec -T postgres pg_isready -U llmtrace
125-
echo "Checking Redis..."
126-
docker compose exec -T redis redis-cli ping
127-
echo "All services healthy."
149+
echo "All services ready."
128150
129151
- name: Run integration tests
130152
env:

0 commit comments

Comments
 (0)