Skip to content

Commit 222bd34

Browse files
pfcoperezpfcoperez
authored
Initial attempt to run Tilt environment in CI (PeerDB-io#4474)
What resources are used and what test are run are given as input parameters. :memo: This is an experiment to use Tilt in CI workflows.
1 parent b50e65a commit 222bd34

1 file changed

Lines changed: 350 additions & 0 deletions

File tree

‎.github/workflows/tilt-flow.yml‎

Lines changed: 350 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,350 @@
1+
name: tilt-flow
2+
3+
# Like flow.yml, but instead of provisioning every service from scratch on the
4+
# runner, it brings the whole stack up through Tilt (repo_root/Tiltfile). Tilt
5+
# drives docker-compose-dev.yml (core PeerDB services) and
6+
# ancillary-docker-compose.yml (test databases + infra), and exposes the
7+
# e2e_*/connector_* test launchers as Tilt resources we can trigger on demand.
8+
#
9+
# This is a manually dispatched workflow: pick a single test resource to run and
10+
# the ancillary services it needs.
11+
12+
on:
13+
workflow_dispatch:
14+
inputs:
15+
test_resources:
16+
description: >-
17+
Space-separated list of test resources to run (Tilt e2e_* or connector_*
18+
resources). They run in parallel (the launchers are allow_parallel).
19+
Examples: "e2e_postgres", "e2e_postgres connector_postgres",
20+
"e2e_mysql-gtid e2e_mysql-pos e2e_mariadb".
21+
type: string
22+
required: true
23+
default: e2e_postgres
24+
ancillary_services:
25+
description: >-
26+
Space-separated ancillary Tilt resources to start before the test.
27+
e2e tests need clickhouse plus their source DB (e.g. "postgres clickhouse",
28+
"mysql-gtid clickhouse", "mongodb clickhouse"); connector tests need just
29+
their DB. Append "toxiproxy openssh" for SSH/chaos tests.
30+
type: string
31+
required: true
32+
default: postgres clickhouse
33+
34+
permissions:
35+
contents: read
36+
37+
concurrency:
38+
group: tilt-flow-${{ github.ref }}-${{ github.event.inputs.test_resources }}
39+
cancel-in-progress: true
40+
41+
env:
42+
# All tilt CLI calls in this repo talk to the control API on this port (see tilt.sh).
43+
TILT_PORT: "10352"
44+
# Pin Tilt for reproducible runs (matches the version used locally).
45+
TILT_VERSION: "0.37.0"
46+
# Don't phone home from CI.
47+
DO_NOT_TRACK: "1"
48+
# How many times to (re)load an unhealthy core PeerDB service before giving up.
49+
# Core services like flow-api/peerdb occasionally fail to initialize on a cold
50+
# start; reloading the individual resource recovers them. Bump this if the
51+
# runner is flaky.
52+
CORE_SERVICE_RECOVERY_ATTEMPTS: "3"
53+
# Bounds (seconds, inclusive) for the randomized wait between core-service
54+
# health polls.
55+
MIN_RECOVERY_TIME: "3"
56+
MAX_RECOVERY_TIME: "5"
57+
58+
jobs:
59+
tilt_flow_test:
60+
runs-on: ubuntu-latest-16-cores
61+
timeout-minutes: 60
62+
steps:
63+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
64+
65+
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
66+
with:
67+
go-version: '1.26.4'
68+
cache-dependency-path: flow/go.sum
69+
70+
# Tilt's `proto-gen` resource runs ./generate-protos.sh (buf generate +
71+
# go generate) on `tilt up`. The flow docker images and the host-run tests
72+
# both consume the resulting flow/generated code, so buf must be on PATH.
73+
- name: Install buf
74+
uses: bufbuild/buf-action@fd21066df7214747548607aaa45548ba2b9bc1ff # v1
75+
with:
76+
setup_only: true
77+
github_token: ${{ github.token }}
78+
79+
- name: install lib-geos and pg_dump
80+
run: |
81+
# No need to update man pages on package install
82+
sudo apt-get remove --purge man-db
83+
# Add PGDG apt repo for latest PostgreSQL client packages
84+
sudo install -d /usr/share/postgresql-common/pgdg
85+
sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
86+
--fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
87+
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \
88+
https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
89+
| sudo tee /etc/apt/sources.list.d/pgdg.list
90+
sudo apt-get update
91+
sudo apt-get install -y libgeos-dev
92+
# pg_dump must be >= the server major version; install v18 so it
93+
# can dump PG 16, 17, and 18 (backward compatible).
94+
sudo apt-get install -y postgresql-client-18
95+
echo /usr/lib/postgresql/18/bin >> $GITHUB_PATH
96+
97+
- run: go mod download
98+
working-directory: ./flow
99+
100+
- name: Install Tilt
101+
run: |
102+
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
103+
tilt version
104+
105+
- name: Prepare environment for Tilt
106+
run: |
107+
# Tilt reads .env at the project root; it is gitignored, so seed it
108+
# from the checked-in template.
109+
if [ ! -f .env ]; then
110+
cp .env.example .env
111+
fi
112+
# The test launchers run `go test` on the host (Tilt local_resource),
113+
# but .env points the DBs at host.docker.internal. Map that name to the
114+
# loopback so the host can reach the docker-compose published ports.
115+
echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts
116+
117+
# ----------------------------------------------------------------------
118+
# Start Tilt. This replaces all of flow.yml's per-service provisioning:
119+
# `tilt up` builds/starts the core PeerDB services (and proto-gen) via
120+
# docker-compose. Ancillary DBs and test launchers are auto_init=False and
121+
# are started explicitly in the steps below.
122+
# ----------------------------------------------------------------------
123+
- name: Start Tilt
124+
run: |
125+
set -euo pipefail
126+
# `tilt up` serves the control API and runs until torn down, so launch
127+
# it in the background and drive it via the CLI in later steps.
128+
nohup tilt up --port "$TILT_PORT" --stream=true > "$RUNNER_TEMP/tilt-up.log" 2>&1 &
129+
echo $! > "$RUNNER_TEMP/tilt.pid"
130+
131+
echo "Waiting for the Tilt API to come up..."
132+
for i in $(seq 1 60); do
133+
if tilt --port "$TILT_PORT" get uiresource >/dev/null 2>&1; then
134+
echo "Tilt API is reachable."
135+
break
136+
fi
137+
if [ "$i" -eq 60 ]; then
138+
echo "::error::Tilt API did not become reachable."
139+
cat "$RUNNER_TEMP/tilt-up.log" || true
140+
exit 1
141+
fi
142+
sleep 2
143+
done
144+
145+
# ----------------------------------------------------------------------
146+
# Bring up the core PeerDB services, recovering any that fail to
147+
# initialize on the first attempt. Core services (e.g. flow-api, peerdb)
148+
# occasionally come up unhealthy on a cold start; when that happens we
149+
# reload the individual resource with `tilt trigger` and wait again, up to
150+
# CORE_SERVICE_RECOVERY_ATTEMPTS times before failing the run.
151+
# ----------------------------------------------------------------------
152+
- name: Bring up core PeerDB services (with recovery)
153+
run: |
154+
set -uo pipefail
155+
156+
# Main PeerDB services that must be healthy before tests run. proto-gen
157+
# feeds the image builds; catalog/flow-api/flow-worker/flow-snapshot-worker
158+
# back the suites; peerdb and minio support peer setup and S3 staging.
159+
MAIN_SERVICES="proto-gen catalog flow-api flow-worker flow-snapshot-worker peerdb minio"
160+
# Per-attempt wait budget (generous, since flow images are built here).
161+
PER_ATTEMPT_TIMEOUT=600
162+
163+
# Classify a resource as "ready", "error", or "pending".
164+
ready_status() {
165+
local res="$1" json upd run
166+
json="$(tilt --port "$TILT_PORT" get uiresource "$res" -o json 2>/dev/null)" || { echo pending; return; }
167+
upd="$(echo "$json" | jq -r '.status.updateStatus // "none"')"
168+
run="$(echo "$json" | jq -r '.status.runtimeStatus // "none"')"
169+
# A failed build/update or a crashed runtime means this attempt failed.
170+
if [ "$upd" = "error" ] || [ "$run" = "error" ]; then echo error; return; fi
171+
# Ready: the update succeeded and the runtime is healthy (or absent,
172+
# e.g. proto-gen which has no long-running process).
173+
if [ "$upd" = "ok" ] && { [ "$run" = "ok" ] || [ "$run" = "not_applicable" ] || [ "$run" = "none" ]; }; then
174+
echo ready; return
175+
fi
176+
echo pending
177+
}
178+
179+
# Randomized backoff (seconds) between health polls, in
180+
# [MIN_RECOVERY_TIME, MAX_RECOVERY_TIME] inclusive.
181+
recovery_sleep() {
182+
local span=$(( MAX_RECOVERY_TIME - MIN_RECOVERY_TIME + 1 ))
183+
[ "$span" -lt 1 ] && span=1
184+
sleep "$(( MIN_RECOVERY_TIME + RANDOM % span ))"
185+
}
186+
187+
# Wait for a resource to become ready; on error or per-attempt timeout,
188+
# reload it (`tilt trigger`) and retry, up to the configured attempts.
189+
wait_or_recover() {
190+
local res="$1" attempt st deadline
191+
for attempt in $(seq 1 "$CORE_SERVICE_RECOVERY_ATTEMPTS"); do
192+
echo "[$res] readiness attempt ${attempt}/${CORE_SERVICE_RECOVERY_ATTEMPTS} (timeout ${PER_ATTEMPT_TIMEOUT}s)"
193+
deadline=$((SECONDS + PER_ATTEMPT_TIMEOUT))
194+
st=pending
195+
while [ "$SECONDS" -lt "$deadline" ]; do
196+
st="$(ready_status "$res")"
197+
case "$st" in
198+
ready) echo "[$res] is Ready."; return 0 ;;
199+
error) echo "::warning::[$res] entered an error state."; break ;;
200+
*) recovery_sleep ;;
201+
esac
202+
done
203+
# Don't reload after the final attempt (nothing left to wait on).
204+
if [ "$attempt" -lt "$CORE_SERVICE_RECOVERY_ATTEMPTS" ]; then
205+
echo "::warning::[$res] not healthy (status: ${st}); reloading the resource."
206+
tilt --port "$TILT_PORT" logs --tail 100 "$res" || true
207+
tilt --port "$TILT_PORT" trigger "$res" || true
208+
fi
209+
done
210+
echo "::error::[$res] failed to become Ready after ${CORE_SERVICE_RECOVERY_ATTEMPTS} attempts."
211+
tilt --port "$TILT_PORT" logs --tail 200 "$res" || true
212+
return 1
213+
}
214+
215+
rc=0
216+
for res in $MAIN_SERVICES; do
217+
wait_or_recover "$res" || rc=1
218+
done
219+
220+
if [ "$rc" -ne 0 ]; then
221+
echo "::error::One or more core PeerDB services could not be recovered."
222+
cat "$RUNNER_TEMP/tilt-up.log" || true
223+
exit 1
224+
fi
225+
echo "Core PeerDB services are up."
226+
227+
# ----------------------------------------------------------------------
228+
# Previous-to-last step: start the requested ancillary services and wait
229+
# until they (and their provisioning steps) are confirmed up and running.
230+
# ----------------------------------------------------------------------
231+
- name: Start requested ancillary services
232+
env:
233+
ANCILLARY_SERVICES: ${{ github.event.inputs.ancillary_services }}
234+
run: |
235+
set -euo pipefail
236+
if [ -z "${ANCILLARY_SERVICES// /}" ]; then
237+
echo "::error::No ancillary services specified."
238+
exit 1
239+
fi
240+
241+
echo "Enabling ancillary services: $ANCILLARY_SERVICES"
242+
# enable accepts multiple resources at once...
243+
tilt --port "$TILT_PORT" enable $ANCILLARY_SERVICES
244+
# ...but enabling is not enough: each must be triggered to start.
245+
for svc in $ANCILLARY_SERVICES; do
246+
echo "Triggering $svc"
247+
tilt --port "$TILT_PORT" trigger "$svc"
248+
done
249+
250+
# Wait until each service is Ready. Data stores also have an autostarted
251+
# provision-<svc> step (gated on the DB) — wait for that too so the
252+
# service is genuinely usable before the test runs.
253+
for svc in $ANCILLARY_SERVICES; do
254+
echo "Waiting for $svc to be Ready..."
255+
if ! tilt --port "$TILT_PORT" wait --for=condition=Ready "uiresource/$svc" --timeout=600s; then
256+
echo "::error::$svc did not become Ready."
257+
tilt --port "$TILT_PORT" logs --tail 200 "$svc" || true
258+
exit 1
259+
fi
260+
if tilt --port "$TILT_PORT" get uiresource "provision-$svc" >/dev/null 2>&1; then
261+
echo "Waiting for provision-$svc to be Ready..."
262+
if ! tilt --port "$TILT_PORT" wait --for=condition=Ready "uiresource/provision-$svc" --timeout=600s; then
263+
echo "::error::provision-$svc did not become Ready."
264+
tilt --port "$TILT_PORT" logs --tail 200 "provision-$svc" || true
265+
exit 1
266+
fi
267+
fi
268+
done
269+
echo "All requested ancillary services are up and running."
270+
271+
# ----------------------------------------------------------------------
272+
# Last step: invoke the requested test resource(s) and report their result.
273+
# The launchers are allow_parallel, so all are triggered up front and then
274+
# polled to completion concurrently.
275+
# ----------------------------------------------------------------------
276+
- name: Run requested test resources
277+
env:
278+
TEST_RESOURCES: ${{ github.event.inputs.test_resources }}
279+
run: |
280+
set -uo pipefail
281+
if [ -z "${TEST_RESOURCES// /}" ]; then
282+
echo "::error::No test resources specified."
283+
exit 1
284+
fi
285+
286+
echo "Enabling and triggering test resources: $TEST_RESOURCES"
287+
# enable accepts multiple resources...
288+
tilt --port "$TILT_PORT" enable $TEST_RESOURCES
289+
# ...but each must be triggered individually to start.
290+
for t in $TEST_RESOURCES; do
291+
echo "Triggering $t"
292+
tilt --port "$TILT_PORT" trigger "$t"
293+
done
294+
295+
# Each launcher is a one-shot local_resource: poll its update status
296+
# until it succeeds (ok) or fails (error). Anything else means it is
297+
# still queued/running. Track each resource's outcome; report a per-test
298+
# log dump as soon as it finishes, then a summary at the end.
299+
TIMEOUT=1800
300+
pending="$TEST_RESOURCES"
301+
failed=""
302+
deadline=$((SECONDS + TIMEOUT))
303+
while [ -n "${pending// /}" ] && [ "$SECONDS" -lt "$deadline" ]; do
304+
still_pending=""
305+
for t in $pending; do
306+
update_status="$(tilt --port "$TILT_PORT" get uiresource "$t" -o json 2>/dev/null | jq -r '.status.updateStatus // "unknown"')"
307+
case "$update_status" in
308+
ok)
309+
echo "Test '$t' succeeded."
310+
echo "----- logs for '$t' -----"
311+
tilt --port "$TILT_PORT" logs "$t" || true
312+
;;
313+
error)
314+
echo "::error::Test '$t' failed."
315+
echo "----- logs for '$t' -----"
316+
tilt --port "$TILT_PORT" logs "$t" || true
317+
failed="$failed $t"
318+
;;
319+
*)
320+
still_pending="$still_pending $t"
321+
;;
322+
esac
323+
done
324+
pending="$still_pending"
325+
[ -n "${pending// /}" ] && sleep 5
326+
done
327+
328+
# Anything still pending hit the overall timeout.
329+
if [ -n "${pending// /}" ]; then
330+
for t in $pending; do
331+
echo "::error::Test '$t' did not finish within ${TIMEOUT}s."
332+
echo "----- logs for '$t' (timed out) -----"
333+
tilt --port "$TILT_PORT" logs "$t" || true
334+
failed="$failed $t"
335+
done
336+
fi
337+
338+
echo "===== Summary ====="
339+
for t in $TEST_RESOURCES; do
340+
case " $failed " in
341+
*" $t "*) echo " $t: FAILED" ;;
342+
*) echo " $t: PASSED" ;;
343+
esac
344+
done
345+
346+
if [ -n "${failed// /}" ]; then
347+
echo "::error::One or more test resources failed:$failed"
348+
exit 1
349+
fi
350+
echo "All requested test resources passed."

0 commit comments

Comments
 (0)