Skip to content

Commit ae375c0

Browse files
kvinwangkvinwang
authored
Merge pull request #1147 from Dstack-TEE/feat/gateway-e2e-compose
test(gateway): run the integration suites under docker compose
2 parents a9db617 + b144cbf commit ae375c0

54 files changed

Lines changed: 4058 additions & 3999 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/gateway-cluster-tests.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
name: Gateway cluster tests
6+
7+
# WaveKV replication between gateway nodes: push and periodic sync, anti-entropy
8+
# repair, bootstrap after losing a store, partition recovery, node identity, and
9+
# the admin RPCs that gate registration. Unit tests cover the store; this runs
10+
# three real gateways and stops, wipes and restarts them.
11+
#
12+
# The nodes authenticate each other for real, so the cluster mTLS path is
13+
# exercised rather than switched off -- which is what the process-based suite
14+
# this replaces could not do.
15+
on:
16+
push:
17+
branches: [ next, 'release/**' ]
18+
paths:
19+
- 'dstack/gateway/**'
20+
- 'dstack/cert-client/**'
21+
- 'dstack/ra-tls/**'
22+
- 'dstack/guest-agent-simulator/**'
23+
- 'sdk/simulator/**'
24+
- 'dstack/crates/mock-attestation/**'
25+
- '.github/workflows/gateway-cluster-tests.yml'
26+
pull_request:
27+
branches: [ next, 'release/**' ]
28+
paths:
29+
- 'dstack/gateway/**'
30+
- 'dstack/cert-client/**'
31+
- 'dstack/ra-tls/**'
32+
- 'dstack/guest-agent-simulator/**'
33+
- 'sdk/simulator/**'
34+
- 'dstack/crates/mock-attestation/**'
35+
- '.github/workflows/gateway-cluster-tests.yml'
36+
37+
# Per workflow, not shared with the other two gateway suites.
38+
#
39+
# A group shared across all three does not serialise them, it drops one: a
40+
# concurrency group holds a single *pending* run, so when the third workflow
41+
# queued behind the first two, the one already waiting was cancelled --
42+
# silently, and reported as a cancelled run rather than a failure. The suites
43+
# no longer need serialising anyway; each brings up its own attestation fixture
44+
# under its own `FIXTURE_NS`, so there is nothing left to collide over.
45+
#
46+
# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the
47+
# run for the commit it replaced.
48+
concurrency:
49+
group: gateway-cluster-${{ github.ref }}
50+
cancel-in-progress: true
51+
52+
permissions:
53+
contents: read
54+
55+
env:
56+
CARGO_TERM_COLOR: always
57+
58+
jobs:
59+
gateway-cluster:
60+
runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }}
61+
# 28 tests, most of which restart nodes and then wait out a 5s sync interval.
62+
timeout-minutes: 60
63+
steps:
64+
- uses: actions/checkout@v5
65+
66+
- name: Install Rust
67+
uses: dtolnay/rust-toolchain@1.92.0
68+
with:
69+
targets: x86_64-unknown-linux-musl
70+
71+
- name: Install musl toolchain
72+
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools
73+
74+
# The gateways create real WireGuard interfaces inside their containers.
75+
- name: Record kernel capabilities
76+
run: |
77+
echo "kernel: $(uname -r)"
78+
sudo modprobe wireguard 2>&1 || echo "no wireguard module available"
79+
echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)"
80+
81+
- name: Cache cargo
82+
uses: actions/cache@v4
83+
with:
84+
path: |
85+
~/.cargo/registry
86+
~/.cargo/git
87+
dstack/target
88+
key: gateway-cluster-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }}
89+
restore-keys: gateway-cluster-${{ runner.os }}-
90+
91+
- name: Cluster suite
92+
working-directory: dstack/gateway/test-run/cluster
93+
run: ./run-cluster-tests.sh
94+
95+
# The suite only materialises node logs for the two tests that assert on
96+
# them, so the mounted directory is close to empty for any other failure.
97+
# Pull them from the daemon instead, by compose service so the shared
98+
# fixture's project-scoped names do not have to be guessed.
99+
- name: Collect node logs on failure
100+
if: failure()
101+
working-directory: dstack/gateway/test-run/cluster
102+
run: |
103+
mkdir -p run/logs
104+
# By project, discovered from the daemon. A bare `docker compose` here
105+
# addresses the project named after this directory (`cluster`), which
106+
# no test ever uses -- every test runs under `cluster-<testname>`, so
107+
# the loop that used `compose config --services` collected nothing.
108+
docker ps -a --filter 'label=com.docker.compose.project' \
109+
--format '{{.Label "com.docker.compose.project"}}' \
110+
| grep -E '^cluster-' | sort -u | while read -r project; do
111+
docker compose -p "$project" -f docker-compose.yml logs --no-color \
112+
> "run/logs/project-$project.log" 2>&1 || true
113+
done
114+
# The suite's own per-test dumps are root-owned inside the bind mount.
115+
docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true
116+
# The attestation fixture is a project of its own now, so it is not in
117+
# this suite's service list and its logs have to be asked for
118+
# separately -- they are where a quote-verification failure explains
119+
# itself.
120+
FIXTURE_NS=dstack-fixture-cluster docker compose -p dstack-fixture-cluster -f ../attestation/fixture.yml logs \
121+
--no-color > "run/logs/fixture.log" 2>&1 || true
122+
docker compose ps -a > run/logs/compose-ps.txt 2>&1 || true
123+
124+
- name: Upload node logs on failure
125+
if: failure()
126+
uses: actions/upload-artifact@v4
127+
with:
128+
name: gateway-cluster-logs
129+
path: dstack/gateway/test-run/cluster/run/logs/
130+
if-no-files-found: ignore
131+
retention-days: 7
132+
133+
- name: Tear down
134+
if: always()
135+
working-directory: dstack/gateway/test-run/cluster
136+
run: ./run-cluster-tests.sh down

‎.github/workflows/gateway-e2e-tests.yml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
name: Gateway e2e tests
6+
7+
# The gateway's certbot/ACME half only exists as a whole: certificate issuance,
8+
# cluster sync of the result, and the dns-persist-01 flow each depend on the
9+
# ones before. Unit tests cover the pieces; this stands up three gateways, a
10+
# Pebble CA, a mock Cloudflare DNS API and a mock attestation collateral
11+
# service, and asserts on what the cluster actually converges to.
12+
#
13+
# The suite verifies peer quotes for real: the gateways check each other against
14+
# development trust anchors derived from the simulator's seed, so it covers the
15+
# cluster mTLS path that a harness with the checks switched off cannot reach.
16+
on:
17+
push:
18+
branches: [ next, 'release/**' ]
19+
paths:
20+
- 'dstack/gateway/**'
21+
- 'dstack/cert-client/**'
22+
- 'dstack/certbot/**'
23+
- 'dstack/ra-tls/**'
24+
- 'dstack/guest-agent-simulator/**'
25+
- 'sdk/simulator/**'
26+
- 'dstack/crates/mock-attestation/**'
27+
- 'tools/mock-cf-dns/**'
28+
- '.github/workflows/gateway-e2e-tests.yml'
29+
pull_request:
30+
branches: [ next, 'release/**' ]
31+
paths:
32+
- 'dstack/gateway/**'
33+
- 'dstack/cert-client/**'
34+
- 'dstack/certbot/**'
35+
- 'dstack/ra-tls/**'
36+
- 'dstack/guest-agent-simulator/**'
37+
- 'sdk/simulator/**'
38+
- 'dstack/crates/mock-attestation/**'
39+
- 'tools/mock-cf-dns/**'
40+
- '.github/workflows/gateway-e2e-tests.yml'
41+
42+
# Per workflow, not shared with the other two gateway suites.
43+
#
44+
# A group shared across all three does not serialise them, it drops one: a
45+
# concurrency group holds a single *pending* run, so when the third workflow
46+
# queued behind the first two, the one already waiting was cancelled --
47+
# silently, and reported as a cancelled run rather than a failure. The suites
48+
# no longer need serialising anyway; each brings up its own attestation fixture
49+
# under its own `FIXTURE_NS`, so there is nothing left to collide over.
50+
#
51+
# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the
52+
# run for the commit it replaced.
53+
concurrency:
54+
group: gateway-e2e-${{ github.ref }}
55+
cancel-in-progress: true
56+
57+
permissions:
58+
contents: read
59+
60+
env:
61+
CARGO_TERM_COLOR: always
62+
63+
jobs:
64+
gateway-e2e:
65+
runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }}
66+
# A cold musl build dominates; the suite itself waits out real ACME orders
67+
# and a 20s cluster-sync settle, so it is minutes rather than seconds.
68+
timeout-minutes: 45
69+
steps:
70+
- uses: actions/checkout@v5
71+
72+
- name: Install Rust
73+
uses: dtolnay/rust-toolchain@1.92.0
74+
with:
75+
# run-e2e.sh builds a static gateway so the image can be alpine.
76+
targets: x86_64-unknown-linux-musl
77+
78+
- name: Install musl toolchain
79+
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools
80+
81+
# The gateways create real WireGuard interfaces inside their containers,
82+
# so the host kernel has to offer the module. Record it: a run that failed
83+
# because the runner image changed under us should say so plainly instead
84+
# of looking like a gateway bug.
85+
- name: Record kernel capabilities
86+
run: |
87+
echo "kernel: $(uname -r)"
88+
sudo modprobe wireguard 2>&1 || echo "no wireguard module available"
89+
echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)"
90+
docker version --format 'docker: {{.Server.Version}}'
91+
docker compose version
92+
93+
- name: Cache cargo
94+
uses: actions/cache@v4
95+
with:
96+
path: |
97+
~/.cargo/registry
98+
~/.cargo/git
99+
dstack/target
100+
key: gateway-e2e-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }}
101+
restore-keys: gateway-e2e-${{ runner.os }}-
102+
103+
# --keep-running so the containers survive for the log step below; the
104+
# script's own EXIT trap would otherwise tear them down and leave nothing
105+
# to collect from exactly the runs worth diagnosing.
106+
- name: Gateway e2e suite
107+
working-directory: dstack/gateway/test-run/e2e
108+
run: ./run-e2e.sh --keep-running
109+
110+
- name: Collect container logs on failure
111+
if: failure()
112+
working-directory: dstack/gateway/test-run/e2e
113+
run: |
114+
# By compose service, not by container name: the shared attestation
115+
# fixture deliberately does not pin one, so that more than one suite
116+
# can be up at a time.
117+
mkdir -p /tmp/gateway-e2e-logs
118+
for svc in $(docker compose config --services); do
119+
docker compose logs --no-color "$svc" > "/tmp/gateway-e2e-logs/$svc.log" 2>&1 || true
120+
done
121+
# The attestation fixture is a project of its own now, so it is not in
122+
# this suite's service list and its logs have to be asked for
123+
# separately -- they are where a quote-verification failure explains
124+
# itself.
125+
FIXTURE_NS=dstack-fixture-e2e docker compose -p dstack-fixture-e2e -f ../attestation/fixture.yml logs \
126+
--no-color > "/tmp/gateway-e2e-logs/fixture.log" 2>&1 || true
127+
docker compose ps -a > /tmp/gateway-e2e-logs/compose-ps.txt 2>&1 || true
128+
129+
- name: Tear down
130+
if: always()
131+
working-directory: dstack/gateway/test-run/e2e
132+
run: ./run-e2e.sh down
133+
134+
- name: Upload logs on failure
135+
if: failure()
136+
uses: actions/upload-artifact@v4
137+
with:
138+
name: gateway-e2e-logs
139+
path: /tmp/gateway-e2e-logs/
140+
if-no-files-found: ignore
141+
retention-days: 7

0 commit comments

Comments
 (0)