|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> |
| 3 | +# SPDX-License-Identifier: Apache-2.0 |
| 4 | +set -euo pipefail |
| 5 | + |
| 6 | +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) |
| 7 | +REPO_DIR=$(cd -- "$SCRIPT_DIR/../.." && pwd) |
| 8 | +STATE_DIR="$SCRIPT_DIR/state" |
| 9 | +ENV_FILE=${DSTACK_E2E_ENV_FILE:-$SCRIPT_DIR/.env} |
| 10 | + |
| 11 | +# Phases are the unit a run can be limited to. Each one is self-contained: it |
| 12 | +# deploys what it needs and asserts on it, so a change that only touches one |
| 13 | +# area does not have to pay for the whole suite. |
| 14 | +# |
| 15 | +# upgrade KMS/Gateway 0.5.8 -> current rolling upgrade (the full suite) |
| 16 | +# certbot current-only KMS + two-node Gateway; ACME account registration, |
| 17 | +# CAA reconciliation, and account rotation |
| 18 | +PHASE=${DSTACK_E2E_PHASE:-upgrade} |
| 19 | +while (( $# )); do |
| 20 | + case "$1" in |
| 21 | + --phase) PHASE=${2:?--phase needs a value}; shift 2 ;; |
| 22 | + --phase=*) PHASE=${1#--phase=}; shift ;; |
| 23 | + -h|--help) |
| 24 | + sed -n '/^# Phases are/,/^PHASE=/p' "${BASH_SOURCE[0]}" | sed 's/^# \?//;$d' |
| 25 | + exit 0 ;; |
| 26 | + *) echo "ERROR: unknown argument $1" >&2; exit 1 ;; |
| 27 | + esac |
| 28 | +done |
| 29 | +case "$PHASE" in |
| 30 | + upgrade|certbot) ;; |
| 31 | + *) echo "ERROR: unknown phase $PHASE (expected upgrade or certbot)" >&2; exit 1 ;; |
| 32 | +esac |
| 33 | +export DSTACK_E2E_PHASE="$PHASE" |
| 34 | + |
| 35 | +setting() { |
| 36 | + local name=$1 fallback=$2 line value |
| 37 | + if [[ -v $name ]]; then |
| 38 | + printf '%s' "${!name}" |
| 39 | + return |
| 40 | + fi |
| 41 | + if [[ -f "$ENV_FILE" ]]; then |
| 42 | + line=$(grep -E "^[[:space:]]*${name}=" "$ENV_FILE" | tail -n1 || true) |
| 43 | + if [[ -n "$line" ]]; then |
| 44 | + value=${line#*=} |
| 45 | + value=${value%$'\r'} |
| 46 | + if [[ "$value" == \"*\" && "$value" == *\" ]]; then |
| 47 | + value=${value:1:${#value}-2} |
| 48 | + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then |
| 49 | + value=${value:1:${#value}-2} |
| 50 | + fi |
| 51 | + printf '%s' "$value" |
| 52 | + return |
| 53 | + fi |
| 54 | + fi |
| 55 | + printf '%s' "$fallback" |
| 56 | +} |
| 57 | + |
| 58 | +OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE \ |
| 59 | + dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d) |
| 60 | +OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE \ |
| 61 | + dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411) |
| 62 | +APP_IMAGE=$(setting DSTACK_E2E_APP_IMAGE nginx:alpine) |
| 63 | +KEEP_STACK=$(setting DSTACK_E2E_KEEP_STACK true) |
| 64 | +CLEAN_STATE=$(setting DSTACK_E2E_UPGRADE_CLEAN_STATE true) |
| 65 | +SKIP_BUILD=$(setting DSTACK_E2E_SKIP_CURRENT_BUILD false) |
| 66 | +# dstack's build metadata deliberately embeds a 20-hex abbreviated revision. |
| 67 | +CURRENT_REV=$(git -C "$REPO_DIR" rev-parse --short=20 HEAD) |
| 68 | +CURRENT_VERSION=$(sed -n 's/^version = "\([^"]*\)"/\1/p' \ |
| 69 | + "$REPO_DIR/dstack/Cargo.toml" | head -n1) |
| 70 | +[[ -n "$CURRENT_VERSION" ]] || { |
| 71 | + echo "ERROR: could not read current workspace version" >&2 |
| 72 | + exit 1 |
| 73 | +} |
| 74 | + |
| 75 | +COMPOSE=(docker compose -f "$SCRIPT_DIR/compose.yml") |
| 76 | +if [[ -f "$ENV_FILE" ]]; then |
| 77 | + COMPOSE=(docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR/compose.yml") |
| 78 | +fi |
| 79 | + |
| 80 | +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } |
| 81 | +die() { log "ERROR: $*" >&2; exit 1; } |
| 82 | +compose() { "${COMPOSE[@]}" "$@"; } |
| 83 | + |
| 84 | +need_bin() { |
| 85 | + [[ -x "$1" ]] || die "missing executable $1" |
| 86 | +} |
| 87 | + |
| 88 | +pull_released_image() { |
| 89 | + local image=$1 component=$2 |
| 90 | + log "pulling released $component image from Docker Hub: $image" |
| 91 | + docker pull "$image" || die "cannot pull released $component image $image" |
| 92 | +} |
| 93 | + |
| 94 | +reset_state() { |
| 95 | + log "resetting Compose stack and E2E state" |
| 96 | + compose down --remove-orphans >/dev/null 2>&1 || true |
| 97 | + docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ |
| 98 | + 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' |
| 99 | +} |
| 100 | + |
| 101 | +build_current_binaries() { |
| 102 | + if [[ "$SKIP_BUILD" == true ]]; then |
| 103 | + log "using prebuilt current musl KMS/Gateway binaries" |
| 104 | + else |
| 105 | + log "building current KMS/Gateway as production-style static musl binaries" |
| 106 | + cargo build --manifest-path "$REPO_DIR/dstack/Cargo.toml" \ |
| 107 | + --release --target x86_64-unknown-linux-musl \ |
| 108 | + -p dstack-kms -p dstack-gateway |
| 109 | + fi |
| 110 | + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" |
| 111 | + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway" |
| 112 | +} |
| 113 | + |
| 114 | +prepare_container_artifacts() { |
| 115 | + local artifact_dir="$STATE_DIR/artifacts/images" |
| 116 | + local context_dir="$STATE_DIR/image-build" |
| 117 | + local rev current_kms_image current_gateway_image |
| 118 | + local old_kms_id old_gateway_id current_kms_id current_gateway_id app_id |
| 119 | + rev=$(git -C "$REPO_DIR" rev-parse --short=16 HEAD) |
| 120 | + current_kms_image="dstack-e2e-kms-current:${rev}" |
| 121 | + current_gateway_image="dstack-e2e-gateway-current:${rev}" |
| 122 | + mkdir -p "$artifact_dir" "$context_dir/kms" "$context_dir/gateway" |
| 123 | + |
| 124 | + cp "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" \ |
| 125 | + "$context_dir/kms/dstack-kms" |
| 126 | + cat > "$context_dir/kms/Dockerfile" <<EOF |
| 127 | +ARG BASE |
| 128 | +FROM \${BASE} |
| 129 | +COPY dstack-kms /usr/local/bin/dstack-kms |
| 130 | +RUN chmod 0755 /usr/local/bin/dstack-kms |
| 131 | +EOF |
| 132 | + docker build --build-arg "BASE=$OLD_KMS_IMAGE" -t "$current_kms_image" "$context_dir/kms" |
| 133 | + |
| 134 | + cp "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway" \ |
| 135 | + "$context_dir/gateway/dstack-gateway" |
| 136 | + cp "$REPO_DIR/dstack/gateway/dstack-app/builder/entrypoint.sh" \ |
| 137 | + "$context_dir/gateway/entrypoint.sh" |
| 138 | + cat > "$context_dir/gateway/Dockerfile" <<EOF |
| 139 | +ARG BASE |
| 140 | +FROM \${BASE} |
| 141 | +COPY dstack-gateway /usr/local/bin/dstack-gateway |
| 142 | +COPY entrypoint.sh /app/entrypoint.sh |
| 143 | +RUN chmod 0755 /usr/local/bin/dstack-gateway /app/entrypoint.sh |
| 144 | +EOF |
| 145 | + docker build --build-arg "BASE=$OLD_GATEWAY_IMAGE" -t "$current_gateway_image" "$context_dir/gateway" |
| 146 | + |
| 147 | + old_kms_id=$(docker image inspect -f '{{.Id}}' "$OLD_KMS_IMAGE") |
| 148 | + old_gateway_id=$(docker image inspect -f '{{.Id}}' "$OLD_GATEWAY_IMAGE") |
| 149 | + current_kms_id=$(docker image inspect -f '{{.Id}}' "$current_kms_image") |
| 150 | + current_gateway_id=$(docker image inspect -f '{{.Id}}' "$current_gateway_image") |
| 151 | + app_id=$(docker image inspect -f '{{.Id}}' "$APP_IMAGE" 2>/dev/null || echo "") |
| 152 | + |
| 153 | + log "saving content-addressed images for import inside CVMs" |
| 154 | + docker save -o "$artifact_dir/kms-current.tar" "$current_kms_image" |
| 155 | + docker save -o "$artifact_dir/gateway-current.tar" "$current_gateway_image" |
| 156 | + # The released images are still the base layers the current binaries are |
| 157 | + # built on, so they are pulled either way; only the upgrade phase boots them. |
| 158 | + if [[ "$PHASE" == upgrade ]]; then |
| 159 | + docker save -o "$artifact_dir/kms-0.5.8.tar" "$OLD_KMS_IMAGE" |
| 160 | + docker save -o "$artifact_dir/gateway-0.5.8.tar" "$OLD_GATEWAY_IMAGE" |
| 161 | + docker save -o "$artifact_dir/app.tar" "$APP_IMAGE" |
| 162 | + fi |
| 163 | + |
| 164 | + cat > "$STATE_DIR/artifacts/images.env" <<EOF |
| 165 | +OLD_KMS_IMAGE=$OLD_KMS_IMAGE |
| 166 | +OLD_KMS_IMAGE_ID=$old_kms_id |
| 167 | +OLD_GATEWAY_IMAGE=$OLD_GATEWAY_IMAGE |
| 168 | +OLD_GATEWAY_IMAGE_ID=$old_gateway_id |
| 169 | +CURRENT_KMS_IMAGE=$current_kms_image |
| 170 | +CURRENT_KMS_IMAGE_ID=$current_kms_id |
| 171 | +CURRENT_GATEWAY_IMAGE=$current_gateway_image |
| 172 | +CURRENT_GATEWAY_IMAGE_ID=$current_gateway_id |
| 173 | +APP_IMAGE=$APP_IMAGE |
| 174 | +APP_IMAGE_ID=$app_id |
| 175 | +EOF |
| 176 | + |
| 177 | + docker run --rm --entrypoint dstack-kms "$OLD_KMS_IMAGE" --version \ |
| 178 | + | tee "$STATE_DIR/work/kms-old.version.txt" |
| 179 | + docker run --rm --entrypoint dstack-kms "$current_kms_image" --version \ |
| 180 | + | tee "$STATE_DIR/work/kms-current.version.txt" |
| 181 | + docker run --rm --entrypoint dstack-gateway "$OLD_GATEWAY_IMAGE" --version \ |
| 182 | + | tee "$STATE_DIR/work/gateway-old.version.txt" |
| 183 | + docker run --rm --entrypoint dstack-gateway "$current_gateway_image" --version \ |
| 184 | + | tee "$STATE_DIR/work/gateway-current.version.txt" |
| 185 | + |
| 186 | + grep -E '^dstack-kms v0\.5\.8 \(git:' "$STATE_DIR/work/kms-old.version.txt" >/dev/null \ |
| 187 | + || die "$OLD_KMS_IMAGE is not KMS 0.5.8" |
| 188 | + grep -E '^dstack-gateway v0\.5\.8 \(git:' "$STATE_DIR/work/gateway-old.version.txt" >/dev/null \ |
| 189 | + || die "$OLD_GATEWAY_IMAGE is not Gateway 0.5.8" |
| 190 | + grep -E "^dstack-kms v${CURRENT_VERSION//./\\.} \\(git:" \ |
| 191 | + "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ |
| 192 | + || die "locally built KMS is not current v$CURRENT_VERSION" |
| 193 | + grep -E "^dstack-gateway v${CURRENT_VERSION//./\\.} \\(git:" \ |
| 194 | + "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ |
| 195 | + || die "locally built Gateway is not current v$CURRENT_VERSION" |
| 196 | + grep -F "$CURRENT_REV" "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ |
| 197 | + || die "KMS binary was not built from current revision $CURRENT_REV" |
| 198 | + grep -F "$CURRENT_REV" "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ |
| 199 | + || die "Gateway binary was not built from current revision $CURRENT_REV" |
| 200 | +} |
| 201 | + |
| 202 | +wait_local_key_provider() { |
| 203 | + local port deadline status |
| 204 | + port=$(setting DSTACK_E2E_KEY_PROVIDER_PORT 13443) |
| 205 | + deadline=$((SECONDS + 120)) |
| 206 | + log "waiting for production SGX Local-Key-Provider on 127.0.0.1:${port}" |
| 207 | + while (( SECONDS < deadline )); do |
| 208 | + status=$(compose ps --format json local-keyprovider 2>/dev/null \ |
| 209 | + | jq -rs 'map(select(.Service == "local-keyprovider"))[0].Health // ""' 2>/dev/null \ |
| 210 | + || true) |
| 211 | + if [[ "$status" == healthy ]]; then |
| 212 | + log "Local-Key-Provider enclave is healthy" |
| 213 | + return 0 |
| 214 | + fi |
| 215 | + sleep 2 |
| 216 | + done |
| 217 | + compose logs --tail=200 aesmd local-keyprovider >&2 || true |
| 218 | + die "Local-Key-Provider did not become healthy; fix SGX/DCAP/PCCS provisioning rather than disabling attestation" |
| 219 | +} |
| 220 | + |
| 221 | +on_exit() { |
| 222 | + local rc=$? |
| 223 | + if (( rc != 0 )); then |
| 224 | + log "E2E failed; recent infrastructure logs follow" |
| 225 | + compose logs --tail=250 auth artifacts vmm runner >&2 || true |
| 226 | + fi |
| 227 | + if [[ "$KEEP_STACK" != true ]]; then |
| 228 | + compose down --remove-orphans >/dev/null 2>&1 || true |
| 229 | + else |
| 230 | + log "leaving stack running for inspection (DSTACK_E2E_KEEP_STACK=true)" |
| 231 | + fi |
| 232 | + exit "$rc" |
| 233 | +} |
| 234 | +trap on_exit EXIT |
| 235 | + |
| 236 | +main() { |
| 237 | + need_bin "$REPO_DIR/dstack/target/release/dstack" |
| 238 | + need_bin "$REPO_DIR/dstack/target/release/dstack-auth" |
| 239 | + need_bin "$REPO_DIR/dstack/target/release/dstack-vmm" |
| 240 | + need_bin "$REPO_DIR/dstack/target/release/supervisor" |
| 241 | + |
| 242 | + pull_released_image "$OLD_KMS_IMAGE" kms |
| 243 | + pull_released_image "$OLD_GATEWAY_IMAGE" gateway |
| 244 | + if [[ "$PHASE" == upgrade ]]; then |
| 245 | + pull_released_image "$APP_IMAGE" application |
| 246 | + fi |
| 247 | + build_current_binaries |
| 248 | + [[ "$CLEAN_STATE" == true ]] && reset_state |
| 249 | + |
| 250 | + log "building E2E infrastructure" |
| 251 | + compose build init-config mock-cf-dns-api aesmd local-keyprovider |
| 252 | + compose run --rm init-config |
| 253 | + prepare_container_artifacts |
| 254 | + |
| 255 | + log "starting authorization, artifact, attestation, ACME and VMM infrastructure" |
| 256 | + compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth artifacts |
| 257 | + wait_local_key_provider |
| 258 | + compose up -d vmm |
| 259 | + |
| 260 | + log "running the $PHASE phase" |
| 261 | + # Keep the complete runner transcript in state/work even though the runner is |
| 262 | + # an ephemeral Compose container. This is especially important for failures |
| 263 | + # during deployment, before a per-VM log file exists. |
| 264 | + compose run --rm --no-deps \ |
| 265 | + -e DSTACK_E2E_PHASE="$PHASE" \ |
| 266 | + -e DSTACK_E2E_CURRENT_VERSION="$CURRENT_VERSION" \ |
| 267 | + -e DSTACK_E2E_CURRENT_REV="$CURRENT_REV" runner \ |
| 268 | + 2>&1 | tee "$STATE_DIR/work/runner.log" |
| 269 | + |
| 270 | + log "$PHASE E2E success" |
| 271 | + log "artifacts: $STATE_DIR/work" |
| 272 | +} |
| 273 | + |
| 274 | +main "$@" |
0 commit comments