Skip to content

Commit dcf3f23

Browse files
richiemcilroyrichiemcilroy
committed
Harden production control-plane deploys
1 parent 28039bf commit dcf3f23

20 files changed

Lines changed: 1765 additions & 119 deletions

‎.env.production.example‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ REND_PLAYER_PLAYBACK_BASE_URL=https://ash-1.play.rend.so
3030
REND_PLAYER_PLAYBACK_COOKIE_DOMAIN=rend.so
3131
REND_API_CORS_ALLOWED_ORIGINS=https://rend.so,https://www.rend.so
3232
REND_API_BIND_ADDR=0.0.0.0:4000
33-
REND_API_AUTO_MIGRATE=true
33+
REND_API_AUTO_MIGRATE=false
3434
REND_API_INLINE_MEDIA_PROCESSING=false
3535
REND_SITE_INTERNAL_TOKEN=replace-me
3636
REND_BILLING_MODE=autumn

‎.github/workflows/release-deploy.yml‎

Lines changed: 191 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,20 @@ on:
4242
required: true
4343
default: true
4444
type: boolean
45+
run_first_host_verifier:
46+
description: "Run scripts/verify-first-host-deploy.sh after deploy."
47+
required: true
48+
default: true
49+
type: boolean
50+
verify_asset_id:
51+
description: "Optional existing hls_ready synthetic/non-customer asset id for deeper verifier playback smoke."
52+
required: false
53+
type: string
54+
verify_control_plane_rollback:
55+
description: "After verification, drill control-plane rollback to the previous slot and re-promote this release."
56+
required: true
57+
default: false
58+
type: boolean
4559

4660
permissions:
4761
contents: read
@@ -125,9 +139,12 @@ jobs:
125139
DEPLOY_CONTROL_PLANE: ${{ github.event_name == 'push' || inputs.deploy_control_plane }}
126140
DEPLOY_EDGES: ${{ github.event_name == 'push' || inputs.deploy_edges }}
127141
RUN_READINESS: ${{ github.event_name == 'push' || inputs.run_readiness }}
142+
RUN_FIRST_HOST_VERIFIER: ${{ github.event_name == 'push' || inputs.run_first_host_verifier }}
143+
VERIFY_CONTROL_PLANE_ROLLBACK: ${{ github.event_name == 'workflow_dispatch' && inputs.verify_control_plane_rollback }}
128144
IMAGE_PLATFORM: ${{ needs.release-images.outputs.image-platform }}
129145
REND_API_BASE_URL: ${{ vars.REND_API_BASE_URL || 'https://api.rend.so' }}
130146
REND_READINESS_EDGES: ${{ vars.REND_READINESS_EDGES }}
147+
REND_VERIFY_ASSET_ID: ${{ inputs.verify_asset_id || vars.REND_VERIFY_ASSET_ID }}
131148
REND_READINESS_API_KEY: ${{ secrets.REND_READINESS_API_KEY }}
132149
REND_EDGE_INTERNAL_TOKEN: ${{ secrets.REND_EDGE_INTERNAL_TOKEN }}
133150
REND_SSH_PRIVATE_KEY: ${{ secrets.REND_SSH_PRIVATE_KEY }}
@@ -141,6 +158,7 @@ jobs:
141158
REND_EDGE_AMS_SSH_HOST: ${{ secrets.REND_EDGE_AMS_SSH_HOST }}
142159
REND_EDGE_AMS_SSH_USER: ${{ secrets.REND_EDGE_AMS_SSH_USER }}
143160
REND_EDGE_AMS_SSH_PORT: ${{ secrets.REND_EDGE_AMS_SSH_PORT }}
161+
DATABASE_URL: ${{ secrets.DATABASE_URL || vars.DATABASE_URL }}
144162
CLICKHOUSE_URL: ${{ secrets.CLICKHOUSE_URL || vars.CLICKHOUSE_URL }}
145163
CLICKHOUSE_DATABASE: ${{ secrets.CLICKHOUSE_DATABASE || vars.CLICKHOUSE_DATABASE || 'rend' }}
146164
CLICKHOUSE_USER: ${{ secrets.CLICKHOUSE_USER || vars.CLICKHOUSE_USER }}
@@ -214,6 +232,36 @@ jobs:
214232
: "${REND_CONTROL_PLANE_SSH_HOST:?REND_CONTROL_PLANE_SSH_HOST is required}"
215233
: "${REND_CONTROL_PLANE_SSH_USER:?REND_CONTROL_PLANE_SSH_USER is required}"
216234
235+
mkdir -p .rend/deploy
236+
monitor_file=".rend/deploy/public-readyz-monitor-${GITHUB_RUN_ID}.jsonl"
237+
monitor_stop_file="${RUNNER_TEMP}/rend-public-readyz-stop"
238+
monitor_failed_file="${RUNNER_TEMP}/rend-public-readyz-failed"
239+
rm -f "$monitor_stop_file" "$monitor_failed_file"
240+
241+
monitor_public_readyz() {
242+
while [[ ! -f "$monitor_stop_file" ]]; do
243+
ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
244+
http_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$REND_API_BASE_URL/readyz" 2>/dev/null || true)"
245+
if [[ "$http_status" == "200" ]]; then
246+
printf '{"ts":"%s","ok":true,"http_status":"%s"}\n' "$ts" "$http_status" >> "$monitor_file"
247+
else
248+
printf '{"ts":"%s","ok":false,"http_status":"%s"}\n' "$ts" "${http_status:-000}" >> "$monitor_file"
249+
touch "$monitor_failed_file"
250+
fi
251+
sleep 2
252+
done
253+
}
254+
255+
stop_monitor() {
256+
touch "$monitor_stop_file"
257+
wait "$monitor_pid" 2>/dev/null || true
258+
}
259+
260+
monitor_public_readyz &
261+
monitor_pid="$!"
262+
trap stop_monitor EXIT
263+
264+
export REND_CONTROL_PLANE_POST_PROMOTION_READY_URL="$REND_API_BASE_URL/readyz"
217265
scripts/deploy-release-over-ssh.sh \
218266
--role control-plane \
219267
--host "$REND_CONTROL_PLANE_SSH_HOST" \
@@ -223,6 +271,13 @@ jobs:
223271
--remote-dir "/tmp/rend-deploy-${GITHUB_RUN_ID}-${GITHUB_SHA::12}-control-plane" \
224272
--manifest "${{ steps.manifest.outputs.path }}"
225273
274+
stop_monitor
275+
trap - EXIT
276+
if [[ -f "$monitor_failed_file" ]]; then
277+
echo "public /readyz monitor observed at least one failed request during control-plane deploy" >&2
278+
exit 1
279+
fi
280+
226281
- name: Deploy edge hosts
227282
if: env.DEPLOY_EDGES == 'true'
228283
run: |
@@ -293,18 +348,18 @@ jobs:
293348
;;
294349
esac
295350
296-
- name: Install playback readiness dependencies
297-
if: env.RUN_READINESS == 'true'
351+
- name: Install deploy verification dependencies
352+
if: env.RUN_READINESS == 'true' || env.RUN_FIRST_HOST_VERIFIER == 'true'
298353
run: |
299354
set -euo pipefail
300355
sudo apt-get update
301-
sudo apt-get install -y ffmpeg
356+
sudo apt-get install -y ffmpeg postgresql-client
302357
303358
- name: Open edge readiness SSH tunnels
304-
if: env.RUN_READINESS == 'true'
359+
if: env.RUN_READINESS == 'true' || env.RUN_FIRST_HOST_VERIFIER == 'true'
305360
run: |
306361
set -euo pipefail
307-
: "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required when RUN_READINESS=true}"
362+
: "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required for deploy verification}"
308363
309364
start_tunnel() {
310365
local label="$1"
@@ -362,6 +417,64 @@ jobs:
362417
363418
echo "REND_READINESS_EDGES=$readiness_edges" >> "$GITHUB_ENV"
364419
420+
- name: Run first-host verifier
421+
if: env.RUN_FIRST_HOST_VERIFIER == 'true'
422+
run: |
423+
set -euo pipefail
424+
: "${REND_READINESS_API_KEY:?REND_READINESS_API_KEY is required when RUN_FIRST_HOST_VERIFIER=true}"
425+
: "${REND_EDGE_INTERNAL_TOKEN:?REND_EDGE_INTERNAL_TOKEN is required when RUN_FIRST_HOST_VERIFIER=true}"
426+
: "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required when RUN_FIRST_HOST_VERIFIER=true}"
427+
: "${DATABASE_URL:?DATABASE_URL Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}"
428+
: "${CLICKHOUSE_URL:?CLICKHOUSE_URL Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}"
429+
: "${CLICKHOUSE_DATABASE:?CLICKHOUSE_DATABASE Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}"
430+
: "${CLICKHOUSE_USER:?CLICKHOUSE_USER Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}"
431+
: "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD Production secret is required when RUN_FIRST_HOST_VERIFIER=true}"
432+
433+
verifier_args=(
434+
--api-base "$REND_API_BASE_URL"
435+
--database-url "$DATABASE_URL"
436+
--clickhouse-url "$CLICKHOUSE_URL"
437+
--clickhouse-database "$CLICKHOUSE_DATABASE"
438+
--clickhouse-user "$CLICKHOUSE_USER"
439+
--clickhouse-password "$CLICKHOUSE_PASSWORD"
440+
--dev-api-key "$REND_READINESS_API_KEY"
441+
--edge-internal-token "$REND_EDGE_INTERNAL_TOKEN"
442+
--rewrite-playback-base
443+
)
444+
445+
if [[ -n "${REND_VERIFY_ASSET_ID:-}" ]]; then
446+
verifier_args+=(--asset-id "$REND_VERIFY_ASSET_ID")
447+
else
448+
if [[ "${RUN_READINESS:-}" != "true" ]]; then
449+
echo "REND_VERIFY_ASSET_ID or RUN_READINESS=true is required for playback coverage" >&2
450+
exit 1
451+
fi
452+
verifier_args+=(--skip-playback)
453+
fi
454+
455+
IFS=',' read -r -a edge_entries <<< "$REND_READINESS_EDGES"
456+
expected_edges=""
457+
for entry in "${edge_entries[@]}"; do
458+
IFS='=' read -r edge_id region public_base private_base extra <<< "$entry"
459+
if [[ -z "$edge_id" || -z "$region" || -z "$public_base" || -n "${extra:-}" ]]; then
460+
echo "invalid REND_READINESS_EDGES entry for first-host verifier" >&2
461+
exit 1
462+
fi
463+
if [[ -z "$private_base" ]]; then
464+
private_base="$public_base"
465+
fi
466+
expected_item="${edge_id}=${region}=${public_base}"
467+
if [[ -z "$expected_edges" ]]; then
468+
expected_edges="$expected_item"
469+
else
470+
expected_edges="${expected_edges},${expected_item}"
471+
fi
472+
verifier_args+=(--edge-base "$public_base" --edge-internal-base "$private_base")
473+
done
474+
verifier_args+=(--expected-edges "$expected_edges")
475+
476+
scripts/verify-first-host-deploy.sh "${verifier_args[@]}"
477+
365478
- name: Run playback readiness gate
366479
if: env.RUN_READINESS == 'true'
367480
run: |
@@ -376,6 +489,79 @@ jobs:
376489
--output ".rend/readiness/playback-readiness-${GITHUB_RUN_ID}.json" \
377490
--latest-output ".rend/readiness/playback-readiness-latest.json"
378491
492+
- name: Verify control-plane rollback drill
493+
if: env.VERIFY_CONTROL_PLANE_ROLLBACK == 'true'
494+
run: |
495+
set -euo pipefail
496+
: "${REND_CONTROL_PLANE_SSH_HOST:?REND_CONTROL_PLANE_SSH_HOST is required}"
497+
: "${REND_CONTROL_PLANE_SSH_USER:?REND_CONTROL_PLANE_SSH_USER is required}"
498+
499+
mkdir -p .rend/deploy
500+
monitor_file=".rend/deploy/public-readyz-monitor-rollback-${GITHUB_RUN_ID}.jsonl"
501+
monitor_stop_file="${RUNNER_TEMP}/rend-rollback-readyz-stop"
502+
monitor_failed_file="${RUNNER_TEMP}/rend-rollback-readyz-failed"
503+
rm -f "$monitor_stop_file" "$monitor_failed_file"
504+
505+
monitor_public_readyz() {
506+
while [[ ! -f "$monitor_stop_file" ]]; do
507+
ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
508+
http_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$REND_API_BASE_URL/readyz" 2>/dev/null || true)"
509+
if [[ "$http_status" == "200" ]]; then
510+
printf '{"ts":"%s","phase":"rollback-drill","ok":true,"http_status":"%s"}\n' "$ts" "$http_status" >> "$monitor_file"
511+
else
512+
printf '{"ts":"%s","phase":"rollback-drill","ok":false,"http_status":"%s"}\n' "$ts" "${http_status:-000}" >> "$monitor_file"
513+
touch "$monitor_failed_file"
514+
fi
515+
sleep 2
516+
done
517+
}
518+
519+
stop_monitor() {
520+
touch "$monitor_stop_file"
521+
wait "$monitor_pid" 2>/dev/null || true
522+
}
523+
524+
monitor_public_readyz &
525+
monitor_pid="$!"
526+
trap stop_monitor EXIT
527+
528+
scripts/deploy-release-over-ssh.sh \
529+
--role control-plane \
530+
--host "$REND_CONTROL_PLANE_SSH_HOST" \
531+
--user "$REND_CONTROL_PLANE_SSH_USER" \
532+
--port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \
533+
--rollback
534+
535+
curl -fsS --retry 12 --retry-delay 5 --retry-all-errors "$REND_API_BASE_URL/readyz" >/dev/null
536+
537+
export REND_CONTROL_PLANE_POST_PROMOTION_READY_URL="$REND_API_BASE_URL/readyz"
538+
scripts/deploy-release-over-ssh.sh \
539+
--role control-plane \
540+
--host "$REND_CONTROL_PLANE_SSH_HOST" \
541+
--user "$REND_CONTROL_PLANE_SSH_USER" \
542+
--port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \
543+
--expected-platform "$IMAGE_PLATFORM" \
544+
--remote-dir "/tmp/rend-deploy-${GITHUB_RUN_ID}-${GITHUB_SHA::12}-control-plane-rollback-drill" \
545+
--manifest "${{ steps.manifest.outputs.path }}"
546+
547+
curl -fsS --retry 12 --retry-delay 5 --retry-all-errors "$REND_API_BASE_URL/readyz" >/dev/null
548+
549+
stop_monitor
550+
trap - EXIT
551+
if [[ -f "$monitor_failed_file" ]]; then
552+
echo "public /readyz monitor observed at least one failed request during control-plane rollback drill" >&2
553+
exit 1
554+
fi
555+
556+
- name: Upload deploy monitor artifact
557+
if: always()
558+
uses: actions/upload-artifact@v4
559+
with:
560+
name: rend-deploy-monitor
561+
path: .rend/deploy/public-readyz-monitor-*.jsonl
562+
if-no-files-found: ignore
563+
retention-days: 14
564+
379565
- name: Upload readiness artifact
380566
if: always() && env.RUN_READINESS == 'true'
381567
uses: actions/upload-artifact@v4

‎docs/deployment-automation.md‎

Lines changed: 70 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,26 +12,56 @@ The GitHub Actions workflow is `.github/workflows/release-deploy.yml`.
1212
3. Write a release manifest with immutable `@sha256:` image refs.
1313
4. Wait on the GitHub `Production` environment approval if that environment is
1414
configured with required reviewers.
15-
5. SSH to the control-plane host and run:
15+
5. SSH to the control-plane host and run a host-side blue/green transaction:
1616
- `scripts/preflight-control-plane-host.sh --skip-bind-port-check`
1717
- `scripts/deploy-control-plane-host.sh --dry-run`
1818
- `scripts/deploy-control-plane-host.sh`
19-
- local `rend-api` `/readyz` and `/healthz`
20-
The deploy helper waits for `rend-api` `/readyz` before starting
21-
`rend-media-worker`.
19+
Before preflight, the deploy wrapper installs the current
20+
`control-plane.compose.yml`, patches an existing concrete Caddyfile to use
21+
the managed upstream snippet, creates that snippet if it is missing, and
22+
leaves any existing upstream target intact. The env sync step also enforces
23+
`REND_API_AUTO_MIGRATE=false` for production serving containers.
24+
`scripts/deploy-release-over-ssh.sh` launches these control-plane commands
25+
through `sudo systemd-run --wait --collect --pipe`, so the transaction runs
26+
as a host-side transient unit. If the GitHub runner or SSH session dies after
27+
the unit starts, the host can still finish promotion rollback/cleanup. The
28+
deploy helper takes a host lock, runs one-shot `rend-api migrate`, starts the
29+
inactive API slot, probes private `/readyz` and `/healthz`, switches the
30+
managed Caddy upstream snippet, reloads Caddy, and rolls back the snippet if
31+
post-promotion checks fail. The previous API slot remains running.
2232
6. SSH to each configured edge host, serially, and run:
2333
- `scripts/preflight-edge-host.sh --skip-bind-port-check`
2434
- `scripts/deploy-edge-host.sh --dry-run`
2535
- `scripts/deploy-edge-host.sh`
2636
- local `rend-edge` `/readyz` and `/healthz`
27-
7. Check public API `/readyz`.
28-
8. Run the synthetic playback readiness gate when the required readiness secrets
37+
7. Monitor public API `/readyz` during the control-plane deploy, store the
38+
redacted JSONL monitor artifact, and check public API `/readyz`.
39+
8. Run `scripts/verify-first-host-deploy.sh` against production targets when
40+
`run_first_host_verifier` is enabled. The workflow reuses the edge SSH
41+
tunnels for private `/readyz`, `/internal/warm`, and `/metrics` checks, then
42+
verifies public deny rules, warmed signed playback, telemetry analytics, and
43+
ClickHouse reachability.
44+
9. Run the synthetic playback readiness gate when the required readiness secrets
2945
and edge targets are configured.
30-
31-
Preflight skips bind-port checks during automated updates because the existing
32-
service should already own the port. It still validates env files, registry
33-
pullability, image platform, host dependencies, managed dependency reachability,
34-
edge control-plane registration/heartbeat, and telemetry ingest.
46+
10. For an explicit production rollback drill, set
47+
`verify_control_plane_rollback=true` on a manual workflow dispatch. The
48+
workflow switches Caddy back to the previous control-plane slot without a
49+
pull/build, checks public `/readyz`, then re-promotes the current digest and
50+
checks public `/readyz` again.
51+
52+
Control-plane preflight treats bound blue/green ports as expected because the
53+
old slot must keep serving. It still validates env files, Caddy upstream wiring,
54+
registry pullability, image platform, host dependencies, and managed dependency
55+
reachability. Edge preflight still validates edge control-plane
56+
registration/heartbeat and telemetry ingest.
57+
58+
## Host Requirements
59+
60+
Control-plane hosts must have `systemd-run` available to run deploy and rollback
61+
transactions as transient host-side units. If `systemd-run` is missing, the
62+
automated control-plane deploy fails before the transaction starts or traffic is
63+
changed. Edge deploys still run as direct SSH commands because the edge path is
64+
not yet transactional.
3565

3666
## Required GitHub Environment
3767

@@ -61,6 +91,9 @@ Set these as `Production` environment secrets unless noted otherwise:
6191
- `CLICKHOUSE_PASSWORD`: production ClickHouse password. `CLICKHOUSE_URL`,
6292
`CLICKHOUSE_DATABASE`, and `CLICKHOUSE_USER` may also be set as secrets instead
6393
of variables.
94+
- `DATABASE_URL`: production Postgres URL used by
95+
`scripts/verify-first-host-deploy.sh` for edge registry checks. Prefer a
96+
secret over a variable.
6497
- `AUTUMN_SECRET_KEY`: live Autumn secret key.
6598

6699
Generate known-hosts entries from a trusted operator machine, verify the
@@ -101,15 +134,37 @@ URLs for the readiness gate. This keeps `/internal/*` and `/metrics` off the
101134
public edge hostnames while still allowing GitHub-hosted runners to run the
102135
synthetic playback checks.
103136

137+
## Optional Variables
138+
139+
- `REND_VERIFY_ASSET_ID`: existing `hls_ready` synthetic/non-customer asset id
140+
for the deeper warmed-playback path in `scripts/verify-first-host-deploy.sh`.
141+
A manual workflow dispatch can override it with `verify_asset_id`. If omitted,
142+
the workflow runs the verifier with `--skip-playback` and relies on the
143+
synthetic playback readiness gate for upload/playback/telemetry proof.
144+
104145
## Rollback
105146

106147
Use the previous `rend-release-manifest` artifact from a successful workflow
107-
run, then run the same host deploy helper manually in this order:
148+
run for edge rollback. For the control plane, switch back to the previous API
149+
slot without pulling/building:
108150

109151
```sh
110152
scripts/deploy-release-over-ssh.sh --role edge --host <edge-host> --user <user> --manifest <previous-manifest>
111-
scripts/deploy-release-over-ssh.sh --role control-plane --host <control-host> --user <user> --manifest <previous-manifest>
153+
scripts/deploy-release-over-ssh.sh --role control-plane --host <control-host> --user <user> --rollback
112154
```
113155

114-
Roll back edges first, then the media worker/API control plane. Treat database
115-
migrations as forward-only unless a tested rollback migration exists.
156+
Roll back edges first, then the media worker/API control plane. The
157+
control-plane rollback assumes the previous API slot is still running. Treat
158+
database migrations as forward-only unless a tested rollback migration exists.
159+
160+
## Remaining Risk
161+
162+
This automation makes the single control-plane host action-safe at the Docker
163+
service/Caddy-upstream layer, but it does not remove the single-host SPOF:
164+
kernel, VM, disk, network, Docker daemon, and Caddy process failures can still
165+
take the host down. Edge deploys are still in-place per edge host; deploy one
166+
edge at a time and keep the other edge serving, then verify with the playback
167+
readiness gate. A full edge blue/green transaction should add per-edge
168+
blue/green ports, a managed Caddy upstream snippet, candidate private
169+
`/readyz`/`/healthz`, and automatic snippet rollback matching the control-plane
170+
pattern.

0 commit comments

Comments
 (0)