deploy-test #23
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: deploy-test | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - develop | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: deploy-test | |
| cancel-in-progress: false | |
| env: | |
| API_IMAGE_REPOSITORY: cixing-api | |
| POSTGRES_IMAGE: postgres:16-alpine | |
| REDIS_IMAGE: redis:7-alpine | |
| MAILPIT_IMAGE: axllent/mailpit:v1.29.7 | |
| NGINX_IMAGE: nginx:1.27-alpine | |
| CERTBOT_IMAGE: certbot/certbot:v2.11.0 | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Validate required secrets | |
| env: | |
| TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }} | |
| TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }} | |
| TEST_SERVER_SSH_KEY: ${{ secrets.TEST_SERVER_SSH_KEY }} | |
| TEST_CONFIG_YAML: ${{ secrets.TEST_CONFIG_YAML }} | |
| TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }} | |
| run: | | |
| set -eu | |
| test -n "$TEST_SERVER_HOST" | |
| test -n "$TEST_SERVER_USERNAME" | |
| test -n "$TEST_SERVER_SSH_KEY" | |
| test -n "$TEST_CONFIG_YAML" | |
| test -n "$TEST_POSTGRES_PASSWORD" | |
| - name: Build API image | |
| run: | | |
| set -eu | |
| IMAGE_LOCAL="${API_IMAGE_REPOSITORY}:${GITHUB_SHA}" | |
| docker build --platform linux/amd64 -f deploy/docker/Dockerfile.api -t "$IMAGE_LOCAL" . | |
| echo "CIXING_IMAGE=$IMAGE_LOCAL" >> "$GITHUB_ENV" | |
| - name: Pull dependency images | |
| run: | | |
| set -eu | |
| docker pull --platform linux/amd64 "$POSTGRES_IMAGE" | |
| docker pull --platform linux/amd64 "$REDIS_IMAGE" | |
| docker pull --platform linux/amd64 "$MAILPIT_IMAGE" | |
| docker pull --platform linux/amd64 "$NGINX_IMAGE" | |
| docker pull --platform linux/amd64 "$CERTBOT_IMAGE" | |
| - name: Save deployment images | |
| run: | | |
| set -eu | |
| mkdir -p /tmp/cixing-deploy/images | |
| docker save "$CIXING_IMAGE" | gzip > /tmp/cixing-deploy/images/cixing-api.tar.gz | |
| docker save "$POSTGRES_IMAGE" | gzip > /tmp/cixing-deploy/images/postgres.tar.gz | |
| docker save "$REDIS_IMAGE" | gzip > /tmp/cixing-deploy/images/redis.tar.gz | |
| docker save "$MAILPIT_IMAGE" | gzip > /tmp/cixing-deploy/images/mailpit.tar.gz | |
| docker save "$NGINX_IMAGE" | gzip > /tmp/cixing-deploy/images/nginx.tar.gz | |
| docker save "$CERTBOT_IMAGE" | gzip > /tmp/cixing-deploy/images/certbot.tar.gz | |
| - name: Prepare deployment files | |
| env: | |
| TEST_CONFIG_YAML: ${{ secrets.TEST_CONFIG_YAML }} | |
| TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }} | |
| run: | | |
| set -eu | |
| mkdir -p /tmp/cixing-deploy | |
| cp deploy/compose/docker-compose.prod.yml /tmp/cixing-deploy/docker-compose.yml | |
| mkdir -p /tmp/cixing-deploy/nginx/entrypoint | |
| cp deploy/nginx/entrypoint/99-gen-conf.sh /tmp/cixing-deploy/nginx/entrypoint/99-gen-conf.sh | |
| printf '%s' "$TEST_CONFIG_YAML" > /tmp/cixing-deploy/config.yaml | |
| { | |
| echo "CIXING_IMAGE=${CIXING_IMAGE}" | |
| echo "POSTGRES_IMAGE=${POSTGRES_IMAGE}" | |
| echo "REDIS_IMAGE=${REDIS_IMAGE}" | |
| echo "MAILPIT_IMAGE=${MAILPIT_IMAGE}" | |
| echo "NGINX_IMAGE=${NGINX_IMAGE}" | |
| echo "CERTBOT_IMAGE=${CERTBOT_IMAGE}" | |
| echo "POSTGRES_PASSWORD=${TEST_POSTGRES_PASSWORD}" | |
| echo "API_PORT=8080" | |
| echo "MAILPIT_UI_PORT=8025" | |
| echo "DOMAIN=test.memento.muxixyz.com" | |
| } > /tmp/cixing-deploy/deploy.env | |
| - name: Configure SSH | |
| env: | |
| TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }} | |
| TEST_SERVER_SSH_KEY: ${{ secrets.TEST_SERVER_SSH_KEY }} | |
| run: | | |
| set -eu | |
| mkdir -p ~/.ssh | |
| printf '%s\n' "$TEST_SERVER_SSH_KEY" > ~/.ssh/id_test_server | |
| chmod 600 ~/.ssh/id_test_server | |
| ssh-keyscan -H "$TEST_SERVER_HOST" >> ~/.ssh/known_hosts | |
| - name: Upload deployment assets | |
| env: | |
| TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }} | |
| TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }} | |
| run: | | |
| set -eu | |
| SCP_OPTS="-o ConnectTimeout=10 -o ServerAliveInterval=15 -i ~/.ssh/id_test_server" | |
| ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" \ | |
| "sudo mkdir -p /opt/cixing/config /opt/cixing/images /opt/cixing/nginx/entrypoint && sudo chown -R $TEST_SERVER_USERNAME /opt/cixing" | |
| # Always upload config + compose + nginx entrypoint (small text files) | |
| scp $SCP_OPTS /tmp/cixing-deploy/docker-compose.yml \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/docker-compose.yml" | |
| scp $SCP_OPTS /tmp/cixing-deploy/deploy.env \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/deploy.env" | |
| scp $SCP_OPTS /tmp/cixing-deploy/config.yaml \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/config/config.yaml" | |
| scp $SCP_OPTS /tmp/cixing-deploy/nginx/entrypoint/99-gen-conf.sh \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/nginx/entrypoint/99-gen-conf.sh" | |
| # Upload dependency images only on first deploy (skip if already present on host) | |
| for img in postgres redis mailpit nginx certbot; do | |
| if ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" \ | |
| "test -f /opt/cixing/images/${img}.tar.gz"; then | |
| echo "skip ${img}.tar.gz (already exists on host)" | |
| else | |
| echo "uploading ${img}.tar.gz" | |
| scp $SCP_OPTS "/tmp/cixing-deploy/images/${img}.tar.gz" \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/" | |
| fi | |
| done | |
| # Always upload cixing-api image (changes every deploy) | |
| echo "uploading cixing-api.tar.gz" | |
| scp $SCP_OPTS /tmp/cixing-deploy/images/cixing-api.tar.gz \ | |
| "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/cixing-api.tar.gz" | |
| - name: Deploy on ECS | |
| env: | |
| TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }} | |
| TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }} | |
| run: | | |
| set -eu | |
| ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" <<'REMOTE' | |
| set -eu | |
| cd /opt/cixing | |
| DOCKER="docker" | |
| if ! docker info >/dev/null 2>&1; then | |
| DOCKER="sudo docker" | |
| fi | |
| compose() { | |
| $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml "$@" | |
| } | |
| cleanup_existing_compose_state() { | |
| echo "cleaning existing compose state" | |
| compose down --remove-orphans 2>/dev/null || true | |
| legacy_container_ids="$($DOCKER ps -aq --filter label=com.docker.compose.project=cixing-prod)" | |
| if [ -n "$legacy_container_ids" ]; then | |
| $DOCKER rm -f $legacy_container_ids >/dev/null 2>&1 || true | |
| fi | |
| legacy_network_ids="$($DOCKER network ls -q --filter label=com.docker.compose.project=cixing-prod)" | |
| if [ -n "$legacy_network_ids" ]; then | |
| $DOCKER network rm $legacy_network_ids >/dev/null 2>&1 || true | |
| fi | |
| if command -v ss >/dev/null 2>&1; then | |
| if ss -lntp 2>/dev/null | grep -q ':80 '; then | |
| echo "port 80 is still occupied before deployment" >&2 | |
| ss -lntp 2>/dev/null | grep ':80 ' >&2 || true | |
| exit 1 | |
| fi | |
| if ss -lntp 2>/dev/null | grep -q ':443 '; then | |
| echo "port 443 is still occupied before deployment" >&2 | |
| ss -lntp 2>/dev/null | grep ':443 ' >&2 || true | |
| exit 1 | |
| fi | |
| if ss -lntp 2>/dev/null | grep -q ':8080 '; then | |
| echo "port 8080 is still occupied before deployment" >&2 | |
| ss -lntp 2>/dev/null | grep ':8080 ' >&2 || true | |
| exit 1 | |
| fi | |
| fi | |
| } | |
| dump_api_diagnostics() { | |
| compose ps | |
| compose logs --tail=200 api | |
| } | |
| cleanup_deployment_artifacts() { | |
| echo "cleaning deployment artifacts" | |
| rm -f /opt/cixing/images/*.tar.gz || true | |
| current_image="$(awk -F= '$1=="CIXING_IMAGE"{print $2}' /opt/cixing/deploy.env)" | |
| if [ -z "$current_image" ]; then | |
| return 0 | |
| fi | |
| for image_ref in $($DOCKER images --format '{{.Repository}}:{{.Tag}}' | grep '^cixing-api:' | grep -F -x -v "$current_image" || true); do | |
| $DOCKER rmi "$image_ref" >/dev/null 2>&1 || true | |
| done | |
| } | |
| api_ready() { | |
| api_container_id="$(compose ps -q api)" | |
| if [ -z "$api_container_id" ]; then | |
| api_state="missing" | |
| api_health="missing" | |
| return 1 | |
| fi | |
| if ! api_state="$($DOCKER inspect --format '{{.State.Status}}' "$api_container_id" 2>/dev/null)"; then | |
| api_container_id="" | |
| api_state="missing" | |
| api_health="missing" | |
| return 1 | |
| fi | |
| if ! api_health="$($DOCKER inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$api_container_id" 2>/dev/null)"; then | |
| api_container_id="" | |
| api_state="missing" | |
| api_health="missing" | |
| return 1 | |
| fi | |
| if [ "$api_state" != "running" ] || [ "$api_health" != "healthy" ]; then | |
| return 1 | |
| fi | |
| if ! $DOCKER exec "$api_container_id" /bin/sh -c 'wget -qO- http://127.0.0.1:8080/healthz >/dev/null'; then | |
| api_health="healthz_failed" | |
| return 1 | |
| fi | |
| return 0 | |
| } | |
| chmod 600 /opt/cixing/deploy.env | |
| chmod 644 /opt/cixing/config/config.yaml | |
| chmod 755 /opt/cixing/nginx/entrypoint/99-gen-conf.sh | |
| cleanup_existing_compose_state | |
| gzip -dc /opt/cixing/images/cixing-api.tar.gz | $DOCKER load | |
| gzip -dc /opt/cixing/images/postgres.tar.gz | $DOCKER load | |
| gzip -dc /opt/cixing/images/redis.tar.gz | $DOCKER load | |
| gzip -dc /opt/cixing/images/mailpit.tar.gz | $DOCKER load | |
| gzip -dc /opt/cixing/images/nginx.tar.gz | $DOCKER load | |
| gzip -dc /opt/cixing/images/certbot.tar.gz | $DOCKER load | |
| echo "starting dependency services" | |
| compose up -d postgres redis mailpit | |
| echo "running database migrations" | |
| compose run --rm -T migrate up </dev/null | |
| echo "database migrations completed" | |
| echo "starting api service" | |
| compose up -d api | |
| echo "starting nginx on port 80" | |
| compose up -d nginx | |
| domain="$(awk -F= '$1=="DOMAIN"{print $2}' /opt/cixing/deploy.env)" | |
| if [ -z "$domain" ]; then | |
| domain="test.memento.muxixyz.com" | |
| fi | |
| if ! compose run --rm -T certbot /bin/sh -c "test -f /etc/letsencrypt/live/$domain/fullchain.pem" </dev/null; then | |
| echo "requesting initial Let's Encrypt certificate for domain=$domain" | |
| # Ensure Nginx is serving the ACME webroot before calling certbot. | |
| compose run --rm -T certbot certonly --webroot -w /var/www/certbot \ | |
| --email "admin@$domain" --agree-tos --no-eff-email \ | |
| -d "$domain" </dev/null | |
| echo "restarting nginx to enable 443" | |
| compose restart nginx | |
| else | |
| echo "existing certificate found for domain=$domain; skipping initial issuance" | |
| fi | |
| echo "starting certbot renewal loop" | |
| compose up -d certbot | |
| attempts=0 | |
| api_state="" | |
| api_health="" | |
| echo "waiting for api readiness" | |
| while [ "$attempts" -lt 30 ]; do | |
| if api_ready; then | |
| break | |
| fi | |
| if [ "$api_state" = "exited" ] || [ "$api_state" = "dead" ]; then | |
| break | |
| fi | |
| attempts=$((attempts + 1)) | |
| sleep 4 | |
| done | |
| if ! api_ready; then | |
| dump_api_diagnostics | |
| if [ "$api_health" = "healthz_failed" ]; then | |
| echo "container /healthz check failed" >&2 | |
| elif [ "$api_state" = "missing" ]; then | |
| echo "api container was not created" >&2 | |
| else | |
| echo "api container state=$api_state health=$api_health" >&2 | |
| fi | |
| exit 1 | |
| fi | |
| echo "api initial readiness check passed: container=$api_container_id state=$api_state health=$api_health" | |
| echo "rechecking api stability after warmup" | |
| sleep 12 | |
| if ! api_ready; then | |
| dump_api_diagnostics | |
| if [ "$api_health" = "healthz_failed" ]; then | |
| echo "api container became unstable after initial readiness: /healthz failed" >&2 | |
| elif [ "$api_state" = "missing" ]; then | |
| echo "api container disappeared after initial readiness" >&2 | |
| else | |
| echo "api container became unstable after initial readiness: state=$api_state health=$api_health" >&2 | |
| fi | |
| exit 1 | |
| fi | |
| final_api_container_id="$(compose ps -q api)" | |
| if [ -z "$final_api_container_id" ]; then | |
| dump_api_diagnostics | |
| echo "api container disappeared before final status output" >&2 | |
| exit 1 | |
| fi | |
| final_api_state="$($DOCKER inspect --format '{{.State.Status}}' "$final_api_container_id")" | |
| final_api_health="$($DOCKER inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$final_api_container_id")" | |
| echo "final api status: container=$final_api_container_id state=$final_api_state health=$final_api_health" | |
| cleanup_deployment_artifacts | |
| compose ps | |
| compose logs --tail=80 api | |
| REMOTE |