fix(deploy): deploy test ECS by uploading images over SSH (#8)

* fix(deploy): upload images to ECS instead of pulling from Docker Hub

* fix(deploy): allow container to read test config

* fix(deploy): verify api health after test deployment

* fix(deploy): verify api stability after test deployment

* fix(deploy): clean stale compose state before test deployment

* fix(deploy): log api deployment status explicitly

* fix(deploy): prevent migrate step from consuming ssh input

* ci(deploy): auto deploy test ECS on develop push

* ci(ops): add test ECS ops workflow

* ci(ops): add test ops workflow and reduce healthcheck noise

* ci(ops): add OSS presign diagnostics

Author: c1trua

.github/workflows/deploy-test.yml

@@ -2,6 +2,9 @@ name: deploy-test
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - develop
 
 permissions:
   contents: read
@@ -11,7 +14,10 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  IMAGE_REPOSITORY: cixing-api
+  API_IMAGE_REPOSITORY: cixing-api
+  POSTGRES_IMAGE: postgres:16-alpine
+  REDIS_IMAGE: redis:7-alpine
+  MAILPIT_IMAGE: axllent/mailpit:v1.29.7
 
 jobs:
   deploy:
@@ -23,42 +29,41 @@ jobs:
 
       - name: Validate required secrets
         env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
           TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }}
           TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }}
           TEST_SERVER_SSH_KEY: ${{ secrets.TEST_SERVER_SSH_KEY }}
           TEST_CONFIG_YAML: ${{ secrets.TEST_CONFIG_YAML }}
           TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
         run: |
           set -eu
-          test -n "$DOCKERHUB_USERNAME"
-          test -n "$DOCKERHUB_TOKEN"
           test -n "$TEST_SERVER_HOST"
           test -n "$TEST_SERVER_USERNAME"
           test -n "$TEST_SERVER_SSH_KEY"
           test -n "$TEST_CONFIG_YAML"
           test -n "$TEST_POSTGRES_PASSWORD"
 
-      - name: Login to Docker Hub
-        env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build API image
         run: |
           set -eu
-          echo "$DOCKERHUB_TOKEN" | docker login docker.io -u "$DOCKERHUB_USERNAME" --password-stdin
+          IMAGE_LOCAL="${API_IMAGE_REPOSITORY}:${GITHUB_SHA}"
+          docker build --platform linux/amd64 -f deploy/docker/Dockerfile.api -t "$IMAGE_LOCAL" .
+          echo "CIXING_IMAGE=$IMAGE_LOCAL" >> "$GITHUB_ENV"
 
-      - name: Build and push image
-        env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      - name: Pull dependency images
         run: |
           set -eu
-          IMAGE_SHA="docker.io/${DOCKERHUB_USERNAME}/${IMAGE_REPOSITORY}:${GITHUB_SHA}"
-          IMAGE_LATEST="docker.io/${DOCKERHUB_USERNAME}/${IMAGE_REPOSITORY}:test-latest"
-          docker build -f deploy/docker/Dockerfile.api -t "$IMAGE_SHA" -t "$IMAGE_LATEST" .
-          docker push "$IMAGE_SHA"
-          docker push "$IMAGE_LATEST"
-          echo "CIXING_IMAGE=$IMAGE_SHA" >> "$GITHUB_ENV"
+          docker pull --platform linux/amd64 "$POSTGRES_IMAGE"
+          docker pull --platform linux/amd64 "$REDIS_IMAGE"
+          docker pull --platform linux/amd64 "$MAILPIT_IMAGE"
+
+      - name: Save deployment images
+        run: |
+          set -eu
+          mkdir -p /tmp/cixing-deploy/images
+          docker save "$CIXING_IMAGE" | gzip > /tmp/cixing-deploy/images/cixing-api.tar.gz
+          docker save "$POSTGRES_IMAGE" | gzip > /tmp/cixing-deploy/images/postgres.tar.gz
+          docker save "$REDIS_IMAGE" | gzip > /tmp/cixing-deploy/images/redis.tar.gz
+          docker save "$MAILPIT_IMAGE" | gzip > /tmp/cixing-deploy/images/mailpit.tar.gz
 
       - name: Prepare deployment files
         env:
@@ -71,6 +76,9 @@ jobs:
           printf '%s' "$TEST_CONFIG_YAML" > /tmp/cixing-deploy/config.yaml
           {
             echo "CIXING_IMAGE=${CIXING_IMAGE}"
+            echo "POSTGRES_IMAGE=${POSTGRES_IMAGE}"
+            echo "REDIS_IMAGE=${REDIS_IMAGE}"
+            echo "MAILPIT_IMAGE=${MAILPIT_IMAGE}"
             echo "POSTGRES_PASSWORD=${TEST_POSTGRES_PASSWORD}"
             echo "API_PORT=8080"
             echo "MAILPIT_UI_PORT=8025"
@@ -87,43 +95,175 @@ jobs:
           chmod 600 ~/.ssh/id_test_server
           ssh-keyscan -H "$TEST_SERVER_HOST" >> ~/.ssh/known_hosts
 
-      - name: Upload deployment files
+      - name: Upload deployment assets
         env:
           TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }}
           TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }}
         run: |
           set -eu
           ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" \
-            "sudo mkdir -p /opt/cixing/config && sudo chown -R $TEST_SERVER_USERNAME /opt/cixing"
+            "sudo mkdir -p /opt/cixing/config /opt/cixing/images && sudo chown -R $TEST_SERVER_USERNAME /opt/cixing"
           scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/docker-compose.yml \
             "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/docker-compose.yml"
           scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/deploy.env \
             "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/deploy.env"
           scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/config.yaml \
             "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/config/config.yaml"
+          scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/images/cixing-api.tar.gz \
+            "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/cixing-api.tar.gz"
+          scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/images/postgres.tar.gz \
+            "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/postgres.tar.gz"
+          scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/images/redis.tar.gz \
+            "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/redis.tar.gz"
+          scp -i ~/.ssh/id_test_server /tmp/cixing-deploy/images/mailpit.tar.gz \
+            "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST:/opt/cixing/images/mailpit.tar.gz"
 
       - name: Deploy on ECS
         env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
           TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }}
           TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }}
         run: |
           set -eu
-          echo "$DOCKERHUB_TOKEN" | ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" \
-            "if docker info >/dev/null 2>&1; then docker login docker.io -u '$DOCKERHUB_USERNAME' --password-stdin; else sudo docker login docker.io -u '$DOCKERHUB_USERNAME' --password-stdin; fi"
           ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" <<'REMOTE'
             set -eu
             cd /opt/cixing
             DOCKER="docker"
             if ! docker info >/dev/null 2>&1; then
               DOCKER="sudo docker"
             fi
-            chmod 600 /opt/cixing/deploy.env /opt/cixing/config/config.yaml
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml pull
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml up -d postgres redis mailpit
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml run --rm migrate up
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml up -d api
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml ps
-            $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml logs --tail=80 api
+            compose() {
+              $DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml "$@"
+            }
+            cleanup_existing_compose_state() {
+              echo "cleaning existing compose state"
+              compose down --remove-orphans 2>/dev/null || true
+              legacy_container_ids="$($DOCKER ps -aq --filter label=com.docker.compose.project=cixing-prod)"
+              if [ -n "$legacy_container_ids" ]; then
+                $DOCKER rm -f $legacy_container_ids >/dev/null 2>&1 || true
+              fi
+              legacy_network_ids="$($DOCKER network ls -q --filter label=com.docker.compose.project=cixing-prod)"
+              if [ -n "$legacy_network_ids" ]; then
+                $DOCKER network rm $legacy_network_ids >/dev/null 2>&1 || true
+              fi
+              if command -v ss >/dev/null 2>&1; then
+                if ss -lntp 2>/dev/null | grep -q ':8080 '; then
+                  echo "port 8080 is still occupied before deployment" >&2
+                  ss -lntp 2>/dev/null | grep ':8080 ' >&2 || true
+                  exit 1
+                fi
+              fi
+            }
+            dump_api_diagnostics() {
+              compose ps
+              compose logs --tail=200 api
+            }
+            cleanup_deployment_artifacts() {
+              echo "cleaning deployment artifacts"
+              rm -f /opt/cixing/images/*.tar.gz || true
+              current_image="$(awk -F= '$1=="CIXING_IMAGE"{print $2}' /opt/cixing/deploy.env)"
+              if [ -z "$current_image" ]; then
+                return 0
+              fi
+              for image_ref in $($DOCKER images --format '{{.Repository}}:{{.Tag}}' | grep '^cixing-api:' | grep -F -x -v "$current_image" || true); do
+                $DOCKER rmi "$image_ref" >/dev/null 2>&1 || true
+              done
+            }
+            api_ready() {
+              api_container_id="$(compose ps -q api)"
+              if [ -z "$api_container_id" ]; then
+                api_state="missing"
+                api_health="missing"
+                return 1
+              fi
+              if ! api_state="$($DOCKER inspect --format '{{.State.Status}}' "$api_container_id" 2>/dev/null)"; then
+                api_container_id=""
+                api_state="missing"
+                api_health="missing"
+                return 1
+              fi
+              if ! api_health="$($DOCKER inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$api_container_id" 2>/dev/null)"; then
+                api_container_id=""
+                api_state="missing"
+                api_health="missing"
+                return 1
+              fi
+              if [ "$api_state" != "running" ] || [ "$api_health" != "healthy" ]; then
+                return 1
+              fi
+              if ! $DOCKER exec "$api_container_id" /bin/sh -c 'wget -qO- http://127.0.0.1:8080/healthz >/dev/null'; then
+                api_health="healthz_failed"
+                return 1
+              fi
+              return 0
+            }
+            chmod 600 /opt/cixing/deploy.env
+            chmod 644 /opt/cixing/config/config.yaml
+            cleanup_existing_compose_state
+            gzip -dc /opt/cixing/images/cixing-api.tar.gz | $DOCKER load
+            gzip -dc /opt/cixing/images/postgres.tar.gz | $DOCKER load
+            gzip -dc /opt/cixing/images/redis.tar.gz | $DOCKER load
+            gzip -dc /opt/cixing/images/mailpit.tar.gz | $DOCKER load
+            echo "starting dependency services"
+            compose up -d postgres redis mailpit
+            echo "running database migrations"
+            compose run --rm -T migrate up </dev/null
+            echo "database migrations completed"
+            echo "starting api service"
+            compose up -d api
+
+            attempts=0
+            api_state=""
+            api_health=""
+            echo "waiting for api readiness"
+            while [ "$attempts" -lt 30 ]; do
+              if api_ready; then
+                break
+              fi
+              if [ "$api_state" = "exited" ] || [ "$api_state" = "dead" ]; then
+                break
+              fi
+              attempts=$((attempts + 1))
+              sleep 4
+            done
+
+            if ! api_ready; then
+              dump_api_diagnostics
+              if [ "$api_health" = "healthz_failed" ]; then
+                echo "container /healthz check failed" >&2
+              elif [ "$api_state" = "missing" ]; then
+                echo "api container was not created" >&2
+              else
+                echo "api container state=$api_state health=$api_health" >&2
+              fi
+              exit 1
+            fi
+
+            echo "api initial readiness check passed: container=$api_container_id state=$api_state health=$api_health"
+            echo "rechecking api stability after warmup"
+            sleep 12
+
+            if ! api_ready; then
+              dump_api_diagnostics
+              if [ "$api_health" = "healthz_failed" ]; then
+                echo "api container became unstable after initial readiness: /healthz failed" >&2
+              elif [ "$api_state" = "missing" ]; then
+                echo "api container disappeared after initial readiness" >&2
+              else
+                echo "api container became unstable after initial readiness: state=$api_state health=$api_health" >&2
+              fi
+              exit 1
+            fi
+
+            final_api_container_id="$(compose ps -q api)"
+            if [ -z "$final_api_container_id" ]; then
+              dump_api_diagnostics
+              echo "api container disappeared before final status output" >&2
+              exit 1
+            fi
+            final_api_state="$($DOCKER inspect --format '{{.State.Status}}' "$final_api_container_id")"
+            final_api_health="$($DOCKER inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$final_api_container_id")"
+            echo "final api status: container=$final_api_container_id state=$final_api_state health=$final_api_health"
+            cleanup_deployment_artifacts
+            compose ps
+            compose logs --tail=80 api
           REMOTE