dagster-oss-deploy.yml

name: Deploy to Dagster OSS on GCP

'on':
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        default: 'production'
        type: choice
        options:
          - production
          - staging

env:
  GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
  GCP_ZONE: ${{ secrets.GCP_ZONE }}
  VM_NAME: ${{ secrets.GCP_VM_NAME }}

jobs:
  deploy:
    if: false  # Temporarily disabled - remove this line when ready to deploy
    name: Deploy Dagster OSS
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd  # v5
        with:
          fetch-depth: 1

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed  # v2
        with:
          credentials_json: ${{ secrets.GCP_SA_KEY }}

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f  # v2
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}

      - name: Prepare deployment files
        run: |
          echo "Preparing deployment package..."
          mkdir -p deploy_temp

          # Copy Docker and Dagster configuration files
          cp docker-compose.yml deploy_temp/
          cp dagster.yaml deploy_temp/
          cp workspace.yaml deploy_temp/

          # Copy macro_agents code
          cp -r macro_agents deploy_temp/

          # Copy dbt_project (needed for dbt assets)
          cp -r dbt_project deploy_temp/

          echo "Deployment package prepared"

      - name: Copy files to GCP VM
        run: |
          echo "Copying files to VM: $VM_NAME in zone $GCP_ZONE..."

          # Create dagster directory on VM if it doesn't exist
          gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='mkdir -p ~/dagster'

          # Copy all files
          gcloud compute scp --recurse deploy_temp/* $VM_NAME:~/dagster/ --zone=$GCP_ZONE

          echo "Files copied successfully"

      - name: Deploy on GCP VM
        run: |
          echo "Deploying Dagster OSS on VM..."

          gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='
            cd ~/dagster

            # Pull latest base images
            docker compose pull dagster_webserver dagster_daemon

            # Stop existing containers
            docker compose down

            # Build user code image
            docker compose build dagster_user_code

            # Start all services
            docker compose up -d

            # Wait for services to start
            sleep 15

            # Check container status
            echo "Container status:"
            docker compose ps

            # Check for any failed containers
            if docker compose ps | grep -q "Exit"; then
              echo "ERROR: Some containers failed to start"
              docker compose logs
              exit 1
            fi
          '

          echo "Deployment completed"

      - name: Health check
        # Probe the Dagster webserver from inside the VM via SSH rather than
        # hitting an external IP over plain HTTP. The Terraform firewall
        # rejects external traffic on port 3000 (access is via IAP), so the
        # old external-IP curl was both insecure and bypassed that intent.
        run: |
          echo "Performing health check via SSH..."
          MAX_RETRIES=10
          RETRY_COUNT=0
          sleep 30

          while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
            if gcloud compute ssh "$VM_NAME" --zone="$GCP_ZONE" --command='curl -f -s -o /dev/null http://localhost:3000'; then
              echo "Dagster UI is healthy (responding on localhost:3000 inside VM)"
              exit 0
            fi
            echo "Attempt $((RETRY_COUNT + 1))/$MAX_RETRIES: Dagster UI not ready yet..."
            RETRY_COUNT=$((RETRY_COUNT + 1))
            sleep 10
          done

          echo "Health check failed - Dagster UI not accessible after $MAX_RETRIES attempts"
          gcloud compute ssh "$VM_NAME" --zone="$GCP_ZONE" --command='docker compose logs --tail=100'
          exit 1

      - name: Verify deployment
        run: |
          echo "Verifying Dagster services..."

          gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='
            cd ~/dagster

            echo "=== Container Health ==="
            docker compose ps

            echo ""
            echo "=== PostgreSQL Status ==="
            docker compose exec -T dagster_postgresql pg_isready -U dagster_user || echo "PostgreSQL not ready"

            echo ""
            echo "=== Recent Webserver Logs ==="
            docker compose logs --tail=20 dagster_webserver

            echo ""
            echo "=== Recent Daemon Logs ==="
            docker compose logs --tail=20 dagster_daemon
          '

      - name: Notify on failure
        if: failure()
        run: |
          echo "Deployment failed! Check the logs above for details."
          echo "SSH command: gcloud compute ssh $VM_NAME --zone=$GCP_ZONE"

      - name: Notify on success
        if: success()
        run: |
          echo "Deployment successful."
          echo "Dagster UI is exposed via IAP (not direct external IP)."
          echo "To view logs: gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='cd ~/dagster && docker compose logs -f'"