feat: cross-CIK predecessor mapping for SEC filing history (#49) (#118) #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to Dagster OSS on GCP | |
| 'on': | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Environment to deploy to' | |
| required: true | |
| default: 'production' | |
| type: choice | |
| options: | |
| - production | |
| - staging | |
| env: | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| GCP_ZONE: ${{ secrets.GCP_ZONE }} | |
| VM_NAME: ${{ secrets.GCP_VM_NAME }} | |
| jobs: | |
| deploy: | |
| if: false # Temporarily disabled - remove this line when ready to deploy | |
| name: Deploy Dagster OSS | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| - name: Prepare deployment files | |
| run: | | |
| echo "Preparing deployment package..." | |
| mkdir -p deploy_temp | |
| # Copy Docker and Dagster configuration files | |
| cp docker-compose.yml deploy_temp/ | |
| cp dagster.yaml deploy_temp/ | |
| cp workspace.yaml deploy_temp/ | |
| # Copy macro_agents code | |
| cp -r macro_agents deploy_temp/ | |
| # Copy dbt_project (needed for dbt assets) | |
| cp -r dbt_project deploy_temp/ | |
| echo "Deployment package prepared" | |
| - name: Copy files to GCP VM | |
| run: | | |
| echo "Copying files to VM: $VM_NAME in zone $GCP_ZONE..." | |
| # Create dagster directory on VM if it doesn't exist | |
| gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='mkdir -p ~/dagster' | |
| # Copy all files | |
| gcloud compute scp --recurse deploy_temp/* $VM_NAME:~/dagster/ --zone=$GCP_ZONE | |
| echo "Files copied successfully" | |
| - name: Deploy on GCP VM | |
| run: | | |
| echo "Deploying Dagster OSS on VM..." | |
| gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command=' | |
| cd ~/dagster | |
| # Pull latest base images | |
| docker compose pull dagster_webserver dagster_daemon | |
| # Stop existing containers | |
| docker compose down | |
| # Build user code image | |
| docker compose build dagster_user_code | |
| # Start all services | |
| docker compose up -d | |
| # Wait for services to start | |
| sleep 15 | |
| # Check container status | |
| echo "Container status:" | |
| docker compose ps | |
| # Check for any failed containers | |
| if docker compose ps | grep -q "Exit"; then | |
| echo "ERROR: Some containers failed to start" | |
| docker compose logs | |
| exit 1 | |
| fi | |
| ' | |
| echo "Deployment completed" | |
| - name: Health check | |
| # Probe the Dagster webserver from inside the VM via SSH rather than | |
| # hitting an external IP over plain HTTP. The Terraform firewall | |
| # rejects external traffic on port 3000 (access is via IAP), so the | |
| # old external-IP curl was both insecure and bypassed that intent. | |
| run: | | |
| echo "Performing health check via SSH..." | |
| MAX_RETRIES=10 | |
| RETRY_COUNT=0 | |
| sleep 30 | |
| while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do | |
| if gcloud compute ssh "$VM_NAME" --zone="$GCP_ZONE" --command='curl -f -s -o /dev/null http://localhost:3000'; then | |
| echo "Dagster UI is healthy (responding on localhost:3000 inside VM)" | |
| exit 0 | |
| fi | |
| echo "Attempt $((RETRY_COUNT + 1))/$MAX_RETRIES: Dagster UI not ready yet..." | |
| RETRY_COUNT=$((RETRY_COUNT + 1)) | |
| sleep 10 | |
| done | |
| echo "Health check failed - Dagster UI not accessible after $MAX_RETRIES attempts" | |
| gcloud compute ssh "$VM_NAME" --zone="$GCP_ZONE" --command='docker compose logs --tail=100' | |
| exit 1 | |
| - name: Verify deployment | |
| run: | | |
| echo "Verifying Dagster services..." | |
| gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command=' | |
| cd ~/dagster | |
| echo "=== Container Health ===" | |
| docker compose ps | |
| echo "" | |
| echo "=== PostgreSQL Status ===" | |
| docker compose exec -T dagster_postgresql pg_isready -U dagster_user || echo "PostgreSQL not ready" | |
| echo "" | |
| echo "=== Recent Webserver Logs ===" | |
| docker compose logs --tail=20 dagster_webserver | |
| echo "" | |
| echo "=== Recent Daemon Logs ===" | |
| docker compose logs --tail=20 dagster_daemon | |
| ' | |
| - name: Notify on failure | |
| if: failure() | |
| run: | | |
| echo "Deployment failed! Check the logs above for details." | |
| echo "SSH command: gcloud compute ssh $VM_NAME --zone=$GCP_ZONE" | |
| - name: Notify on success | |
| if: success() | |
| run: | | |
| echo "Deployment successful." | |
| echo "Dagster UI is exposed via IAP (not direct external IP)." | |
| echo "To view logs: gcloud compute ssh $VM_NAME --zone=$GCP_ZONE --command='cd ~/dagster && docker compose logs -f'" |