Hubcap Scheduler #5093
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hubcap Scheduler | |
| permissions: | |
| contents: read | |
| on: | |
| # TODO: Re-enable when you move from heroku to github actions | |
| schedule: | |
| # Run every hour at :00 for production | |
| - cron: '0 * * * *' | |
| pull_request: | |
| # Run on PRs in dry-run mode for testing | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Environment to run against' | |
| required: true | |
| default: 'test' | |
| type: choice | |
| options: | |
| - 'test' | |
| - 'production' | |
| dry_run: | |
| description: 'Run in dry-run mode (no PRs created)' | |
| required: false | |
| default: 'true' | |
| type: choice | |
| options: | |
| - 'true' | |
| - 'false' | |
| jobs: | |
| hubcap: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| environment: ${{ github.event.inputs.environment || (github.event_name == 'pull_request' && 'test') || 'production' }} | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Install dependencies with uv | |
| run: | | |
| uv sync | |
| # Install dev requirements for PR testing | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| uv sync --extra test | |
| fi | |
| - name: Install dbt fusion | |
| run: | | |
| curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update | |
| - name: Configure environment | |
| env: | |
| HUBCAP_CONFIG: ${{ secrets.HUBCAP_CONFIG }} | |
| INPUT_DRY_RUN: ${{ github.event.inputs.dry_run || (github.event_name == 'pull_request' && 'true') || 'false' }} | |
| ENVIRONMENT: ${{ github.event.inputs.environment || (github.event_name == 'pull_request' && 'test') || 'production' }} | |
| FUSION_BINARY: '/home/runner/.local/bin/dbt' | |
| run: | | |
| echo "Using $ENVIRONMENT environment configuration" | |
| # Create config based on event type | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| # Use test config for PR events (secrets may not be available for forks) | |
| cat > base_config.json << EOF | |
| { | |
| "org": "dbt-labs", | |
| "repo": "hub.getdbt.com-test", | |
| "push_branches": false, | |
| "one_branch_per_repo": true, | |
| "user": { | |
| "name": "github-actions", | |
| "email": "github-actions@github.com", | |
| "token": "dummy-token" | |
| } | |
| } | |
| EOF | |
| else | |
| # Use production config from secrets | |
| echo "$HUBCAP_CONFIG" > base_config.json | |
| fi | |
| # Add Fusion binary path from env var | |
| jq --arg fusion_path "$FUSION_BINARY" '.fusion_binary_path = $fusion_path' base_config.json > base_config_fusion.json | |
| # Modify config for dry run if requested | |
| if [ "$INPUT_DRY_RUN" = "true" ]; then | |
| echo "Enabling dry-run mode (push_branches = false)" | |
| jq '.push_branches = false' base_config_fusion.json > config.json | |
| else | |
| echo "Running in live mode (push_branches = true)" | |
| cp base_config_fusion.json config.json | |
| fi | |
| # Set the CONFIG environment variable for hubcap.py | |
| echo "CONFIG<<EOF" >> $GITHUB_ENV | |
| cat config.json >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Run hubcap | |
| run: | | |
| echo "Starting hubcap execution..." | |
| uv run hubcap 2>&1 | tee hubcap.log | |
| - name: Check execution results | |
| if: always() | |
| run: | | |
| echo "=== Execution Summary ===" | |
| # Check for errors | |
| # Space after ERROR differentiates error logs from parse errors | |
| if grep -q 'ERROR ' hubcap.log; then | |
| echo "❌ Found ERROR events in execution:" | |
| grep 'ERROR ' hubcap.log | |
| echo "EXECUTION_STATUS=failed" >> $GITHUB_ENV | |
| else | |
| echo "✅ No ERROR-level log entries found" | |
| fi | |
| # Check for successful completion | |
| if grep -q 'hubcap execution completed successfully' hubcap.log; then | |
| echo "✅ Hubcap completed successfully" | |
| echo "EXECUTION_STATUS=success" >> $GITHUB_ENV | |
| elif grep -q 'No packages have new versions to update' hubcap.log; then | |
| echo "✅ Hubcap completed successfully (no updates needed)" | |
| echo "EXECUTION_STATUS=success" >> $GITHUB_ENV | |
| else | |
| echo "❌ Hubcap did not complete successfully" | |
| echo "EXECUTION_STATUS=failed" >> $GITHUB_ENV | |
| fi | |
| # Summary of what was processed | |
| if grep -q 'Pushing branches:' hubcap.log; then | |
| echo "📝 Branches processed:" | |
| grep 'Pushing branches:' hubcap.log | |
| fi | |
| - name: Upload execution artifacts | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 | |
| with: | |
| name: hubcap-execution-${{ github.event.inputs.environment || 'production' }}-${{ github.run_number }} | |
| path: hubcap.log | |
| retention-days: 30 | |
| - name: Fail job if execution failed | |
| if: always() && env.EXECUTION_STATUS == 'failed' | |
| run: | | |
| echo "❌ Hubcap execution failed - check logs above" | |
| exit 1 |