Alt Sources Dry Run #538
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Alt Sources Dry Run | |
| on: | |
| schedule: | |
| - cron: '0 2 * * *' # Supplemental early-morning verification | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: 'Skip publishing (always true for this workflow)' | |
| required: false | |
| default: 'true' | |
| force_publish: | |
| description: 'Force publish even if sorteo/fecha and amounts are unchanged' | |
| required: false | |
| default: 'false' | |
| jobs: | |
| dry-run: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements.txt' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Restore pipeline state | |
| id: restore-state | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: pipeline_state | |
| key: dry-run-state-${{ github.ref_name }}-${{ github.run_number }} | |
| restore-keys: | | |
| dry-run-state-${{ github.ref_name }}- | |
| - name: Execute dry-run pipeline (All sources) | |
| env: | |
| ALT_SOURCE_URLS: ${{ vars.ALT_SOURCE_URLS }} | |
| GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} | |
| run: | | |
| mkdir -p artifacts pipeline_state logs | |
| FORCE_PUBLISH="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.force_publish || 'false' }}" | |
| if [ "${FORCE_PUBLISH}" = "true" ]; then FORCE_FLAG="--force-publish"; else FORCE_FLAG=""; fi | |
| python -m polla_app run \ | |
| --sources pozos \ | |
| --retries 2 \ | |
| --timeout 25 \ | |
| --no-fail-fast \ | |
| --raw-dir artifacts/raw \ | |
| --normalized artifacts/normalized.jsonl \ | |
| --comparison-report artifacts/comparison_report.json \ | |
| --summary artifacts/run_summary.json \ | |
| --state-file pipeline_state/last_run.jsonl \ | |
| --log-file logs/run.jsonl \ | |
| --mismatch-threshold 0.3 \ | |
| --include-pozos \ | |
| ${FORCE_FLAG} | |
| python -m polla_app publish \ | |
| --normalized artifacts/normalized.jsonl \ | |
| --comparison-report artifacts/comparison_report.json \ | |
| --summary artifacts/run_summary.json \ | |
| --worksheet "Normalized" \ | |
| --discrepancy-tab "Discrepancies" \ | |
| --dry-run | |
| - name: Save pipeline state | |
| uses: actions/cache/save@v3 | |
| if: always() | |
| with: | |
| path: pipeline_state | |
| key: dry-run-state-${{ github.ref_name }}-${{ github.run_number }} | |
| - name: Upload dry-run artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dry-run-artifacts | |
| path: | | |
| artifacts/ | |
| logs/ | |