WWF discovery #474
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: WWF discovery | |
| on: | |
| schedule: | |
| - cron: "40 * * * *" | |
| workflow_dispatch: | |
| inputs: | |
| audit_only: | |
| description: "Write fidelity artifacts without OCR or submission" | |
| required: true | |
| type: boolean | |
| default: true | |
| concurrency: | |
| group: pipeline-trigger | |
| cancel-in-progress: false | |
| jobs: | |
| discover-wwf: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout scheduler repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| cache: pip | |
| cache-dependency-path: requirements-ocr-worker.txt | |
| - name: Install OCR worker dependencies | |
| run: pip install -r requirements-ocr-worker.txt | |
| - name: Run WWF discovery audit | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.audit_only }} | |
| run: python scripts/discover_wwf_candidates.py --audit-dir artifacts/wwf | |
| env: | |
| WWF_MIN_NOTICE_YEAR: "2026" | |
| WWF_MAX_CANDIDATES_PER_RUN: "50" | |
| WWF_MAX_DETAILS_PER_RUN: "20" | |
| WWF_FETCH_MAX_ATTEMPTS: "3" | |
| WWF_FETCH_BACKOFF_SECONDS: "2" | |
| WWF_FETCH_TIMEOUT_SECONDS: "30" | |
| - name: Upload WWF fidelity artifacts | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.audit_only && always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wwf-fidelity-${{ github.run_id }} | |
| path: artifacts/wwf | |
| if-no-files-found: error | |
| - name: Run WWF discovery, download, OCR, and submission | |
| if: ${{ github.event_name != 'workflow_dispatch' || !inputs.audit_only }} | |
| run: python scripts/discover_wwf_candidates.py | |
| env: | |
| RENDER_APP_URL: ${{ secrets.RENDER_APP_URL }} | |
| PIPELINE_SECRET: ${{ secrets.PIPELINE_SECRET }} | |
| WWF_MIN_NOTICE_YEAR: "2026" | |
| WWF_MAX_CANDIDATES_PER_RUN: "50" | |
| WWF_MAX_DETAILS_PER_RUN: "20" | |
| WWF_FETCH_MAX_ATTEMPTS: "3" | |
| WWF_FETCH_BACKOFF_SECONDS: "2" | |
| WWF_FETCH_TIMEOUT_SECONDS: "30" | |
| SCRAPE_MAX_PDF_BYTES: "15000000" | |
| KREUZBERG_PADDLE_LANGUAGE: "latin" | |
| KREUZBERG_PADDLE_MODEL_TIER: "tiny" | |
| KREUZBERG_USE_GPU: "false" | |
| KREUZBERG_FORCE_OCR_DEFAULT: "false" | |
| KREUZBERG_EXTRACTION_TIMEOUT_SECONDS: "120" | |
| FLAGS_use_mkldnn: "0" | |
| PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT: "0" |