pipeline.yml

name: Daily Job Market Pipeline

on:
  schedule:
    - cron: "0 14 * * *"   # 14:00 UTC = 7:00 AM PST / 10:00 AM EST
  workflow_dispatch:

jobs:
  run-pipeline:
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Download existing DB from HuggingFace
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          HF_REPO: ${{ secrets.HF_REPO }}
        run: python scripts/download_db.py

      - name: Run pipeline
        env:
          ADZUNA_APP_ID: ${{ secrets.ADZUNA_APP_ID }}
          ADZUNA_APP_KEY: ${{ secrets.ADZUNA_APP_KEY }}
        run: python run_pipeline.py

      - name: Update README with today's market stats
        run: python scripts/update_readme.py

      - name: Upload updated DB to HuggingFace
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          HF_REPO: ${{ secrets.HF_REPO }}
        run: python scripts/upload_db.py

      - name: Commit updated README
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add README.md
          git diff --staged --quiet || git commit -m "auto: market report $(date -u '+%Y-%m-%d')"
          git push