Update All Data (Optimized) #61
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update All Data (Optimized) | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 3 */4 * *" | |
| concurrency: | |
| group: "data-update" | |
| cancel-in-progress: false | |
| jobs: | |
| update-all: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| actions: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # ===== Node.js Setup with Caching ===== | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v3 | |
| with: | |
| version: 8 | |
| - name: Install Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Install wrangler | |
| run: pnpm i -D wrangler@latest | |
| # ===== Python Setup with Caching ===== | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| cache-dependency-path: "scripts/requirements.txt" | |
| - name: Install Python dependencies | |
| run: | | |
| cd scripts | |
| pip install -r requirements.txt | |
| pip install requests | |
| # ===== Data Update Steps ===== | |
| - name: Fetch data, Update D1, and Generate Recent Updates | |
| run: pnpm run update-data-remote | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| - name: Update Typesense Index | |
| env: | |
| TYPESENSE_API_KEY: ${{ secrets.PRIVATE_TYPESENSE_API_KEY }} | |
| TYPESENSE_HOST: ${{ secrets.TYPESENSE_HOST }} | |
| TYPESENSE_PROTOCOL: ${{ secrets.TYPESENSE_PROTOCOL }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| run: | | |
| echo "🚀 Starting Typesense update..." | |
| if [ -f "src/lib/data/data.csv" ]; then | |
| python3 scripts/search_sync.py src/lib/data/data.csv "$TYPESENSE_PROTOCOL" "$TYPESENSE_HOST" "$TYPESENSE_API_KEY" | |
| echo "✅ Typesense update completed" | |
| else | |
| echo "❌ src/lib/data/data.csv not found!" | |
| exit 1 | |
| fi | |
| # ===== R Setup with Caching and Pre-compiled Binaries ===== | |
| - name: Setup R | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: "4.3.0" | |
| use-public-rspm: true # USE PRE-COMPILED BINARIES! | |
| - name: Setup Pandoc | |
| uses: r-lib/actions/setup-pandoc@v2 | |
| - name: Cache R packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.R_LIBS_USER }} | |
| key: ${{ runner.os }}-r-4.3.0-tidyverse-${{ hashFiles('analysis/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-r-4.3.0-tidyverse- | |
| ${{ runner.os }}-r-4.3.0- | |
| - name: Install R dependencies | |
| run: | | |
| cd analysis | |
| # Use Posit Package Manager for pre-compiled Linux binaries | |
| R -e "install.packages(c('rmarkdown', 'tidyverse', 'jsonlite', 'here'), repos='https://packagemanager.posit.co/cran/__linux__/jammy/latest')" | |
| - name: Build Notebook | |
| run: pnpm run build-notebook | |
| # ===== Commit Changes ===== | |
| - name: Commit and Push Changes | |
| id: commit | |
| run: | | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add src/lib/data/data.csv static/summary_stats.json static/current_movies.json static/notebook/index.html src/lib/data/charts/*.json | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| git commit -m "chore: update data [skip ci]" | |
| git push | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Trigger production deploy | |
| if: steps.commit.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh workflow run cf-pages-publish.yaml --ref ${{ github.ref_name }} |