quarto-publish.yml

# ╔══════════════════════════════════════════════════════════════════════╗
# ║  Github Page – Optimized Build & Deploy                             ║
# ║                                                                      ║
# ║  Key optimizations over the original workflow:                       ║
# ║  1. FREEZE CACHE FIX: babelquarto renders in a temp dir and only    ║
# ║     copies _site back – the _freeze it creates is discarded. We     ║
# ║     pre-render all non-frozen files first so that workspace _freeze  ║
# ║     is fully populated before babelquarto copies it to its temp dir. ║
# ║     This turns a ~90 min render into ~20 min on subsequent runs.    ║
# ║  2. CHANGED-FILE DETECTION: Detects which QMDs changed and only     ║
# ║     pre-renders those files, enabling fail-fast (catch errors in     ║
# ║     ~2 min instead of ~90 min).                                      ║
# ║  3. CONDITIONAL JULIA SETUP: Skips the ~9 min Julia installation    ║
# ║     when freeze already covers Julia files and none changed.         ║
# ║  4. SMART CONCURRENCY: Groups by PR number or branch so updated     ║
# ║     pushes cancel stale in-progress builds.                          ║
# ║  5. RESOURCE CONSERVATION: Failed builds still save their freeze    ║
# ║     progress, so the next run picks up where it left off.            ║
# ╚══════════════════════════════════════════════════════════════════════╝

on:
  push:
    branches: [main, master]
    paths-ignore:
      - '*.md'
      - 'LICENSE'
      - 'scripts/**'
      - '.github/scripts/**'
      - '.github/workflows/auto-translate.yml'
      - '.github/workflows/compress-images.yml'
      - '.github/workflows/generate-skills.yml'
      - '.github/workflows/skill-command.yml'
      - '.github/workflows/translate-command.yml'
      - '**/*.zh.qmd'
  pull_request:
    branches: [main, master]
    paths-ignore:
      - '*.md'
  workflow_dispatch:
    inputs:
      force_full_rebuild:
        description: 'Force full rebuild (delete freeze cache and re-execute all files)'
        type: boolean
        default: false

name: Github Page

env:
  RUST_BACKTRACE: 1

jobs:
  build-deploy:
    runs-on: ubuntu-latest
    timeout-minutes: 180
    # Group by PR number for PRs, by branch ref for pushes.
    # New pushes/commits cancel the previous in-progress build in the same group.
    concurrency:
      group: quarto-build-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
      cancel-in-progress: true
    env:
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
      TMP_DIR: /mnt/TMP
    permissions:
      contents: write
      pages: write
      id-token: write
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # ── Detect changed files ─────────────────────────────────
      # Determines which QMDs changed so we can:
      #  (a) pre-render only those files (fail-fast)
      #  (b) skip Julia/Python setup when unnecessary
      - name: Detect changed QMD files
        id: changes
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE_SHA="${{ github.event.pull_request.base.sha }}"
          elif [ "${{ github.event_name }}" = "push" ]; then
            BASE_SHA="${{ github.event.before }}"
          else
            BASE_SHA=""
          fi

          # Handle first push, force push, or manual dispatch
          if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then
            echo "changed_qmds=" >> "$GITHUB_OUTPUT"
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
            echo "need_julia=true" >> "$GITHUB_OUTPUT"
            echo "need_python=true" >> "$GITHUB_OUTPUT"
            echo "ℹ️ No base SHA available – treating as full build"
            exit 0
          fi

          CHANGED_QMDS=$(git diff --name-only --diff-filter=ACMR "$BASE_SHA" HEAD -- '*.qmd' 2>/dev/null | tr '\n' ' ' || echo "")
          echo "changed_qmds=$CHANGED_QMDS" >> "$GITHUB_OUTPUT"

          if [ -n "$CHANGED_QMDS" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          fi

          JULIA_CHANGED=$(echo "$CHANGED_QMDS" | grep -c 'Julia/' || true)
          PYTHON_CHANGED=$(echo "$CHANGED_QMDS" | grep -c 'Python/' || true)
          echo "need_julia=$( [ "$JULIA_CHANGED" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT"
          echo "need_python=$( [ "$PYTHON_CHANGED" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT"

          echo "📋 Changed QMDs: ${CHANGED_QMDS:-none}"
          echo "   Julia changes: $JULIA_CHANGED | Python changes: $PYTHON_CHANGED"

      # ── Restore freeze cache (CRITICAL for performance) ──────
      # The freeze cache stores pre-computed knitr outputs for all QMD files.
      # It is populated by the "Pre-render all non-frozen QMD files" step
      # (which writes directly to the workspace _freeze directory), NOT by
      # babelquarto itself (babelquarto renders in a temp dir and never
      # syncs _freeze back to the workspace).
      #
      # Key strategy:
      #   - Primary key includes commit SHA (unique per commit)
      #   - run_attempt in the key ensures retries create a new cache entry
      #     (cache entries are immutable). Retries still restore from the
      #     previous attempt via the restore-keys prefix match.
      #   - restore-keys prefix ensures we always get the latest cache
      #   - A separate save step (with if: always()) preserves progress
      #     even when the build fails partway through
      - name: Restore freeze cache
        id: freeze-restore
        if: inputs.force_full_rebuild != true
        uses: actions/cache/restore@v4
        with:
          path: _freeze
          key: freeze-${{ runner.os }}-${{ github.sha }}-${{ github.run_attempt }}
          restore-keys: |
            freeze-${{ runner.os }}-${{ github.sha }}-
            freeze-${{ runner.os }}-

      - name: Clear freeze for forced full rebuild
        if: inputs.force_full_rebuild == true
        run: rm -rf _freeze

      - name: Report freeze cache status
        id: freeze-check
        run: |
          if [ -d "_freeze" ]; then
            FREEZE_COUNT=$(find _freeze -name "*.json" -type f 2>/dev/null | wc -l)
            echo "✅ Freeze cache available: $FREEZE_COUNT frozen output(s)"
            echo "has_freeze=true" >> "$GITHUB_OUTPUT"

            # Check per-language coverage
            JULIA_FROZEN=$(find _freeze -path "*/Julia/*" -name "*.json" 2>/dev/null | wc -l)
            PYTHON_FROZEN=$(find _freeze -path "*/Python/*" -name "*.json" 2>/dev/null | wc -l)
            echo "julia_frozen=$JULIA_FROZEN" >> "$GITHUB_OUTPUT"
            echo "python_frozen=$PYTHON_FROZEN" >> "$GITHUB_OUTPUT"
            echo "   Julia frozen: $JULIA_FROZEN | Python frozen: $PYTHON_FROZEN"
          else
            echo "⚠️ No freeze cache – full execution required"
            echo "has_freeze=false" >> "$GITHUB_OUTPUT"
            echo "julia_frozen=0" >> "$GITHUB_OUTPUT"
            echo "python_frozen=0" >> "$GITHUB_OUTPUT"
          fi

      # ── Quarto Setup ────────────────────────────────────────
      - name: Set up Quarto
        uses: quarto-dev/quarto-actions/setup@v2
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tinytex: true

      # ── R Setup ──────────────────────────────────────────────
      - uses: r-lib/actions/setup-r@v2
        with:
          use-public-rspm: true

      - name: Cache R packages
        uses: actions/cache@v4
        with:
          path: |
            ~/.local/share/renv
            ~/.cache/R/renv
            /usr/local/lib/R/site-library
            ~/R
          key: r-packages-${{ runner.os }}-${{ hashFiles('DESCRIPTION') }}
          restore-keys: |
            r-packages-${{ runner.os }}-

      - name: Install system dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libcurl4-openssl-dev

      - uses: r-lib/actions/setup-r-dependencies@v2
        env:
          PKG_SYSREQS: "true"

      # ── Python Setup ─────────────────────────────────────────
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then
            pip install -r requirements.txt
          fi

      # ── Julia Setup (conditional) ────────────────────────────
      # Julia setup takes ~9 min. Skip it when:
      #  - No Julia QMDs changed AND freeze already covers Julia files
      # Always install when: Julia files changed, no freeze, or force rebuild.
      - name: Set up Julia
        id: setup-julia
        if: >-
          steps.changes.outputs.need_julia == 'true' ||
          steps.freeze-check.outputs.julia_frozen == '0' ||
          inputs.force_full_rebuild == true
        uses: julia-actions/setup-julia@v2
        with:
          version: '1'

      - name: Cache Julia packages
        if: steps.setup-julia.outcome == 'success'
        uses: actions/cache@v4
        with:
          path: ~/.julia
          key: julia-packages-${{ runner.os }}-${{ hashFiles('Julia/Project.toml') }}
          restore-keys: |
            julia-packages-${{ runner.os }}-

      - name: Install Julia packages
        if: steps.setup-julia.outcome == 'success'
        run: |
          # Find the QuartoNotebookRunner version required by this Quarto installation.
          # Quarto bundles its own julia-engine Project.toml that pins an exact QNR version.
          JULIA_ENGINE_PROJ=$(find /opt/quarto -name "Project.toml" -path "*/julia-engine/*" 2>/dev/null | head -1)
          QNR_VERSION=""
          if [ -n "$JULIA_ENGINE_PROJ" ]; then
            QNR_VERSION=$(grep -oP 'QuartoNotebookRunner = "=([^"]+)"' "$JULIA_ENGINE_PROJ" \
              | grep -oP '[0-9]+\.[0-9]+\.[0-9]+' || true)
            echo "Quarto requires QuartoNotebookRunner=${QNR_VERSION} (from ${JULIA_ENGINE_PROJ})"
          fi

          # Create a dedicated QNR environment.  Setting QUARTO_JULIA_PROJECT to this
          # path tells Quarto to skip ensureQuartoNotebookRunnerEnvironment (which calls
          # Pkg.update() and needs network at render time) and simply verify QNR loads.
          QNR_ENV="${HOME}/.julia/environments/quarto-julia-env"
          mkdir -p "${QNR_ENV}"

          if [ -n "$QNR_VERSION" ]; then
            julia --project="${QNR_ENV}" -e "
              using Pkg
              Pkg.add(name=\"QuartoNotebookRunner\", version=\"${QNR_VERSION}\")
              Pkg.instantiate()
              using QuartoNotebookRunner
              println(\"QuartoNotebookRunner \$(pkgversion(QuartoNotebookRunner)) ready\")
            "
          else
            julia --project="${QNR_ENV}" -e '
              using Pkg
              Pkg.add("QuartoNotebookRunner")
              Pkg.instantiate()
              using QuartoNotebookRunner
              println("QuartoNotebookRunner $(pkgversion(QuartoNotebookRunner)) ready")
            '
          fi

          # Export so the render step can pick it up.
          echo "QUARTO_JULIA_PROJECT=${QNR_ENV}" >> "$GITHUB_ENV"

          # Pre-instantiate the tutorial project environment (Julia/Project.toml).
          # QNR workers use JULIA_PROJECT=@. and cd into the notebook directory,
          # so they automatically pick up Julia/Project.toml for CairoMakie etc.
          julia --project=Julia -e '
            using Pkg
            Pkg.resolve()
            Pkg.instantiate()
            using CairoMakie, DataFrames, Statistics
            println("Tutorial packages ready")
          '

      # ── Dependency Scanning ──────────────────────────────────
      - name: Scan and update dependencies
        run: |
          Rscript update-dependencies.R
          python manage-dependencies.py --update

      - name: Commit updated dependency files
        if: github.event_name != 'pull_request'
        run: |
          git config --local user.name "$GITHUB_ACTOR"
          git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com"
          git add DESCRIPTION requirements.txt Julia/Project.toml
          git commit -m 'chore: update dependencies' || echo "No changes to commit"
          git push origin || echo "No changes to push"

      # ── Render ───────────────────────────────────────────────
      - name: Create temporary directory
        run: |
          sudo mkdir -p ${{ env.TMP_DIR }}
          sudo chown -R $USER:$USER ${{ env.TMP_DIR }}

      # Pre-render Julia notebooks BEFORE the R-heavy main render to avoid OOM.
      # babelquarto loads R + hundreds of packages (~1.5 GB); adding CairoMakie
      # (~1-2 GB peak) would exceed the runner's RAM limit.
      - name: Pre-render Julia notebooks
        if: steps.setup-julia.outcome == 'success'
        env:
          QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
          TMPDIR: ${{ env.TMP_DIR }}
          TMP: ${{ env.TMP_DIR }}
          TEMP: ${{ env.TMP_DIR }}
        run: |
          echo "Pre-rendering Julia tutorials (no R process = lower peak memory)..."
          failed=0
          for qmd in Julia/*.qmd; do
            echo "  → $qmd"
            if quarto render "$qmd" 2>&1; then
              echo "  ✓ OK: $qmd"
            else
              echo "  ✗ Failed: $qmd"
              failed=1
            fi
          done
          julia_log="${HOME}/.cache/quarto/julia/julia_server_log.txt"
          if [ -f "$julia_log" ]; then
            echo ""
            echo "=== QNR server log ==="
            tail -50 "$julia_log"
            echo "======================"
          fi
          [ $failed -eq 0 ] || exit 1
          echo "All Julia notebooks pre-rendered. Freeze cache populated."

      # Pre-render ONLY the changed QMD files for early failure detection.
      # If a changed file has a syntax error or broken dependency, this step
      # fails in ~2 min instead of waiting ~90 min for the full site render.
      # Successfully rendered files populate the freeze cache for the next step.
      - name: Pre-render changed QMD files (fail-fast)
        if: steps.changes.outputs.has_changes == 'true'
        env:
          CHANGED_QMDS: ${{ steps.changes.outputs.changed_qmds }}
          TMPDIR: ${{ env.TMP_DIR }}
          TMP: ${{ env.TMP_DIR }}
          TEMP: ${{ env.TMP_DIR }}
          QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
        run: |
          echo "Pre-rendering changed files for early failure detection..."
          for qmd in $CHANGED_QMDS; do
            # Julia files are pre-rendered separately (OOM prevention)
            if [[ "$qmd" == Julia/* ]]; then continue; fi
            if [ ! -f "$qmd" ]; then continue; fi

            echo "  → $qmd"
            quarto render "$qmd" 2>&1 || { echo "❌ FAILED: $qmd"; exit 1; }
            echo "  ✓ OK: $qmd"

            # Also render the bilingual counterpart so freeze covers both
            if [[ "$qmd" == *.zh.qmd ]]; then
              counterpart="${qmd%.zh.qmd}.qmd"
            else
              counterpart="${qmd%.qmd}.zh.qmd"
            fi
            if [ -f "$counterpart" ] && ! echo "$CHANGED_QMDS" | grep -qF "$counterpart"; then
              echo "  → $counterpart (bilingual counterpart)"
              quarto render "$counterpart" 2>&1 || { echo "❌ FAILED: $counterpart"; exit 1; }
              echo "  ✓ OK: $counterpart"
            fi
          done
          echo "✅ All changed files pre-rendered successfully."

      # ── Pre-render ALL non-frozen files (freeze-cache fix) ─────
      # ROOT CAUSE: babelquarto::render_website() renders in a temp dir
      # (withr::local_tempdir()), creates _freeze there, copies only _site
      # back to the workspace, and then deletes the temp dir. The workspace
      # _freeze is never updated, so the save step finds only the few
      # entries from the changed-file pre-render step above (e.g. 6 Julia
      # entries), not the 500+ entries from the full render.
      #
      # FIX: pre-render every non-frozen .qmd file here (workspace _freeze
      # gets populated), then babelquarto copies the full _freeze into its
      # temp dir and uses it – skipping all code execution and only running
      # pandoc, cutting the render from ~90 min down to ~20 min.
      #
      # Timing estimates:
      #   First run (cold cache):  pre-render all ~60–80 min +
      #                            babelquarto (pandoc only) ~20 min +
      #                            setup ~15 min  ≈ 95–115 min total
      #   Subsequent runs (warm):  pre-render 0 files + babelquarto ~20 min
      #                            + setup ~15 min  ≈ 35 min total
      # Timeout is 180 min to give first-run cold-cache builds a safe margin.
      - name: Pre-render all non-frozen QMD files
        env:
          TMPDIR: ${{ env.TMP_DIR }}
          TMP: ${{ env.TMP_DIR }}
          TEMP: ${{ env.TMP_DIR }}
          QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
        run: |
          echo "Pre-rendering non-frozen QMD files to populate workspace _freeze..."
          RENDERED=0
          SKIPPED=0
          FAILED=0

          while IFS= read -r -d '' qmd; do
            rel="${qmd#./}"

            # Julia files are handled by the dedicated Julia pre-render step
            [[ "$rel" == Julia/* ]] && { SKIPPED=$((SKIPPED+1)); continue; }

            # If a _freeze entry already exists for this file, skip it.
            # Quarto mirrors the source path: _freeze/<path-without-ext>/
            freeze_dir="_freeze/${rel%.qmd}"
            if [ -d "$freeze_dir" ]; then
              SKIPPED=$((SKIPPED+1))
              continue
            fi

            echo "  → $rel"
            if quarto render "$qmd" 2>&1; then
              RENDERED=$((RENDERED+1))
            else
              echo "  ⚠️ Warning: failed to render $rel (non-fatal)"
              FAILED=$((FAILED+1))
            fi
          done < <(find . -name "*.qmd" \
                       -not -path "./_freeze/*" \
                       -not -path "./_site/*" \
                       -not -path "./.quarto/*" \
                       -print0 | sort -z)

          echo ""
          echo "📊 Pre-render summary: $RENDERED rendered, $SKIPPED skipped (frozen/Julia), $FAILED failed"
          [ $FAILED -gt 0 ] && echo "::warning::$FAILED QMD file(s) failed to pre-render"
          # Count JSON outputs for a richer diagnostic (one per frozen code chunk)
          TOTAL_FROZEN=$(find _freeze -name "*.json" -type f 2>/dev/null | wc -l)
          echo "❄️  Freeze cache now has $TOTAL_FROZEN frozen output(s) (JSON entries)"

      # ── Generate skill ZIP ───────────────────────────────────
      # Must run BEFORE babelquarto so the zip exists at project root when
      # babelquarto copies the project to its temp dir for rendering.
      # Quarto then copies it to _site/ via the resources entry in _quarto.yml,
      # ensuring quarto publish gh-pages reliably includes it.
      - name: Generate unified skill ZIP package
        run: |
          # Stage files with correct names inside the zip
          SKILL_TMP=$(mktemp -d)
          cp SKILL.md "${SKILL_TMP}/SKILL.md"
          cp files/gallery_data.csv "${SKILL_TMP}/gallery_data.csv"
          cp files/gallery_data_zh.csv "${SKILL_TMP}/gallery_data_zh.csv"
          cp files/bizard-skill-readme.md "${SKILL_TMP}/README.md"
          (cd "${SKILL_TMP}" && zip -r - SKILL.md gallery_data.csv gallery_data_zh.csv README.md) > bizard-skill.zip
          rm -rf "${SKILL_TMP}"
          echo "✅ Created bizard-skill.zip at project root ($(du -h bizard-skill.zip | cut -f1))"

      - name: Render website with Babelquarto
        env:
          TMPDIR: ${{ env.TMP_DIR }}
          TMP: ${{ env.TMP_DIR }}
          TEMP: ${{ env.TMP_DIR }}
          R_COMPILE_PKGS: 0
          QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
        run: Rscript -e 'babelquarto::render_website()'

      # ── Fix zh page figure files ──────────────────────────────
      # ROOT CAUSE: babelquarto renames <name>.zh.qmd → <name>.qmd before
      # rendering the zh pass, but the frozen chunk HTML still references
      # <name>.zh_files/figure-html/ (the path from when the file was named
      # <name>.zh.qmd).  The actual figure PNGs are never placed at that path
      # because code is not re-executed (freeze is used).  The English figures
      # live at _site/<category>/<name>_files/ and are identical (zh .qmd
      # files share the same code blocks as their English counterparts).
      # FIX: after babelquarto finishes, copy the English <name>_files/ tree
      # into _site/zh/<category>/<name>.zh_files/ for every zh HTML page
      # that references a missing *.zh_files/ directory.
      - name: Fix zh page figure files
        run: |
          echo "Copying figure files into zh output directories..."
          COPIED=0
          while IFS= read -r -d '' zh_html; do
            zh_dir=$(dirname "$zh_html")
            name=$(basename "$zh_html" .html)

            # Relative path from _site/zh (may be empty for root-level pages)
            rel="${zh_dir#_site/zh}"
            rel="${rel#/}"

            # Corresponding English _files directory
            if [ -z "$rel" ]; then
              eng_files="_site/${name}_files"
            else
              eng_files="_site/${rel}/${name}_files"
            fi

            zh_files="${zh_dir}/${name}.zh_files"

            if [ -d "$eng_files" ] && [ ! -d "$zh_files" ]; then
              cp -r "$eng_files" "$zh_files"
              echo "  ✓ Copied: $eng_files → $zh_files"
              COPIED=$((COPIED+1))
            fi
          done < <(find _site/zh -name "*.html" -print0)
          echo "✅ Done: copied figure files for ${COPIED} zh pages"

      # ── Save freeze cache (ALWAYS – even on failure) ─────────
      # This is the key to reducing wasted work across consecutive
      # failed builds. Even if the render fails on file #400,
      # files #1-#399 have their freeze saved for the next run.
      - name: Save freeze cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: _freeze
          key: freeze-${{ runner.os }}-${{ github.sha }}-${{ github.run_attempt }}

      # ── Deploy ───────────────────────────────────────────────
      - name: Upload Quarto artifacts
        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'
        uses: actions/upload-pages-artifact@v3
        with:
          path: "_site"

      - name: Deploy to GitHub Pages
        if: github.event_name != 'pull_request'
        uses: quarto-dev/quarto-actions/publish@v2
        with:
          target: gh-pages
          render: false
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Cleanup temporary directory
        if: always()
        run: |
          TMP_PATH="${{ env.TMP_DIR }}"
          if [ -n "$TMP_PATH" ] && [[ "$TMP_PATH" =~ ^/mnt/TMP ]] && [ -d "$TMP_PATH" ]; then
            sudo rm -rf "$TMP_PATH" || echo "Warning: cleanup of $TMP_PATH failed (non-critical)"
            echo "Removed $TMP_PATH directory"
          else
            echo "Skipping cleanup - validation failed (path: $TMP_PATH)"
          fi