Add contributor and fix n_sets calculation #762

Workflow file for this run

.github/workflows/quarto-publish.yml at 7d48a90

	# ╔══════════════════════════════════════════════════════════════════════╗
	# ║ Github Page – Optimized Build & Deploy ║
	# ║ ║
	# ║ Key optimizations over the original workflow: ║
	# ║ 1. FREEZE CACHE FIX: babelquarto renders in a temp dir and only ║
	# ║ copies _site back – the _freeze it creates is discarded. We ║
	# ║ pre-render all non-frozen files first so that workspace _freeze ║
	# ║ is fully populated before babelquarto copies it to its temp dir. ║
	# ║ This turns a ~90 min render into ~20 min on subsequent runs. ║
	# ║ 2. CHANGED-FILE DETECTION: Detects which QMDs changed and only ║
	# ║ pre-renders those files, enabling fail-fast (catch errors in ║
	# ║ ~2 min instead of ~90 min). ║
	# ║ 3. CONDITIONAL JULIA SETUP: Skips the ~9 min Julia installation ║
	# ║ when freeze already covers Julia files and none changed. ║
	# ║ 4. SMART CONCURRENCY: Groups by PR number or branch so updated ║
	# ║ pushes cancel stale in-progress builds. ║
	# ║ 5. RESOURCE CONSERVATION: Failed builds still save their freeze ║
	# ║ progress, so the next run picks up where it left off. ║
	# ╚══════════════════════════════════════════════════════════════════════╝

	on:
	push:
	branches: [main, master]
	paths-ignore:
	- '*.md'
	- 'LICENSE'
	- 'scripts/**'
	- '.github/scripts/**'
	- '.github/workflows/auto-translate.yml'
	- '.github/workflows/compress-images.yml'
	- '.github/workflows/generate-skills.yml'
	- '.github/workflows/skill-command.yml'
	- '.github/workflows/translate-command.yml'
	pull_request:
	branches: [main, master]
	paths-ignore:
	- '*.md'
	workflow_dispatch:
	inputs:
	force_full_rebuild:
	description: 'Force full rebuild (delete freeze cache and re-execute all files)'
	type: boolean
	default: false

	name: Github Page

	env:
	RUST_BACKTRACE: 1

	jobs:
	build-deploy:
	runs-on: ubuntu-latest
	timeout-minutes: 180
	# Group by PR number for PRs, by branch ref for pushes.
	# New pushes/commits cancel the previous in-progress build in the same group.
	concurrency:
	group: quarto-build-${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true
	env:
	GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
	TMP_DIR: /mnt/TMP
	permissions:
	contents: write
	pages: write
	id-token: write
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	# ── Detect changed files ─────────────────────────────────
	# Determines which QMDs changed so we can:
	# (a) pre-render only those files (fail-fast)
	# (b) skip Julia/Python setup when unnecessary
	- name: Detect changed QMD files
	id: changes
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	BASE_SHA="${{ github.event.pull_request.base.sha }}"
	elif [ "${{ github.event_name }}" = "push" ]; then
	BASE_SHA="${{ github.event.before }}"
	else
	BASE_SHA=""
	fi

	# Handle first push, force push, or manual dispatch
	if [ -z "$BASE_SHA" ] \|\| [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then
	echo "changed_qmds=" >> "$GITHUB_OUTPUT"
	echo "has_changes=false" >> "$GITHUB_OUTPUT"
	echo "need_julia=true" >> "$GITHUB_OUTPUT"
	echo "need_python=true" >> "$GITHUB_OUTPUT"
	echo "ℹ️ No base SHA available – treating as full build"
	exit 0
	fi

	CHANGED_QMDS=$(git diff --name-only --diff-filter=ACMR "$BASE_SHA" HEAD -- '*.qmd' 2>/dev/null \| tr '\n' ' ' \|\| echo "")
	echo "changed_qmds=$CHANGED_QMDS" >> "$GITHUB_OUTPUT"

	if [ -n "$CHANGED_QMDS" ]; then
	echo "has_changes=true" >> "$GITHUB_OUTPUT"
	else
	echo "has_changes=false" >> "$GITHUB_OUTPUT"
	fi

	JULIA_CHANGED=$(echo "$CHANGED_QMDS" \| grep -c 'Julia/' \|\| true)
	PYTHON_CHANGED=$(echo "$CHANGED_QMDS" \| grep -c 'Python/' \|\| true)
	echo "need_julia=$( [ "$JULIA_CHANGED" -gt 0 ] && echo true \|\| echo false )" >> "$GITHUB_OUTPUT"
	echo "need_python=$( [ "$PYTHON_CHANGED" -gt 0 ] && echo true \|\| echo false )" >> "$GITHUB_OUTPUT"

	echo "📋 Changed QMDs: ${CHANGED_QMDS:-none}"
	echo " Julia changes: $JULIA_CHANGED \| Python changes: $PYTHON_CHANGED"

	# ── Restore freeze cache (CRITICAL for performance) ──────
	# The freeze cache stores pre-computed knitr outputs for all QMD files.
	# It is populated by the "Pre-render all non-frozen QMD files" step
	# (which writes directly to the workspace _freeze directory), NOT by
	# babelquarto itself (babelquarto renders in a temp dir and never
	# syncs _freeze back to the workspace).
	#
	# Key strategy:
	# - Primary key includes commit SHA (unique per commit)
	# - run_attempt in the key ensures retries create a new cache entry
	# (cache entries are immutable). Retries still restore from the
	# previous attempt via the restore-keys prefix match.
	# - restore-keys prefix ensures we always get the latest cache
	# - A separate save step (with if: always()) preserves progress
	# even when the build fails partway through
	- name: Restore freeze cache
	id: freeze-restore
	if: inputs.force_full_rebuild != true
	uses: actions/cache/restore@v4
	with:
	path: _freeze
	key: freeze-${{ runner.os }}-${{ github.sha }}-${{ github.run_attempt }}
	restore-keys: \|
	freeze-${{ runner.os }}-${{ github.sha }}-
	freeze-${{ runner.os }}-

	- name: Clear freeze for forced full rebuild
	if: inputs.force_full_rebuild == true
	run: rm -rf _freeze

	- name: Report freeze cache status
	id: freeze-check
	run: \|
	if [ -d "_freeze" ]; then
	FREEZE_COUNT=$(find _freeze -name "*.json" -type f 2>/dev/null \| wc -l)
	echo "✅ Freeze cache available: $FREEZE_COUNT frozen output(s)"
	echo "has_freeze=true" >> "$GITHUB_OUTPUT"

	# Check per-language coverage
	JULIA_FROZEN=$(find _freeze -path "/Julia/" -name "*.json" 2>/dev/null \| wc -l)
	PYTHON_FROZEN=$(find _freeze -path "/Python/" -name "*.json" 2>/dev/null \| wc -l)
	echo "julia_frozen=$JULIA_FROZEN" >> "$GITHUB_OUTPUT"
	echo "python_frozen=$PYTHON_FROZEN" >> "$GITHUB_OUTPUT"
	echo " Julia frozen: $JULIA_FROZEN \| Python frozen: $PYTHON_FROZEN"
	else
	echo "⚠️ No freeze cache – full execution required"
	echo "has_freeze=false" >> "$GITHUB_OUTPUT"
	echo "julia_frozen=0" >> "$GITHUB_OUTPUT"
	echo "python_frozen=0" >> "$GITHUB_OUTPUT"
	fi

	# ── Quarto Setup ────────────────────────────────────────
	- name: Set up Quarto
	uses: quarto-dev/quarto-actions/setup@v2
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	tinytex: true

	# ── R Setup ──────────────────────────────────────────────
	- uses: r-lib/actions/setup-r@v2
	with:
	use-public-rspm: true

	- name: Cache R packages
	uses: actions/cache@v4
	with:
	path: \|
	~/.local/share/renv
	~/.cache/R/renv
	/usr/local/lib/R/site-library
	~/R
	key: r-packages-${{ runner.os }}-${{ hashFiles('DESCRIPTION') }}
	restore-keys: \|
	r-packages-${{ runner.os }}-

	- name: Install system dependencies
	run: \|
	sudo apt-get update -y
	sudo apt-get install -y libcurl4-openssl-dev

	- uses: r-lib/actions/setup-r-dependencies@v2
	env:
	PKG_SYSREQS: "true"

	# ── Python Setup ─────────────────────────────────────────
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	cache: 'pip'

	- name: Install Python dependencies
	run: \|
	python -m pip install --upgrade pip
	if [ -f requirements.txt ]; then
	pip install -r requirements.txt
	fi

	# ── Julia Setup (conditional) ────────────────────────────
	# Julia setup takes ~9 min. Skip it when:
	# - No Julia QMDs changed AND freeze already covers Julia files
	# Always install when: Julia files changed, no freeze, or force rebuild.
	- name: Set up Julia
	id: setup-julia
	if: >-
	steps.changes.outputs.need_julia == 'true' \|\|
	steps.freeze-check.outputs.julia_frozen == '0' \|\|
	inputs.force_full_rebuild == true
	uses: julia-actions/setup-julia@v2
	with:
	version: '1'

	- name: Cache Julia packages
	if: steps.setup-julia.outcome == 'success'
	uses: actions/cache@v4
	with:
	path: ~/.julia
	key: julia-packages-${{ runner.os }}-${{ hashFiles('Julia/Project.toml') }}
	restore-keys: \|
	julia-packages-${{ runner.os }}-

	- name: Install Julia packages
	if: steps.setup-julia.outcome == 'success'
	run: \|
	# Find the QuartoNotebookRunner version required by this Quarto installation.
	# Quarto bundles its own julia-engine Project.toml that pins an exact QNR version.
	JULIA_ENGINE_PROJ=$(find /opt/quarto -name "Project.toml" -path "/julia-engine/" 2>/dev/null \| head -1)
	QNR_VERSION=""
	if [ -n "$JULIA_ENGINE_PROJ" ]; then
	QNR_VERSION=$(grep -oP 'QuartoNotebookRunner = "=([^"]+)"' "$JULIA_ENGINE_PROJ" \
	\| grep -oP '[0-9]+\.[0-9]+\.[0-9]+' \|\| true)
	echo "Quarto requires QuartoNotebookRunner=${QNR_VERSION} (from ${JULIA_ENGINE_PROJ})"
	fi

	# Create a dedicated QNR environment. Setting QUARTO_JULIA_PROJECT to this
	# path tells Quarto to skip ensureQuartoNotebookRunnerEnvironment (which calls
	# Pkg.update() and needs network at render time) and simply verify QNR loads.
	QNR_ENV="${HOME}/.julia/environments/quarto-julia-env"
	mkdir -p "${QNR_ENV}"

	if [ -n "$QNR_VERSION" ]; then
	julia --project="${QNR_ENV}" -e "
	using Pkg
	Pkg.add(name=\"QuartoNotebookRunner\", version=\"${QNR_VERSION}\")
	Pkg.instantiate()
	using QuartoNotebookRunner
	println(\"QuartoNotebookRunner \$(pkgversion(QuartoNotebookRunner)) ready\")
	"
	else
	julia --project="${QNR_ENV}" -e '
	using Pkg
	Pkg.add("QuartoNotebookRunner")
	Pkg.instantiate()
	using QuartoNotebookRunner
	println("QuartoNotebookRunner $(pkgversion(QuartoNotebookRunner)) ready")
	'
	fi

	# Export so the render step can pick it up.
	echo "QUARTO_JULIA_PROJECT=${QNR_ENV}" >> "$GITHUB_ENV"

	# Pre-instantiate the tutorial project environment (Julia/Project.toml).
	# QNR workers use JULIA_PROJECT=@. and cd into the notebook directory,
	# so they automatically pick up Julia/Project.toml for CairoMakie etc.
	julia --project=Julia -e '
	using Pkg
	Pkg.resolve()
	Pkg.instantiate()
	using CairoMakie, DataFrames, Statistics
	println("Tutorial packages ready")
	'

	# ── Dependency Scanning ──────────────────────────────────
	- name: Scan and update dependencies
	run: \|
	Rscript update-dependencies.R
	python manage-dependencies.py --update

	- name: Commit updated dependency files
	if: github.event_name != 'pull_request'
	run: \|
	git config --local user.name "$GITHUB_ACTOR"
	git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com"
	git add DESCRIPTION requirements.txt Julia/Project.toml
	git commit -m 'chore: update dependencies' \|\| echo "No changes to commit"
	git push origin \|\| echo "No changes to push"

	# ── Render ───────────────────────────────────────────────
	- name: Create temporary directory
	run: \|
	sudo mkdir -p ${{ env.TMP_DIR }}
	sudo chown -R $USER:$USER ${{ env.TMP_DIR }}

	# Pre-render Julia notebooks BEFORE the R-heavy main render to avoid OOM.
	# babelquarto loads R + hundreds of packages (~1.5 GB); adding CairoMakie
	# (~1-2 GB peak) would exceed the runner's RAM limit.
	- name: Pre-render Julia notebooks
	if: steps.setup-julia.outcome == 'success'
	env:
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	run: \|
	echo "Pre-rendering Julia tutorials (no R process = lower peak memory)..."
	failed=0
	for qmd in Julia/*.qmd; do
	echo " → $qmd"
	if quarto render "$qmd" 2>&1; then
	echo " ✓ OK: $qmd"
	else
	echo " ✗ Failed: $qmd"
	failed=1
	fi
	done
	julia_log="${HOME}/.cache/quarto/julia/julia_server_log.txt"
	if [ -f "$julia_log" ]; then
	echo ""
	echo "=== QNR server log ==="
	tail -50 "$julia_log"
	echo "======================"
	fi
	[ $failed -eq 0 ] \|\| exit 1
	echo "All Julia notebooks pre-rendered. Freeze cache populated."

	# Pre-render ONLY the changed QMD files for early failure detection.
	# If a changed file has a syntax error or broken dependency, this step
	# fails in ~2 min instead of waiting ~90 min for the full site render.
	# Successfully rendered files populate the freeze cache for the next step.
	- name: Pre-render changed QMD files (fail-fast)
	if: steps.changes.outputs.has_changes == 'true'
	env:
	CHANGED_QMDS: ${{ steps.changes.outputs.changed_qmds }}
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	run: \|
	echo "Pre-rendering changed files for early failure detection..."
	for qmd in $CHANGED_QMDS; do
	# Julia files are pre-rendered separately (OOM prevention)
	if [[ "$qmd" == Julia/* ]]; then continue; fi
	if [ ! -f "$qmd" ]; then continue; fi

	echo " → $qmd"
	quarto render "$qmd" 2>&1 \|\| { echo "❌ FAILED: $qmd"; exit 1; }
	echo " ✓ OK: $qmd"

	# Also render the bilingual counterpart so freeze covers both
	if [[ "$qmd" == *.zh.qmd ]]; then
	counterpart="${qmd%.zh.qmd}.qmd"
	else
	counterpart="${qmd%.qmd}.zh.qmd"
	fi
	if [ -f "$counterpart" ] && ! echo "$CHANGED_QMDS" \| grep -qF "$counterpart"; then
	echo " → $counterpart (bilingual counterpart)"
	quarto render "$counterpart" 2>&1 \|\| { echo "❌ FAILED: $counterpart"; exit 1; }
	echo " ✓ OK: $counterpart"
	fi
	done
	echo "✅ All changed files pre-rendered successfully."

	# ── Pre-render ALL non-frozen files (freeze-cache fix) ─────
	# ROOT CAUSE: babelquarto::render_website() renders in a temp dir
	# (withr::local_tempdir()), creates _freeze there, copies only _site
	# back to the workspace, and then deletes the temp dir. The workspace
	# _freeze is never updated, so the save step finds only the few
	# entries from the changed-file pre-render step above (e.g. 6 Julia
	# entries), not the 500+ entries from the full render.
	#
	# FIX: pre-render every non-frozen .qmd file here (workspace _freeze
	# gets populated), then babelquarto copies the full _freeze into its
	# temp dir and uses it – skipping all code execution and only running
	# pandoc, cutting the render from ~90 min down to ~20 min.
	#
	# Timing estimates:
	# First run (cold cache): pre-render all ~60–80 min +
	# babelquarto (pandoc only) ~20 min +
	# setup ~15 min ≈ 95–115 min total
	# Subsequent runs (warm): pre-render 0 files + babelquarto ~20 min
	# + setup ~15 min ≈ 35 min total
	# Timeout is 180 min to give first-run cold-cache builds a safe margin.
	- name: Pre-render all non-frozen QMD files
	env:
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	run: \|
	echo "Pre-rendering non-frozen QMD files to populate workspace _freeze..."
	RENDERED=0
	SKIPPED=0
	FAILED=0

	while IFS= read -r -d '' qmd; do
	rel="${qmd#./}"

	# Julia files are handled by the dedicated Julia pre-render step
	[[ "$rel" == Julia/* ]] && { SKIPPED=$((SKIPPED+1)); continue; }

	# If a _freeze entry already exists for this file, skip it.
	# Quarto mirrors the source path: _freeze/<path-without-ext>/
	freeze_dir="_freeze/${rel%.qmd}"
	if [ -d "$freeze_dir" ]; then
	SKIPPED=$((SKIPPED+1))
	continue
	fi

	echo " → $rel"
	if quarto render "$qmd" 2>&1; then
	RENDERED=$((RENDERED+1))
	else
	echo " ⚠️ Warning: failed to render $rel (non-fatal)"
	FAILED=$((FAILED+1))
	fi
	done < <(find . -name "*.qmd" \
	-not -path "./_freeze/*" \
	-not -path "./_site/*" \
	-not -path "./.quarto/*" \
	-print0 \| sort -z)

	echo ""
	echo "📊 Pre-render summary: $RENDERED rendered, $SKIPPED skipped (frozen/Julia), $FAILED failed"
	[ $FAILED -gt 0 ] && echo "::warning::$FAILED QMD file(s) failed to pre-render"
	# Count JSON outputs for a richer diagnostic (one per frozen code chunk)
	TOTAL_FROZEN=$(find _freeze -name "*.json" -type f 2>/dev/null \| wc -l)
	echo "❄️ Freeze cache now has $TOTAL_FROZEN frozen output(s) (JSON entries)"

	- name: Render website with Babelquarto
	env:
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	R_COMPILE_PKGS: 0
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	run: Rscript -e 'babelquarto::render_website()'

	- name: Generate unified skill ZIP package
	run: \|
	mkdir -p _site/
	# Stage files with correct names for the zip
	SKILL_TMP=$(mktemp -d)
	cp SKILL.md "${SKILL_TMP}/SKILL.md"
	cp files/gallery_data.csv "${SKILL_TMP}/gallery_data.csv"
	cp files/gallery_data_zh.csv "${SKILL_TMP}/gallery_data_zh.csv"
	cp files/bizard-skill-readme.md "${SKILL_TMP}/README.md"
	(cd "${SKILL_TMP}" && zip -r - SKILL.md gallery_data.csv gallery_data_zh.csv README.md) > _site/bizard-skill.zip
	rm -rf "${SKILL_TMP}"
	echo "✅ Created _site/bizard-skill.zip ($(du -h _site/bizard-skill.zip \| cut -f1))"

	# ── Save freeze cache (ALWAYS – even on failure) ─────────
	# This is the key to reducing wasted work across consecutive
	# failed builds. Even if the render fails on file #400,
	# files #1-#399 have their freeze saved for the next run.
	- name: Save freeze cache
	if: always()
	uses: actions/cache/save@v4
	with:
	path: _freeze
	key: freeze-${{ runner.os }}-${{ github.sha }}-${{ github.run_attempt }}

	# ── Deploy ───────────────────────────────────────────────
	- name: Upload Quarto artifacts
	if: github.ref == 'refs/heads/main' \|\| github.ref == 'refs/heads/master' \|\| github.event_name == 'pull_request'
	uses: actions/upload-pages-artifact@v3
	with:
	path: "_site"

	- name: Deploy to GitHub Pages
	if: github.event_name != 'pull_request'
	uses: quarto-dev/quarto-actions/publish@v2
	with:
	target: gh-pages
	render: false
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Cleanup temporary directory
	if: always()
	run: \|
	TMP_PATH="${{ env.TMP_DIR }}"
	if [ -n "$TMP_PATH" ] && [[ "$TMP_PATH" =~ ^/mnt/TMP ]] && [ -d "$TMP_PATH" ]; then
	sudo rm -rf "$TMP_PATH" \|\| echo "Warning: cleanup of $TMP_PATH failed (non-critical)"
	echo "Removed $TMP_PATH directory"
	else
	echo "Skipping cleanup - validation failed (path: $TMP_PATH)"
	fi

Navigation Menu

Uh oh!

Add contributor and fix n_sets calculation #762

Workflow file

Add contributor and fix n_sets calculation #762

Uh oh!

Workflow file for this run