Skip to content

Commit 4d1c9e2

Browse files
sosiristsengsosiristseng
authored
dynamic matrix (#143)
1 parent 6ea0d5f commit 4d1c9e2

File tree

5 files changed

+151
-174
lines changed

5 files changed

+151
-174
lines changed

‎.github/ci.jl‎

Lines changed: 22 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
1-
using Distributed
2-
using Tables
3-
using MarkdownTables
4-
using SHA
1+
using JSON
2+
using Literate
53

6-
@everywhere begin
7-
ENV["GKSwstype"] = "100"
8-
using Literate, JSON
4+
ENV["GKSwstype"] = "100"
5+
6+
function main(; dopostproc=true)
7+
file = get(ENV, "NB", "test.ipynb")
8+
cachedir = get(ENV, "NBCACHE", ".cache")
9+
if endswith(file, ".jl")
10+
run_literate(file; cachedir, dopostproc)
11+
elseif endswith(file, ".ipynb")
12+
lit = to_literate(file)
13+
run_literate(lit; cachedir, dopostproc)
14+
end
915
end
1016

1117
# Post-process Jupyter notebook
12-
@everywhere function postprocess(nbpath)
13-
oldfilesize = filesize(nbpath)
14-
nb = open(JSON.parse, nbpath, "r")
18+
function postprocess(ipynb)
19+
oldfilesize = filesize(ipynb)
20+
nb = open(JSON.parse, ipynb, "r")
1521
for cell in nb["cells"]
1622
!haskey(cell, "outputs") && continue
1723
for output in cell["outputs"]
@@ -31,34 +37,14 @@ end
3137
end
3238
end
3339
end
34-
rm(nbpath; force=true)
35-
write(nbpath, JSON.json(nb, 2))
36-
@info "$(nbpath) is processed. The original size is $(Base.format_bytes(oldfilesize)). The new size is $(Base.format_bytes(filesize(nbpath)))."
37-
return nbpath
38-
end
39-
40-
# Remove cached notebook and sha files if there is no corresponding notebook
41-
function clean_cache(cachedir)
42-
for (root, _, files) in walkdir(cachedir)
43-
for file in files
44-
fn, ext = splitext(file)
45-
if ext == ".sha"
46-
target = joinpath(joinpath(splitpath(root)[2:end]), fn)
47-
nb = target * ".ipynb"
48-
lit = target * ".jl"
49-
if !isfile(nb) && !isfile(lit)
50-
cachepath = joinpath(root, fn)
51-
@info "Notebook $(nb) or $(lit) not found. Removing $(cachepath) SHA and notebook."
52-
rm(cachepath * ".sha"; force=true)
53-
rm(cachepath * ".ipynb"; force=true)
54-
end
55-
end
56-
end
57-
end
40+
rm(ipynb; force=true)
41+
write(ipynb, JSON.json(nb, 2))
42+
@info "The original size is $(Base.format_bytes(oldfilesize)). The new size is $(Base.format_bytes(filesize(ipynb)))."
43+
return ipynb
5844
end
5945

6046
# Convert a Jupyter notebook into a Literate notebook. Adapted from https://github.com/JuliaInterop/NBInclude.jl.
61-
function to_literate(nbpath; shell_or_help=r"^\s*[;?]")
47+
function to_literate(nbpath; shell_or_help = r"^\s*[;?]")
6248
nb = open(JSON.parse, nbpath, "r")
6349
jlpath = splitext(nbpath)[1] * ".jl"
6450
open(jlpath, "w") do io
@@ -80,39 +66,7 @@ function to_literate(nbpath; shell_or_help=r"^\s*[;?]")
8066
return jlpath
8167
end
8268

83-
# List notebooks without caches in a file tree
84-
function list_notebooks(basedir, cachedir)
85-
list = String[]
86-
for (root, _, files) in walkdir(basedir)
87-
for file in files
88-
name, ext = splitext(file)
89-
if ext == ".ipynb" || ext == ".jl"
90-
nb = joinpath(root, file)
91-
shaval = read(nb, String) |> sha256 |> bytes2hex
92-
@info "$(nb) SHA256 = $(shaval)"
93-
shafilename = joinpath(cachedir, root, name * ".sha")
94-
if isfile(shafilename) && read(shafilename, String) == shaval
95-
@info "$(nb) cache hits and will not be executed."
96-
else
97-
@info "$(nb) cache misses. Writing hash to $(shafilename)."
98-
mkpath(dirname(shafilename))
99-
write(shafilename, shaval)
100-
if ext == ".ipynb"
101-
litnb = to_literate(nb)
102-
rm(nb; force=true)
103-
push!(list, litnb)
104-
elseif ext == ".jl"
105-
push!(list, nb)
106-
end
107-
end
108-
end
109-
end
110-
end
111-
return list
112-
end
113-
114-
# Run a Literate notebook
115-
@everywhere function run_literate(file, cachedir; dopostproc=true)
69+
function run_literate(file; cachedir = ".cache", dopostproc=true)
11670
outpath = joinpath(abspath(pwd()), cachedir, dirname(file))
11771
mkpath(outpath)
11872
ipynb = Literate.notebook(file, dirname(file); mdstrings=true, execute=true)
@@ -121,37 +75,4 @@ end
12175
return ipynb
12276
end
12377

124-
function main(;
125-
basedir=get(ENV, "DOCDIR", "docs"),
126-
cachedir=get(ENV, "NBCACHE", ".cache"),
127-
dopostproc=true)
128-
129-
mkpath(cachedir)
130-
clean_cache(cachedir)
131-
litnbs = list_notebooks(basedir, cachedir)
132-
133-
if !isempty(litnbs)
134-
## Execute literate notebooks in worker process(es)
135-
ts_lit = pmap(litnbs; on_error=identity) do nb
136-
@elapsed run_literate(nb, cachedir; dopostproc)
137-
end
138-
rmprocs(workers()) ## Remove worker processes to release some memory
139-
failed = false
140-
for (nb, t) in zip(litnbs, ts_lit)
141-
if t isa ErrorException
142-
println("Notebook: ", nb, "failed with error: \n", t.msg)
143-
failed = true
144-
end
145-
end
146-
147-
if failed
148-
error("Please check error(s).")
149-
else
150-
## Print execution result
151-
Tables.table([litnbs ts_lit]; header=["Notebook", "Elapsed (s)"]) |> markdown_table(String) |> print
152-
end
153-
end
154-
end
155-
156-
# Run code
15778
main()

‎.github/workflows/ci.yml‎

Lines changed: 127 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build notebooks and publish
1+
name: CI with dynamic parallel matrix
22

33
on:
44
workflow_dispatch:
@@ -15,15 +15,19 @@ env:
1515
JULIA_CI: 'true'
1616
JULIA_CONDAPKG_BACKEND: 'Null'
1717
JULIA_CONDAPKG_OFFLINE: 'true'
18-
# JULIA_CPU_TARGET: 'generic;icelake-server,clone_all;znver3,clone_all'
19-
JULIA_NUM_THREADS: '2'
18+
JULIA_CPU_TARGET: 'generic;icelake-server,clone_all;znver3,clone_all'
19+
JULIA_NUM_THREADS: 'auto'
2020
NBCACHE: '.cache'
21-
LITERATE_PROC: '6'
2221
PY_VER: '3.14'
2322

23+
2424
jobs:
25-
CI:
26-
runs-on: self-hosted
25+
setup:
26+
runs-on: ubuntu-latest
27+
outputs:
28+
matrix: ${{ steps.set-matrix.outputs.matrix }}
29+
hash: ${{ steps.hash.outputs.value }}
30+
ver: ${{ steps.julia-version.outputs.resolved }}
2731
steps:
2832
- name: Checkout repository
2933
uses: actions/checkout@v6
@@ -38,89 +42,145 @@ jobs:
3842
run: uv pip install --system -r requirements.txt
3943
- name: Read Julia version
4044
id: julia-version
41-
run: python -c 'import tomllib; from pathlib import Path; print("value=", tomllib.loads(Path("Manifest.toml").read_text())["julia_version"], sep="")' >> "$GITHUB_OUTPUT"
45+
run: python -c 'import tomllib; from pathlib import Path; print("resolved=", tomllib.loads(Path("Manifest.toml").read_text())["julia_version"], sep="")' >> "$GITHUB_OUTPUT"
4246
- name: Get environment hash
4347
id: hash
4448
run: |
4549
echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT"
46-
- name: Cache executed notebooks
50+
echo "ver=${{ runner.os }}-julia-${{ steps.julia-version.outputs.resolved }}" >> "$GITHUB_OUTPUT"
51+
- name: Cache Julia packages
4752
uses: actions/cache@v5
48-
id: cache-nb
53+
id: cache-julia
4954
with:
50-
path: |
51-
${{ env.NBCACHE }}/**/*.ipynb
52-
${{ env.NBCACHE }}/**/*.sha
53-
key: notebook-${{ steps.hash.outputs.value }}-${{ hashFiles('docs/**') }}
55+
path: ~/.julia
56+
key: ${{ runner.os }}-julia-${{ steps.julia-version.outputs.resolved }}-${{ steps.hash.outputs.value }}
5457
restore-keys: |
55-
notebook-${{ steps.hash.outputs.value }}-
58+
${{ runner.os }}-julia-${{ steps.julia-version.outputs.resolved }}-
5659
- name: Setup Julia
5760
uses: julia-actions/setup-julia@v2
58-
if: ${{ steps.cache-nb.outputs.cache-hit != 'true' }}
61+
if: ${{ steps.cache-julia.outputs.cache-hit != 'true' }}
5962
with:
60-
version: ${{ steps.julia-version.outputs.value }}
63+
version: ${{ steps.julia-version.outputs.resolved }}
6164
arch: ${{ runner.arch }}
62-
- name: Restore Julia packages
63-
uses: actions/cache/restore@v5
64-
if: ${{ steps.cache-nb.outputs.cache-hit != 'true' && runner.environment == 'github-hosted'}}
65-
id: cache-julia
66-
with:
67-
path: ~/.julia
68-
key: ${{ runner.os }}-julia-${{ steps.julia-version.outputs.value }}-${{ steps.hash.outputs.value }}
69-
restore-keys: |
70-
${{ runner.os }}-julia-${{ steps.julia-version.outputs.value }}
7165
- name: Install Julia packages
72-
if: ${{ steps.cache-nb.outputs.cache-hit != 'true' && (runner.environment == 'self-hosted' || steps.cache-julia.outputs.cache-hit != 'true') }}
66+
if: ${{ steps.cache-julia.outputs.cache-hit != 'true' }}
7367
shell: julia --color=yes --project=@. {0}
7468
run: |
7569
using Pkg, Dates
7670
Pkg.instantiate()
7771
Pkg.precompile()
78-
if ENV["RUNNER_ENVIRONMENT"] == "github-hosted"
79-
Pkg.gc(;collect_delay=Day(0))
80-
end
81-
- name: Save Julia packages
82-
uses: actions/cache/save@v5
83-
if: ${{ steps.cache-nb.outputs.cache-hit != 'true' && runner.environment == 'github-hosted' && steps.cache-julia.outputs.cache-hit != 'true' }}
84-
with:
85-
path: ~/.julia
86-
key: ${{ steps.cache-julia.outputs.cache-primary-key }}
87-
- name: Run notebooks
88-
if: ${{ steps.cache-nb.outputs.cache-hit != 'true' }}
89-
run: julia --project=@. --color=yes -p ${{ env.LITERATE_PROC }} .github/ci.jl
90-
- name: Copy back built notebooks
91-
run: |
92-
cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
93-
find docs/ -type f -name "*.jl" -delete
94-
- name: Upload notebook
95-
uses: actions/upload-artifact@v6
96-
with:
97-
name: notebooks
98-
path: docs/**/*.ipynb
99-
- name: Upload figures
100-
uses: actions/upload-artifact@v6
101-
with:
102-
name: figures
103-
path: |
104-
docs/**/*.png
105-
docs/**/*.pdf
106-
- name: Setup Quarto
107-
if: ${{ runner.environment == 'github-hosted' }}
108-
uses: quarto-dev/quarto-actions/setup@v2
109-
- name: Render Quarto Project
110-
run: quarto render docs/
111-
- name: Upload pages artifact
112-
uses: actions/upload-pages-artifact@v4
113-
if: ${{ github.ref == 'refs/heads/main' }}
114-
with:
115-
path: docs/_site/
72+
Pkg.gc(;collect_delay=Day(0))
73+
- name: List notebooks as a JSON array
74+
id: set-matrix
75+
run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT"
76+
77+
execute:
78+
needs: setup
79+
strategy:
80+
max-parallel: 10
81+
fail-fast: false
82+
matrix:
83+
notebook: ${{ fromJSON(needs.setup.outputs.matrix) }}
84+
runs-on: ubuntu-latest
85+
env:
86+
NB: docs/${{ matrix.notebook }}
87+
steps:
88+
- name: Checkout
89+
uses: actions/checkout@v6
90+
- name: Cache notebook
91+
uses: actions/cache@v5
92+
id: nb-cache
93+
with:
94+
path: ${{ env.NBCACHE }}
95+
key: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
96+
- name: Setup Python
97+
uses: actions/setup-python@v6
98+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
99+
id: setup-python
100+
with:
101+
python-version: ${{ env.PY_VER }}
102+
- name: Install the latest version of uv
103+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
104+
uses: astral-sh/setup-uv@v7
105+
- name: Install Python dependencies
106+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
107+
run: uv pip install --system -r requirements.txt
108+
- name: Setup Julia
109+
uses: julia-actions/setup-julia@v2
110+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
111+
with:
112+
version: ${{ needs.setup.outputs.ver }}
113+
arch: ${{ runner.arch }}
114+
- name: Restore Julia packages
115+
uses: actions/cache/restore@v5
116+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
117+
with:
118+
path: ~/.julia
119+
key: ${{ runner.os }}-julia-${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}
120+
- name: Execute notebook
121+
if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
122+
run: julia --project=@. .github/ci.jl
123+
- name: Convert artifact Name
124+
id: art
125+
run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
126+
- name: Upload Notebook
127+
uses: actions/upload-artifact@v6
128+
with:
129+
name: notebook-${{ steps.art.outputs.name }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
130+
path: ${{ env.NBCACHE }}
131+
include-hidden-files: true
132+
retention-days: 1
133+
134+
render:
135+
needs: execute
136+
runs-on: ubuntu-latest
137+
steps:
138+
- name: Checkout
139+
uses: actions/checkout@v6
140+
- name: Download notebooks
141+
uses: actions/download-artifact@v7
142+
with:
143+
path: ${{ env.NBCACHE }}/
144+
pattern: notebook-*
145+
merge-multiple: true
146+
- name: Copy back built notebooks
147+
run: |
148+
cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
149+
find docs/ -type f -name "*.jl" -delete
150+
- name: Setup Quarto
151+
uses: quarto-dev/quarto-actions/setup@v2
152+
- name: Render Quarto Project
153+
run: quarto render docs --to html
154+
- name: Upload artifact for GH pages
155+
uses: actions/upload-pages-artifact@v4
156+
if: ${{ github.ref == 'refs/heads/main' }}
157+
with:
158+
path: docs/_site/
159+
160+
# CI conclusion for GitHub status check
161+
# Adaped from https://brunoscheufler.com/blog/2022-04-09-the-required-github-status-check-that-wasnt
162+
CI:
163+
needs: render
164+
if: always()
165+
runs-on: ubuntu-slim
166+
steps:
167+
- run: |
168+
if [[ ${{ needs.render.result }} == "success" ]]; then
169+
echo "Tests passed"
170+
exit 0
171+
else
172+
echo "Tests failed"
173+
exit 1
174+
fi
116175
176+
# Deployment job
117177
deploy:
118178
name: Deploy to GitHub pages
119-
needs: CI
120-
if: ${{ github.ref == 'refs/heads/main'}}
179+
needs: render
180+
if: ${{ github.ref == 'refs/heads/main' }}
121181
# Grant GITHUB_TOKEN the permissions required to make a Pages deployment
122182
permissions:
123-
pages: write # to deploy to Pages
183+
pages: write # to deploy to Pages
124184
id-token: write # to verify the deployment originates from an appropriate source
125185
environment:
126186
name: github-pages

0 commit comments

Comments
 (0)