Skip to content

ci(docs): run all docs on cpu runners #1595

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
avik-pal merged 1 commit into from
Dec 9, 2025
Merged

ci(docs): run all docs on cpu runners #1595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 79 additions & 78 deletions .github/workflows/Documentation.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,89 +33,89 @@ concurrency:
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

jobs:
tutorial-cuda:
if: ${{ !contains(github.event.head_commit.message, '[skip docs]') }}
runs-on: ubuntu-latest
steps:
- name: Trigger CUDA Tutorial Build
id: buildkite-build
uses: buildkite/trigger-pipeline-action@v2.4.1
with:
buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
pipeline: "julialang/lux-dot-jl"
commit: ${{ github.event.pull_request.head.sha }}
branch: ${{ github.event.pull_request.head.ref }}
message: |
Triggered from PR #${{ github.event.pull_request.number }}: ${{ github.event.pull_request.title }}
send_pull_request: true
pull_request_base_branch: ${{ github.event.pull_request.base.ref }}
build_env_vars: >-
{
"LUX_CUDA_TUTORIALS": "1",
"GITHUB_PR_NUMBER": "${{ github.event.pull_request.number }}"
}

- name: Download CUDA Tutorial Artifacts
uses: EnricoMi/download-buildkite-artifact-action@v1
with:
buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
buildkite_build_url: ${{ steps.buildkite-build.outputs.url }}
ignore_build_states: blocked,canceled,skipped,not_run
ignore_job_states: timed_out,failed
output_path: artifacts
poll_interval: 30

- name: "Combine CUDA Tutorial Artifacts"
run: |
set -e

SOURCE_DIR="artifacts"
OUTPUT_DIR="artifacts_combined"

mkdir -p "$OUTPUT_DIR"

# Check if source directory exists
if [ ! -d "$SOURCE_DIR" ]; then
echo "Error: Source directory '$SOURCE_DIR' not found"
exit 1
fi

BUILD_DIRS=$(find "$SOURCE_DIR" -mindepth 1 -type d -name "Tutorial-Build-*")

[ -n "$BUILD_DIRS" ] || { echo "Error: No Tutorial-Build directories found"; exit 1; }

echo "$BUILD_DIRS" | while read dir; do
echo "Processing: $dir"
rsync -av --ignore-existing "$dir/" "$OUTPUT_DIR/"
done

# Verify
[ "$(ls -A $OUTPUT_DIR)" ] || { echo "Error: Merge produced empty directory"; exit 1; }

echo "✓ Merge complete!"
shell: bash

- name: Upload Tutorial Artifacts
uses: actions/upload-artifact@v5
with:
name: "tutorial-cuda"
path: |
artifacts_combined/docs/src/tutorials/beginner/**/*
artifacts_combined/docs/src/tutorials/intermediate/**/*
artifacts_combined/docs/src/tutorials/advanced/**/*
artifacts_combined/tutorial_deps/*
artifacts_combined/**/*.cov
artifacts_combined/docs/src/public/examples/**/*
retention-days: 90
if-no-files-found: error
# tutorial-cuda:
# if: ${{ !contains(github.event.head_commit.message, '[skip docs]') }}
# runs-on: ubuntu-latest
# steps:
# - name: Trigger CUDA Tutorial Build
# id: buildkite-build
# uses: buildkite/trigger-pipeline-action@v2.4.1
# with:
# buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
# pipeline: "julialang/lux-dot-jl"
# commit: ${{ github.event.pull_request.head.sha }}
# branch: ${{ github.event.pull_request.head.ref }}
# message: |
# Triggered from PR #${{ github.event.pull_request.number }}: ${{ github.event.pull_request.title }}
# send_pull_request: true
# pull_request_base_branch: ${{ github.event.pull_request.base.ref }}
# build_env_vars: >-
# {
# "LUX_CUDA_TUTORIALS": "1",
# "GITHUB_PR_NUMBER": "${{ github.event.pull_request.number }}"
# }

# - name: Download CUDA Tutorial Artifacts
# uses: EnricoMi/download-buildkite-artifact-action@v1
# with:
# buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
# buildkite_build_url: ${{ steps.buildkite-build.outputs.url }}
# ignore_build_states: blocked,canceled,skipped,not_run
# ignore_job_states: timed_out,failed
# output_path: artifacts
# poll_interval: 30

# - name: "Combine CUDA Tutorial Artifacts"
# run: |
# set -e

# SOURCE_DIR="artifacts"
# OUTPUT_DIR="artifacts_combined"

# mkdir -p "$OUTPUT_DIR"

# # Check if source directory exists
# if [ ! -d "$SOURCE_DIR" ]; then
# echo "Error: Source directory '$SOURCE_DIR' not found"
# exit 1
# fi

# BUILD_DIRS=$(find "$SOURCE_DIR" -mindepth 1 -type d -name "Tutorial-Build-*")

# [ -n "$BUILD_DIRS" ] || { echo "Error: No Tutorial-Build directories found"; exit 1; }

# echo "$BUILD_DIRS" | while read dir; do
# echo "Processing: $dir"
# rsync -av --ignore-existing "$dir/" "$OUTPUT_DIR/"
# done

# # Verify
# [ "$(ls -A $OUTPUT_DIR)" ] || { echo "Error: Merge produced empty directory"; exit 1; }

# echo "✓ Merge complete!"
# shell: bash

# - name: Upload Tutorial Artifacts
# uses: actions/upload-artifact@v5
# with:
# name: "tutorial-cuda"
# path: |
# artifacts_combined/docs/src/tutorials/beginner/**/*
# artifacts_combined/docs/src/tutorials/intermediate/**/*
# artifacts_combined/docs/src/tutorials/advanced/**/*
# artifacts_combined/tutorial_deps/*
# artifacts_combined/**/*.cov
# artifacts_combined/docs/src/public/examples/**/*
# retention-days: 90
# if-no-files-found: error

tutorial-cpu:
if: ${{ !contains(github.event.head_commit.message, '[skip docs]') }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
group: [0, 1, 2, 3]
group: [0, 1, 2, 3, 4, 5]
steps:
- uses: actions/checkout@v6
- name: Collect Workflow Telemetry
Expand All @@ -133,7 +133,7 @@ jobs:
run: julia --color=yes --project=docs --threads=auto docs/tutorials.jl
env:
TUTORIAL_BACKEND_GROUP: "CPU"
BUILDKITE_PARALLEL_JOB_COUNT: 4
BUILDKITE_PARALLEL_JOB_COUNT: 6
BUILDKITE_PARALLEL_JOB: ${{ matrix.group }}
LD_LIBRARY_PATH: ""
- name: Upload Tutorial Artifacts
Expand All @@ -153,7 +153,8 @@ jobs:
documentation:
if: ${{ !contains(github.event.head_commit.message, '[skip docs]') }}
runs-on: ubuntu-latest
needs: [tutorial-cpu, tutorial-cuda]
# needs: [tutorial-cpu, tutorial-cuda]
needs: [tutorial-cpu]
steps:
- uses: actions/checkout@v6
with:
Expand Down
26 changes: 13 additions & 13 deletions docs/tutorials.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
#! format: off
const BEGINNER_TUTORIALS = [
# tutorial name device should_run?
("Basics", "CPU", true),
("PolynomialFitting", "CUDA", true),
("SimpleRNN", "CUDA", true),
("SimpleChains", "CUDA", false),
("OptimizationIntegration", "CPU", true),
# tutorial name device should_run?
("Basics", "CPU", true),
("PolynomialFitting", "CPU", true),
("SimpleRNN", "CPU", true),
("SimpleChains", "CPU", false),
("OptimizationIntegration", "CPU", true),
]
const INTERMEDIATE_TUTORIALS = [
("NeuralODE", "CUDA", false),
("NeuralODE", "CPU", false),
("BayesianNN", "CPU", false),
("HyperNet", "CUDA", true),
("PINN2DPDE", "CUDA", true),
("ConvolutionalVAE", "CUDA", true),
("GCN_Cora", "CUDA", true),
("HyperNet", "CPU", true),
("PINN2DPDE", "CPU", true),
("ConvolutionalVAE", "CPU", true),
("GCN_Cora", "CPU", true),
("RealNVP", "CPU", false),
("LSTMEncoderDecoder", "CPU", true),
("CIFAR10/conv_mixer.jl", "CPU", false),
("CIFAR10/simple_cnn.jl", "CUDA", false),
("CIFAR10/simple_cnn.jl", "CPU", false),
("CIFAR10/resnet20.jl", "CPU", false),
]
const ADVANCED_TUTORIALS = [
Expand Down Expand Up @@ -112,7 +112,7 @@ asyncmap(TUTORIALS_BUILDING; ntasks=NTASKS) do (i, (d, (input_path, should_run))

withenv(
"JULIA_NUM_THREADS" => "$(Threads.nthreads())",
"JULIA_CUDA_HARD_MEMORY_LIMIT" => "$(100 ÷ NTASKS)%",
"JULIA_CPU_HARD_MEMORY_LIMIT" => "$(100 ÷ NTASKS)%",
"JULIA_PKG_PRECOMPILE_AUTO" => "0",
"JULIA_DEBUG" => "Literate",
) do
Expand Down
Loading