Merge branch 'main' into ci/perf #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Tests | ||
|
Check failure on line 1 in .github/workflows/performance.yml
|
||
| on: # yamllint disable-line rule:truthy | ||
| push: | ||
| branches: | ||
| - master | ||
| schedule: | ||
| - cron: "30 6 * * 1" # every Monday at 6:30 AM UTC | ||
| workflow_dispatch: | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| integration-tests-slurm: | ||
| name: Performance testing on a SLURM cluster | ||
| runs-on: self-hosted | ||
| timeout-minutes: 10 | ||
| if: github.actor == 'jacob-chmura' || github.actor == 'shenyanghuang' | ||
| env: | ||
| TGM_CI_PERF_LOG_BASE: $SLURM_TMPDIR/tgm_ci_perf | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v4 | ||
| - name: Set up python | ||
| id: setup-python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version-file: ".python-version" | ||
| - name: Set up uv | ||
| run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh | ||
| - name: Restore uv cache | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: /tmp/.uv-cache | ||
| key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | ||
| restore-keys: | | ||
| uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | ||
| uv-${{ runner.os }} | ||
| - name: Install dependencies | ||
| run: uv sync --group dev | ||
| - name: Run Performance Benchmarks | ||
| run: | | ||
| LOG_BASE=${TGM_CI_PERF_LOG_BASE:-$HOME/tgm_ci_perf} | ||
| SCRIPT_PATH=$(mktemp) | ||
| cat > "$SCRIPT_PATH" <<'EOF' | ||
| #!/bin/bash | ||
| #SBATCH --job-name=tgm_benchmarks | ||
| #SBATCH --output=$HOME/tgm_ci_perf/benchmarks.out | ||
| #SBATCH --error=$HOME/tgm_ci_perf/benchmarks.err | ||
| #SBATCH --cpus-per-task=4 | ||
| #SBATCH --mem=16G | ||
| #SBATCH --gres=gpu:a100l:1 | ||
| #SBATCH --time=00:30:00 | ||
| set -euo pipefail | ||
| module load python/3.10 | ||
| module load cudatoolkit/11.7 | ||
| echo "===== JOB INFO =====" | ||
| echo "Job ID: $SLURM_JOB_ID" | ||
| echo "Job Name: $SLURM_JOB_NAME" | ||
| echo "Node List: $SLURM_NODELIST" | ||
| echo "Num CPUs: $SLURM_CPUS_PER_TASK" | ||
| echo "GPU(s): ${{CUDA_VISIBLE_DEVICES:-None}}" | ||
| echo "Memory: ${{SLURM_MEM_PER_NODE:-N/A}}" | ||
| echo "Start Time: $(date)" | ||
| echo "====================" | ||
| cd "$GITHUB_WORKSPACE" | ||
| export TGM_CI_PERF_LOG_BASE=${TGM_CI_PERF_LOG_BASE:-$HOME/tgm_ci_perf} | ||
| uv run pytest -m --benchmark-only --benchmark-json=$TGM_CI_PERF_LOG_BASE/benchmarks.json | ||
| EOF | ||
| chmod +x "$SCRIPT_PATH" | ||
| sbatch "$SCRIPT_PATH" | ||
| - name: Wait for benchmarks to finish | ||
| run: | | ||
| echo "Waiting for job to finish..." | ||
| sleep 60 # TODO: Add polling here | ||
| - name: Get Latest CI Run Log Path | ||
| if: always() | ||
| id: get-log-dir | ||
| run: | | ||
| # Parse path + ID written by the test runner (2 lines in ~/tgm_ci_perf/latest_path.txt) | ||
| mapfile -t lines < "$TGM_CI_PERF_LOG_BASE/latest_path.txt" | ||
| LOG_PATH="${lines[0]}" | ||
| CI_RUN_DIR="${lines[1]}" | ||
| echo "log_path=$LOG_PATH" >> "$GITHUB_OUTPUT" | ||
| echo "ci_run_dir=$CI_RUN_DIR" >> "$GITHUB_OUTPUT" | ||
| echo "Resolved log path: $LOG_PATH" | ||
| echo "Resolved run dir: $CI_RUN_DIR" | ||
| - name: Upload benchmark results | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ steps.get-log-dir.outputs.ci_run_dir }}-benchmarks | ||
| path: $TGM_CI_PERF_LOG_BASE/benchmarks.json | ||
| - name: Minimize uv cache | ||
| run: uv cache prune --ci | ||
