Bump the production-dependencies group across 1 directory with 18 updates #1786
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| macos-tests: | |
| if: >- | |
| github.event.pull_request.draft == false && | |
| contains(github.event.pull_request.labels.*.name, 'macos-test') | |
| name: macOS-tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| - {os: macos-latest, architecture: arm64, python-version: '3.11'} | |
| # - {os: macos-latest, architecture: arm64, python-version: '3.12'} | |
| # the reason why we commented out 3.12 is that it hits github rate limit for some modules (e.g., knn-vc) | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 # no need for the history | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install ffmpeg | |
| run: | | |
| brew install ffmpeg | |
| shell: bash | |
| - name: Verify ffmpeg | |
| run: | | |
| ffmpeg -version | |
| shell: bash | |
| - name: Install dependencies with uv | |
| run: | | |
| uv sync --all-extras --group dev | |
| shell: bash | |
| - name: check files | |
| run: | | |
| ls /Users/runner/work/senselab/senselab/src/tests/data_for_testing/ | |
| shell: bash | |
| - name: check pwd | |
| run: pwd | |
| shell: bash | |
| - name: Verify FFmpeg Installation | |
| run: | | |
| echo "PATH: $PATH" | |
| which ffmpeg | |
| ffmpeg -version | |
| shell: bash | |
| - name: Run unit tests | |
| id: run-tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| # torchcodec needs FFmpeg shared libraries at runtime | |
| DYLD_LIBRARY_PATH: /opt/homebrew/lib:${{ env.DYLD_LIBRARY_PATH }} | |
| run: | | |
| uv run pytest -n auto \ | |
| --junitxml=pytest.xml \ | |
| --cov-report=term-missing:skip-covered \ | |
| --cov-report=xml:coverage.xml \ | |
| --cov=src src/tests \ | |
| --log-level=DEBUG \ | |
| --verbose | |
| shell: bash | |
| pre-commit: | |
| if: github.event.pull_request.draft == false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: # no need for the history | |
| fetch-depth: 1 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies with uv | |
| run: uv sync --all-extras --group dev | |
| - name: Run pre-commit | |
| run: uv run pre-commit run --all-files | |
| shell: bash | |
| tutorial-cpu-tests: | |
| if: >- | |
| github.event.pull_request.draft == false && | |
| contains(github.event.pull_request.labels.*.name, 'test-tutorials') | |
| name: tutorial-cpu-tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install FFmpeg | |
| run: bash scripts/install-ffmpeg.sh | |
| - name: Install dependencies | |
| run: uv sync --all-extras --group dev | |
| - name: Run all tutorial notebooks (CPU) | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| uv run python -c " | |
| import json, subprocess, sys | |
| manifest = json.load(open('tutorials/manifest.json')) | |
| failed = [] | |
| for t in manifest['tutorials']: | |
| if t.get('skip'): | |
| print(f'SKIP ({t.get(\"skip_reason\",\"manual\")}): {t[\"path\"]}') | |
| continue | |
| timeout = t.get('timeout_cpu', t.get('timeout', 600)) | |
| print(f'RUN: {t[\"path\"]} (timeout={timeout}s)') | |
| r = subprocess.run( | |
| ['uv', 'run', 'papermill', t['path'], '/dev/null', | |
| '--cwd', '.', '-k', 'python3', | |
| '--execution-timeout', str(timeout)], | |
| capture_output=True, text=True, timeout=timeout + 120 | |
| ) | |
| if r.returncode != 0: | |
| print(f'FAIL: {t[\"path\"]}') | |
| print(r.stderr[-500:] if r.stderr else '') | |
| failed.append(t['path']) | |
| else: | |
| print(f'PASS: {t[\"path\"]}') | |
| if failed: | |
| print(f'\n{len(failed)} tutorial(s) failed:') | |
| for f in failed: print(f' {f}') | |
| sys.exit(1) | |
| print(f'\nAll tutorials passed (CPU)!') | |
| " | |
| cpu-tests: | |
| if: github.event.pull_request.draft == false | |
| name: cpu-tests (${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ${{ contains(github.event.pull_request.labels.*.name, 'full-matrix') && fromJSON('["3.11","3.12","3.13","3.14"]') || fromJSON('["3.12"]') }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install FFmpeg shared libraries | |
| run: bash scripts/install-ffmpeg.sh | |
| - name: Install dependencies with uv | |
| run: uv sync --all-extras --group dev | |
| - name: Run CPU tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| uv run pytest src/tests \ | |
| --junitxml=pytest.xml \ | |
| --cov-report=term-missing:skip-covered \ | |
| --cov-report=xml:coverage.xml \ | |
| --cov=src \ | |
| --log-level=DEBUG \ | |
| --verbose | |
| # ── EC2 GPU Runner Jobs ─────────────────────────────────────────────── | |
| # Ephemeral GPU instances via machulav/ec2-github-runner. | |
| # Triggered by 'ec2-gpu-test' label after macOS-tests and pre-commit pass. | |
| # See .github/EC2_GPU_RUNNER.md for setup instructions. | |
| # ── EC2 GPU Tests (single instance, all extras) ─────────────────────── | |
| # PR labels override defaults: gpu-instance:<type>, gpu-family:<family>, | |
| # gpu-multi, gpu-ondemand:true. See EC2_GPU_RUNNER.md for details. | |
| start-runner: | |
| if: >- | |
| github.event.pull_request.draft == false && | |
| contains(github.event.pull_request.labels.*.name, 'ec2-gpu-test') | |
| name: start-runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Parse GPU config labels | |
| id: gpu-config | |
| run: | | |
| LABELS='${{ toJSON(github.event.pull_request.labels.*.name) }}' | |
| INSTANCE=$(echo "$LABELS" | jq -r '[.[] | select(startswith("gpu-instance:"))] | if length > 0 then .[0] | split(":")[1] else "" end') | |
| MULTI_GPU=$(echo "$LABELS" | jq -r '[.[] | select(. == "gpu-multi")] | if length > 0 then "true" else "" end') | |
| if [ -z "$INSTANCE" ] && [ -n "$MULTI_GPU" ]; then INSTANCE="g5.12xlarge"; fi | |
| if [ -z "$INSTANCE" ]; then | |
| FAMILY=$(echo "$LABELS" | jq -r '[.[] | select(startswith("gpu-family:"))] | if length > 0 then .[0] | split(":")[1] else "" end') | |
| if [ -n "$FAMILY" ]; then | |
| case "$FAMILY" in | |
| g4dn) INSTANCE="g4dn.xlarge" ;; g5) INSTANCE="g5.xlarge" ;; g6) INSTANCE="g6.xlarge" ;; | |
| p3) INSTANCE="p3.2xlarge" ;; p4d) INSTANCE="p4d.24xlarge" ;; p5) INSTANCE="p5.48xlarge" ;; esac | |
| fi | |
| fi | |
| ONDEMAND=$(echo "$LABELS" | jq -r '[.[] | select(. == "gpu-ondemand:true")] | if length > 0 then "true" else "" end') | |
| MARKET=$([ -n "$ONDEMAND" ] && echo "" || echo "spot") | |
| echo "instance=${INSTANCE}" >> $GITHUB_OUTPUT | |
| echo "market_type=${MARKET}" >> $GITHUB_OUTPUT | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2.5.2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| ec2-instance-type: ${{ steps.gpu-config.outputs.instance || vars.AWS_INSTANCE_TYPE }} | |
| market-type: ${{ steps.gpu-config.outputs.market_type }} | |
| availability-zones-config: ${{ vars.AWS_AZ_CONFIG }} | |
| gpu-tests: | |
| name: gpu-tests (${{ matrix.python-version }}) | |
| needs: start-runner | |
| runs-on: ${{ needs.start-runner.outputs.label }} | |
| defaults: | |
| run: | |
| shell: bash -eo pipefail {0} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ${{ contains(github.event.pull_request.labels.*.name, 'full-matrix') && fromJSON('["3.11","3.12","3.13","3.14"]') || fromJSON('["3.12"]') }} | |
| env: | |
| EC2_USER_HOME: /home/ec2-user | |
| UV_CACHE_DIR: /scratch/uv-cache | |
| HF_HOME: /scratch/huggingface | |
| HF_DATASETS_CACHE: /scratch/huggingface/datasets | |
| TRANSFORMERS_CACHE: /scratch/huggingface/transformers | |
| TORCH_HOME: /scratch/torch | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Prepare scratch storage | |
| run: | | |
| set -ex | |
| # Mount instance store if not already mounted | |
| if ! mountpoint -q /scratch; then | |
| sudo mkfs.ext4 -F /dev/nvme1n1 2>/dev/null || true | |
| sudo mkdir -p /scratch | |
| sudo mount /dev/nvme1n1 /scratch || true | |
| sudo chown $(whoami) /scratch | |
| fi | |
| mkdir -p "$UV_CACHE_DIR" "$HF_HOME" "$TORCH_HOME" | |
| - name: Install uv | |
| run: | | |
| if ! command -v uv &>/dev/null; then | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| fi | |
| export PATH="${EC2_USER_HOME}/.local/bin:$PATH" | |
| uv --version | |
| - name: Install FFmpeg shared libraries | |
| run: | | |
| bash scripts/install-ffmpeg.sh | |
| - name: Set up environment (Python ${{ matrix.python-version }}) | |
| run: | | |
| set -euxo pipefail | |
| export PATH="${EC2_USER_HOME}/.local/bin:$PATH" | |
| uv sync --python ${{ matrix.python-version }} --python-preference only-managed --all-extras --group dev | |
| # torchcodec needs libpython on LD_LIBRARY_PATH. | |
| # Search uv-managed Python, system lib dirs for libpython. | |
| PYTHON_LIB_DIR=$(uv run python -c "import pathlib,sys;v=f'{sys.version_info.major}.{sys.version_info.minor}';exe=pathlib.Path(sys.executable).resolve();candidates=[exe.parent.parent/'lib',pathlib.Path('/usr/lib/x86_64-linux-gnu'),pathlib.Path('/usr/lib64'),pathlib.Path('/usr/local/lib')];print(next((str(d) for d in candidates if list(d.glob(f'libpython{v}*'))),str(exe.parent.parent/'lib')))") | |
| echo "PYTHON_LIB_DIR=${PYTHON_LIB_DIR}" >> $GITHUB_ENV | |
| - name: Verify GPU access | |
| env: | |
| LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }} | |
| run: | | |
| export PATH="${EC2_USER_HOME}/.local/bin:$PATH" | |
| nvidia-smi | |
| uv run python -c " | |
| import torch | |
| assert torch.cuda.is_available(), 'CUDA not available' | |
| n = torch.cuda.device_count() | |
| print(f'GPUs: {n}') | |
| for i in range(n): | |
| print(f' [{i}] {torch.cuda.get_device_name(i)}') | |
| print(f'CUDA: {torch.version.cuda}') | |
| print(f'PyTorch: {torch.__version__}') | |
| " | |
| - name: Run unit tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }} | |
| run: | | |
| export PATH="${EC2_USER_HOME}/.local/bin:$PATH" | |
| uv run pytest src/tests \ | |
| --junitxml=pytest.xml \ | |
| --cov-report=term-missing:skip-covered \ | |
| --cov-report=xml:coverage.xml \ | |
| --cov=src \ | |
| --log-level=DEBUG \ | |
| --verbose | |
| - name: Run all tutorial notebooks (GPU) | |
| if: >- | |
| matrix.python-version == '3.12' && | |
| contains(github.event.pull_request.labels.*.name, 'test-tutorials') | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }} | |
| run: | | |
| export PATH="${EC2_USER_HOME}/.local/bin:$PATH" | |
| uv run python -c " | |
| import json, subprocess, sys | |
| manifest = json.load(open('tutorials/manifest.json')) | |
| failed = [] | |
| for t in manifest['tutorials']: | |
| if t.get('skip'): | |
| print(f'SKIP: {t[\"path\"]} ({t.get(\"skip_reason\", \"manual\")})') | |
| continue | |
| timeout = t.get('timeout_gpu', t.get('timeout', 600)) | |
| print(f'RUN: {t[\"path\"]} (timeout={timeout}s)') | |
| r = subprocess.run( | |
| ['uv', 'run', 'papermill', t['path'], '/dev/null', | |
| '--cwd', '.', '-k', 'python3', | |
| '--execution-timeout', str(timeout)], | |
| capture_output=True, text=True, timeout=timeout + 120 | |
| ) | |
| if r.returncode != 0: | |
| print(f'FAIL: {t[\"path\"]}') | |
| print(r.stderr[-500:] if r.stderr else '') | |
| failed.append(t['path']) | |
| else: | |
| print(f'PASS: {t[\"path\"]}') | |
| if failed: | |
| print(f'{len(failed)} tutorial(s) failed:') | |
| for f in failed: print(f' {f}') | |
| sys.exit(1) | |
| print('All tutorials passed (GPU)!') | |
| " | |
| stop-runner: | |
| name: stop-runner | |
| needs: [start-runner, gpu-tests] | |
| runs-on: ubuntu-latest | |
| if: ${{ always() && needs.start-runner.result == 'success' }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2.5.2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |