tests.yaml

name: tests

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  macos-tests:
    if: >-
      github.event.pull_request.draft == false &&
      contains(github.event.pull_request.labels.*.name, 'macos-test')
    name: macOS-tests
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        include:
        - {os: macos-latest, architecture: arm64, python-version: '3.11'}
        # - {os: macos-latest, architecture: arm64, python-version: '3.12'}
        # the reason why we commented out 3.12 is that it hits github rate limit for some modules (e.g., knn-vc)
    steps:
    - uses: actions/checkout@v5
      with:
        fetch-depth: 1   # no need for the history
    - name: Install uv
      uses: astral-sh/setup-uv@v5
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install ffmpeg
      run: |
        brew install ffmpeg
      shell: bash
    - name: Verify ffmpeg
      run: |
        ffmpeg -version
      shell: bash
    - name: Install dependencies with uv
      run: |
        uv sync --all-extras --group dev
      shell: bash
    - name: check files
      run: |
        ls /Users/runner/work/senselab/senselab/src/tests/data_for_testing/
      shell: bash
    - name: check pwd
      run: pwd
      shell: bash
    - name: Verify FFmpeg Installation
      run: |
        echo "PATH: $PATH"
        which ffmpeg
        ffmpeg -version
      shell: bash
    - name: Run unit tests
      id: run-tests
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
        # torchcodec needs FFmpeg shared libraries at runtime
        DYLD_LIBRARY_PATH: /opt/homebrew/lib:${{ env.DYLD_LIBRARY_PATH }}
      run: |
        uv run pytest -n auto \
          --junitxml=pytest.xml \
          --cov-report=term-missing:skip-covered \
          --cov-report=xml:coverage.xml \
          --cov=src src/tests \
          --log-level=DEBUG \
          --verbose
      shell: bash

  pre-commit:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
      with:  # no need for the history
        fetch-depth: 1
    - name: Install uv
      uses: astral-sh/setup-uv@v5
    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: '3.11'
    - name: Install dependencies with uv
      run: uv sync --all-extras --group dev
    - name: Run pre-commit
      run: uv run pre-commit run --all-files
      shell: bash

  tutorial-cpu-tests:
    if: >-
      github.event.pull_request.draft == false &&
      contains(github.event.pull_request.labels.*.name, 'test-tutorials')
    name: tutorial-cpu-tests
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
      with:
        fetch-depth: 1
    - name: Install uv
      uses: astral-sh/setup-uv@v5
    - name: Set up Python 3.12
      uses: actions/setup-python@v6
      with:
        python-version: '3.12'
    - name: Install FFmpeg
      run: bash scripts/install-ffmpeg.sh
    - name: Install dependencies
      run: uv sync --all-extras --group dev
    - name: Run all tutorial notebooks (CPU)
      env:
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
      run: |
        uv run python -c "
        import json, subprocess, sys
        manifest = json.load(open('tutorials/manifest.json'))
        failed = []
        for t in manifest['tutorials']:
            if t.get('skip'):
                print(f'SKIP ({t.get(\"skip_reason\",\"manual\")}): {t[\"path\"]}')
                continue
            timeout = t.get('timeout_cpu', t.get('timeout', 600))
            print(f'RUN: {t[\"path\"]} (timeout={timeout}s)')
            r = subprocess.run(
                ['uv', 'run', 'papermill', t['path'], '/dev/null',
                 '--cwd', '.', '-k', 'python3',
                 '--execution-timeout', str(timeout)],
                capture_output=True, text=True, timeout=timeout + 120
            )
            if r.returncode != 0:
                print(f'FAIL: {t[\"path\"]}')
                print(r.stderr[-500:] if r.stderr else '')
                failed.append(t['path'])
            else:
                print(f'PASS: {t[\"path\"]}')
        if failed:
            print(f'\n{len(failed)} tutorial(s) failed:')
            for f in failed: print(f'  {f}')
            sys.exit(1)
        print(f'\nAll tutorials passed (CPU)!')
        "

  cpu-tests:
    if: github.event.pull_request.draft == false
    name: cpu-tests (${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ contains(github.event.pull_request.labels.*.name, 'full-matrix') && fromJSON('["3.11","3.12","3.13","3.14"]') || fromJSON('["3.12"]') }}
    steps:
    - uses: actions/checkout@v5
      with:
        fetch-depth: 1
    - name: Install uv
      uses: astral-sh/setup-uv@v5
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install FFmpeg shared libraries
      run: bash scripts/install-ffmpeg.sh
    - name: Install dependencies with uv
      run: uv sync --all-extras --group dev
    - name: Run CPU tests
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
      run: |
        uv run pytest src/tests \
          --junitxml=pytest.xml \
          --cov-report=term-missing:skip-covered \
          --cov-report=xml:coverage.xml \
          --cov=src \
          --log-level=DEBUG \
          --verbose

  # ── EC2 GPU Runner Jobs ───────────────────────────────────────────────
  # Ephemeral GPU instances via machulav/ec2-github-runner.
  # Triggered by 'ec2-gpu-test' label after macOS-tests and pre-commit pass.
  # See .github/EC2_GPU_RUNNER.md for setup instructions.

  # ── EC2 GPU Tests (single instance, all extras) ───────────────────────
  # PR labels override defaults: gpu-instance:<type>, gpu-family:<family>,
  # gpu-multi, gpu-ondemand:true. See EC2_GPU_RUNNER.md for details.

  start-runner:
    if: >-
      github.event.pull_request.draft == false &&
      contains(github.event.pull_request.labels.*.name, 'ec2-gpu-test')
    name: start-runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
    - name: Parse GPU config labels
      id: gpu-config
      run: |
        LABELS='${{ toJSON(github.event.pull_request.labels.*.name) }}'
        INSTANCE=$(echo "$LABELS" | jq -r '[.[] | select(startswith("gpu-instance:"))] | if length > 0 then .[0] | split(":")[1] else "" end')
        MULTI_GPU=$(echo "$LABELS" | jq -r '[.[] | select(. == "gpu-multi")] | if length > 0 then "true" else "" end')
        if [ -z "$INSTANCE" ] && [ -n "$MULTI_GPU" ]; then INSTANCE="g5.12xlarge"; fi
        if [ -z "$INSTANCE" ]; then
          FAMILY=$(echo "$LABELS" | jq -r '[.[] | select(startswith("gpu-family:"))] | if length > 0 then .[0] | split(":")[1] else "" end')
          if [ -n "$FAMILY" ]; then
            case "$FAMILY" in
              g4dn) INSTANCE="g4dn.xlarge" ;; g5) INSTANCE="g5.xlarge" ;; g6) INSTANCE="g6.xlarge" ;;
              p3) INSTANCE="p3.2xlarge" ;; p4d) INSTANCE="p4d.24xlarge" ;; p5) INSTANCE="p5.48xlarge" ;; esac
          fi
        fi
        ONDEMAND=$(echo "$LABELS" | jq -r '[.[] | select(. == "gpu-ondemand:true")] | if length > 0 then "true" else "" end')
        MARKET=$([ -n "$ONDEMAND" ] && echo "" || echo "spot")
        echo "instance=${INSTANCE}" >> $GITHUB_OUTPUT
        echo "market_type=${MARKET}" >> $GITHUB_OUTPUT
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v6
      with:
        aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
        aws-region: ${{ vars.AWS_REGION }}
    - name: Start EC2 runner
      id: start-ec2-runner
      uses: machulav/ec2-github-runner@v2.5.2
      with:
        mode: start
        github-token: ${{ secrets.GH_TOKEN }}
        ec2-instance-type: ${{ steps.gpu-config.outputs.instance || vars.AWS_INSTANCE_TYPE }}
        market-type: ${{ steps.gpu-config.outputs.market_type }}
        availability-zones-config: ${{ vars.AWS_AZ_CONFIG }}

  gpu-tests:
    name: gpu-tests (${{ matrix.python-version }})
    needs: start-runner
    runs-on: ${{ needs.start-runner.outputs.label }}
    defaults:
      run:
        shell: bash -eo pipefail {0}
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ contains(github.event.pull_request.labels.*.name, 'full-matrix') && fromJSON('["3.11","3.12","3.13","3.14"]') || fromJSON('["3.12"]') }}
    env:
      EC2_USER_HOME: /home/ec2-user
      UV_CACHE_DIR: /scratch/uv-cache
      HF_HOME: /scratch/huggingface
      HF_DATASETS_CACHE: /scratch/huggingface/datasets
      TRANSFORMERS_CACHE: /scratch/huggingface/transformers
      TORCH_HOME: /scratch/torch
    steps:
    - uses: actions/checkout@v5
      with:
        fetch-depth: 1
    - name: Prepare scratch storage
      run: |
        set -ex
        # Mount instance store if not already mounted
        if ! mountpoint -q /scratch; then
          sudo mkfs.ext4 -F /dev/nvme1n1 2>/dev/null || true
          sudo mkdir -p /scratch
          sudo mount /dev/nvme1n1 /scratch || true
          sudo chown $(whoami) /scratch
        fi
        mkdir -p "$UV_CACHE_DIR" "$HF_HOME" "$TORCH_HOME"
    - name: Install uv
      run: |
        if ! command -v uv &>/dev/null; then
          curl -LsSf https://astral.sh/uv/install.sh | sh
        fi
        export PATH="${EC2_USER_HOME}/.local/bin:$PATH"
        uv --version
    - name: Install FFmpeg shared libraries
      run: |
        bash scripts/install-ffmpeg.sh
    - name: Set up environment (Python ${{ matrix.python-version }})
      run: |
        set -euxo pipefail
        export PATH="${EC2_USER_HOME}/.local/bin:$PATH"
        uv sync --python ${{ matrix.python-version }} --python-preference only-managed --all-extras --group dev
        # torchcodec needs libpython on LD_LIBRARY_PATH.
        # Search uv-managed Python, system lib dirs for libpython.
        PYTHON_LIB_DIR=$(uv run python -c "import pathlib,sys;v=f'{sys.version_info.major}.{sys.version_info.minor}';exe=pathlib.Path(sys.executable).resolve();candidates=[exe.parent.parent/'lib',pathlib.Path('/usr/lib/x86_64-linux-gnu'),pathlib.Path('/usr/lib64'),pathlib.Path('/usr/local/lib')];print(next((str(d) for d in candidates if list(d.glob(f'libpython{v}*'))),str(exe.parent.parent/'lib')))")
        echo "PYTHON_LIB_DIR=${PYTHON_LIB_DIR}" >> $GITHUB_ENV
    - name: Verify GPU access
      env:
        LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }}
      run: |
        export PATH="${EC2_USER_HOME}/.local/bin:$PATH"
        nvidia-smi
        uv run python -c "
        import torch
        assert torch.cuda.is_available(), 'CUDA not available'
        n = torch.cuda.device_count()
        print(f'GPUs: {n}')
        for i in range(n):
            print(f'  [{i}] {torch.cuda.get_device_name(i)}')
        print(f'CUDA: {torch.version.cuda}')
        print(f'PyTorch: {torch.__version__}')
        "
    - name: Run unit tests
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
        LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }}
      run: |
        export PATH="${EC2_USER_HOME}/.local/bin:$PATH"
        uv run pytest src/tests \
          --junitxml=pytest.xml \
          --cov-report=term-missing:skip-covered \
          --cov-report=xml:coverage.xml \
          --cov=src \
          --log-level=DEBUG \
          --verbose
    - name: Run all tutorial notebooks (GPU)
      if: >-
        matrix.python-version == '3.12' &&
        contains(github.event.pull_request.labels.*.name, 'test-tutorials')
      env:
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
        LD_LIBRARY_PATH: /opt/miniforge/lib:${{ env.PYTHON_LIB_DIR }}
      run: |
        export PATH="${EC2_USER_HOME}/.local/bin:$PATH"
        uv run python -c "
        import json, subprocess, sys
        manifest = json.load(open('tutorials/manifest.json'))
        failed = []
        for t in manifest['tutorials']:
            if t.get('skip'):
                print(f'SKIP: {t[\"path\"]} ({t.get(\"skip_reason\", \"manual\")})')
                continue
            timeout = t.get('timeout_gpu', t.get('timeout', 600))
            print(f'RUN: {t[\"path\"]} (timeout={timeout}s)')
            r = subprocess.run(
                ['uv', 'run', 'papermill', t['path'], '/dev/null',
                 '--cwd', '.', '-k', 'python3',
                 '--execution-timeout', str(timeout)],
                capture_output=True, text=True, timeout=timeout + 120
            )
            if r.returncode != 0:
                print(f'FAIL: {t[\"path\"]}')
                print(r.stderr[-500:] if r.stderr else '')
                failed.append(t['path'])
            else:
                print(f'PASS: {t[\"path\"]}')
        if failed:
            print(f'{len(failed)} tutorial(s) failed:')
            for f in failed: print(f'  {f}')
            sys.exit(1)
        print('All tutorials passed (GPU)!')
        "

  stop-runner:
    name: stop-runner
    needs: [start-runner, gpu-tests]
    runs-on: ubuntu-latest
    if: ${{ always() && needs.start-runner.result == 'success' }}
    steps:
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v6
      with:
        aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
        aws-region: ${{ vars.AWS_REGION }}
    - name: Stop EC2 runner
      uses: machulav/ec2-github-runner@v2.5.2
      with:
        mode: stop
        github-token: ${{ secrets.GH_TOKEN }}
        label: ${{ needs.start-runner.outputs.label }}
        ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}