selfdrive_tests.yaml

name: selfdrive
on:
  push:
    branches-ignore:
      - 'testing-closet*'
  pull_request:

env:
  BASE_IMAGE: openpilot-base
  CL_BASE_IMAGE: openpilot-base-cl
  DOCKER_REGISTRY: ghcr.io/commaai
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
  BUILD: |
      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c

  BUILD_CL: |
      docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $CL_BASE_IMAGE /bin/sh -c

  UNIT_TEST: coverage run --append -m unittest discover

jobs:
  # TODO: once actions/cache supports read only mode, use the cache for all jobs
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        submodules: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build devel
      run: |
        TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
        cp Dockerfile.openpilot_base $STRIPPED_DIR
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
                          python -m unittest discover selfdrive/car"

  build_all:
    name: build all
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot with all flags
      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"
    - name: Cleanup scons cache
      run: |
          ${{ env.RUN }} "scons -j$(nproc) --extras --test && \
                          rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --extras --test --cache-populate"

  #build_mac:
  #  name: build macos
  #  runs-on: macos-latest
  #  timeout-minutes: 60
  #  steps:
  #  - uses: actions/checkout@v3
  #    with:
  #      submodules: true
  #  - name: Determine pre-existing Homebrew packages
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
  #      echo 'EOF' >> $GITHUB_ENV
  #  - name: Cache dependencies
  #    id: dependency-cache
  #    uses: actions/cache@v2
  #    with:
  #      path: |
  #        ~/.pyenv
  #        ~/.local/share/virtualenvs/
  #        /usr/local/Cellar
  #        ~/github_brew_cache_entries.txt
  #        /tmp/scons_cache
  #      key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
  #      restore-keys: macos-
  #  - name: Brew link restored dependencies
  #    run: |
  #      if [ -f ~/github_brew_cache_entries.txt ]; then
  #        while read pkg; do
  #          brew link --force "$pkg" # `--force` for keg-only packages
  #        done < ~/github_brew_cache_entries.txt
  #      else
  #        echo "Cache entries not found"
  #      fi
  #  - name: Install dependencies
  #    run: ./tools/mac_setup.sh
  #  - name: Build openpilot
  #    run: |
  #      source tools/openpilot_env.sh
  #      pipenv run selfdrive/manager/build.py
  #
  #      # cleanup scons cache
  #      rm -rf /tmp/scons_cache/
  #      pipenv run scons -j$(nproc) --cache-populate
  #  - name: Remove pre-existing Homebrew packages for caching
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      cd /usr/local/Cellar
  #      new_cellar=$(ls -1)
  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
  #          rm -rf "$pkg"
  #        fi
  #      done
  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt

  build_webcam:
    name: build webcam
    runs-on: ubuntu-20.04
    timeout-minutes: 90
    env:
      IMAGE_NAME: openpilotwebcamci
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: |
        eval "$BUILD"
        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
        docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
    - name: Build openpilot
      run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
    - name: Push to container registry
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest

  docker_push:
    name: docker push
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
    - name: Build CL Docker image
      run: eval "$BUILD_CL"
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest

  static_analysis:
    name: static analysis
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
      run: ${{ env.RUN }} "pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run valgrind
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
                        FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt

  unit_tests:
    name: unit tests
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - name: Get current date
      id: date
      run: echo "::set-output name=time::$(date +'%s')"
    - name: Output timestamp
      run: echo $TIMESTAMP
      env:
        TIMESTAMP: ${{ steps.date.outputs.time }}
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
        restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run unit tests
      run: |
        ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
                        scons -j$(nproc) --test && \
                        $UNIT_TEST common && \
                        $UNIT_TEST opendbc/can && \
                        $UNIT_TEST selfdrive/boardd && \
                        $UNIT_TEST selfdrive/controls && \
                        $UNIT_TEST selfdrive/monitoring && \
                        $UNIT_TEST selfdrive/loggerd && \
                        $UNIT_TEST selfdrive/car && \
                        $UNIT_TEST selfdrive/locationd && \
                        $UNIT_TEST selfdrive/athena && \
                        $UNIT_TEST selfdrive/thermald && \
                        $UNIT_TEST selfdrive/hardware/tici && \
                        $UNIT_TEST selfdrive/modeld && \
                        $UNIT_TEST tools/lib/tests && \
                        ./common/tests/test_util && \
                        ./common/tests/test_swaglog && \
                        ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                        ./selfdrive/loggerd/tests/test_logger &&\
                        ./system/proclogd/tests/test_proclog && \
                        ./selfdrive/ui/replay/tests/test_replay && \
                        ./selfdrive/camerad/test/ae_gray_test && \
                        coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

  process_replay:
    name: process replay
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run replay
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
                        FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        coverage xml"
    - name: Print diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
                        CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
  
  model_replay_onnx:
    name: model replay onnx
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Pull LFS
      run: git lfs pull
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/model_replay.py') }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      # Sim docker is needed to get the intel OPENCL drivers
      run: eval "$BUILD_CL"
    - name: Run replay
      run: |
        ${{ env.RUN_CL }} "scons -j$(nproc) && \
                            ONNXCPU=1 FILEREADER_CACHE=1 CI=1 coverage run \
                            selfdrive/test/process_replay/model_replay.py -j$(nproc) && \
                            coverage xml"

  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test longitudinal
      run: |
        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
                        scons -j$(nproc) && \
                        cd selfdrive/test/longitudinal_maneuvers && \
                        coverage run ./test_longitudinal.py && \
                        coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: longitudinal
        path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/

  test_cars:
    name: cars
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1, 2, 3, 4]
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test car models
      run: |
        ${{ env.RUN }} "scons -j$(nproc) --test && \
                        FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \
                        coverage xml && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 5
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

  docs:
    name: build docs
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build docker container
      run: |
        docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
        docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
        DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
    - name: Push docker container
      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/openpilot-docs:latest