bris-inference/.github/workflows/tox.yml at f7df0af4d35993d1eaeafe9a9eefebb81a57f064 · metno/bris-inference · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
# This workflow will install Python dependencies, run tests using tox
name: Simple Tox Tests

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10"]

    steps:
      # Clean out runner data we don't use, should free up ~15 GB
      - name: "Make space on runner by removing tools we don't need using rmz, a fast version of rm."
        run: |
          df -h
          DOWNLOAD_URL="https://github.com/SUPERCILEX/fuc/releases/download/3.1.1/x86_64-unknown-linux-gnu-rmz"
          curl --location --output rmz "$DOWNLOAD_URL"
          chmod +x rmz
          sudo mv rmz /usr/bin/rmz
          rmz --version
          sudo rmz --force \
            /usr/local/lib/android \
            /opt/ghc \
            "${HOME}/.rustup" \
            "${HOME}/.cargo" \
            "${HOME}/.dotnet" \
            /opt/containerd \
            /opt/hostedtoolcache \
            /opt/microsoft \
            /opt/az \
            /opt/pipx* \
            /opt/google \
            /opt/mssql-tools || true
          df -h

      - uses: actions/checkout@v4

      # https://github.com/actions/cache?tab=readme-ov-file
      - name: Restore tox dir cache and saves after run
        id: restore-tox
        uses: actions/cache@v4
        env:
          cache-name: cache-tox-data
        with:
          path: .tox
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-

      # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip' # caching pip dependencies

      - name: Install python dependencies
        run: |
          python -m pip install tox

      - name: Run ruff format check (<50M)
        run: |
          tox -e ruff_checkformat

      - name: Run ruff check (<50M)
        run: |
          tox -e ruff_check

      - name: Fetch dataset (175M)
        run: |
          tox -e trainingdata

      - name: Run tox unit tests (6G)
        run: |
          tox -e py310

      - name: Run tox security check (<50M)
        run: |
          tox -e bandit

      - name: Run inference
        run: |
          tox -e inference_CI
      # Keeping this makes our cache ~7G, causing it to be evicted often. Rather caching pip, so this goes OK fast.
      - name: Don't cache inference_CI tox dir
        run: rmz --force .tox/inference_CI

      - name: Run inference_multi_CI
        run: |
          tox -e inference_multi_CI
      # Keeping this makes our cache ~7G, causing it to be evicted often. Rather caching pip, so this goes OK fast.
      - name: Don't cache inference_multi_CI tox dir
        run: rmz --force .tox/inference_multi_CI

      # Show disk usage
      - run: df -h
      - run: du -hsc .tox ~/.cache/pip