diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 4846d2c2..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Python package - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10"] - - steps: - # Clean out runner data we don't use, should free up ~15 GB - - run: df -h - - name: "node-cleanup" - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a - - run: df -h - - - uses: actions/checkout@v4 - - - name: Cache tox dir - id: cache-tox - uses: actions/cache@v4 - env: - cache-name: cache-tox-data - with: - path: .tox - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - if: ${{ steps.cache-tox-data.outputs.cache-hit != 'true' }} - name: List the state of tox - continue-on-error: true - run: ls -lha .tox/py310/ - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' # caching pip dependencies - - - name: Install python dependencies - run: | - python -m pip install tox - - - name: Run ruff format check - run: | - tox -e ruff_checkformat - - - name: Run ruff check - run: | - tox -e ruff_check - - - name: Fetch dataset - run: | - tox -e trainingdata - - - name: Run tox unit tests - run: | - tox -e py310 - - - name: Run tox security check - run: | - tox -e bandit - - - name: Run inference - run: | - tox -e inference_CI - - - name: Run inference_multi_CI - run: | - tox -e inference_multi_CI - - # To warn us when we get close to disk capacity again - - run: df -h diff --git a/.github/workflows/run_extensive_test.yml b/.github/workflows/run_extensive_test.yml new file mode 100644 index 00000000..429774e4 --- /dev/null +++ b/.github/workflows/run_extensive_test.yml @@ -0,0 +1,75 @@ +# This workflow will install Python dependencies, run tests using tox +name: Extensive Tox Tests + +on: workflow_dispatch + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + + steps: + # Clean out runner data we don't use, should free up ~15 GB + - name: "Make space on runner by removing tools we don't need using rmz, a fast version of rm." + run: | + df -h + DOWNLOAD_URL="https://github.com/SUPERCILEX/fuc/releases/download/3.1.1/x86_64-unknown-linux-gnu-rmz" + time curl --location --output rmz "$DOWNLOAD_URL" + chmod +x rmz + sudo mv rmz /usr/bin/rmz + rmz --version + + sudo time rmz --force \ + /usr/share/swift || true + # Can be deleted: https://dev.to/mathio/squeezing-disk-space-from-github-actions-runners-an-engineers-guide-3pjg + df -h + + - uses: actions/checkout@v4 + + # https://github.com/actions/cache + - name: Restore tox dir cache and save after run + id: restore-tox + uses: actions/cache@v4 + env: + cache-name: cache-tox-data + with: + path: .tox + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('pyproject.toml', 'tox.ini', '.tox/*/CACHEDIR.TAG') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + cache-dependency-path: | + pyproject.toml + tox.ini + + - name: Install python dependencies + run: | + python -m pip install tox + + - name: Fetch dataset (175M) + run: | + tox -e trainingdata + + - name: Run inference_CI + run: | + tox -e inference_CI + + - name: Run inference_multi_CI + run: | + tox -e inference_multi_CI + + # Show disk usage + - run: df -h + - run: du -hsc .tox/* ~/.cache/pip diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml new file mode 100644 index 00000000..e320da69 --- /dev/null +++ b/.github/workflows/run_test.yml @@ -0,0 +1,72 @@ +# This workflow will install Python dependencies, run tests using tox +name: Simple Tox Tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v4 + + # https://github.com/actions/cache + - name: Restore tox dir cache and save after run + id: restore-tox + uses: actions/cache@v4 + env: + cache-name: cache-tox-data + with: + path: .tox + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('pyproject.toml', 'tox.ini', '.tox/*/CACHEDIR.TAG') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + cache-dependency-path: | + pyproject.toml + tox.ini + + - name: Install python dependencies + run: | + python -m pip install tox + + - name: Run ruff format check (<50M) + run: | + tox -e ruff_checkformat + + - name: Run ruff check (<50M) + run: | + tox -e ruff_check + + - name: Fetch dataset (175M) + run: | + tox -e trainingdata + + - name: Run tox unit tests (6G) + run: | + tox -e py310 + + - name: Run tox security check (<50M) + run: | + tox -e bandit + + # Show disk usage + - run: df -h + - run: du -hsc .tox/* ~/.cache/pip diff --git a/tox.ini b/tox.ini index 9a2a247f..31b81a47 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ commands = # Store downloaded data, so we don't have to download it again next run. cp -af {envtmpdir}/main.zip {toxworkdir}/main.zip # Unpack and move data to correct location. - unzip {envtmpdir}/main.zip -d {envtmpdir}/ + unzip -qq {envtmpdir}/main.zip -d {envtmpdir}/ cp -af {envtmpdir}/bris-test-data-main/bris_random_data.zarr {toxworkdir}/