open-tyndp/.github/workflows/test.yaml at ba4ed7c5ef90c4183b9b3668f937bf38b43c882b · open-energy-transition/open-tyndp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Test workflows

on:
  push:
    branches:
    - master
  pull_request:
  schedule:
  - cron: "0 5 * * 1-6"
  - cron: "0 5 * * 0"
  workflow_dispatch:

# Cancel any in-progress runs when a new run is triggered
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  run-tests:
    name: OS
    runs-on: ${{ matrix.os }}-latest
    strategy:
      fail-fast: false
      matrix:
        # Run windows only on scheduled runs on Sundays, otherwise ignore
        os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu"]') }}

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v6

    - uses: dorny/paths-filter@v3
      id: filter
      with:
        filters: |
          src:
            - 'scripts/**'
            - 'rules/**'
            - 'data/**'
            - 'Snakefile'
            - 'config/**'
            - 'test/**'
            - 'pixi.toml'
            - 'pixi.lock'
            - '.github/workflows/test.yaml'

    - name: Free up disk space
      run: |
        echo "Initial disk space"
        df -h
        echo "Free up disk space"
        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
        sudo docker image prune --all --force
        sudo docker builder prune -a --force
        echo "Final disk space"
        df -h
    - name: Skip - no source changes
      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
      run: echo "Skipping tests because no source code changes detected"

    - name: Setup Pixi
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      uses: prefix-dev/setup-pixi@v0.9.3
      with:
        pixi-version: v0.59.0
        cache: true
        # Do not cache in branches
        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
        environments: open-tyndp

    - name: Setup cache keys
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      run: |
        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts

    - uses: actions/cache@v5
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      with:
        path: |
          data
          cutouts
        key: data-cutouts-${{ env.WEEK }}

    - name: Run pylint check on scripts
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      # check for undefined variables to reuse functions across scripts
      run: |
        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*

    - name: Run snakemake test workflows (sb)
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      env:
        SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
        SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
      run: |
        pixi run -e open-tyndp tyndp-sb-test

    - name: Run snakemake test workflows (cba)
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      env:
        SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
        SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
      run: |
        pixi run -e open-tyndp tyndp-cba-test

    - name: Run snakemake test workflows (upstream)
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      env:
        SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
        SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
      run: |
        pixi run -e open-tyndp integration-tests

    - name: Run unit tests
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
      run: |
        pixi run -e open-tyndp unit-tests

    - name: Upload artifacts
      if: always()
      uses: actions/upload-artifact@v6
      with:
        name: results-${{ matrix.os }}
        path: |
          logs
          .snakemake/log
          results
        retention-days: 3

    - name: Show remaining disk space
      if: always()
      run: df -h