Merge branch 'main' into 4095-add-observations-to-sea-ice-sensitivity-recipe-retry

Author: NParsonsMO

.circleci/config.yml

@@ -1,11 +1,7 @@
 ---
 version: 2.1
 
-orbs:
-  coverage-reporter: codacy/coverage-reporter@13.13.0
-
 commands:
-
   check_changes:
     steps:
       - run:
@@ -41,6 +37,7 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
+            pip check
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
@@ -75,7 +72,7 @@ commands:
       - run:
           name: Install git+ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - check_changes
@@ -95,7 +92,7 @@ commands:
             conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
             mamba list >> /logs/conda.txt
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            pip install --no-deps << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
             if [[ "<<parameters.flags>>" != *'--editable'* ]]
             then
               rm -r esmvaltool
@@ -135,7 +132,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
             mkdir /logs
-            pip install .[test] > /logs/install.txt 2>&1
+            pip install --no-deps .[test] > /logs/install.txt 2>&1
       - run:
           name: Check Python code style and mistakes
           command: |
@@ -146,15 +143,15 @@ jobs:
           name: Remove source code to test the installed software
           command: rm -r esmvaltool
       - test_and_report
+      - run:
+          # Documentation available at https://docs.codacy.com/coverage-reporter/
+          name: Upload coverage report to Codacy
+          command: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -s -r test-reports/coverage.xml
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
             - /root/.cache/pip
             - .pytest_cache
-      - coverage-reporter/send_report:
-          coverage-reports: 'test-reports/coverage.xml'
-          project-token: $CODACY_PROJECT_TOKEN
-          skip: true  # skip if project-token is not defined (i.e. on a fork)
 
   test_installation_from_source_test_mode:
     # Test installation from source
@@ -186,7 +183,7 @@ jobs:
       - run:
           name: Install git and ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - run:
@@ -205,23 +202,29 @@ jobs:
             mkdir /logs
             conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
+            # Temporarily install intake-esgf here until there is a new release
+            # that includes https://github.com/esgf2-us/intake-esgf/pull/160.
+            pip install git+https://github.com/esgf2-us/intake-esgf@main
             mamba list >> /logs/conda.txt
-            pip install --editable .[develop]
+            pip install --no-deps --editable .[develop]
             git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
-            pip install --editable $HOME/ESMValCore[develop]
+            pip install --no-deps --editable $HOME/ESMValCore[develop]
       - log_versions
       - test_and_report
       - run:
           name: Run recipes
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mkdir -p ~/climate_data
-            esmvaltool config get_config_user
-            echo "search_esgf: when_missing" >> ~/.config/esmvaltool/config-user.yml
-            cat ~/.config/esmvaltool/config-user.yml
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
-              esmvaltool run "$recipe"
+              esmvaltool run --max-parallel-tasks=2 "$recipe"
             done
       - store_artifacts:
           path: /root/esmvaltool_output
@@ -236,7 +239,7 @@ jobs:
     # Test building documentation
     docker:
       - image: condaforge/miniforge3:latest
-    resource_class: medium
+    resource_class: medium+
     steps:
       - checkout
       - run:
@@ -246,7 +249,8 @@ jobs:
             # Install
             conda env create -n esmvaltool -f environment_python.yml
             conda activate esmvaltool
-            pip install .[doc]
+            pip install --no-deps .[doc]
+            pip check
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
@@ -269,7 +273,7 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.11' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.12' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
             # Log versions

.codacy.yml

@@ -2,14 +2,7 @@
 ---
 engines:
   coverage:
-    enabled: false
     exclude_paths: ["tests"]
-  metrics:
-    enabled: true
-  duplication:
-    enabled: true
-  ruff:
-    enabled: true
 
 exclude_paths:
   [

.github/dependabot.yml

@@ -12,3 +12,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    labels:
+      - dependencies
+      - github_actions
+      - automatedPR

.github/workflows/check-rtw.yml

@@ -36,7 +36,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job
       # can access it
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Cylc
         uses: cylc/setup-cylc@v1

.github/workflows/check_links.yml

@@ -0,0 +1,42 @@
+name: Check Links for Documentation
+
+on:
+  # push:
+  #   branches:
+  #     - link-checker
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 00 1 * *" # monthly
+
+jobs:
+  linkChecker:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write # Required for peter-evans/create-issue-from-file
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
+
+        with:
+          #  This token is included to avoid github.com requests rate limiting
+          token: ${{ secrets.TOKEN_GITHUB }}
+          fail: false
+          args: >
+            ./doc/sphinx/**/*.rst
+            --max-concurrency 1 --no-progress
+            --accept "200..=204, 302, 429"
+
+      - name: Set current date
+        run: echo "REPORT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Create Issue From File
+        if: steps.lychee.outputs.exit_code != 0
+        uses: peter-evans/create-issue-from-file@v6
+        with:
+          title: "🔗 Link Checker Report - ${{ env.REPORT_DATE }}"
+          content-filepath: ./lychee/out.md
+          labels: documentation, github_actions

.github/workflows/check_links_in_pr.yml

@@ -0,0 +1,45 @@
+name: Link Checker for Pull requests
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  changedFiles:
+    runs-on: ubuntu-latest
+    outputs:
+      files: ${{ steps.changed-doc-files.outputs.all_changed_files }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Get changed doc files
+        id: changed-doc-files
+        uses: tj-actions/changed-files@v47
+        with:
+          # Avoid using single or double quotes for multiline patterns
+          files: |
+            **.rst
+          matrix: true
+
+  linkChecker:
+    runs-on: ubuntu-latest
+    needs: changedFiles
+    if: ${{ needs.changedFiles.outputs.files != '' && toJSON(fromJSON(needs.changedFiles.outputs.files)) != '[]' }}
+    strategy:
+      matrix:
+        file: ${{ fromJSON(needs.changedFiles.outputs.files) }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+      - name: Link Checker for changed files
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
+        with:
+          #  This token is included to avoid github.com requests rate limiting
+          token: ${{ secrets.TOKEN_GITHUB }}
+          # Fail check on broken links
+          fail: true
+          args: >
+            ${{ matrix.file }}
+            --max-concurrency 1 --no-progress
+            --accept "200..=204, 429"

.github/workflows/citation_file_validator.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.event_name != 'schedule' || github.repository == 'ESMValGroup/ESMValTool' # avoid scheduled GHAs in forks
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/cffconvert-github-action@2.0.0

.github/workflows/create-condalock-file.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: 'ubuntu-latest'
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
@@ -73,7 +73,7 @@ jobs:
       # Automated PR
       # see https://github.com/marketplace/actions/create-pull-request
       - name: Create Automated PR if conda lock file has changed
-        uses: peter-evans/create-pull-request@v7
+        uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Updating Linux condalock file

.github/workflows/cron_esmvalbot_test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: 'ubuntu-latest'
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Create empty commit on branch
@@ -30,7 +30,7 @@ jobs:
       # as comment in the PR
       # see https://github.com/marketplace/actions/create-pull-request
       - name: Create Auto PR
-        uses: peter-evans/create-pull-request@v7
+        uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: empty message

.github/workflows/install-from-conda.yml

@@ -7,7 +7,7 @@ on:
     branches:
       - main
   schedule:
-    - cron: '0 4 * * *'
+    - cron: "0 4 * * *"
 
 # Required shell entrypoint to have properly configured bash shell
 defaults:
@@ -17,11 +17,11 @@ defaults:
 jobs:
   linux:
     runs-on: "ubuntu-latest"
-    if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
+    if: github.repository == 'ESMValGroup/ESMValTool' # avoid GAs in forks
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11", "3.12", "3.13"]
+        python-version: ["3.12", "3.13"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: conda-incubator/setup-miniconda@v3
@@ -40,11 +40,12 @@ jobs:
         run: mamba install esmvaltool 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
       - name: Verify installation
         run: |
+          pip check
           esmvaltool --help
           esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v7
         with:
           name: Conda_Install_Linux_python_${{ matrix.python-version }}
           path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -83,7 +84,7 @@ jobs:
 #      - run: esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
 #      - name: Upload artifacts
 #        if: ${{ always() }}  # upload artifacts even if fail
-#        uses: actions/upload-artifact@v5
+#        uses: actions/upload-artifact@v7
 #        with:
 #          name: Conda_Install_OSX_python_${{ matrix.python-version }}
 #          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}