update main

Author: Liza Malinina

.circleci/config.yml

@@ -5,7 +5,6 @@ orbs:
   coverage-reporter: codacy/coverage-reporter@13.13.0
 
 commands:
-
   check_changes:
     steps:
       - run:
@@ -41,10 +40,11 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mamba --version
+            pip check
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
+            esmvaltool develop compare -- --help
             ncl -V
             cdo --version
       - store_test_results:
@@ -75,7 +75,7 @@ commands:
       - run:
           name: Install git+ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - check_changes
@@ -95,8 +95,7 @@ commands:
             conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
             mamba list >> /logs/conda.txt
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            pip install --no-deps << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
             if [[ "<<parameters.flags>>" != *'--editable'* ]]
             then
               rm -r esmvaltool
@@ -136,8 +135,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
             mkdir /logs
-            pip install .[test] > /logs/install.txt 2>&1
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            pip install --no-deps .[test] > /logs/install.txt 2>&1
       - run:
           name: Check Python code style and mistakes
           command: |
@@ -154,9 +152,9 @@ jobs:
             - /root/.cache/pip
             - .pytest_cache
       - coverage-reporter/send_report:
-          coverage-reports: 'test-reports/coverage.xml'
+          coverage-reports: "test-reports/coverage.xml"
           project-token: $CODACY_PROJECT_TOKEN
-          skip: true  # skip if project-token is not defined (i.e. on a fork)
+          skip: true # skip if project-token is not defined (i.e. on a fork)
 
   test_installation_from_source_test_mode:
     # Test installation from source
@@ -188,7 +186,7 @@ jobs:
       - run:
           name: Install git and ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - run:
@@ -205,26 +203,24 @@ jobs:
             # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
+            # Temporarily install intake-esgf here until it is an ESMValCore dependency in v2.14.
+            echo "  - intake-esgf" >> environment.yml
             conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
             mamba list >> /logs/conda.txt
-            pip install --editable .[develop]
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            pip install --no-deps --editable .[develop]
             git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
-            pip install --editable $HOME/ESMValCore[develop]
+            pip install --no-deps --editable $HOME/ESMValCore[develop]
       - log_versions
       - test_and_report
       - run:
           name: Run recipes
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mkdir -p ~/climate_data
-            esmvaltool config get_config_user
-            echo "search_esgf: when_missing" >> ~/.config/esmvaltool/config-user.yml
-            cat ~/.config/esmvaltool/config-user.yml
+            esmvaltool config copy data-esmvalcore-esgf.yml
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
-              esmvaltool run "$recipe"
+              esmvaltool run --max-parallel-tasks=2 "$recipe"
             done
       - store_artifacts:
           path: /root/esmvaltool_output
@@ -249,7 +245,8 @@ jobs:
             # Install
             conda env create -n esmvaltool -f environment_python.yml
             conda activate esmvaltool
-            pip install .[doc]
+            pip install --no-deps .[doc]
+            pip check
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
@@ -272,11 +269,9 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.11' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.12' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
-            # install the Julia dependencies
-            esmvaltool install Julia > /logs/install_Julia.txt 2>&1
             # Log versions
             mamba env export > /logs/environment.yml
             # Test installation

.circleci/install_triggers

@@ -1,6 +1,5 @@
 ^\.circleci/
 ^environment\.yml$
-^esmvaltool/install/
 ^pyproject.toml$
 ^setup\.py$
 ^setup\.cfg$

.codacy.yml

@@ -1,27 +1,15 @@
 # codacy configuration file
-
 ---
-
 engines:
-    coverage:
-        enabled: false
-        exclude_paths: [
-            'tests',
-        ]
-    metrics:
-        enabled: true
-    duplication:
-        enabled: true
-    prospector:
-        enabled: true
-    pylint:
-        enabled: true
-        python_version: 3
+  coverage:
+    exclude_paths: ["tests"]
 
-exclude_paths: [
-    'archive/**',
-    'doc/sphinx/**',
-    'esmvaltool/cmor/tables/**',
-    'tests/**',
-    'esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py',
-]
+exclude_paths:
+  [
+    "archive/**",
+    "doc/sphinx/**",
+    "esmvaltool/cmor/tables/**",
+    "tests/**",
+    "esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py",
+    "esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py",
+  ]

.github/dependabot.yml

@@ -0,0 +1,14 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`
+    # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/check-rtw.yml

@@ -36,7 +36,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job
       # can access it
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Setup Cylc
         uses: cylc/setup-cylc@v1

.github/workflows/check_links.yml

@@ -0,0 +1,42 @@
+name: Check Links for Documentation
+
+on:
+  # push:
+  #   branches:
+  #     - link-checker
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 00 1 * *" # monthly
+
+jobs:
+  linkChecker:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write # Required for peter-evans/create-issue-from-file
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
+
+        with:
+          #  This token is included to avoid github.com requests rate limiting
+          token: ${{ secrets.TOKEN_GITHUB }}
+          fail: false
+          args: >
+            ./doc/sphinx/**/*.rst
+            --max-concurrency 1 --no-progress
+            --accept "200..=204, 302, 429"
+
+      - name: Set current date
+        run: echo "REPORT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Create Issue From File
+        if: steps.lychee.outputs.exit_code != 0
+        uses: peter-evans/create-issue-from-file@v6
+        with:
+          title: "🔗 Link Checker Report - ${{ env.REPORT_DATE }}"
+          content-filepath: ./lychee/out.md
+          labels: documentation, github_actions

.github/workflows/check_links_in_pr.yml

@@ -0,0 +1,45 @@
+name: Link Checker for Pull requests
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  changedFiles:
+    runs-on: ubuntu-latest
+    outputs:
+      files: ${{ steps.changed-doc-files.outputs.all_changed_files }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Get changed doc files
+        id: changed-doc-files
+        uses: tj-actions/changed-files@v47
+        with:
+          # Avoid using single or double quotes for multiline patterns
+          files: |
+            **.rst
+          matrix: true
+
+  linkChecker:
+    runs-on: ubuntu-latest
+    needs: changedFiles
+    if: ${{ needs.changedFiles.outputs.files != '' && toJSON(fromJSON(needs.changedFiles.outputs.files)) != '[]' }}
+    strategy:
+      matrix:
+        file: ${{ fromJSON(needs.changedFiles.outputs.files) }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+      - name: Link Checker for changed files
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
+        with:
+          #  This token is included to avoid github.com requests rate limiting
+          token: ${{ secrets.TOKEN_GITHUB }}
+          # Fail check on broken links
+          fail: true
+          args: >
+            ${{ matrix.file }}
+            --max-concurrency 1 --no-progress
+            --accept "200..=204, 429"

.github/workflows/citation_file_validator.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.event_name != 'schedule' || github.repository == 'ESMValGroup/ESMValTool' # avoid scheduled GHAs in forks
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/cffconvert-github-action@2.0.0

.github/workflows/create-condalock-file.yml

@@ -21,28 +21,20 @@ jobs:
     runs-on: 'ubuntu-latest'
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.12"
+          python-version: "3.13"
           miniforge-version: "latest"
           use-mamba: true
       - name: Show conda config
         run: |
           conda update -n base -c conda-forge conda
           conda --version
-          # setup-miniconda@v3 installs an old conda and mamba
-          # forcing a modern mamba updates both mamba and conda
-          # unpin mamba after conda-lock=3 release
-          # see github.com/ESMValGroup/ESMValTool/issues/3782
-          conda install -c conda-forge "mamba>=1.4.8,<2"
-          conda config --show-sources
-          conda config --show
-          conda --version
           mamba --version
       - name: Python info
         run: |
@@ -69,7 +61,7 @@ jobs:
           which python
           pip --version
       - name: Install ESMValTool
-        run: pip install -e .[develop]
+        run: pip install --no-deps -e .[develop]
       - name: Check ESMValTool version
         run: esmvaltool --help
       - name: Check ESMValTool help
@@ -81,7 +73,7 @@ jobs:
       # Automated PR
       # see https://github.com/marketplace/actions/create-pull-request
       - name: Create Automated PR if conda lock file has changed
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Updating Linux condalock file

.github/workflows/cron_esmvalbot_test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: 'ubuntu-latest'
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Create empty commit on branch
@@ -30,7 +30,7 @@ jobs:
       # as comment in the PR
       # see https://github.com/marketplace/actions/create-pull-request
       - name: Create Auto PR
-        uses: peter-evans/create-pull-request@v7
+        uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: empty message