Merge branch 'main' into extend_esa_cci_ozone_cmorizer

Author: axel-lauer

.circleci/config.yml

@@ -41,10 +41,12 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mamba --version
+            # Add existing Julia to PATH
+            export PATH=/root/.juliaup/bin:$PATH
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
+            esmvaltool develop compare -- --help
             ncl -V
             cdo --version
       - store_test_results:
@@ -86,16 +88,19 @@ commands:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
           name: Install dependencies
+          no_output_timeout: 30m
           command: |
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             mamba --version
-            mamba install -c conda-forge "mamba>=2"  # mamba<2 can't solve env
-            mamba env create -n esmvaltool -f environment.yml
+            conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
             mamba list >> /logs/conda.txt
             pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            curl -fsSL https://install.julialang.org | sh -s -- --yes
+            export PATH=/root/.juliaup/bin:$PATH
+            julia --version
             esmvaltool install Julia > /logs/install_julia.txt 2>&1
             if [[ "<<parameters.flags>>" != *'--editable'* ]]
             then
@@ -137,6 +142,9 @@ jobs:
             conda activate esmvaltool
             mkdir /logs
             pip install .[test] > /logs/install.txt 2>&1
+            curl -fsSL https://install.julialang.org | sh -s -- --yes
+            export PATH=/root/.juliaup/bin:$PATH
+            julia --version
             esmvaltool install Julia > /logs/install_julia.txt 2>&1
       - run:
           name: Check Python code style and mistakes
@@ -198,17 +206,21 @@ jobs:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
           name: Install
+          no_output_timeout: 30m
           command: |
             # Install according to instructions on readthedocs with the
             # development version of ESMValTool and ESMValCore:
             # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba install -c conda-forge "mamba>=2"  # mamba<2 can't solve env
-            mamba env create -n esmvaltool -f environment.yml
+            conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
             mamba list >> /logs/conda.txt
             pip install --editable .[develop]
+            # Install Julia from source and add to PATH
+            curl -fsSL https://install.julialang.org | sh -s -- --yes
+            export PATH=/root/.juliaup/bin:$PATH
+            julia --version
             esmvaltool install Julia > /logs/install_julia.txt 2>&1
             git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
             pip install --editable $HOME/ESMValCore[develop]
@@ -275,6 +287,10 @@ jobs:
             mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.11' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
+            # install Julia from source and add to PATH
+            curl -fsSL https://install.julialang.org | sh -s -- --yes
+            export PATH=/root/.juliaup/bin:$PATH
+            julia --version
             # install the Julia dependencies
             esmvaltool install Julia > /logs/install_Julia.txt 2>&1
             # Log versions

.codacy.yml

@@ -19,8 +19,10 @@ engines:
         python_version: 3
 
 exclude_paths: [
+    'archive/**',
     'doc/sphinx/**',
     'esmvaltool/cmor/tables/**',
     'tests/**',
-    'esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py'
+    'esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py',
+    'esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py',
 ]

.github/workflows/create-condalock-file.yml

@@ -28,21 +28,13 @@ jobs:
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.12"
+          python-version: "3.13"
           miniforge-version: "latest"
           use-mamba: true
       - name: Show conda config
         run: |
           conda update -n base -c conda-forge conda
           conda --version
-          # setup-miniconda@v3 installs an old conda and mamba
-          # forcing a modern mamba updates both mamba and conda
-          # unpin mamba after conda-lock=3 release
-          # see github.com/ESMValGroup/ESMValTool/issues/3782
-          conda install -c conda-forge "mamba>=1.4.8,<2"
-          conda config --show-sources
-          conda config --show
-          conda --version
           mamba --version
       - name: Python info
         run: |

.github/workflows/install-from-conda.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]  # add "3.13" when pkg built
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: conda-incubator/setup-miniconda@v3
@@ -57,7 +57,7 @@ jobs:
 #    if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
 #    strategy:
 #      matrix:
-#        python-version: ["3.10", "3.11"]
+#        python-version: ["3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:

.github/workflows/install-from-condalock-file.yml

@@ -31,7 +31,7 @@ jobs:
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:

.github/workflows/install-from-source.yml

@@ -20,7 +20,7 @@ jobs:
     if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -61,7 +61,7 @@ jobs:
 #    if: github.repository == 'ESMValGroup/ESMValTool'  # avoid GAs in forks
 #    strategy:
 #      matrix:
-#        python-version: ["3.10", "3.11"]
+#        python-version: ["3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:

.github/workflows/pypi-build-and-deploy.yml

@@ -17,10 +17,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.12
+      - name: Set up Python 3.13
         uses: actions/setup-python@v1
         with:
-          python-version: "3.12"
+          python-version: "3.13"
       - name: Install pep517
         run: >-
           python -m

.github/workflows/run-tests-monitor.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
@@ -49,6 +49,8 @@ jobs:
         run: pip install pytest-monitor
       - name: Install ESMValTool
         run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Install Julia from source
+        run: curl -fsSL https://install.julialang.org | sh -s -- --yes
       - name: Install Julia dependencies
         run: esmvaltool install Julia
       - name: Run tests
@@ -68,7 +70,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

.github/workflows/stale.yml

@@ -0,0 +1,35 @@
+name: Close stale issues and pull requests
+
+on:
+  schedule:
+    # Run once a day
+    - cron: "30 1 * * *"
+
+jobs:
+  stale:
+    if: "github.repository == 'ESMValGroup/ESMValTool'"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 1080
+          days-before-pr-stale: 360
+          days-before-close: 60
+          ascending: true
+          operations-per-run: 300
+          stale-issue-message: |
+            In order to maintain a backlog of relevant issues, we automatically label them as stale after 1080 days of inactivity.
+
+            If this issue is still important to you, please comment below to remove the stale label. Otherwise, this issue will be automatically closed in 60 days.
+          stale-pr-message: |
+            In order to maintain a backlog of relevant pull requests, we automatically label them as stale after 360 days of inactivity.
+
+            If this pull request is still important to you, please comment below to remove the stale label. Otherwise, this pull request will be automatically closed in 60 days. If this pull request only suffers from a lack of reviewers, please have a look at [our documentation](https://docs.esmvaltool.org/en/latest/community/review.html#how-do-i-request-a-review-of-my-pull-request) and follow the steps described there.
+          close-issue-message: |
+            This stale issue has been automatically closed due to a lack of community activity.
+
+            If you still care about this issue, please re-open it. If you do not have sufficient permissions for this, tag the @<span></span>ESMValGroup/technical-lead-development-team and @<span></span>ESMValGroup/scientific-lead-development-team in a comment below including a short message that this issue is still relevant to you. They will re-open it on your behalf.
+          close-pr-message: |
+            This stale pull request has been automatically closed due to a lack of community activity.
+
+            If you still care about this pull request, please re-open it. If you do not have sufficient permissions for this, tag the @<span></span>ESMValGroup/technical-lead-development-team and @<span></span>ESMValGroup/scientific-lead-development-team in a comment below including a short message that this pull request is still relevant to you. They will re-open it on your behalf.

.github/workflows/test-development.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
@@ -48,6 +48,8 @@ jobs:
           python -V 2>&1 | tee develop_test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
       - name: Install ESMValTool
         run: pip install -e .[develop] 2>&1 | tee develop_test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Install Julia from source
+        run: curl -fsSL https://install.julialang.org | sh -s -- --yes
       - name: Install Julia dependencies
         run: esmvaltool install Julia
       - name: Install development version of ESMValCore