Merge remote-tracking branch 'origin/main' into cmorize_glass

Author: bettina-gier

.circleci/config.yml

@@ -2,56 +2,48 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@13.11.16
+  pixi: prefix-dev/pixi@0.1.0
 
-commands:
-
-  check_changes:
+jobs:
+  tests:
+    # Run tests
+    docker:
+      - image: cimg/base:current
+    resource_class: large
     steps:
+      - checkout
+      - pixi/pixi_setup
       - run:
-          name: Check whether or not installation tests are needed
+          name: Check dependencies are compatible and installed
           command: |
-            if (test "$CIRCLE_BRANCH" = main ||
-              git --no-pager diff --name-only origin/main... |
-              grep -q -E -f .circleci/install_triggers)
-            then
-              echo Running installation tests
-            else
-              echo Skipping installation tests
-              circleci step halt
-            fi
-
-  log_versions:
-    steps:
+            pixi run --as-is pip check
+            # The following command should not install any new dependencies
+            # because they are already in the conda environment, but this is
+            # not checked.
+            pixi run --as-is pip install --dry-run .[all]
       - run:
-          name: Log versions
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            esmvaltool version
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
-
-  test_and_report:
-    steps:
+          name: Check code formatting and mistakes
+          command: pixi run --as-is pre-commit run --all
       - run:
           name: Run tests
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            pytest -n 4 --junitxml=test-reports/report.xml
+            pixi run --as-is test -n 4 --durations=10 --junitxml=test-reports/report.xml
+      - run:
+          name: Check the command line interface
+          command: |
+            source <(pixi shell-hook --as-is)
             esmvaltool version
             esmvaltool -- --help
-            ncl -V
-            cdo --version
+            esmvaltool develop compare -- --help
       - store_test_results:
           path: test-reports/report.xml
-      - store_artifacts:
-          path: /logs
+      - run:
+          # Documentation available at https://docs.codacy.com/coverage-reporter/
+          name: Upload coverage report to Codacy
+          command: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -s -r test-reports/coverage.xml
       - run:
           name: Compress pytest artifacts
-          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-root/ .
+          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-$(whoami)/ .
           when: always
       - store_artifacts:
           path: pytest.tar.gz
@@ -62,197 +54,106 @@ commands:
       - store_artifacts:
           path: test-reports.tar.gz
 
-  test_installation_from_source:
-    parameters:
-      extra:
-        type: string
-      flags:
-        type: string
-        default: ""
-    steps:
-      - run:
-          name: Install git+ssh
-          environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
-          command: apt update && apt install -y git ssh
-      - checkout
-      - check_changes
-      - run:
-          name: Generate cache key
-          command: date '+%Y-%V' | tee cache_key.txt
-      - restore_cache:
-          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-      - run:
-          name: Install dependencies
-          command: |
-            # Install
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
-            conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
-            if [[ "<<parameters.flags>>" != *'--editable'* ]]
-            then
-              rm -r esmvaltool
-            fi
-      - log_versions
-      - run:
-          name: Lint source code
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            flake8 -j 4
-      - test_and_report
-      - save_cache:
-          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-          paths:
-            - /opt/conda/pkgs
-            - /root/.cache/pip
-            - .pytest_cache
-
-jobs:
-  run_tests:
-    # Run tests
+  test_esmvaltool_base_installation:
+    # Test running the example recipe with minimal dependencies.
     docker:
-      - image: esmvalgroup/esmvaltool:development
-    resource_class: large
+      - image: cimg/base:current
+    resource_class: medium
     steps:
       - checkout
       - run:
           name: Generate cache key
-          command: date '+%Y-%V' | tee cache_key.txt
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
       - restore_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - pixi/pixi_setup:
+          env: esmvaltool-base
       - run:
-          name: Install dependencies
+          name: Check dependencies are compatible
+          command: pixi run --as-is --environment esmvaltool-base pip check
+      - run:
+          name: Run example recipe
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            conda activate esmvaltool
-            pip install .[test] > /logs/install.txt 2>&1
-            # Run flake8 check with source code (it doesn't work without it)
-            flake8 -j 4
-            # Update/install Julia dependencies
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
-            # Remove source to test installed software
-            rm -r esmvaltool
-      - test_and_report
+            source <(pixi shell-hook --as-is --environment esmvaltool-base)
+            esmvaltool version
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
+            esmvaltool run --max-parallel-tasks=2 testing/recipe_python_for_CI.yml
+      - store_artifacts:
+          path: /root/esmvaltool_output
       - save_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
-            - /root/.cache/pip
-            - .pytest_cache
-      - coverage-reporter/send_report:
-          coverage-reports: 'test-reports/coverage.xml'
-          project-token: $CODACY_PROJECT_TOKEN
-          skip: true  # skip if project-token is not defined (i.e. on a fork)
-
-  test_installation_from_source_test_mode:
-    # Test installation from source
-    docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: large
-    steps:
-      - test_installation_from_source:
-          extra: test
-
-  test_installation_from_source_develop_mode:
-    # Test development installation
-    docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: large
-    steps:
-      - test_installation_from_source:
-          extra: develop
-          flags: "--editable"
+            - /root/climate_data
 
   test_upstream_development:
     # Test running recipes with the development version of ESMValCore. The
     # purpose of this test to discover backward-incompatible changes early on in
     # the development cycle.
     docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: large
+      - image: cimg/base:current
+    resource_class: medium
     steps:
-      - run:
-          name: Install git and ssh
-          environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
-          command: apt update && apt install -y git ssh
       - checkout
       - run:
           name: Generate cache key
-          command: echo $(date '+%Y')-$(expr $(date '+%V') / 2) | tee cache_key.txt
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
       - restore_cache:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - pixi/pixi_setup:
+          env: esmvalcore-dev
       - run:
-          name: Install
+          name: Run tests
           command: |
-            # Install according to instructions on readthedocs with the
-            # development version of ESMValTool and ESMValCore:
-            # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
-            conda activate esmvaltool
-            pip install --editable .[develop]
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
-            git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
-            pip install --editable $HOME/ESMValCore[develop]
-      - log_versions
-      - test_and_report
+            pixi run --as-is --environment esmvalcore-dev test -n 2 --junitxml=test-reports/report.xml
+      - store_test_results:
+          path: test-reports/report.xml
       - run:
           name: Run recipes
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            mkdir -p ~/climate_data
-            esmvaltool config get_config_user
-            echo "offline: false" >> ~/.esmvaltool/config-user.yml
-            cat ~/.esmvaltool/config-user.yml
-            esmvaltool run examples/recipe_python.yml
+            source <(pixi shell-hook --as-is --environment esmvalcore-dev)
+            esmvaltool version
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
-              esmvaltool run "$recipe"
+              esmvaltool run --max-parallel-tasks=2 "$recipe"
             done
       - store_artifacts:
           path: /root/esmvaltool_output
       - save_cache:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
-            - /opt/conda/pkgs
-            - /root/.cache/pip
             - /root/climate_data
 
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: small
+      - image: cimg/base:current
+    resource_class: medium
     steps:
       - checkout
+      - pixi/pixi_setup
       - run:
-          command: |
-            mkdir /logs
-            . /opt/conda/etc/profile.d/conda.sh
-            # Install
-            mamba env create
-            conda activate esmvaltool
-            pip install .[doc]
-            # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
-            # Test building documentation
-            MPLBACKEND=Agg sphinx-build -W doc/sphinx/source doc/spinx/build
-      - store_artifacts:
-          path: /logs
+          name: Build documentation
+          command: pixi run --as-is doc --fail-on-warning --show-traceback
 
   test_installation_from_conda:
     # Test conda package installation
     docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: medium
+      - image: condaforge/miniforge3:latest
+    resource_class: large
     steps:
       - run:
           command: |
@@ -261,11 +162,9 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.10' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.13' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
-            # install the Julia dependencies
-            esmvaltool install Julia > /logs/install_Julia.txt 2>&1
             # Log versions
             mamba env export > /logs/environment.yml
             # Test installation
@@ -279,9 +178,7 @@ jobs:
 workflows:
   commit:
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
-      - test_installation_from_source_develop_mode
+      - tests
   nightly:
     triggers:
       - schedule:
@@ -291,9 +188,8 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
-      - test_installation_from_source_develop_mode
+      - tests
+      - test_esmvaltool_base_installation
       - test_upstream_development
       - build_documentation
       - test_installation_from_conda