Merge branch 'main' into recipe_seaice_areaextents_sh

Author: flicj191

.circleci/config.yml

@@ -1,11 +1,7 @@
 ---
 version: 2.1
 
-orbs:
-  coverage-reporter: codacy/coverage-reporter@13.13.0
-
 commands:
-
   check_changes:
     steps:
       - run:
@@ -41,10 +37,11 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mamba --version
+            pip check
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
+            esmvaltool develop compare -- --help
             ncl -V
             cdo --version
       - store_test_results:
@@ -75,7 +72,7 @@ commands:
       - run:
           name: Install git+ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - check_changes
@@ -86,14 +83,16 @@ commands:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
           name: Install dependencies
+          no_output_timeout: 30m
           command: |
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            mamba --version
+            conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            mamba list >> /logs/conda.txt
+            pip install --no-deps << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
             if [[ "<<parameters.flags>>" != *'--editable'* ]]
             then
               rm -r esmvaltool
@@ -104,7 +103,7 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            flake8 -j 4
+            pre-commit run --all
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -133,32 +132,31 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
             mkdir /logs
-            pip install .[test] > /logs/install.txt 2>&1
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            pip install --no-deps .[test] > /logs/install.txt 2>&1
       - run:
           name: Check Python code style and mistakes
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            flake8 -j 4
+            pre-commit run --all
       - run:
           name: Remove source code to test the installed software
           command: rm -r esmvaltool
       - test_and_report
+      - run:
+          # Documentation available at https://docs.codacy.com/coverage-reporter/
+          name: Upload coverage report to Codacy
+          command: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -s -r test-reports/coverage.xml
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
             - /root/.cache/pip
             - .pytest_cache
-      - coverage-reporter/send_report:
-          coverage-reports: 'test-reports/coverage.xml'
-          project-token: $CODACY_PROJECT_TOKEN
-          skip: true  # skip if project-token is not defined (i.e. on a fork)
 
   test_installation_from_source_test_mode:
     # Test installation from source
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -167,7 +165,7 @@ jobs:
   test_installation_from_source_develop_mode:
     # Test development installation
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -179,13 +177,13 @@ jobs:
     # purpose of this test to discover backward-incompatible changes early on in
     # the development cycle.
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - run:
           name: Install git and ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - run:
@@ -195,31 +193,38 @@ jobs:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
           name: Install
+          no_output_timeout: 30m
           command: |
             # Install according to instructions on readthedocs with the
             # development version of ESMValTool and ESMValCore:
             # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            conda env create -n esmvaltool -f environment.yml --verbose
             conda activate esmvaltool
-            pip install --editable .[develop]
-            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            # Temporarily install intake-esgf here until there is a new release
+            # that includes https://github.com/esgf2-us/intake-esgf/pull/160.
+            pip install git+https://github.com/esgf2-us/intake-esgf@main
+            mamba list >> /logs/conda.txt
+            pip install --no-deps --editable .[develop]
             git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
-            pip install --editable $HOME/ESMValCore[develop]
+            pip install --no-deps --editable $HOME/ESMValCore[develop]
       - log_versions
       - test_and_report
       - run:
           name: Run recipes
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            mkdir -p ~/climate_data
-            esmvaltool config get_config_user
-            echo "search_esgf: when_missing" >> ~/.esmvaltool/config-user.yml
-            cat ~/.esmvaltool/config-user.yml
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
-              esmvaltool run "$recipe"
+              esmvaltool run --max-parallel-tasks=2 "$recipe"
             done
       - store_artifacts:
           path: /root/esmvaltool_output
@@ -233,18 +238,19 @@ jobs:
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge:latest
-    resource_class: medium
+      - image: condaforge/miniforge3:latest
+    resource_class: medium+
     steps:
       - checkout
       - run:
           command: |
             mkdir /logs
             . /opt/conda/etc/profile.d/conda.sh
             # Install
-            mamba env create
+            conda env create -n esmvaltool -f environment_python.yml
             conda activate esmvaltool
-            pip install .[doc]
+            pip install --no-deps .[doc]
+            pip check
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
@@ -257,7 +263,7 @@ jobs:
   test_installation_from_conda:
     # Test conda package installation
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - run:
@@ -267,11 +273,9 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.11' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.12' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
-            # install the Julia dependencies
-            esmvaltool install Julia > /logs/install_Julia.txt 2>&1
             # Log versions
             mamba env export > /logs/environment.yml
             # Test installation

.circleci/install_triggers

@@ -1,6 +1,5 @@
 ^\.circleci/
 ^environment\.yml$
-^esmvaltool/install/
 ^pyproject.toml$
 ^setup\.py$
 ^setup\.cfg$

.codacy.yml

@@ -1,25 +1,15 @@
 # codacy configuration file
-
 ---
-
 engines:
-    coverage:
-        enabled: true
-        exclude_paths: [
-            'tests',
-        ]
-    metrics:
-        enabled: true
-    duplication:
-        enabled: true
-    prospector:
-        enabled: true
-    pylint:
-        enabled: true
-        python_version: 3
+  coverage:
+    exclude_paths: ["tests"]
 
-exclude_paths: [
-    'doc/sphinx/**',
-    'esmvaltool/cmor/tables/**',
-    'tests/**'
-]
+exclude_paths:
+  [
+    "archive/**",
+    "doc/sphinx/**",
+    "esmvaltool/cmor/tables/**",
+    "tests/**",
+    "esmvaltool/utils/recipe_test_workflow/app/configure/bin/test_configure.py",
+    "esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py",
+  ]

.dockerignore

@@ -7,4 +7,3 @@ ESMValTool.egg-info
 
 !.git
 !.zenodo.json
-

.editorconfig

@@ -27,4 +27,3 @@ indent_size = 2
 
 [*.{md,Rmd}]
 trim_trailing_whitespace = false
-

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 esmvaltool/cmorizers @ESMValGroup/obs-maintainers
 .github/workflows @valeriupredoi
+esmvaltool/utils/recipe_test_workflow/ @alistairsellar @ehogan

.github/dependabot.yml

@@ -0,0 +1,18 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`
+    # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - dependencies
+      - github_actions
+      - automatedPR

.github/workflows/check-rtw.yml

@@ -0,0 +1,79 @@
+# This workflow performs various validation steps for Cylc and Rose.
+name: Check Recipe Test Workflow (RTW)
+
+# Controls when the action will run
+on:
+  # Triggers the workflow on push events
+  push:
+    paths:
+      - esmvaltool/utils/recipe_test_workflow/**
+
+  # Schedule this workflow to run at 04:00 every 10 days
+  schedule:
+    - cron: '0 4 */10 * *'
+
+# Common variables are defined here
+env:
+  RTW_ROOT_DIR: esmvaltool/utils/recipe_test_workflow
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+# A workflow run is made up of one or more jobs that can run
+# sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "check-rtw"
+  check-rtw:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    if: github.event_name != 'schedule' || github.repository == 'ESMValGroup/ESMValTool' # avoid scheduled GHAs in forks
+
+    # Steps represent a sequence of tasks that will be executed as part
+    # of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job
+      # can access it
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Cylc
+        uses: cylc/setup-cylc@v1
+        with:
+          cylc_rose: true
+
+      - name: Validate Cylc workflow
+        run: |
+          cd ${RTW_ROOT_DIR}
+          cylc validate . -O metoffice
+
+      - name: Run Cylc configuration linter
+        run: |
+          cd ${RTW_ROOT_DIR}
+          cylc lint
+
+      - name: Validate format of Rose configuration files
+        run: |
+          cd ${RTW_ROOT_DIR}
+          output="$(rose config-dump)"
+          msg="Run 'rose config-dump' to re-dump the Rose configuration files"
+          msg="${msg} in the common format, then commit the changes."
+          # The '-z' option returns true if 'output' is empty.
+          if [[ -z "${output}" ]]; then true; else echo "${msg}" && exit 1; fi
+
+      - name: Validate Rose configuration metadata
+        run: |
+          cd ${RTW_ROOT_DIR}
+          rose metadata-check -C meta/
+
+      - name: Run Rose configuration validation macros
+        run: |
+          cd ${RTW_ROOT_DIR}
+          rose macro -V
+
+      - name: Lint shell scripts
+        run: |
+          cd ${RTW_ROOT_DIR}
+          output=$(find . -name "*.sh" -exec shellcheck {} \;)
+          if [ "$output" ]; then echo "${output}" && exit 1; fi