Add an option to install with minimal dependencies (#4517)

Author: bouweandela

.circleci/config.yml

@@ -14,8 +14,13 @@ jobs:
       - checkout
       - pixi/pixi_setup
       - run:
-          name: Check dependencies are compatible
-          command: pixi run --as-is pip check
+          name: Check dependencies are compatible and installed
+          command: |
+            pixi run --as-is pip check
+            # The following command should not install any new dependencies
+            # because they are already in the conda environment, but this is
+            # not checked.
+            pixi run --as-is pip install --dry-run .[all]
       - run:
           name: Check code formatting and mistakes
           command: pixi run --as-is pre-commit run --all
@@ -49,35 +54,71 @@ jobs:
       - store_artifacts:
           path: test-reports.tar.gz
 
+  test_esmvaltool_base_installation:
+    # Test running the example recipe with minimal dependencies.
+    docker:
+      - image: cimg/base:current
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
+      - restore_cache:
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - pixi/pixi_setup:
+          env: esmvaltool-base
+      - run:
+          name: Check dependencies are compatible
+          command: pixi run --as-is --environment esmvaltool-base pip check
+      - run:
+          name: Run example recipe
+          command: |
+            source <(pixi shell-hook --as-is --environment esmvaltool-base)
+            esmvaltool version
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
+            esmvaltool run --max-parallel-tasks=2 testing/recipe_python_for_CI.yml
+      - store_artifacts:
+          path: /root/esmvaltool_output
+      - save_cache:
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /root/climate_data
+
   test_upstream_development:
     # Test running recipes with the development version of ESMValCore. The
     # purpose of this test to discover backward-incompatible changes early on in
     # the development cycle.
     docker:
       - image: cimg/base:current
-    resource_class: large
+    resource_class: medium
     steps:
       - checkout
       - run:
           name: Generate cache key
-          command: echo $(date '+%Y')-$(expr $(date '+%V') / 2) | tee cache_key.txt
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
       - restore_cache:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - pixi/pixi_setup:
           env: esmvalcore-dev
       - run:
           name: Run tests
           command: |
-            pixi run --as-is --environment esmvalcore-dev test -n 4 --junitxml=test-reports/report.xml
+            pixi run --as-is --environment esmvalcore-dev test -n 2 --junitxml=test-reports/report.xml
       - store_test_results:
           path: test-reports/report.xml
       - run:
           name: Run recipes
           command: |
             source <(pixi shell-hook --as-is --environment esmvalcore-dev)
-            # Temporarily install intake-esgf here until there is a new release
-            # that includes https://github.com/esgf2-us/intake-esgf/pull/160.
-            pip install git+https://github.com/esgf2-us/intake-esgf@main
             esmvaltool version
             esmvaltool config copy data-intake-esgf.yml
             python -c '
@@ -100,7 +141,7 @@ jobs:
     # Test building documentation
     docker:
       - image: cimg/base:current
-    resource_class: medium+
+    resource_class: medium
     steps:
       - checkout
       - pixi/pixi_setup
@@ -148,6 +189,7 @@ workflows:
                 - main
     jobs:
       - tests
+      - test_esmvaltool_base_installation
       - test_upstream_development
       - build_documentation
       - test_installation_from_conda

.github/workflows/test.yml

@@ -32,13 +32,14 @@ jobs:
           - test-py312
           - test-py313
           - test-py314
+          - esmvaltool-base-dev
     name: ${{ matrix.os }} ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11  # v0.9.6
+      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
         with:
           frozen: true
           environments: ${{ matrix.environment }}
@@ -47,7 +48,15 @@ jobs:
       - name: Check code quality
         run: pre-commit run -a
       - name: Run tests
-        run: pytest -n 2 -m "not installation"
+        if: matrix.environment != 'esmvaltool-base-dev'
+        run: pytest -n 2
+      - name: Run tests
+        if: matrix.environment == 'esmvaltool-base-dev'
+        run: >-
+          pytest
+          -n 2
+          --ignore=tests/sample_data/
+          --ignore=tests/unit/cmorizers/test_cmorization_interface.py
       - name: Upload artifacts
         if: ${{ always() }} # upload artifacts even if fail
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

doc/sphinx/source/community/code_documentation.rst

@@ -522,22 +522,42 @@ See this `statement <https://www.apache.org/licenses/GPL-compatibility.html>`__
 by the authors of the Apache 2.0 license for more information.
 
 When adding or removing dependencies, please consider applying the changes in
-the following locations in ``pyproject.toml``:
+the following two locations in ``pyproject.toml``:
 
-- ``dependencies``
-  contains dependencies that can be installed from `PyPI <https://pypi.org/>`__
-- ``[tool.pixi.dependencies]``
-  contains dependencies that can be installed from `conda-forge <https://conda-forge.org/>`__
+- ``[project.optional-dependencies]``
+  the ``all`` item contains all dependencies needed to run the diagnostics
+  and CMORizer scripts, with the names used on `PyPI <https://pypi.org/>`__.
+- ``[tool.pixi.feature.all]``
+  the same list of dependencies as under ``[project.optional-dependencies]``,
+  but with the names used on `conda-forge <https://conda-forge.org/>`__.
 
 It is strongly preferred that those two lists are kept in sync, apart from
 differences in how packages are named. Run the command ``pixi lock --no-install``
 after making changes to the dependencies to update the ``pixi.lock`` file, which
 is used to make sure that the same versions of packages are installed for
 all ESMValTool developers.
 
-In addition to these core dependencies, there are also optional dependencies
-for diagnostics that need R or NCL. These are listed in the sections
-``[tool.pixi.feature.r]`` and ``[tool.pixi.feature.ncl]`` in respectively.
+There is also a minimal set of dependencies that are required to run the example
+recipe ``examples/recipe_python.yml``. Please do not add new dependencies to
+this list unless they are really needed to support the core functionality of
+ESMValTool. These can be used by users who want to run ESMValTool with a minimal
+set of dependencies and want to manage which dependencies of diagnostics and/or
+CMORizer scripts get installed themselves. The minimal dependencies are listed
+in the following two locations in ``pyproject.toml``:
+
+- ``dependencies``
+  contains a minimal set of dependencies, those required to run the example
+  recipe, with the names used on `PyPI <https://pypi.org/>`__
+- ``[tool.pixi.dependencies]``
+  the same list of dependencies as under ``dependencies``, but with the names
+  used on `conda-forge <https://conda-forge.org/>`__
+
+It is strongly preferred that those two lists are kept in sync, apart from
+differences in how packages are named.
+
+In addition to the core dependencies mentioned above, there are further optional
+dependencies for diagnostics that need R or NCL. These are listed in the sections
+``[tool.pixi.feature.r]`` and ``[tool.pixi.feature.ncl]`` respectively.
 
 There are also three feature groups for development dependencies:
 

doc/sphinx/source/quickstart/installation.rst

@@ -165,14 +165,14 @@ message to the console.
 Installation of subpackages
 ---------------------------
 
-The diagnostics bundled in ESMValTool are scripts in four different programming
-languages: Python, NCL, and R.
+The diagnostics bundled in ESMValTool are scripts in three different programming
+languages: Python, R, and NCL.
 
 There are three language specific packages available:
 
-* ``esmvaltool-ncl``
 * ``esmvaltool-python``
 * ``esmvaltool-r``
+* ``esmvaltool-ncl``
 
 The main ``esmvaltool`` package contains all three subpackages listed above.
 If you only need to run a recipe with diagnostics in some of these languages, it