Add earth2grid as optional dependency of fme (#1520)

Alternative to #1301 to specify earth2grid as an optional 'healpix'
dependency. Also adds the healpix requirements to our CI environments
*except* the `cpu-very-fast` tests which we can use to ensure `fme` can
still be installed and used without the healpix extra.

Resolve #1542

---------

Co-authored-by: Anna Dodson <[email protected]>

Author: oliverwm1

.github/workflows/docs.yaml

@@ -37,13 +37,17 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}-${{ hashFiles('constraints.txt') }}
+          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}-${{ hashFiles('healpix-requirements.txt') }}-${{ hashFiles('constraints.txt') }}
       - name: Install dependencies
         run: |
           python -m pip install uv
-          uv pip install --system -c constraints.txt -e .[docs]
+          # can't use --system here because it doesn't work with --no-build-isolation and a healpix dependency
+          uv venv --python 3.10
+          uv pip install -c constraints.txt -e .[docs]
+          uv pip install --no-build-isolation -c constraints.txt -e .[docs,healpix]
       - name: Build docs
         run: |
+          source .venv/bin/activate
           cd docs && make doctest html
       - name: Upload to GitHub Pages
         uses: actions/upload-pages-artifact@v3

.github/workflows/tests.yaml

@@ -25,15 +25,19 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}-${{ hashFiles('constraints.txt') }}
+          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}-${{ hashFiles('healpix-requirements.txt') }}-${{ hashFiles('constraints.txt') }}
       - name: Install dependencies
         run: |
           python -m pip install uv
-          uv pip install --system -c constraints.txt -e .[dev]
+          # can't use --system here because it doesn't work with --no-build-isolation and a healpix dependency
+          uv venv --python 3.10
+          uv pip install -c constraints.txt -e .[dev]
+          uv pip install --no-build-isolation -c constraints.txt -e .[dev,healpix]
       - name: Run pytest
         run: |
+          source .venv/bin/activate
           make test
-  cpu-very-fast:
+  cpu-very-fast-no-healpix:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -69,15 +73,19 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}-${{ hashFiles('constraints.txt') }}
+          key: ${{ env.CURRENT_WEEK }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}-${{ hashFiles('healpix-requirements.txt') }}-${{ hashFiles('constraints.txt') }}
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y make git
+          sudo apt-get install -y make git build-essential
           python -m pip install uv
-          uv pip install --system -c constraints.txt -e .[dev]
+          # can't use --system here because it doesn't work with --no-build-isolation and a healpix dependency
+          uv venv --python 3.10
+          uv pip install -c constraints.txt -e .[dev]
+          uv pip install --no-build-isolation -c constraints.txt -e .[dev,healpix]
       - name: Run CUDA check and pytest
         run: |
+            source .venv/bin/activate
             export PATH=/usr/local/nvidia/bin:${PATH}
             export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
             python3 fme/require_gpu.py

Makefile

@@ -7,6 +7,12 @@ DEPLOY_TARGET ?= pypi
 BEAKER_WORKSPACE = ai2/ace
 CURRENT_DATE = $(shell date +'%Y-%m-%d')
 
+ifeq ($(shell uname), Linux)
+	CONDA_PACKAGES=gxx_linux-64 pip
+else
+	CONDA_PACKAGES=pip
+endif
+
 build_docker_image:
 	DOCKER_BUILDKIT=1 docker build --platform=linux/amd64 -f docker/Dockerfile -t $(IMAGE):$(VERSION) --target production .
 
@@ -37,9 +43,10 @@ test_image:
 
 # recommended to deactivate current conda environment before running this
 create_environment:
-	conda create -n $(ENVIRONMENT_NAME) python=3.10 pip
+	conda create -n $(ENVIRONMENT_NAME) python=3.10 $(CONDA_PACKAGES)
 	conda run --no-capture-output -n $(ENVIRONMENT_NAME) python -m pip install uv
 	conda run --no-capture-output -n $(ENVIRONMENT_NAME) uv pip install -c constraints.txt -e .[dev,docs]
+	conda run --no-capture-output -n $(ENVIRONMENT_NAME) uv pip install --no-build-isolation -c constraints.txt -e .[dev,docs,healpix]
 	conda run --no-capture-output -n $(ENVIRONMENT_NAME) uv pip install -r analysis-deps.txt
 
 test:

docker/Dockerfile

@@ -12,11 +12,13 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c
 # install python deps
 COPY requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install -r /tmp/requirements.txt
+COPY healpix-requirements.txt /tmp/healpix-requirements.txt
+RUN python3 -m pip install --no-build-isolation -r /tmp/healpix-requirements.txt
 
 FROM base AS production
 
 # copy local code and install
-COPY fme ${FME_DIR}/fme
+COPY . ${FME_DIR}
 RUN cd $FME_DIR && pip install --no-deps -e .
 
 # copy after install so editing scripts does not trigger reinstall

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = { file = "requirements.txt" }
 optional-dependencies.dev = { file = "dev-requirements.txt" }
 optional-dependencies.docs = { file = "docs/requirements.txt" }
 optional-dependencies.deploy = { file = "deploy-requirements.txt" }
+optional-dependencies.healpix = { file = "healpix-requirements.txt" }
 
 [tool.setuptools.packages]
 find = {}
@@ -39,6 +40,7 @@ cache-keys = [
     { file = "requirements.txt" },
     { file = "dev-requirements.txt" },
     { file = "docs/requirements.txt" },
+    { file = "healpix-requirements.txt" },
 ]
 
 [tool.ruff.lint]