Update to 1.12.0

.github/workflows/build_workflow.yml

@@ -23,7 +23,7 @@ jobs:
         shell: bash -l {0}
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.11", "3.12", "3.13"]
         mpi: ["hpc", "nompi", "mpich", "openmpi"]
       fail-fast: false
     steps:

e3sm_supported_machines/bootstrap.py

@@ -75,7 +75,7 @@ def get_env_setup(args, config, machine):
 
     if machine is not None and compiler is not None:
         conda_mpi = 'hpc'
-        env_suffix = f'_{machine}'
+        env_suffix = '_compute'
     else:
         conda_mpi = mpi
         env_suffix = '_login'
@@ -110,12 +110,6 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version,
         os.chdir(build_dir)
 
     env_name = f'e3sm_unified_{version}{env_suffix}'
-
-    # add the compiler and MPI library to the spack env name
-    spack_env = f'{env_name}_{compiler}_{mpi}'
-    # spack doesn't like dots
-    spack_env = spack_env.replace('.', '_')
-
     env_path = os.path.join(conda_base, 'envs', env_name)
 
     if conda_mpi in ['nompi', 'hpc']:
@@ -183,7 +177,7 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version,
     else:
         print(f'{env_name} already exists')
 
-    return env_path, env_name, activate_env, channels, spack_env
+    return env_path, env_name, activate_env, channels
 
 
 def install_mache_from_branch(activate_env, fork, branch):
@@ -196,7 +190,7 @@ def install_mache_from_branch(activate_env, fork, branch):
 
 
 def build_sys_ilamb_esmpy(config, machine, compiler, mpi, template_path,
-                          activate_env, channels, spack_base, spack_env):
+                          activate_env, channels, spack_base):
 
     mpi4py_version = config.get('e3sm_unified', 'mpi4py')
     ilamb_version = config.get('e3sm_unified', 'ilamb')
@@ -224,7 +218,7 @@ def build_sys_ilamb_esmpy(config, machine, compiler, mpi, template_path,
     modules = f'{activate_env_lines}\n{modules}'
 
     spack_view = f'{spack_base}/var/spack/environments/' \
-                 f'{spack_env}/.spack-env/view'
+                 f'e3sm_spack_env/.spack-env/view'
     script = template.render(
         mpicc=mpicc, modules=modules, template_path=template_path,
         mpi4py_version=mpi4py_version, build_mpi4py=str(build_mpi4py),
@@ -248,10 +242,10 @@ def build_sys_ilamb_esmpy(config, machine, compiler, mpi, template_path,
     return esmf_mk
 
 
-def build_spack_env(config, machine, compiler, mpi, spack_env, tmpdir):
+def build_spack_env(config, machine, compiler, mpi, env_name, tmpdir):
 
     base_path = config.get('e3sm_unified', 'base_path')
-    spack_base = f'{base_path}/spack/{spack_env}'
+    spack_base = f'{base_path}/{env_name}/{machine}/'
 
     if config.has_option('e3sm_unified', 'use_e3sm_hdf5_netcdf'):
         use_e3sm_hdf5_netcdf = config.getboolean('e3sm_unified',
@@ -274,9 +268,9 @@ def build_spack_env(config, machine, compiler, mpi, spack_env, tmpdir):
             continue
         value = section[option]
         if value != '':
-            specs.append(f'"{value}"')
+            specs.append(f'{value}')
 
-    make_spack_env(spack_path=spack_base, env_name=spack_env,
+    make_spack_env(base_path=spack_base, env_name='e3sm_spack_env',
                    spack_specs=specs, compiler=compiler, mpi=mpi,
                    machine=machine, tmpdir=tmpdir, include_e3sm_lapack=True,
                    include_e3sm_hdf5_netcdf=use_e3sm_hdf5_netcdf,
@@ -375,10 +369,6 @@ def check_env(script_filename, env_name, conda_mpi, machine):
         command = f'{activate} && python -c "import {import_name}"'
         test_command(command, os.environ, import_name)
 
-    # an extra check because the lack of ESMFRegrid is a problem for e3sm_diags
-    command = f'{activate} && python -c "from regrid2 import ESMFRegrid"'
-    test_command(command, os.environ, 'cdms2')
-
     for command in commands:
         package = command[0]
         command_str = ' '.join(command)
@@ -402,6 +392,7 @@ def main():
     template_path = f'{source_path}/templates'
 
     version = args.version
+    env_name = f'e3sm_unified_{version}'.replace('.', '_')
 
     machine = args.machine
     print(f'arg: {machine}')
@@ -418,7 +409,9 @@ def main():
     else:
         is_test = not config.getboolean('e3sm_unified', 'release')
 
-    conda_base = get_conda_base(args.conda_base, config, shared=True)
+    conda_base = get_conda_base(
+        args.conda_base, config, machine, env_name, shared=True
+    )
     conda_base = os.path.abspath(conda_base)
 
     source_activation_scripts = \
@@ -439,7 +432,7 @@ def main():
     nompi_suffix = '_login'
     # first, make environment for login nodes.  We're using no-MPI from
     # conda-forge for now
-    env_path, env_nompi, activate_env, _, _ = build_env(
+    conda_env_path, env_nompi, activate_env, _ = build_env(
         is_test, recreate, nompi_compiler, mpi, 'nompi', version,
         python, conda_base, nompi_suffix, nompi_suffix, activate_base,
         args.local_conda_build, config)
@@ -452,9 +445,14 @@ def main():
     if not is_test:
         # make a symlink to the environment
         link = os.path.join(conda_base, 'envs', 'e3sm_unified_latest')
-        check_call(f'ln -sfn {env_path} {link}')
-
-    env_path, env_name, activate_env, channels, spack_env = build_env(
+        check_call(f'ln -sfn {conda_env_path} {link}')
+
+    (
+        conda_env_path,
+        conda_env_name,
+        activate_env,
+        channels
+    ) = build_env(
         is_test, recreate, compiler, mpi, conda_mpi, version,
         python, conda_base, activ_suffix, env_suffix, activate_base,
         args.local_conda_build, config)
@@ -463,11 +461,11 @@ def main():
                     env_vars=['export HDF5_USE_FILE_LOCKING=FALSE'])
 
     if compiler is not None:
-        spack_base = build_spack_env(config, machine, compiler, mpi, spack_env,
+        spack_base = build_spack_env(config, machine, compiler, mpi, env_name,
                                      args.tmpdir)
         esmf_mk = build_sys_ilamb_esmpy(config, machine, compiler, mpi,
                                         template_path, activate_env, channels,
-                                        spack_base, spack_env)
+                                        spack_base)
         sys_info['env_vars'].append(esmf_mk)
     else:
         spack_base = None
@@ -476,14 +474,14 @@ def main():
     for ext in ['sh', 'csh']:
         if compiler is not None:
             spack_script = get_spack_script(
-                spack_path=spack_base, env_name=spack_env, compiler=compiler,
-                mpi=mpi, shell=ext, machine=machine)
+                spack_path=spack_base, env_name="e3sm_spack_env",
+                compiler=compiler, mpi=mpi, shell=ext, machine=machine)
         else:
             spack_script = ''
 
         script_filename = write_load_e3sm_unified(
             template_path, activ_path, conda_base, is_test, version,
-            activ_suffix, env_name, env_nompi, sys_info, ext, machine,
+            activ_suffix, conda_env_name, env_nompi, sys_info, ext, machine,
             spack_script)
         if ext == 'sh':
             test_script_filename = script_filename
@@ -493,7 +491,7 @@ def main():
             link = os.path.join(activ_path, link)
             check_call(f'ln -sfn {script_filename} {link}')
 
-    check_env(test_script_filename, env_name, conda_mpi, machine)
+    check_env(test_script_filename, conda_env_name, conda_mpi, machine)
 
     commands = f'{activate_base} && conda clean -y -p -t'
     check_call(commands)
@@ -502,8 +500,8 @@ def main():
     if spack_base is not None:
         paths.append(spack_base)
     group = config.get('e3sm_unified', 'group')
-    update_permissions(paths, group, show_progress=True,
-                       group_writable=False, other_readable=True)
+    #update_permissions(paths, group, show_progress=True,
+    #                   group_writable=False, other_readable=True)
 
 
 if __name__ == '__main__':

e3sm_supported_machines/default.cfg

@@ -12,7 +12,7 @@ recreate = False
 suffix =
 
 # the python version
-python = 3.10
+python = 3.13
 
 # the MPI version (nompi, mpich or openmpi)
 mpi = nompi
@@ -22,7 +22,7 @@ ilamb = 2.7.2
 
 # the version of mache to use during deployment (should match the version used
 # in the package itself)
-mache = 1.28.0
+mache = 1.31.0
 
 # the version of mpi4py to build if using system compilers and MPI
 mpi4py = 4.0.1
@@ -39,13 +39,13 @@ xesmf = None
 # spack package specs
 [spack_specs]
 
-esmf = [email protected].0+mpi+netcdf~pnetcdf~external-parallelio
-hdf5 = [email protected].3+cxx+fortran+hl+mpi+shared
+esmf = [email protected].1+mpi+netcdf~pnetcdf~external-parallelio
+hdf5 = [email protected].6+cxx+fortran+hl+mpi+shared
 moab = [email protected]+mpi+hdf5+netcdf+pnetcdf+metis+parmetis+tempest
-nco = [email protected].2+openmp
+nco = [email protected].4+openmp
 netcdf_c = [email protected]+mpi~parallel-netcdf
 netcdf_fortran = [email protected]
 parallel_netcdf = [email protected]
 # parallelio = [email protected]+fortran+mpi~pnetcdf
-tempestextremes = tempestextremes@2.2.3+mpi
+tempestextremes = tempestextremes@2.3.1+mpi
 tempestremap = [email protected]

e3sm_supported_machines/deploy_e3sm_unified.py

@@ -84,8 +84,16 @@ def main():
     source_path = os.getcwd()
 
     config = get_config(args.config_file)
+    version = args.version
 
-    conda_base = get_conda_base(args.conda_base, config, shared=False)
+    env_name = f'e3sm_unified_{version}'.replace('.', '_')
+
+    if args.machine is None:
+        raise ValueError("Please specify what machine you are deploying on")
+
+    conda_base = get_conda_base(
+        args.conda_base, config, args.machine, env_name, shared=False
+    )
     conda_base = os.path.abspath(conda_base)
 
     source_activation_scripts = \

e3sm_supported_machines/shared.py

@@ -127,14 +127,14 @@ def install_miniforge3(conda_base, activate_base):
     check_call(commands)
 
 
-def get_conda_base(conda_base, config, shared):
+def get_conda_base(conda_base, config, machine, env_name, shared):
     if shared:
-        conda_base = os.path.join(
-            config.get('e3sm_unified', 'base_path'), 'base')
+        base_path = config.get('e3sm_unified', 'base_path')
+        conda_base = os.path.join(base_path, env_name, machine, 'conda')
     elif conda_base is None:
         if config.has_option('e3sm_unified', 'base_path'):
-            conda_base = os.path.abspath(os.path.join(
-                config.get('e3sm_unified', 'base_path'), 'base'))
+            base_path = config.get('e3sm_unified', 'base_path')
+            conda_base = os.path.join(base_path, env_name, machine, 'conda')
         elif 'CONDA_EXE' in os.environ:
             # if this is a test, assume we're the same base as the
             # environment currently active

recipes/e3sm-unified/build_packages.py

@@ -8,10 +8,10 @@
 
 from shared import get_rc_dev_labels, get_version_from_meta
 
-DEV_PYTHON_VERSIONS = ["3.10"]
+DEV_PYTHON_VERSIONS = ["3.13"]
 DEV_MPI_VERSIONS = ["nompi", "hpc"]
 
-RELEASE_PYTHON_VERSIONS = ["3.9", "3.10"]
+RELEASE_PYTHON_VERSIONS = ["3.11", "3.12", "3.13"]
 RELEASE_MPI_VERSIONS = ["nompi", "mpich", "openmpi", "hpc"]
 
 
@@ -31,8 +31,12 @@ def generate_matrix_files(dev, python_versions, mpi_versions):
             mpi_versions = RELEASE_MPI_VERSIONS
     matrix_files = []
     for python in python_versions:
+        if float(python) >= 3.13:
+            python_build_str = f"{python}.* *_cp{''.join(python.split('.'))}"
+        else:
+            python_build_str = f"{python}.* *_cpython"
         for mpi in mpi_versions:
-            script = template.render(python=python, mpi=mpi)
+            script = template.render(python=python_build_str, mpi=mpi)
             filename = f"configs/mpi_{mpi}_python{python}.yaml"
             with open(filename, "w") as handle:
                 handle.write(script)

recipes/e3sm-unified/configs/template.yaml

@@ -1,5 +1,5 @@
 python:
-- {{ python }}.* *_cpython
+- {{ python }}
 
 mpi:
 - {{ mpi }}

recipes/e3sm-unified/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "E3SM-Unified" %}
-{% set version = "1.11.1" %}
+{% set version = "1.12.0rc2" %}
 {% set build = 0 %}
 
 package:
@@ -44,18 +44,18 @@ requirements:
     - ipython
     - jupyter
     - livvkit 3.1.0
-    - mache 1.28.0
+    - mache 1.31.0
     - moab 5.5.1 {{ mpi_prefix }}_tempest_*  # [mpi != 'hpc']
-    - mosaic 1.1.0  # [py>=310]
-    - mpas-analysis 1.13.0
-    - mpas_tools 0.36.0
-    - nco 5.3.2  # [mpi != 'hpc']
-    - pcmdi_metrics 3.8.2
+    - mosaic 1.1.0
+    - mpas-analysis 1.14.0rc01
+    - mpas_tools 1.2.2
+    - nco 5.3.4  # [mpi != 'hpc']
+    # can not be installed until dependency on genutils, cdutil, cdms2, cdp
+    # - pcmdi_metrics 3.8.2
     - squadgen 1.2.2  # [linux]
     - tempest-remap 2.2.0  # [mpi != 'hpc']
-    - tempest-extremes 2.2.3 {{ mpi_prefix }}_*  # [mpi != 'hpc']
-    - uxarray 2024.11.1  # [py<310]
-    - uxarray >=2024.12.0  # [py>=310]
+    - tempest-extremes 2.3.1 {{ mpi_prefix }}_*  # [mpi != 'hpc']
+    - uxarray >=2024.12.0
     - xcdat 0.8.0
     - zppy 3.0.0
     - zppy-interfaces 0.1.2
@@ -75,24 +75,17 @@ requirements:
     - blas
     - bottleneck
     - cartopy >=0.17.0
-    - cdat_info 8.2.1
-    - cdms2 3.1.5
-    - cdtime 3.1.4
-    - cdutil 8.2.1
     - cmocean
-    # the last version before python 3.9 support was dropped
-    - dask 2024.8.0  # [py<310]
-    - dask 2024.11.2  # [py>=310]
+    - dask 2024.11.2
     - dogpile.cache
     - eofs
-    - esmf 8.8.0 {{ mpi_prefix }}_*
-    - esmpy 8.8.0
+    - esmf 8.8.1 {{ mpi_prefix }}_*
+    - esmpy 8.8.1
     - f90nml
     - ffmpeg
-    - genutil 8.2.1
     - globus-sdk
     - gsw
-    - hdf5 1.14.3 {{ mpi_prefix }}_*
+    - hdf5 1.14.6 {{ mpi_prefix }}_*
     - ipygany
     - libnetcdf 4.9.2 {{ mpi_prefix }}_*
     - lxml
@@ -114,9 +107,7 @@ requirements:
     - progressbar2
     - proj 9.5.1
     - pyevtk
-    # the last version before python 3.9 support was dropped
-    - pyproj 3.6.1  # [py<310]
-    - pyproj 3.7.0  # [py>=310]
+    - pyproj 3.7.0
     - pyremap
     - pytest
     - pywavelets
@@ -128,10 +119,8 @@ requirements:
     - shapely
     - sympy >=0.7.6
     - tabulate
-    - windspharm  # [py<=310]
-    # the last version before python 3.9 support was dropped
-    - xarray 2024.7.0  # [py<310]
-    - xarray 2025.1.1  # [py>=310]
+    - windspharm
+    - xarray 2025.1.1
     - xesmf 0.8.8
 
     # addition ilamb 2.7 dependencies, for system MPI builds