Merge pull request #104 from xylar/v1.7.0

Update to E3SM-Unified v1.7.0

Author: xylar

README.md

@@ -23,6 +23,5 @@ conda create -n e3sm-unified-nompi -c conda-forge -c defaults -c e3sm \
     python=3.9 "e3sm-unified=*=nompi_*"
 ```
 
- The following packages are only available for linux and not OSX:
- - processflow
+ The following package is only available for linux and not OSX:
  - zstash

e3sm-unified-test.sh

@@ -45,7 +45,7 @@ conda activate e3sm-unified
 check_last_rc "Could not source e3sm-unified conda environment\n"
 
 # Try to run some simple help commands
-for cmd in "e3sm_diags" "processflow" "mpas_analysis"
+for cmd in "e3sm_diags" "mpas_analysis"
 do
     $cmd --help
     check_last_rc "$cmd help failed\n"

e3sm_supported_machines/bootstrap.py

@@ -7,8 +7,9 @@
 from importlib.resources import path
 from configparser import ConfigParser
 
-from mache import discover_machine, MachineInfo
-from mache.spack import make_spack_env, get_spack_script
+from mache import discover_machine
+from mache.spack import make_spack_env, get_spack_script, \
+    get_modules_env_vars_and_mpi_compilers
 from mache.version import __version__ as mache_version
 from mache.permissions import update_permissions
 from shared import parse_args, check_call, install_miniconda, get_conda_base
@@ -116,8 +117,7 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version,
         if nco_dev:
             channels = f'{channels} -c conda-forge/label/nco_dev'
         channels = f'{channels} -c conda-forge/label/e3sm_dev ' \
-                   f'-c conda-forge -c defaults -c e3sm/label/e3sm_dev ' \
-                   f'-c e3sm'
+                   f'-c conda-forge -c defaults -c e3sm/label/e3sm_dev -c e3sm'
     else:
         channels = '--override-channels -c conda-forge -c defaults -c e3sm'
 
@@ -149,24 +149,23 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version,
     return env_path, env_name, activate_env, channels, spack_env
 
 
-def build_sys_ilamb(config, machine_info, compiler, mpi, template_path,
+def build_sys_ilamb(config, machine, compiler, mpi, template_path,
                     activate_env, channels):
 
     mpi4py_version = config.get('e3sm_unified', 'mpi4py')
     ilamb_version = config.get('e3sm_unified', 'ilamb')
     build_mpi4py = str(mpi4py_version != 'None')
     build_ilamb = str(ilamb_version != 'None')
 
-    mpicc, _, _, mod_commands, _ = \
-        machine_info.get_modules_and_mpi_compilers(compiler, mpi)
+    mpicc, _, _, modules = \
+        get_modules_env_vars_and_mpi_compilers(machine, compiler, mpi,
+                                               shell='sh')
 
     script_filename = 'build.bash'
 
     with open(f'{template_path}/build.template', 'r') as f:
         template = Template(f.read())
 
-    modules = '\n'.join(mod_commands)
-
     # need to activate the conda environment to install mpi4py and ilamb, and
     # possibly for compilers and MPI library (if not on a supported machine)
     activate_env_lines = activate_env.replace('; ', '\n')
@@ -185,7 +184,7 @@ def build_sys_ilamb(config, machine_info, compiler, mpi, template_path,
     check_call(command)
 
 
-def build_spack_env(config, machine, compiler, mpi, spack_env):
+def build_spack_env(config, machine, compiler, mpi, spack_env, tmpdir):
 
     base_path = config.get('e3sm_unified', 'base_path')
     spack_base = f'{base_path}/spack/spack_for_mache_{mache_version}'
@@ -199,7 +198,7 @@ def build_spack_env(config, machine, compiler, mpi, spack_env):
 
     make_spack_env(spack_path=spack_base, env_name=spack_env,
                    spack_specs=specs, compiler=compiler, mpi=mpi,
-                   machine=machine)
+                   machine=machine, tmpdir=tmpdir, include_e3sm_lapack=True)
 
     return spack_base
 
@@ -282,8 +281,7 @@ def check_env(script_filename, env_name, conda_mpi, machine):
     commands = [['mpas_analysis', '-h'],
                 ['livv', '--version'],
                 ['globus', '--help'],
-                ['zstash', '--help'],
-                ['processflow', '-v']]
+                ['zstash', '--help']]
 
     if machine is None:
         # on HPC machines, these only work on compute nodes because of mpi4py
@@ -324,11 +322,6 @@ def main():
         machine = discover_machine()
         print(f'discovered: {machine}')
 
-    if machine is not None:
-        machine_info = MachineInfo(machine=machine)
-    else:
-        machine_info = None
-
     config = get_config(args.config_file, machine)
 
     if args.release:
@@ -376,8 +369,9 @@ def main():
                     env_vars=['export HDF5_USE_FILE_LOCKING=FALSE'])
 
     if compiler is not None:
-        spack_base = build_spack_env(config, machine, compiler, mpi, spack_env)
-        build_sys_ilamb(config, machine_info, compiler, mpi, template_path,
+        spack_base = build_spack_env(config, machine, compiler, mpi, spack_env,
+                                     args.tmpdir, )
+        build_sys_ilamb(config, machine, compiler, mpi, template_path,
                         activate_env, channels)
     else:
         spack_base = None

e3sm_supported_machines/default.cfg

@@ -22,7 +22,7 @@ ilamb = 2.6
 
 # the version of mache to use during deployment (should match the version used
 # in the package itself)
-mache = 1.4.1
+mache = 1.5.0
 
 # the version of mpi4py to build if using system compilers
 mpi4py = 3.1.3
@@ -31,11 +31,11 @@ mpi4py = 3.1.3
 # spack package specs
 [spack_specs]
 
-esmf = esmf@8.1.1+mpi+netcdf~pio+pnetcdf
+esmf = esmf@8.2.0+mpi+netcdf~pio+pnetcdf
 hdf5 = [email protected]+cxx+fortran+hl+mpi+shared
 moab = [email protected]+mpi+hdf5+netcdf+pnetcdf+metis+parmetis+tempest
-nco = nco@5.0.6+openmp
+nco = nco@5.1.0+openmp
 netcdf_c = [email protected]+mpi~parallel-netcdf
-netcdf_fortran = [email protected].3
-tempestextremes = [email protected]+mpi
+netcdf_fortran = [email protected].4
+tempestextremes = [email protected].1+mpi
 tempestremap = [email protected]

e3sm_supported_machines/shared.py

@@ -13,8 +13,8 @@
 
 def parse_args(bootstrap):
     parser = argparse.ArgumentParser(
-        description='Deploy a compass conda environment')
-    parser.add_argument("--version", dest="version", default="1.6.0",
+        description='Deploy E3SM-Unified')
+    parser.add_argument("--version", dest="version", default="1.7.0",
                         help="The version of E3SM-Unified to deploy")
     parser.add_argument("--conda", dest="conda_base",
                         help="Path for the  conda base")
@@ -38,6 +38,9 @@ def parse_args(bootstrap):
                              "build")
     parser.add_argument("--use_local", dest="use_local", action='store_true',
                         help="Use locally built conda packages (for testing).")
+    parser.add_argument("--tmpdir", dest="tmpdir",
+                        help="A temporary directory for building spack "
+                             "packages")
     if bootstrap:
         parser.add_argument("--local_conda_build", dest="local_conda_build",
                             type=str,

recipes/e3sm-unified/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "E3SM-Unified" %}
-{% set version = "1.6.0" %}
+{% set version = "1.7.0" %}
 {% set build = 0 %}
 
 package:
@@ -30,24 +30,25 @@ requirements:
   run:
     ### main packages ###
     - python
-    - e3sm_diags 2.6.1
-    - e3sm_to_cmip 1.8.1
-    - geometric_features 0.5.0
+    - e3sm_diags 2.7.0
+    - e3sm_to_cmip 1.8.2
+    - geometric_features 0.6.0
     - globus-cli
     - ilamb 2.6  # [mpi != 'nompi' and mpi != 'hpc']
     - ipython
     - jupyter
-    - livvkit 3.0.0
-    - mache 1.4.1
+    - livvkit 3.0.1
+    - mache 1.5.0
     - moab 5.3.1 {{ mpi_prefix }}_tempest_*  # [mpi != 'hpc']
-    - mpas-analysis 1.6.1
-    - mpas_tools 0.12.0
-    - nco 5.0.6  # [mpi != 'hpc']
-    - processflow 2.2.5  # [linux]
+    - mpas-analysis 1.7.0
+    - mpas_tools 0.14.0
+    - nco 5.1.0  # [mpi != 'hpc']
+    - pcmdi_metrics 2.3.1
     - tempest-remap 2.1.1  # [mpi != 'hpc']
-    - tempest-extremes 2.2 {{ mpi_prefix }}_*  # [mpi != 'hpc']
-    - zstash 1.2.0  # [linux]
-    - zppy 2.0.0
+    - tempest-extremes 2.2.1 {{ mpi_prefix }}_*  # [mpi != 'hpc']
+    - xcdat 0.3.0
+    - zstash 1.2.1  # [linux]
+    - zppy 2.1.0
     ### dependencies ###
     - {{ mpi }}  # [mpi != 'nompi' and mpi != 'hpc']
     - blas
@@ -58,9 +59,10 @@ requirements:
     - cdtime 3.1.4
     - cdutil 8.2.1
     - cmocean
-    - dask 2022.01.1
+    - dask 2022.5.2
+    - dogpile.cache
     - eofs
-    - esmf 8.1.1 {{ mpi_prefix }}_*  # [mpi != 'hpc']
+    - esmf 8.2.0 {{ mpi_prefix }}_*  # [mpi != 'hpc']
     - f90nml
     - ffmpeg
     - genutil 8.2.1
@@ -76,16 +78,16 @@ requirements:
     - nb_conda
     - nb_conda_kernels
     - ncview 2.1.8
-    - netcdf4 1.5.7 nompi_*
+    - netcdf4 1.5.8 nompi_*
     - numpy >1.13
     - openssh  # [mpi == 'openmpi']
     - output_viewer 1.3.3
     - pillow
     - plotly
     - progressbar2
-    - proj 8.2.1
+    - proj 9.0.0
     - pyevtk
-    - pyproj 3.3.0
+    - pyproj 3.3.1
     - pyremap
     - pytest
     - pywavelets
@@ -94,7 +96,7 @@ requirements:
     - shapely
     - sympy >=0.7.6
     - tabulate
-    - xarray 0.21.1
+    - xarray 2022.3.0
     - xesmf
 
     # addition ilamb 2.6 dependencies, for system MPI builds
@@ -120,7 +122,6 @@ test:
   commands:
     - mpas_analysis --version
     - livv --version
-    - processflow -v  # [linux]
     - e3sm_diags --help
     - zstash --help  # [linux]
     - ilamb-fetch -h  # [mpi != 'nompi' and mpi != 'hpc']