Add support for spack v1.0.0

MANIFEST.in

@@ -1,5 +1,6 @@
 include mache/cime_machine_config/*.xml
 include mache/machines/*.cfg
+include mache/spack/config.cfg
 include mache/spack/templates/*.yaml
 include mache/spack/templates/*.template
 include mache/spack/templates/*.sh

mache/spack/config.cfg

@@ -0,0 +1,26 @@
+[spack]
+# the tag OR branch OR commit to checkout the spack/spack-packages repo out at.
+# Only ONE of the three can be used a time.
+tag = v1.0.0
+branch
+commit
+
+[spack_packages]
+# this must be a https url and the double back slash must be escaped
+remote = https://github.com/spack/spack-packages.git
+
+# the tag OR branch OR commit to checkout the spack/spack-packages repo out at.
+# Only ONE of the three can be used a time.
+tag
+branch = develop
+commit
+#commit = 5bb9d2cbbcfedd4728d6c1457efbe68be3c61d7b
+
+[e3sm_spack_packages]
+# this must be a https url and the double back slash must be escaped
+remote = https://github.com/andrewdnolan/spack-packages.git
+
+# the tag or branch or commit are optional arguments
+tag
+branch = open_PR_rebase
+commit

mache/spack/env.py

@@ -1,3 +1,4 @@
+import configparser
 import os
 import subprocess
 from importlib import resources as importlib_resources
@@ -6,11 +7,10 @@
 
 from mache.machine_info import MachineInfo, discover_machine
 from mache.spack.shared import _get_modules, _get_yaml_data
-from mache.version import __version__
 
 
 def make_spack_env(
-    spack_path,
+    base_path,
     env_name,
     spack_specs,
     compiler,
@@ -31,8 +31,8 @@ def make_spack_env(
 
     Parameters
     ----------
-    spack_path : str
-        The base path where spack has been (or will be) cloned
+    base_path : str
+        TBD
 
     env_name : str
         The name of the spack environment to be created or recreated
@@ -78,6 +78,14 @@ def make_spack_env(
         has been installed.
     """
 
+    (
+        spack_tag,
+        spack_pkgs_remote,
+        e3sm_pkgs_remote,
+        spack_pkgs_update_flag,
+        e3sm_pkgs_update_flag,
+    ) = _get_spack_vars_and_flags()
+
     if machine is None:
         machine = discover_machine()
         if machine is None:
@@ -139,16 +147,32 @@ def make_spack_env(
     )
     with open(str(path)) as fp:
         template = Template(fp.read())
+
     if tmpdir is not None:
         if not os.path.exists(tmpdir):
             os.mkdir(tmpdir)
 
         modules = f'{modules}\nexport TMPDIR={tmpdir}'
 
+    ###########################################################################
+    # TODO:
+    #   - if file paths are set this way, we need to add the machine in there
+    #       - check if `base_path` contains machine, otherwise add it
+    ###########################################################################
+    spack_path = f'{base_path}/spack/spack_for_{compiler}_{mpi}'
+    spack_pkgs_path = f'{base_path}/spack-packages'
+    e3sm_pkgs_path = f'{base_path}/e3sm-spack-packages'
+
     template_args = dict(
         modules=modules,
-        version=__version__,
         spack_path=spack_path,
+        spack_tag=spack_tag,
+        spack_pkgs_remote=spack_pkgs_remote,
+        spack_pkgs_path=spack_pkgs_path,
+        spack_pkgs_update_flag=spack_pkgs_update_flag,
+        e3sm_pkgs_remote=e3sm_pkgs_remote,
+        e3sm_pkgs_path=e3sm_pkgs_path,
+        e3sm_pkgs_update_flag=e3sm_pkgs_update_flag,
         env_name=env_name,
         yaml_filename=yaml_filename,
         custom_spack=custom_spack,
@@ -167,6 +191,66 @@ def make_spack_env(
     subprocess.check_call(f'env -i bash -l {build_filename}', shell=True)
 
 
+def _get_spack_vars_and_flags():
+    """
+    Get the variables and flags needed to checkout the spack, spack-packages
+    and our e3sm-spack-pkgs repositories
+    """
+    config = configparser.ConfigParser(allow_no_value=True)
+    cfg_path = importlib_resources.files('mache.spack') / 'config.cfg'
+    config.read(str(cfg_path))
+
+    spack_tag = config.get('spack', 'tag')
+    spack_pkgs_remote = config.get('spack_packages', 'remote')
+    e3sm_pkgs_remote = config.get('e3sm_spack_packages', 'remote')
+
+    spack_pkgs_update_flag = get_spack_repo_update_flag(
+        config['spack_packages']
+    )
+    if spack_pkgs_update_flag is None:
+        raise ValueError('')
+
+    e3sm_pkgs_update_flag = get_spack_repo_update_flag(
+        config['e3sm_spack_packages']
+    )
+
+    return (
+        spack_tag,
+        spack_pkgs_remote,
+        e3sm_pkgs_remote,
+        spack_pkgs_update_flag,
+        e3sm_pkgs_update_flag,
+    )
+
+
+def get_spack_repo_update_flag(config):
+    """
+    Format the string passed to `spack repo update` based on the value
+    provided in the config file. Ensures if a value is provided, only
+    one of: branch, tag, or commit are used.
+    """
+
+    checkout_vars = dict(
+        tag=config.get('tag'),
+        commit=config.get('commit'),
+        branch=config.get('branch'),
+    )
+
+    non_none = {k: v for k, v in checkout_vars.items() if v is not None}
+
+    if len(non_none) == 1:
+        key, value = next(iter(non_none.items()))
+        return f'--{key} {value}'
+    elif len(non_none) == 0:
+        return None
+    else:
+        raise ValueError(f"""
+        Only one of {list(checkout_vars)} can be provided.
+        But {list(non_none)} were specified in {config.name} section of
+        the `config.cfg` file.
+        """)
+
+
 def get_modules_env_vars_and_mpi_compilers(
     machine,
     compiler,

mache/spack/templates/build_spack_env.template

@@ -6,14 +6,43 @@ set -e
 
 if [ -d {{ spack_path }} ]; then
   cd {{ spack_path }}
-  git fetch origin
-  git reset --hard origin/spack_for_mache_{{ version }}
+  git fetch origin tag {{ spack_tag }} --no-tags
+  git reset --hard {{ spack_tag }}
 else
-  git clone -b spack_for_mache_{{ version }} [email protected]:E3SM-Project/spack.git {{ spack_path }}
+  git clone -b {{ spack_tag }} [email protected]:spack/spack.git {{ spack_path }}
   cd {{ spack_path }}
 fi
 source share/spack/setup-env.sh
 
+spack repo remove --scope site builtin >& /dev/null || true
+spack repo remove --scope site e3sm-spack-pkgs >& /dev/null || true
+
+spack repo add \
+    --scope site \
+    --name builtin \
+    {{ spack_pkgs_remote }} \
+    {{ spack_pkgs_path }}
+
+spack repo update \
+    --scope site \
+    {{ spack_pkgs_update_flag }} \
+    builtin
+
+spack repo add \
+    --scope site \
+    --name e3sm-spack-pkgs \
+    {{ e3sm_pkgs_remote }} \
+    {{ e3sm_pkgs_path }}
+
+{%- if e3sm_pkgs_update_flag is defined %}
+
+spack repo update \
+    --scope site  \
+    {{ e3sm_pkgs_update_flag }} \
+    e3sm-spack-pkgs
+
+{%- endif %}
+
 {%- if spack_mirror is defined %}
     spack mirror remove spack_mirror >& /dev/null || true
     spack mirror add spack_mirror file://{{ spack_mirror }}

mache/spack/templates/chrysalis_gnu_openmpi.yaml

@@ -20,7 +20,6 @@ spack:
     unify: true
   packages:
     all:
-      compiler: [{{ compiler }}]
       providers:
         mpi: [{{ mpi }}%{{ compiler }}]
 {%- if e3sm_lapack %}
@@ -104,6 +103,13 @@ spack:
             c: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gcc
             cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/g++
             fortran: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran
+            f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran
+            fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran
+          flags: {}
+          operating_system: rhel8
+          target: x86_64
+          environment: {}
+          extra_rpaths: []
       buildable: false
     openmpi:
       externals:
@@ -122,40 +128,26 @@ spack:
 {%- if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: [email protected]+cxx+fortran+hl+mpi
+      - spec: [email protected]+cxx+fortran+hl+mpi%{{ compiler }}
         modules:
         - hdf5/1.10.7-ol6xuae
       buildable: false
     netcdf-c:
       externals:
-      - spec: [email protected]+mpi~parallel-netcdf
+      - spec: [email protected]+mpi~parallel-netcdf%{{ compiler }}
         modules:
         - netcdf-c/4.7.4-pfocec2
       buildable: false
     netcdf-fortran:
       externals:
-      - spec: [email protected]
+      - spec: [email protected]%{{ compiler }}
         modules:
         - netcdf-fortran/4.5.3-va3hoor
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: [email protected]+cxx+fortran
+      - spec: [email protected]+cxx+fortran%{{ compiler }}
         modules:
         - parallel-netcdf/1.11.0-d7h4ysd
       buildable: false
 {%- endif %}
-  compilers:
-  - compiler:
-      spec: {{ compiler }}
-      paths:
-        cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gcc
-        cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/g++
-        f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran
-        fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran
-      flags: {}
-      operating_system: rhel8
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []

mache/spack/templates/chrysalis_intel_openmpi.yaml

@@ -1,4 +1,4 @@
-{%- set compiler = "intel@20.0.4" %}
+{%- set compiler = "intel[email protected]" %}
 {%- set mpi = "[email protected]" %}
 spack:
   specs:
@@ -20,7 +20,6 @@ spack:
     unify: true
   packages:
     all:
-      compiler: [{{ compiler }}]
       providers:
         mpi: [{{ mpi }}%{{ compiler }}]
 {%- if e3sm_lapack %}
@@ -93,12 +92,23 @@ spack:
         modules:
           - perl/5.32.0-bsnc6lt
       buildable: false
-    intel:
+    intel-oneapi-compilers-classic:
       externals:
       - spec: {{ compiler }}
         prefix: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g
         modules:
         - intel/20.0.4-kodw73g
+        extra_attributes:
+          compilers:
+            cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icc
+            cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icpc
+            f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort
+            fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort
+          flags: {}
+          operating_system: rhel8
+          target: x86_64
+          environment: {}
+          extra_rpaths: []
       buildable: false
     openmpi:
       externals:
@@ -117,40 +127,26 @@ spack:
 {%- if e3sm_hdf5_netcdf %}
     hdf5:
       externals:
-      - spec: [email protected]+cxx+fortran+hl+mpi
+      - spec: [email protected]+cxx+fortran+hl+mpi%{{ compiler }}
         modules:
         - hdf5/1.10.7-4cghwvq
       buildable: false
     netcdf-c:
       externals:
-      - spec: [email protected]+mpi~parallel-netcdf
+      - spec: [email protected]+mpi~parallel-netcdf%{{ compiler }}
         modules:
         - netcdf-c/4.7.4-4qjdadt
       buildable: false
     netcdf-fortran:
       externals:
-      - spec: [email protected]
+      - spec: [email protected]%{{ compiler }}
         modules:
         - netcdf-fortran/4.5.3-qozrykr
       buildable: false
     parallel-netcdf:
       externals:
-      - spec: [email protected]+cxx+fortran
+      - spec: [email protected]+cxx+fortran%{{ compiler }}
         modules:
         - parallel-netcdf/1.11.0-icrpxty
       buildable: false
 {%- endif %}
-  compilers:
-  - compiler:
-      spec: {{ compiler }}
-      paths:
-        cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icc
-        cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icpc
-        f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort
-        fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort
-      flags: {}
-      operating_system: rhel8
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []