diff --git a/MANIFEST.in b/MANIFEST.in index 588719c8..6f8525d7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include mache/cime_machine_config/*.xml include mache/machines/*.cfg +include mache/spack/config.cfg include mache/spack/templates/*.yaml include mache/spack/templates/*.template include mache/spack/templates/*.sh diff --git a/mache/spack/config.cfg b/mache/spack/config.cfg new file mode 100644 index 00000000..966108e6 --- /dev/null +++ b/mache/spack/config.cfg @@ -0,0 +1,26 @@ +[spack] +# the tag OR branch OR commit to checkout the spack/spack-packages repo out at. +# Only ONE of the three can be used a time. +tag = v1.0.0 +branch +commit + +[spack_packages] +# this must be a https url and the double back slash must be escaped +remote = https://github.com/spack/spack-packages.git + +# the tag OR branch OR commit to checkout the spack/spack-packages repo out at. +# Only ONE of the three can be used a time. +tag +branch = develop +commit +#commit = 5bb9d2cbbcfedd4728d6c1457efbe68be3c61d7b + +[e3sm_spack_packages] +# this must be a https url and the double back slash must be escaped +remote = https://github.com/andrewdnolan/spack-packages.git + +# the tag or branch or commit are optional arguments +tag +branch = open_PR_rebase +commit diff --git a/mache/spack/env.py b/mache/spack/env.py index 222568fa..725a59c2 100644 --- a/mache/spack/env.py +++ b/mache/spack/env.py @@ -1,3 +1,4 @@ +import configparser import os import subprocess from importlib import resources as importlib_resources @@ -6,11 +7,10 @@ from mache.machine_info import MachineInfo, discover_machine from mache.spack.shared import _get_modules, _get_yaml_data -from mache.version import __version__ def make_spack_env( - spack_path, + base_path, env_name, spack_specs, compiler, @@ -31,8 +31,8 @@ def make_spack_env( Parameters ---------- - spack_path : str - The base path where spack has been (or will be) cloned + base_path : str + TBD env_name : str The name of the spack environment to be created or recreated @@ -78,6 +78,14 @@ def make_spack_env( has been installed. """ + ( + spack_tag, + spack_pkgs_remote, + e3sm_pkgs_remote, + spack_pkgs_update_flag, + e3sm_pkgs_update_flag, + ) = _get_spack_vars_and_flags() + if machine is None: machine = discover_machine() if machine is None: @@ -139,16 +147,32 @@ def make_spack_env( ) with open(str(path)) as fp: template = Template(fp.read()) + if tmpdir is not None: if not os.path.exists(tmpdir): os.mkdir(tmpdir) modules = f'{modules}\nexport TMPDIR={tmpdir}' + ########################################################################### + # TODO: + # - if file paths are set this way, we need to add the machine in there + # - check if `base_path` contains machine, otherwise add it + ########################################################################### + spack_path = f'{base_path}/spack/spack_for_{compiler}_{mpi}' + spack_pkgs_path = f'{base_path}/spack-packages' + e3sm_pkgs_path = f'{base_path}/e3sm-spack-packages' + template_args = dict( modules=modules, - version=__version__, spack_path=spack_path, + spack_tag=spack_tag, + spack_pkgs_remote=spack_pkgs_remote, + spack_pkgs_path=spack_pkgs_path, + spack_pkgs_update_flag=spack_pkgs_update_flag, + e3sm_pkgs_remote=e3sm_pkgs_remote, + e3sm_pkgs_path=e3sm_pkgs_path, + e3sm_pkgs_update_flag=e3sm_pkgs_update_flag, env_name=env_name, yaml_filename=yaml_filename, custom_spack=custom_spack, @@ -167,6 +191,66 @@ def make_spack_env( subprocess.check_call(f'env -i bash -l {build_filename}', shell=True) +def _get_spack_vars_and_flags(): + """ + Get the variables and flags needed to checkout the spack, spack-packages + and our e3sm-spack-pkgs repositories + """ + config = configparser.ConfigParser(allow_no_value=True) + cfg_path = importlib_resources.files('mache.spack') / 'config.cfg' + config.read(str(cfg_path)) + + spack_tag = config.get('spack', 'tag') + spack_pkgs_remote = config.get('spack_packages', 'remote') + e3sm_pkgs_remote = config.get('e3sm_spack_packages', 'remote') + + spack_pkgs_update_flag = get_spack_repo_update_flag( + config['spack_packages'] + ) + if spack_pkgs_update_flag is None: + raise ValueError('') + + e3sm_pkgs_update_flag = get_spack_repo_update_flag( + config['e3sm_spack_packages'] + ) + + return ( + spack_tag, + spack_pkgs_remote, + e3sm_pkgs_remote, + spack_pkgs_update_flag, + e3sm_pkgs_update_flag, + ) + + +def get_spack_repo_update_flag(config): + """ + Format the string passed to `spack repo update` based on the value + provided in the config file. Ensures if a value is provided, only + one of: branch, tag, or commit are used. + """ + + checkout_vars = dict( + tag=config.get('tag'), + commit=config.get('commit'), + branch=config.get('branch'), + ) + + non_none = {k: v for k, v in checkout_vars.items() if v is not None} + + if len(non_none) == 1: + key, value = next(iter(non_none.items())) + return f'--{key} {value}' + elif len(non_none) == 0: + return None + else: + raise ValueError(f""" + Only one of {list(checkout_vars)} can be provided. + But {list(non_none)} were specified in {config.name} section of + the `config.cfg` file. + """) + + def get_modules_env_vars_and_mpi_compilers( machine, compiler, diff --git a/mache/spack/templates/build_spack_env.template b/mache/spack/templates/build_spack_env.template index 1b81c15a..6c9b5619 100644 --- a/mache/spack/templates/build_spack_env.template +++ b/mache/spack/templates/build_spack_env.template @@ -6,14 +6,43 @@ set -e if [ -d {{ spack_path }} ]; then cd {{ spack_path }} - git fetch origin - git reset --hard origin/spack_for_mache_{{ version }} + git fetch origin tag {{ spack_tag }} --no-tags + git reset --hard {{ spack_tag }} else - git clone -b spack_for_mache_{{ version }} git@github.com:E3SM-Project/spack.git {{ spack_path }} + git clone -b {{ spack_tag }} git@github.com:spack/spack.git {{ spack_path }} cd {{ spack_path }} fi source share/spack/setup-env.sh +spack repo remove --scope site builtin >& /dev/null || true +spack repo remove --scope site e3sm-spack-pkgs >& /dev/null || true + +spack repo add \ + --scope site \ + --name builtin \ + {{ spack_pkgs_remote }} \ + {{ spack_pkgs_path }} + +spack repo update \ + --scope site \ + {{ spack_pkgs_update_flag }} \ + builtin + +spack repo add \ + --scope site \ + --name e3sm-spack-pkgs \ + {{ e3sm_pkgs_remote }} \ + {{ e3sm_pkgs_path }} + +{%- if e3sm_pkgs_update_flag is defined %} + +spack repo update \ + --scope site \ + {{ e3sm_pkgs_update_flag }} \ + e3sm-spack-pkgs + +{%- endif %} + {%- if spack_mirror is defined %} spack mirror remove spack_mirror >& /dev/null || true spack mirror add spack_mirror file://{{ spack_mirror }} diff --git a/mache/spack/templates/chrysalis_gnu_openmpi.yaml b/mache/spack/templates/chrysalis_gnu_openmpi.yaml index cf01d6a5..b0e33229 100644 --- a/mache/spack/templates/chrysalis_gnu_openmpi.yaml +++ b/mache/spack/templates/chrysalis_gnu_openmpi.yaml @@ -20,7 +20,6 @@ spack: unify: true packages: all: - compiler: [{{ compiler }}] providers: mpi: [{{ mpi }}%{{ compiler }}] {%- if e3sm_lapack %} @@ -104,6 +103,13 @@ spack: c: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gcc cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/g++ fortran: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran + f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran + fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + environment: {} + extra_rpaths: [] buildable: false openmpi: externals: @@ -122,40 +128,26 @@ spack: {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.10.7+cxx+fortran+hl+mpi + - spec: hdf5@1.10.7+cxx+fortran+hl+mpi%{{ compiler }} modules: - hdf5/1.10.7-ol6xuae buildable: false netcdf-c: externals: - - spec: netcdf-c@4.7.4+mpi~parallel-netcdf + - spec: netcdf-c@4.7.4+mpi~parallel-netcdf%{{ compiler }} modules: - netcdf-c/4.7.4-pfocec2 buildable: false netcdf-fortran: externals: - - spec: netcdf-fortran@4.5.3 + - spec: netcdf-fortran@4.5.3%{{ compiler }} modules: - netcdf-fortran/4.5.3-va3hoor buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.11.0+cxx+fortran + - spec: parallel-netcdf@1.11.0+cxx+fortran%{{ compiler }} modules: - parallel-netcdf/1.11.0-d7h4ysd buildable: false {%- endif %} - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gcc - cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/g++ - f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran - fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/gcc-11.2.0-bgddrif/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/mache/spack/templates/chrysalis_intel_openmpi.yaml b/mache/spack/templates/chrysalis_intel_openmpi.yaml index eee43fbb..9d1959e3 100644 --- a/mache/spack/templates/chrysalis_intel_openmpi.yaml +++ b/mache/spack/templates/chrysalis_intel_openmpi.yaml @@ -1,4 +1,4 @@ -{%- set compiler = "intel@20.0.4" %} +{%- set compiler = "intel-oneapi-compilers-classic@19.1.3.304" %} {%- set mpi = "openmpi@4.1.6" %} spack: specs: @@ -20,7 +20,6 @@ spack: unify: true packages: all: - compiler: [{{ compiler }}] providers: mpi: [{{ mpi }}%{{ compiler }}] {%- if e3sm_lapack %} @@ -93,12 +92,23 @@ spack: modules: - perl/5.32.0-bsnc6lt buildable: false - intel: + intel-oneapi-compilers-classic: externals: - spec: {{ compiler }} prefix: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g modules: - intel/20.0.4-kodw73g + extra_attributes: + compilers: + cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icc + cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icpc + f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort + fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort + flags: {} + operating_system: rhel8 + target: x86_64 + environment: {} + extra_rpaths: [] buildable: false openmpi: externals: @@ -117,40 +127,26 @@ spack: {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.10.7+cxx+fortran+hl+mpi + - spec: hdf5@1.10.7+cxx+fortran+hl+mpi%{{ compiler }} modules: - hdf5/1.10.7-4cghwvq buildable: false netcdf-c: externals: - - spec: netcdf-c@4.7.4+mpi~parallel-netcdf + - spec: netcdf-c@4.7.4+mpi~parallel-netcdf%{{ compiler }} modules: - netcdf-c/4.7.4-4qjdadt buildable: false netcdf-fortran: externals: - - spec: netcdf-fortran@4.5.3 + - spec: netcdf-fortran@4.5.3%{{ compiler }} modules: - netcdf-fortran/4.5.3-qozrykr buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.11.0+cxx+fortran + - spec: parallel-netcdf@1.11.0+cxx+fortran%{{ compiler }} modules: - parallel-netcdf/1.11.0-icrpxty buildable: false {%- endif %} - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icc - cxx: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/icpc - f77: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort - fc: /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/intel-20.0.4-kodw73g/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort - flags: {} - operating_system: rhel8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/mache/spack/templates/pm-cpu_gnu_mpich.yaml b/mache/spack/templates/pm-cpu_gnu_mpich.yaml index 7aa599c9..a2ba7762 100644 --- a/mache/spack/templates/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/templates/pm-cpu_gnu_mpich.yaml @@ -18,7 +18,6 @@ spack: unify: when_possible packages: all: - compiler: [{{ compiler }}] providers: mpi: [{{ mpi }}%{{ compiler }}] lapack: [cray-libsci@23.12.5] @@ -117,9 +116,16 @@ spack: - libfabric/1.22.0 extra_attributes: compilers: - c: cc + cc: cc cxx: CC - fortran: ftn + f77: ftn + fc: ftn + flags: {} + target: x86_64 + operating_system: sles15 + environment: + prepend_path: + PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" buildable: false cray-mpich: externals: @@ -137,43 +143,22 @@ spack: {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared%{{ compiler }} prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared%{{ compiler }} prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf%{{ compiler }} prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false netcdf-fortran: externals: - - spec: netcdf-fortran@4.5.3 + - spec: netcdf-fortran@4.5.3%{{ compiler }} prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 buildable: false {%- endif %} - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 - - cray-libsci/23.12.5 - - craype-accel-host - - craype/2.7.30 - - libfabric/1.22.0 - environment: - prepend_path: - PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" diff --git a/mache/spack/templates/pm-cpu_intel_mpich.yaml b/mache/spack/templates/pm-cpu_intel_mpich.yaml index 7cbe8432..15782327 100644 --- a/mache/spack/templates/pm-cpu_intel_mpich.yaml +++ b/mache/spack/templates/pm-cpu_intel_mpich.yaml @@ -1,4 +1,4 @@ -{%- set compiler = "intel@2023.2.0" %} +{%- set compiler = "intel-oneapi-compilers-classic@2023.2.0" %} {%- set mpi = "cray-mpich@8.1.28" %} spack: specs: @@ -18,7 +18,6 @@ spack: unify: when_possible packages: all: - compiler: [{{ compiler }}] providers: mpi: [{{ mpi }}%{{ compiler }}] lapack: [cray-libsci@23.12.5] @@ -105,7 +104,7 @@ spack: - spec: xz@5.2.3 prefix: /usr buildable: false - intel: + intel-oneapi-compilers-classic: externals: - spec: {{ compiler }} modules: @@ -115,6 +114,14 @@ spack: - craype-accel-host - craype/2.7.30 - libfabric/1.22.0 + extra_attributes: + compilers: + cc: cc + cxx: CC + f77: ftn + fc: ftn + operating_system: sles15 + target: x86_64 buildable: false cray-mpich: externals: @@ -132,43 +139,25 @@ spack: {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared + - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared%{{ compiler }} prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/intel/2023.2 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared + - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared%{{ compiler }} prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/2023.2 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf + - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf%{{ compiler }} prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/intel/2023.2 buildable: false netcdf-fortran: externals: - - spec: netcdf-fortran@4.5.3 + - spec: netcdf-fortran@4.5.3%{{ compiler }} prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/intel/2023.2 buildable: false {%- endif %} - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - PrgEnv-intel/8.5.0 - - intel/2023.2.0 - - cray-libsci/23.12.5 - - craype-accel-host - - craype/2.7.30 - - libfabric/1.22.0 - environment: - prepend_path: - PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig" + env_vars: + prepend_path: + PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"