diff --git a/docs/users_guide/spack/build.md b/docs/users_guide/spack/build.md index d053bf78..420f37a7 100644 --- a/docs/users_guide/spack/build.md +++ b/docs/users_guide/spack/build.md @@ -116,7 +116,6 @@ spack_script = get_spack_script( config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template ) ``` @@ -161,7 +160,6 @@ mpicc, mpicxx, mpifc, mod_env_commands = get_modules_env_vars_and_mpi_compilers( shell='sh', # or 'csh' include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template ) ``` diff --git a/mache/cime_machine_config/config_machines.xml b/mache/cime_machine_config/config_machines.xml index 44998bbc..bf37c0a1 100644 --- a/mache/cime_machine_config/config_machines.xml +++ b/mache/cime_machine_config/config_machines.xml @@ -219,35 +219,35 @@ PrgEnv-gnu/8.5.0 - gcc-native/12.3 - cray-libsci/23.12.5 + gcc-native/13.2 + cray-libsci/24.07.0 PrgEnv-intel/8.5.0 - intel/2023.2.0 + intel/2024.1.0 PrgEnv-nvidia - nvidia/24.5 - cray-libsci/23.12.5 + nvidia/25.5 + cray-libsci/24.07.0 PrgEnv-aocc aocc/4.1.0 - cray-libsci/23.12.5 + cray-libsci/24.07.0 craype-accel-host - craype/2.7.30 - cray-mpich/8.1.28 + craype/2.7.32 + cray-mpich/8.1.30 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 - cmake/3.24.3 + cray-parallel-netcdf/1.12.3.13 + cmake/3.30.2 @@ -406,23 +406,23 @@ PrgEnv-gnu/8.5.0 - gcc-native/12.3 + gcc-native/13.2 PrgEnv-nvidia - nvidia/24.5 + nvidia/25.5 - cudatoolkit/12.4 + cudatoolkit/12.9 craype-accel-nvidia80 - cudatoolkit/12.4 + cudatoolkit/12.9 craype-accel-nvidia80 - gcc-native-mixed/12.3 + gcc-native-mixed/13.2 @@ -434,13 +434,13 @@ - cray-libsci/23.12.5 - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 - cmake/3.24.3 + cray-libsci/24.07.0 + craype/2.7.32 + cray-mpich/8.1.30 + cray-hdf5-parallel/1.14.3.1 + cray-netcdf-hdf5parallel/4.9.0.13 + cray-parallel-netcdf/1.12.3.13 + cmake/3.30.2 diff --git a/mache/machines/anvil.cfg b/mache/machines/anvil.cfg index c38f649b..0ebdaf07 100644 --- a/mache/machines/anvil.cfg +++ b/mache/machines/anvil.cfg @@ -63,18 +63,6 @@ partitions = acme-small, acme-medium, acme-large qos = regular, acme_high -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/machines/aurora.cfg b/mache/machines/aurora.cfg index 6ca07523..0a68cb9c 100644 --- a/mache/machines/aurora.cfg +++ b/mache/machines/aurora.cfg @@ -49,18 +49,6 @@ account = E3SM_Dec # queues (default is the first) queues = prod, debug -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/machines/chicoma-cpu.cfg b/mache/machines/chicoma-cpu.cfg index ea1e2d82..2bb4de9b 100644 --- a/mache/machines/chicoma-cpu.cfg +++ b/mache/machines/chicoma-cpu.cfg @@ -40,18 +40,6 @@ partitions = standard, gpu qos = standard, debug -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/machines/chrysalis.cfg b/mache/machines/chrysalis.cfg index c6d2bd47..51d11b7b 100644 --- a/mache/machines/chrysalis.cfg +++ b/mache/machines/chrysalis.cfg @@ -56,19 +56,6 @@ cores_per_node = 128 # available partition(s) (default is the first) partitions = compute, debug, Shigh - -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = True - - # config options related to synchronizing files [sync] diff --git a/mache/machines/compy.cfg b/mache/machines/compy.cfg index 6f4a34ec..461afc6d 100644 --- a/mache/machines/compy.cfg +++ b/mache/machines/compy.cfg @@ -63,18 +63,6 @@ partitions = slurm qos = regular -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = True - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/machines/dane.cfg b/mache/machines/dane.cfg index a5c77d08..2ee57c3f 100644 --- a/mache/machines/dane.cfg +++ b/mache/machines/dane.cfg @@ -53,18 +53,6 @@ cores_per_node = 112 partitions = pbatch, pdebug -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = True - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/machines/frontier.cfg b/mache/machines/frontier.cfg index 7cc486ce..3bf5a748 100644 --- a/mache/machines/frontier.cfg +++ b/mache/machines/frontier.cfg @@ -56,14 +56,6 @@ partitions = batch # Config options related to spack environments [spack] -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - # whether the machine uses cray compilers cray_compilers = True diff --git a/mache/machines/pm-cpu.cfg b/mache/machines/pm-cpu.cfg index 7661f144..b6a9b91a 100644 --- a/mache/machines/pm-cpu.cfg +++ b/mache/machines/pm-cpu.cfg @@ -62,17 +62,10 @@ constraints = cpu # quality of service (default is the first) qos = regular, debug, premium + # Config options related to spack environments [spack] -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - # whether the machine uses cray compilers cray_compilers = True diff --git a/mache/machines/pm-gpu.cfg b/mache/machines/pm-gpu.cfg index eaee4abc..6766ed0e 100644 --- a/mache/machines/pm-gpu.cfg +++ b/mache/machines/pm-gpu.cfg @@ -65,17 +65,10 @@ constraints = gpu # quality of service (default is the first) qos = regular, debug, premium + # Config options related to spack environments [spack] -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = False - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - # whether the machine uses cray compilers cray_compilers = True diff --git a/mache/machines/ruby.cfg b/mache/machines/ruby.cfg index 28745850..b6a9955f 100644 --- a/mache/machines/ruby.cfg +++ b/mache/machines/ruby.cfg @@ -53,18 +53,6 @@ cores_per_node = 56 partitions = pbatch, pdebug -# Config options related to spack environments -[spack] - -# whether to load modules from the spack yaml file before loading the spack -# environment -modules_before = True - -# whether to load modules from the spack yaml file after loading the spack -# environment -modules_after = False - - # config options related to synchronizing files [sync] diff --git a/mache/spack/config_machines.py b/mache/spack/config_machines.py index 434f535c..6c54e074 100644 --- a/mache/spack/config_machines.py +++ b/mache/spack/config_machines.py @@ -1,5 +1,6 @@ import re from collections import defaultdict +from importlib import resources as importlib_resources from lxml import etree @@ -104,13 +105,13 @@ def config_to_shell_script(config, shell_type): script_lines.append('') if e3sm_hdf5_netcdf_modules: - script_lines.append('{% if e3sm_hdf5_netcdf %}') + script_lines.append('{%- if e3sm_hdf5_netcdf %}') script_lines.extend( _convert_module_commands_to_script_lines( e3sm_hdf5_netcdf_modules, shell_type ) ) - script_lines.append('{% endif %}') + script_lines.append('{%- endif %}') script_lines.append('') script_lines.extend(_convert_env_vars_to_script_lines(config, shell_type)) @@ -121,7 +122,7 @@ def config_to_shell_script(config, shell_type): def extract_spack_from_config_machines( - machine, compiler, mpilib, shell, output + machine, compiler, mpilib, shell, output=None ): """ Extract machine configuration from XML and write it to a shell script. @@ -136,10 +137,18 @@ def extract_spack_from_config_machines( MPI library name. shell : str Shell script type ('sh' or 'csh'). - output : str + output : str, optional Output file to write the shell script. + + Returns + ------- + script: str + The generated shell script as a string. """ - config_filename = 'mache/cime_machine_config/config_machines.xml' + config_filename = ( + importlib_resources.files('mache.cime_machine_config') + / 'config_machines.xml' + ) config = extract_machine_config(config_filename, machine, compiler, mpilib) if config is None: @@ -149,8 +158,11 @@ def extract_spack_from_config_machines( ) script = config_to_shell_script(config, shell) - with open(output, 'w') as f: - f.write(script) + if output is not None: + with open(output, 'w') as f: + f.write(script) + + return script def _convert_module_commands_to_script_lines(module_commands, shell_type): @@ -249,12 +261,12 @@ def _convert_env_vars_to_script_lines(config, shell_type): script_lines.append('') if e3sm_hdf5_netcdf_env_vars: - script_lines.append('{% if e3sm_hdf5_netcdf %}') + script_lines.append('{%- if e3sm_hdf5_netcdf %}') for name, value in e3sm_hdf5_netcdf_env_vars: if shell_type == 'sh': script_lines.append(f'export {name}="{value}"') elif shell_type == 'csh': script_lines.append(f'setenv {name} "{value}"') - script_lines.append('{% endif %}') + script_lines.append('{%- endif %}') return script_lines diff --git a/mache/spack/env.py b/mache/spack/env.py index 222568fa..a76fffc1 100644 --- a/mache/spack/env.py +++ b/mache/spack/env.py @@ -5,7 +5,7 @@ from jinja2 import Template from mache.machine_info import MachineInfo, discover_machine -from mache.spack.shared import _get_modules, _get_yaml_data +from mache.spack.shared import _get_yaml_data from mache.version import __version__ @@ -89,12 +89,6 @@ def make_spack_env( if config_file is not None: config.read(config_file) - section = config['spack'] - - with_modules = section.getboolean('modules_before') or section.getboolean( - 'modules_after' - ) - yaml_data = _get_yaml_data( machine, compiler, @@ -109,11 +103,7 @@ def make_spack_env( with open(yaml_filename, 'w') as handle: handle.write(yaml_data) - if with_modules: - mods = _get_modules(yaml_data) - modules = f'module purge\n{mods}' - else: - modules = '' + modules = '' for shell_filename in [f'{machine}.sh', f'{machine}_{compiler}_{mpi}.sh']: # load modules, etc. for this machine @@ -174,7 +164,6 @@ def get_modules_env_vars_and_mpi_compilers( shell, include_e3sm_lapack=False, include_e3sm_hdf5_netcdf=False, - yaml_template=None, ): """ Get the non-spack modules, environment variables and compiler names for a @@ -203,11 +192,6 @@ def get_modules_env_vars_and_mpi_compilers( Whether to include the same hdf5, netcdf-c, netcdf-fortran and pnetcdf as used in E3SM - yaml_template : str, optional - A jinja template for a yaml file to be used for the environment instead - of the mache template. This allows you to use compilers and other - modules that differ from E3SM. - Returns ------- mpicc : str @@ -236,27 +220,10 @@ def get_modules_env_vars_and_mpi_compilers( if config.has_section('spack'): section = config['spack'] - with_modules = section.getboolean( - 'modules_before' - ) or section.getboolean('modules_after') if config.has_option('spack', 'cray_compilers'): cray_compilers = section.getboolean('cray_compilers') - else: - with_modules = False mod_env_commands = 'module purge\n' - if with_modules: - yaml_data = _get_yaml_data( - machine, - compiler, - mpi, - include_e3sm_lapack, - include_e3sm_hdf5_netcdf, - specs=[], - yaml_template=yaml_template, - ) - mods = _get_modules(yaml_data) - mod_env_commands = f'{mod_env_commands}\n{mods}\n' for shell_filename in [ f'{machine}.{shell}', diff --git a/mache/spack/script.py b/mache/spack/script.py index 9cc28fad..a3206001 100644 --- a/mache/spack/script.py +++ b/mache/spack/script.py @@ -3,7 +3,7 @@ from jinja2 import Template from mache.machine_info import MachineInfo, discover_machine -from mache.spack.shared import _get_modules, _get_yaml_data +from mache.spack.config_machines import extract_spack_from_config_machines def get_spack_script( @@ -16,7 +16,6 @@ def get_spack_script( config_file=None, include_e3sm_lapack=False, include_e3sm_hdf5_netcdf=False, - yaml_template=None, ): """ Build a snippet of a load script for the given spack environment @@ -53,11 +52,6 @@ def get_spack_script( Whether to include the same hdf5, netcdf-c, netcdf-fortran and pnetcdf as used in E3SM - yaml_template : str, optional - A jinja template for a yaml file to be used for the environment instead - of the mache template. This allows you to use compilers and other - modules that differ from E3SM. - Returns ------- load_script : str @@ -78,35 +72,17 @@ def get_spack_script( if config_file is not None: config.read(config_file) - section = config['spack'] - - modules_before = section.getboolean('modules_before') - modules_after = section.getboolean('modules_after') - - yaml_data = _get_yaml_data( - machine, - compiler, - mpi, - include_e3sm_lapack, - include_e3sm_hdf5_netcdf, - specs=[], - yaml_template=yaml_template, - ) - - if modules_before or modules_after: - load_script = 'module purge\n' - if modules_before: - mods = _get_modules(yaml_data) - load_script = f'{load_script}\n{mods}\n' - else: - load_script = '' - - load_script = ( - f'{load_script}' + load_script_template = ( f'source {spack_path}/share/spack/setup-env.{shell}\n' f'spack env activate {env_name}' ) + # start with the shell script from the config_machines.xml for the + # given machine, compiler, and mpi + load_script_template += '\n' + extract_spack_from_config_machines( + machine, compiler, mpi, shell + ) + for shell_filename in [ f'{machine}.{shell}', f'{machine}_{compiler}_{mpi}.{shell}', @@ -117,18 +93,18 @@ def get_spack_script( ) try: with open(str(path)) as fp: - template = Template(fp.read()) + script_template = fp.read() except FileNotFoundError: # there's nothing to add, which is fine continue - shell_script = template.render( - e3sm_lapack=include_e3sm_lapack, - e3sm_hdf5_netcdf=include_e3sm_hdf5_netcdf, - ) - load_script = f'{load_script}\n{shell_script}' - if modules_after: - mods = _get_modules(yaml_data) - load_script = f'{load_script}\n{mods}' + # append a template if one exists + load_script_template += '\n' + script_template + + # render the jinja template + load_script = Template(load_script_template).render( + e3sm_lapack=include_e3sm_lapack, + e3sm_hdf5_netcdf=include_e3sm_hdf5_netcdf, + ) return load_script diff --git a/mache/spack/shared.py b/mache/spack/shared.py index 7d47459f..5a3787d9 100644 --- a/mache/spack/shared.py +++ b/mache/spack/shared.py @@ -1,6 +1,5 @@ from importlib import resources as importlib_resources -import yaml from jinja2 import Template @@ -38,21 +37,3 @@ def _get_yaml_data( e3sm_hdf5_netcdf=include_e3sm_hdf5_netcdf, ) return yaml_data - - -def _get_modules(yaml_string): - """Get a list of modules from a yaml file""" - yaml_data = yaml.safe_load(yaml_string) - mods = [] - if 'spack' in yaml_data and 'packages' in yaml_data['spack']: - package_data = yaml_data['spack']['packages'] - for package in package_data.values(): - if 'externals' in package: - for item in package['externals']: - if 'modules' in item: - for mod in item['modules']: - mods.append(f'module load {mod}') - - mods_str = '\n'.join(mods) - - return mods_str diff --git a/mache/spack/templates/anvil_gnu_mvapich.csh b/mache/spack/templates/anvil_gnu_mvapich.csh deleted file mode 100644 index 50307692..00000000 --- a/mache/spack/templates/anvil_gnu_mvapich.csh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh -setenv MODULEPATH $MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0-xhxgy33 \ - intel-mkl/2020.4.304-d6zw4xa \ - mvapich2/2.2-verbs-ppznoge - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf/4.4.1-ve2zfkw \ - netcdf-cxx/4.2-2rkopdl \ - netcdf-fortran/4.4.4-thtylny \ - parallel-netcdf/1.11.0-c22b2bn -{%- endif %} - -setenv UCX_TLS "self,sm,ud" -setenv UCX_UD_MLX5_RX_QUEUE_LEN "16384" -setenv MV2_ENABLE_AFFINITY "0" -setenv MV2_SHOW_CPU_BINDING "1" -setenv MV2_HOMOGENEOUS_CLUSTER "1" diff --git a/mache/spack/templates/anvil_gnu_mvapich.sh b/mache/spack/templates/anvil_gnu_mvapich.sh deleted file mode 100644 index 71b03fed..00000000 --- a/mache/spack/templates/anvil_gnu_mvapich.sh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh -export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0-xhxgy33 \ - intel-mkl/2020.4.304-d6zw4xa \ - mvapich2/2.2-verbs-ppznoge - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf/4.4.1-ve2zfkw \ - netcdf-cxx/4.2-2rkopdl \ - netcdf-fortran/4.4.4-thtylny \ - parallel-netcdf/1.11.0-c22b2bn -{%- endif %} - -export UCX_TLS="self,sm,ud" -export UCX_UD_MLX5_RX_QUEUE_LEN="16384" -export MV2_ENABLE_AFFINITY="0" -export MV2_SHOW_CPU_BINDING="1" -export MV2_HOMOGENEOUS_CLUSTER="1" diff --git a/mache/spack/templates/anvil_gnu_openmpi.csh b/mache/spack/templates/anvil_gnu_openmpi.csh deleted file mode 100644 index 36505512..00000000 --- a/mache/spack/templates/anvil_gnu_openmpi.csh +++ /dev/null @@ -1,24 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh -setenv MODULEPATH $MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0-xhxgy33 \ - intel-mkl/2020.4.304-d6zw4xa \ - openmpi/4.1.1-x5n4m36 - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-mtfptpl \ - netcdf-cxx/4.2-osp27dq \ - netcdf-fortran/4.4.4-5yd6dos \ - parallel-netcdf/1.11.0-a7ohxsg -{%- endif %} - -setenv UCX_TLS "self,sm,ud" -setenv UCX_UD_MLX5_RX_QUEUE_LEN "16384" diff --git a/mache/spack/templates/anvil_gnu_openmpi.sh b/mache/spack/templates/anvil_gnu_openmpi.sh deleted file mode 100644 index d9fc4604..00000000 --- a/mache/spack/templates/anvil_gnu_openmpi.sh +++ /dev/null @@ -1,24 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh -export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0-xhxgy33 \ - intel-mkl/2020.4.304-d6zw4xa \ - openmpi/4.1.1-x5n4m36 - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-mtfptpl \ - netcdf-cxx/4.2-osp27dq \ - netcdf-fortran/4.4.4-5yd6dos \ - parallel-netcdf/1.11.0-a7ohxsg -{%- endif %} - -export UCX_TLS="self,sm,ud" -export UCX_UD_MLX5_RX_QUEUE_LEN="16384" diff --git a/mache/spack/templates/anvil_intel_impi.csh b/mache/spack/templates/anvil_intel_impi.csh deleted file mode 100644 index e9171165..00000000 --- a/mache/spack/templates/anvil_intel_impi.csh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh -setenv MODULEPATH $MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - intel-mpi/2019.9.304-i42whlw - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-blyisdg \ - netcdf-cxx/4.2-gkqc6fq \ - netcdf-fortran/4.4.4-eanrh5t \ - parallel-netcdf/1.11.0-y3nmmej -{%- endif %} - -setenv UCX_TLS "self,sm,ud" -setenv UCX_UD_MLX5_RX_QUEUE_LEN "16384" -setenv KMP_AFFINITY "granularity=core,balanced" -setenv KMP_HOT_TEAMS_MODE "1" diff --git a/mache/spack/templates/anvil_intel_impi.sh b/mache/spack/templates/anvil_intel_impi.sh deleted file mode 100644 index aa881e35..00000000 --- a/mache/spack/templates/anvil_intel_impi.sh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh -export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - intel-mpi/2019.9.304-i42whlw - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-blyisdg \ - netcdf-cxx/4.2-gkqc6fq \ - netcdf-fortran/4.4.4-eanrh5t \ - parallel-netcdf/1.11.0-y3nmmej -{%- endif %} - -export UCX_TLS="self,sm,ud" -export UCX_UD_MLX5_RX_QUEUE_LEN="16384" -export KMP_AFFINITY="granularity=core,balanced" -export KMP_HOT_TEAMS_MODE="1" diff --git a/mache/spack/templates/anvil_intel_mvapich.csh b/mache/spack/templates/anvil_intel_mvapich.csh deleted file mode 100644 index 6cb97a36..00000000 --- a/mache/spack/templates/anvil_intel_mvapich.csh +++ /dev/null @@ -1,30 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh -setenv MODULEPATH $MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - mvapich2/2.3.6-verbs-x4iz7lq - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-gei7x7w \ - netcdf-cxx/4.2-db2f5or \ - netcdf-fortran/4.4.4-b4ldb3a \ - parallel-netcdf/1.11.0-kj4jsvt -{%- endif %} - -setenv UCX_TLS "self,sm,ud" -setenv UCX_UD_MLX5_RX_QUEUE_LEN "16384" -setenv MV2_ENABLE_AFFINITY "0" -setenv MV2_SHOW_CPU_BINDING "1" -setenv MV2_HOMOGENEOUS_CLUSTER "1" -setenv KMP_AFFINITY "granularity=core,balanced" -setenv KMP_HOT_TEAMS_MODE "1" diff --git a/mache/spack/templates/anvil_intel_mvapich.sh b/mache/spack/templates/anvil_intel_mvapich.sh deleted file mode 100644 index b7bcdfc2..00000000 --- a/mache/spack/templates/anvil_intel_mvapich.sh +++ /dev/null @@ -1,30 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh -export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - mvapich2/2.3.6-verbs-x4iz7lq - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-gei7x7w \ - netcdf-cxx/4.2-db2f5or \ - netcdf-fortran/4.4.4-b4ldb3a \ - parallel-netcdf/1.11.0-kj4jsvt -{%- endif %} - -export UCX_TLS="self,sm,ud" -export UCX_UD_MLX5_RX_QUEUE_LEN="16384" -export MV2_ENABLE_AFFINITY="0" -export MV2_SHOW_CPU_BINDING="1" -export MV2_HOMOGENEOUS_CLUSTER="1" -export KMP_AFFINITY="granularity=core,balanced" -export KMP_HOT_TEAMS_MODE="1" diff --git a/mache/spack/templates/anvil_intel_openmpi.csh b/mache/spack/templates/anvil_intel_openmpi.csh deleted file mode 100644 index 927b2332..00000000 --- a/mache/spack/templates/anvil_intel_openmpi.csh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh -setenv MODULEPATH $MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - openmpi/4.1.1-v3b3npd - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-smyuxme \ - netcdf-cxx/4.2-kfb2aag \ - netcdf-fortran/4.4.4-mablvyc \ - parallel-netcdf/1.11.0-x4n5s7k -{%- endif %} - -setenv UCX_TLS "self,sm,ud" -setenv UCX_UD_MLX5_RX_QUEUE_LEN "16384" -setenv KMP_AFFINITY "granularity=core,balanced" -setenv KMP_HOT_TEAMS_MODE "1" diff --git a/mache/spack/templates/anvil_intel_openmpi.sh b/mache/spack/templates/anvil_intel_openmpi.sh deleted file mode 100644 index 4d232613..00000000 --- a/mache/spack/templates/anvil_intel_openmpi.sh +++ /dev/null @@ -1,27 +0,0 @@ -source /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh -export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core - -module purge - -module use \ - /lcrc/group/e3sm/soft/modulefiles/anvil - -module load \ - cmake/3.26.3-nszudya \ - gcc/8.2.0 \ - intel/20.0.4-lednsve \ - intel-mkl/2020.4.304-voqlapk \ - openmpi/4.1.1-v3b3npd - -{%- if e3sm_hdf5_netcdf %} -module load \ - netcdf-c/4.4.1-smyuxme \ - netcdf-cxx/4.2-kfb2aag \ - netcdf-fortran/4.4.4-mablvyc \ - parallel-netcdf/1.11.0-x4n5s7k -{%- endif %} - -export UCX_TLS="self,sm,ud" -export UCX_UD_MLX5_RX_QUEUE_LEN="16384" -export KMP_AFFINITY="granularity=core,balanced" -export KMP_HOT_TEAMS_MODE="1" diff --git a/mache/spack/templates/aurora_oneapi-ifx_mpich.csh b/mache/spack/templates/aurora_oneapi-ifx_mpich.csh deleted file mode 100644 index d8e6669e..00000000 --- a/mache/spack/templates/aurora_oneapi-ifx_mpich.csh +++ /dev/null @@ -1,22 +0,0 @@ -source /usr/share/lmod/lmod/init/csh - -module load \ - cmake/3.30.5 \ - oneapi/release/2025.0.5 - -setenv LD_LIBRARY_PATH "/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:${LD_LIBRARY_PATH}" -setenv PATH "/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:${PATH}" -setenv FI_CXI_DEFAULT_CQ_SIZE "131072" -setenv FI_CXI_CQ_FILL_PERCENT "20" -setenv LIBOMPTARGET_DEBUG "0" -setenv MPIR_CVAR_ENABLE_GPU "0" -setenv GPU_TILE_COMPACT " " -setenv RANKS_BIND "core" -setenv KMP_AFFINITY "granularity=core,balanced" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -setenv NETCDF_C_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -setenv NETCDF_FORTRAN_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -setenv PNETCDF_PATH "/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002" -{%- endif %} diff --git a/mache/spack/templates/aurora_oneapi-ifx_mpich.sh b/mache/spack/templates/aurora_oneapi-ifx_mpich.sh deleted file mode 100644 index 0f494d34..00000000 --- a/mache/spack/templates/aurora_oneapi-ifx_mpich.sh +++ /dev/null @@ -1,22 +0,0 @@ -source /usr/share/lmod/lmod/init/sh - -module load \ - cmake/3.30.5 \ - oneapi/release/2025.0.5 - -export LD_LIBRARY_PATH="/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:${LD_LIBRARY_PATH}" -export PATH="/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:${PATH}" -export FI_CXI_DEFAULT_CQ_SIZE="131072" -export FI_CXI_CQ_FILL_PERCENT="20" -export LIBOMPTARGET_DEBUG="0" -export MPIR_CVAR_ENABLE_GPU="0" -export GPU_TILE_COMPACT=" " -export RANKS_BIND="core" -export KMP_AFFINITY="granularity=core,balanced" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -export NETCDF_C_PATH="/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -export NETCDF_FORTRAN_PATH="/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002" -export PNETCDF_PATH="/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002" -{%- endif %} diff --git a/mache/spack/templates/chicoma-cpu_gnu_mpich.csh b/mache/spack/templates/chicoma-cpu_gnu_mpich.csh deleted file mode 100644 index c5cf0609..00000000 --- a/mache/spack/templates/chicoma-cpu_gnu_mpich.csh +++ /dev/null @@ -1,57 +0,0 @@ -source /usr/share/lmod/lmod/init/csh - -module rm cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - cray-netcdf \ - cray-hdf5 \ - gcc \ - gcc-native \ - intel \ - intel-oneapi \ - nvidia \ - aocc \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - craype \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc - -# we must load cray-libsci for gcc to work -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.29.6 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -setenv PERL5LIB /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 -setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -setenv MPICH_COLL_SYNC MPI_Bcast - -# for standalone MPAS builds -setenv GNU_CRAY_LDFLAGS "-Wl,--enable-new-dtags" - -setenv LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/templates/chicoma-cpu_gnu_mpich.sh b/mache/spack/templates/chicoma-cpu_gnu_mpich.sh deleted file mode 100644 index a7850e97..00000000 --- a/mache/spack/templates/chicoma-cpu_gnu_mpich.sh +++ /dev/null @@ -1,57 +0,0 @@ -source /usr/share/lmod/lmod/init/sh - -module rm cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - cray-netcdf \ - cray-hdf5 \ - gcc \ - gcc-native \ - intel \ - intel-oneapi \ - nvidia \ - aocc \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - craype \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc - -# we must load cray-libsci for gcc to work -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.29.6 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -export PERL5LIB=/usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 -export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -export MPICH_COLL_SYNC=MPI_Bcast - -export LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" - -# for standalone MPAS builds -export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" diff --git a/mache/spack/templates/chicoma-cpu_nvidia_mpich.csh b/mache/spack/templates/chicoma-cpu_nvidia_mpich.csh deleted file mode 100644 index 72d4bc80..00000000 --- a/mache/spack/templates/chicoma-cpu_nvidia_mpich.csh +++ /dev/null @@ -1,55 +0,0 @@ -source /usr/share/lmod/lmod/init/csh - - -module rm cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - cray-netcdf \ - cray-hdf5 \ - gcc \ - gcc-native \ - intel \ - intel-oneapi \ - nvidia \ - aocc \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - craype \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc - -module load PrgEnv-nvidia/8.5.0 \ - nvidia/24.7 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.29.6 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -setenv PERL5LIB /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 -setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -setenv MPICH_COLL_SYNC MPI_Bcast - -setenv LD_LIBRARY_PATH $CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH -setenv BLA_VENDOR NVHPC diff --git a/mache/spack/templates/chicoma-cpu_nvidia_mpich.sh b/mache/spack/templates/chicoma-cpu_nvidia_mpich.sh deleted file mode 100644 index 32bace26..00000000 --- a/mache/spack/templates/chicoma-cpu_nvidia_mpich.sh +++ /dev/null @@ -1,54 +0,0 @@ -source /usr/share/lmod/lmod/init/sh - -module rm cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - cray-netcdf \ - cray-hdf5 \ - gcc \ - gcc-native \ - intel \ - intel-oneapi \ - nvidia \ - aocc \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - craype \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc - -module load PrgEnv-nvidia/8.5.0 \ - nvidia/24.7 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.29.6 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -export PERL5LIB=/usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 -export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -export MPICH_COLL_SYNC=MPI_Bcast - -export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH -export BLA_VENDOR=NVHPC diff --git a/mache/spack/templates/chrysalis_gnu_openmpi.csh b/mache/spack/templates/chrysalis_gnu_openmpi.csh deleted file mode 100644 index 80abd929..00000000 --- a/mache/spack/templates/chrysalis_gnu_openmpi.csh +++ /dev/null @@ -1,7 +0,0 @@ -module load cmake - -setenv OMPI_MCA_sharedfp "^lockedfile,individual" -setenv UCX_TLS "^xpmem" -## purposefully omitting OMP variables that slow down MPAS-Ocean runs -#setenv OMP_STACKSIZE 128M -#setenv OMP_PLACES cores diff --git a/mache/spack/templates/chrysalis_gnu_openmpi.sh b/mache/spack/templates/chrysalis_gnu_openmpi.sh deleted file mode 100644 index c7117b1d..00000000 --- a/mache/spack/templates/chrysalis_gnu_openmpi.sh +++ /dev/null @@ -1,7 +0,0 @@ -module load cmake - -export OMPI_MCA_sharedfp="^lockedfile,individual" -export UCX_TLS="^xpmem" -## purposefully omitting OMP variables that slow down MPAS-Ocean runs -#export OMP_STACKSIZE=128M -#export OMP_PLACES=cores diff --git a/mache/spack/templates/chrysalis_intel_impi.csh b/mache/spack/templates/chrysalis_intel_impi.csh deleted file mode 100644 index 1b30a52d..00000000 --- a/mache/spack/templates/chrysalis_intel_impi.csh +++ /dev/null @@ -1,5 +0,0 @@ -setenv UCX_TLS "^xpmem" -setenv OMPI_MCA_sharedfp "^lockedfile,individual" -setenv OMP_STACKSIZE 128M -setenv KMP_AFFINITY "granularity=core,balanced" -setenv KMP_AFFINITY "granularity=thread,balanced" diff --git a/mache/spack/templates/chrysalis_intel_impi.sh b/mache/spack/templates/chrysalis_intel_impi.sh deleted file mode 100644 index 40782dac..00000000 --- a/mache/spack/templates/chrysalis_intel_impi.sh +++ /dev/null @@ -1,5 +0,0 @@ -export UCX_TLS="^xpmem" -export OMPI_MCA_sharedfp="^lockedfile,individual" -export OMP_STACKSIZE=128M -export KMP_AFFINITY="granularity=core,balanced" -export KMP_AFFINITY="granularity=thread,balanced" diff --git a/mache/spack/templates/chrysalis_intel_openmpi.csh b/mache/spack/templates/chrysalis_intel_openmpi.csh deleted file mode 100644 index 1b30a52d..00000000 --- a/mache/spack/templates/chrysalis_intel_openmpi.csh +++ /dev/null @@ -1,5 +0,0 @@ -setenv UCX_TLS "^xpmem" -setenv OMPI_MCA_sharedfp "^lockedfile,individual" -setenv OMP_STACKSIZE 128M -setenv KMP_AFFINITY "granularity=core,balanced" -setenv KMP_AFFINITY "granularity=thread,balanced" diff --git a/mache/spack/templates/chrysalis_intel_openmpi.sh b/mache/spack/templates/chrysalis_intel_openmpi.sh deleted file mode 100644 index 40782dac..00000000 --- a/mache/spack/templates/chrysalis_intel_openmpi.sh +++ /dev/null @@ -1,5 +0,0 @@ -export UCX_TLS="^xpmem" -export OMPI_MCA_sharedfp="^lockedfile,individual" -export OMP_STACKSIZE=128M -export KMP_AFFINITY="granularity=core,balanced" -export KMP_AFFINITY="granularity=thread,balanced" diff --git a/mache/spack/templates/compy_gnu_openmpi.csh b/mache/spack/templates/compy_gnu_openmpi.csh deleted file mode 100644 index 11b19804..00000000 --- a/mache/spack/templates/compy_gnu_openmpi.csh +++ /dev/null @@ -1 +0,0 @@ -setenv OMPI_MCA_btl "^openib" diff --git a/mache/spack/templates/compy_gnu_openmpi.sh b/mache/spack/templates/compy_gnu_openmpi.sh deleted file mode 100644 index 9d342c8a..00000000 --- a/mache/spack/templates/compy_gnu_openmpi.sh +++ /dev/null @@ -1 +0,0 @@ -export OMPI_MCA_btl="^openib" diff --git a/mache/spack/templates/compy_intel_impi.csh b/mache/spack/templates/compy_intel_impi.csh deleted file mode 100644 index 36044008..00000000 --- a/mache/spack/templates/compy_intel_impi.csh +++ /dev/null @@ -1 +0,0 @@ -setenv LD_LIBRARY_PATH "/share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64:${LD_LIBRARY_PATH}" diff --git a/mache/spack/templates/compy_intel_impi.sh b/mache/spack/templates/compy_intel_impi.sh deleted file mode 100644 index 7303b7a1..00000000 --- a/mache/spack/templates/compy_intel_impi.sh +++ /dev/null @@ -1 +0,0 @@ -export LD_LIBRARY_PATH="/share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64:${LD_LIBRARY_PATH}" diff --git a/mache/spack/templates/frontier_crayamd-mphipcc_mpich.csh b/mache/spack/templates/frontier_crayamd-mphipcc_mpich.csh deleted file mode 100644 index 5cb4e9a4..00000000 --- a/mache/spack/templates/frontier_crayamd-mphipcc_mpich.csh +++ /dev/null @@ -1,49 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module switch Core Core/24.00 -module switch PrgEnv-cray PrgEnv-amd/8.3.3 -module switch amd amd/5.4.0 -module load cray-libsci/22.12.1.1 - -module load \ - craype-accel-amd-gfx90a \ - rocm/5.4.0 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv MPICH_GPU_SUPPORT_ENABLED "1" -setenv GPUS_PER_NODE "--gpus-per-node=8" -setenv GPU_BIND_ARGS "--gpu-bind=closest" -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/amdclang-15.0.0/lib/pkgconfig:${CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_crayamd-mphipcc_mpich.sh b/mache/spack/templates/frontier_crayamd-mphipcc_mpich.sh deleted file mode 100644 index f60442fe..00000000 --- a/mache/spack/templates/frontier_crayamd-mphipcc_mpich.sh +++ /dev/null @@ -1,49 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module switch Core Core/24.00 -module switch PrgEnv-cray PrgEnv-amd/8.3.3 -module switch amd amd/5.4.0 -module load cray-libsci/22.12.1.1 - -module load \ - craype-accel-amd-gfx90a \ - rocm/5.4.0 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export MPICH_GPU_SUPPORT_ENABLED="1" -export GPUS_PER_NODE="--gpus-per-node=8" -export GPU_BIND_ARGS="--gpu-bind=closest" -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/amdclang-15.0.0/lib/pkgconfig:${CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_crayamd_mpich.csh b/mache/spack/templates/frontier_crayamd_mpich.csh deleted file mode 100644 index 2bff069f..00000000 --- a/mache/spack/templates/frontier_crayamd_mpich.csh +++ /dev/null @@ -1,44 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module switch Core Core/24.00 -module switch PrgEnv-cray PrgEnv-amd/8.3.3 -module switch amd amd/5.4.0 -module load cray-libsci/22.12.1.1 - -module load \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/amdclang-15.0.0/lib/pkgconfig:${CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_crayamd_mpich.sh b/mache/spack/templates/frontier_crayamd_mpich.sh deleted file mode 100644 index 09f6cf4c..00000000 --- a/mache/spack/templates/frontier_crayamd_mpich.sh +++ /dev/null @@ -1,44 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module switch Core Core/24.00 -module switch PrgEnv-cray PrgEnv-amd/8.3.3 -module switch amd amd/5.4.0 -module load cray-libsci/22.12.1.1 - -module load \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/amdclang-15.0.0/lib/pkgconfig:${CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craycray-mphipcc_mpich.csh b/mache/spack/templates/frontier_craycray-mphipcc_mpich.csh deleted file mode 100644 index b455db15..00000000 --- a/mache/spack/templates/frontier_craycray-mphipcc_mpich.csh +++ /dev/null @@ -1,48 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module load \ - Core/24.00 \ - cpe/22.12 \ - cce/15.0.1 \ - craype/2.7.20 \ - craype-accel-amd-gfx90a \ - rocm/5.4.0 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv MPICH_GPU_SUPPORT_ENABLED "1" -setenv GPUS_PER_NODE "--gpus-per-node=8" -setenv GPU_BIND_ARGS "--gpu-bind=closest" -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/crayclang-15.0.1/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craycray-mphipcc_mpich.sh b/mache/spack/templates/frontier_craycray-mphipcc_mpich.sh deleted file mode 100644 index b1746103..00000000 --- a/mache/spack/templates/frontier_craycray-mphipcc_mpich.sh +++ /dev/null @@ -1,48 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module load \ - Core/24.00 \ - cpe/22.12 \ - cce/15.0.1 \ - craype/2.7.20 \ - craype-accel-amd-gfx90a \ - rocm/5.4.0 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export MPICH_GPU_SUPPORT_ENABLED="1" -export GPUS_PER_NODE="--gpus-per-node=8" -export GPU_BIND_ARGS="--gpu-bind=closest" -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/crayclang-15.0.1/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craycray_mpich.csh b/mache/spack/templates/frontier_craycray_mpich.csh deleted file mode 100644 index bf97289a..00000000 --- a/mache/spack/templates/frontier_craycray_mpich.csh +++ /dev/null @@ -1,43 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module load \ - Core/24.00 \ - cpe/22.12 \ - cce/15.0.1 \ - craype/2.7.20 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/crayclang-15.0.1/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craycray_mpich.sh b/mache/spack/templates/frontier_craycray_mpich.sh deleted file mode 100644 index e19c33b0..00000000 --- a/mache/spack/templates/frontier_craycray_mpich.sh +++ /dev/null @@ -1,43 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module load \ - Core/24.00 \ - cpe/22.12 \ - cce/15.0.1 \ - craype/2.7.20 \ - libunwind/1.5.0 \ - libfabric/1.20.1 \ - cray-mpich/8.1.26 \ - subversion/1.14.1 \ - git/2.36.1 \ - cmake/3.23.2 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.12.2.1 \ - cray-netcdf-hdf5parallel/4.9.0.1 \ - cray-parallel-netcdf/1.12.3.1 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/crayclang-15.0.1/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craygnu-mphipcc_mpich.csh b/mache/spack/templates/frontier_craygnu-mphipcc_mpich.csh deleted file mode 100644 index 0d394586..00000000 --- a/mache/spack/templates/frontier_craygnu-mphipcc_mpich.csh +++ /dev/null @@ -1,45 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module load \ - Core/25.03 \ - PrgEnv-gnu \ - cpe/24.11 \ - libunwind/1.8.1 \ - subversion \ - git \ - cmake \ - craype-accel-amd-gfx90a \ - rocm/6.2.4 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.14.3.3 \ - cray-netcdf-hdf5parallel/4.9.0.15 \ - cray-parallel-netcdf/1.12.3.15 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv MPICH_GPU_SUPPORT_ENABLED "1" -setenv GPUS_PER_NODE "--gpus-per-node=8" -setenv GPU_BIND_ARGS "--gpu-bind=closest" -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/gcc-native-13.2/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craygnu-mphipcc_mpich.sh b/mache/spack/templates/frontier_craygnu-mphipcc_mpich.sh deleted file mode 100644 index 9d423af7..00000000 --- a/mache/spack/templates/frontier_craygnu-mphipcc_mpich.sh +++ /dev/null @@ -1,45 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module load \ - Core/25.03 \ - PrgEnv-gnu \ - cpe/24.11 \ - libunwind/1.8.1 \ - subversion \ - git \ - cmake \ - craype-accel-amd-gfx90a \ - rocm/6.2.4 - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.14.3.3 \ - cray-netcdf-hdf5parallel/4.9.0.15 \ - cray-parallel-netcdf/1.12.3.15 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export MPICH_GPU_SUPPORT_ENABLED="1" -export GPUS_PER_NODE="--gpus-per-node=8" -export GPU_BIND_ARGS="--gpu-bind=closest" -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/gcc-native-13.2/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craygnu_mpich.csh b/mache/spack/templates/frontier_craygnu_mpich.csh deleted file mode 100644 index 3e700c88..00000000 --- a/mache/spack/templates/frontier_craygnu_mpich.csh +++ /dev/null @@ -1,40 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/csh - -module reset - -module load \ - Core/25.03 \ - PrgEnv-gnu \ - cpe/24.11 \ - libunwind/1.8.1 \ - subversion \ - git \ - cmake - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.14.3.3 \ - cray-netcdf-hdf5parallel/4.9.0.15 \ - cray-parallel-netcdf/1.12.3.15 -{%- endif %} - -setenv HDF5_ROOT "" -setenv MPICH_GPU_SUPPORT_ENABLED "0" -setenv MPICH_VERSION_DISPLAY "1" -setenv MPICH_OFI_CXI_COUNTER_REPORT "2" -setenv LD_LIBRARY_PATH "${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -setenv SKIP_BLAS "True" -setenv GPUS_PER_NODE " " -setenv NTASKS_PER_GPU " " -setenv GPU_BIND_ARGS " " -setenv PKG_CONFIG_PATH "/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/gcc-native-13.2/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH "${NETCDF_DIR}" -setenv NETCDF_C_PATH "${NETCDF_DIR}" -setenv NETCDF_FORTRAN_PATH "${NETCDF_DIR}" -setenv PNETCDF_PATH "${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/frontier_craygnu_mpich.sh b/mache/spack/templates/frontier_craygnu_mpich.sh deleted file mode 100644 index 096ae1ab..00000000 --- a/mache/spack/templates/frontier_craygnu_mpich.sh +++ /dev/null @@ -1,40 +0,0 @@ -source /opt/cray/pe/lmod/lmod/init/sh - -module reset - -module load \ - Core/25.03 \ - PrgEnv-gnu \ - cpe/24.11 \ - libunwind/1.8.1 \ - subversion \ - git \ - cmake - -module rm \ - darshan-runtime &> /dev/null - -{%- if e3sm_hdf5_netcdf %} -module load \ - cray-hdf5-parallel/1.14.3.3 \ - cray-netcdf-hdf5parallel/4.9.0.15 \ - cray-parallel-netcdf/1.12.3.15 -{%- endif %} - -export HDF5_ROOT="" -export MPICH_GPU_SUPPORT_ENABLED="0" -export MPICH_VERSION_DISPLAY="1" -export MPICH_OFI_CXI_COUNTER_REPORT="2" -export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" -export SKIP_BLAS="True" -export GPUS_PER_NODE=" " -export NTASKS_PER_GPU=" " -export GPU_BIND_ARGS=" " -export PKG_CONFIG_PATH="/lustre/orion/cli115/world-shared/frontier/3rdparty/protobuf/21.6/gcc-native-13.2/lib/pkgconfig:${PKG_CONFIG_PATH}" - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH="${NETCDF_DIR}" -export NETCDF_C_PATH="${NETCDF_DIR}" -export NETCDF_FORTRAN_PATH="${NETCDF_DIR}" -export PNETCDF_PATH="${PNETCDF_DIR}" -{%- endif %} diff --git a/mache/spack/templates/pm-cpu.sh b/mache/spack/templates/pm-cpu.sh new file mode 100644 index 00000000..5a9f8432 --- /dev/null +++ b/mache/spack/templates/pm-cpu.sh @@ -0,0 +1,4 @@ +if [ -z "${NERSC_HOST:-}" ]; then + # happens when building spack environment + export NERSC_HOST="perlmutter" +fi diff --git a/mache/spack/templates/pm-cpu_gnu_mpich.csh b/mache/spack/templates/pm-cpu_gnu_mpich.csh deleted file mode 100644 index 5a45bfa8..00000000 --- a/mache/spack/templates/pm-cpu_gnu_mpich.csh +++ /dev/null @@ -1,57 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -setenv MPICH_MPIIO_DVS_MAXNODES 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -## Not needed -# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv FI_MR_CACHE_MONITOR kdreg2 -setenv MPICH_COLL_SYNC MPI_Bcast -setenv GATOR_INITIAL_MB 4000MB -setenv BLA_VENDOR Generic diff --git a/mache/spack/templates/pm-cpu_gnu_mpich.sh b/mache/spack/templates/pm-cpu_gnu_mpich.sh deleted file mode 100644 index 68963fc5..00000000 --- a/mache/spack/templates/pm-cpu_gnu_mpich.sh +++ /dev/null @@ -1,62 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -export MPICH_MPIIO_DVS_MAXNODES=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -## Not needed -# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export FI_MR_CACHE_MONITOR=kdreg2 - -if [ -z "${NERSC_HOST:-}" ]; then - # happens when building spack environment - export NERSC_HOST="perlmutter" -fi -export MPICH_COLL_SYNC=MPI_Bcast -export GATOR_INITIAL_MB=4000MB -export BLA_VENDOR=Generic diff --git a/mache/spack/templates/pm-cpu_gnu_mpich.yaml b/mache/spack/templates/pm-cpu_gnu_mpich.yaml index 7aa599c9..b437ca0b 100644 --- a/mache/spack/templates/pm-cpu_gnu_mpich.yaml +++ b/mache/spack/templates/pm-cpu_gnu_mpich.yaml @@ -1,5 +1,5 @@ -{%- set compiler = "gcc@12.3" %} -{%- set mpi = "cray-mpich@8.1.28" %} +{%- set compiler = "gcc@13.2" %} +{%- set mpi = "cray-mpich@8.1.30" %} spack: specs: - {{ compiler }} @@ -35,8 +35,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.24.3 - prefix: /global/common/software/nersc9/cmake/3.24.3 + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 buildable: false findutils: externals: @@ -110,10 +110,10 @@ spack: - spec: {{ compiler }} modules: - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 - - cray-libsci/23.12.5 + - gcc-native/13.2 + - cray-libsci/24.07.0 - craype-accel-host - - craype/2.7.30 + - craype/2.7.32 - libfabric/1.22.0 extra_attributes: compilers: @@ -126,13 +126,13 @@ spack: - spec: {{ mpi }}%{{ compiler }} modules: - libfabric/1.22.0 - - cray-mpich/8.1.28 + - cray-mpich/8.1.30 buildable: false cray-libsci: externals: - - spec: cray-libsci@23.12.5 + - spec: cray-libsci@24.07.0 modules: - - cray-libsci/23.12.5 + - cray-libsci/24.07.0 buildable: false {%- if e3sm_hdf5_netcdf %} hdf5: @@ -142,8 +142,8 @@ spack: buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 + - spec: parallel-netcdf@1.12.3.13+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.13/gnu/12.3 buildable: false netcdf-c: externals: @@ -169,10 +169,10 @@ spack: target: x86_64 modules: - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 - - cray-libsci/23.12.5 + - gcc-native/13.2 + - cray-libsci/24.07.0 - craype-accel-host - - craype/2.7.30 + - craype/2.7.32 - libfabric/1.22.0 environment: prepend_path: diff --git a/mache/spack/templates/pm-cpu_intel_mpich.csh b/mache/spack/templates/pm-cpu_intel_mpich.csh deleted file mode 100644 index 3e946865..00000000 --- a/mache/spack/templates/pm-cpu_intel_mpich.csh +++ /dev/null @@ -1,56 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-intel/8.5.0 \ - intel/2023.2.0 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -setenv MPICH_MPIIO_DVS_MAXNODES 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -## Not needed -# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv FI_MR_CACHE_MONITOR kdreg2 -setenv MPICH_COLL_SYNC MPI_Bcast -setenv GATOR_INITIAL_MB 4000MB -setenv BLA_VENDOR Intel10_64_dyn diff --git a/mache/spack/templates/pm-cpu_intel_mpich.sh b/mache/spack/templates/pm-cpu_intel_mpich.sh deleted file mode 100644 index a3910504..00000000 --- a/mache/spack/templates/pm-cpu_intel_mpich.sh +++ /dev/null @@ -1,61 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-intel/8.5.0 \ - intel/2023.2.0 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -export MPICH_MPIIO_DVS_MAXNODES=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -## Not needed -# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export FI_MR_CACHE_MONITOR=kdreg2 - -if [ -z "${NERSC_HOST:-}" ]; then - # happens when building spack environment - export NERSC_HOST="perlmutter" -fi -export MPICH_COLL_SYNC=MPI_Bcast -export GATOR_INITIAL_MB=4000MB -export BLA_VENDOR=Intel10_64_dyn diff --git a/mache/spack/templates/pm-cpu_intel_mpich.yaml b/mache/spack/templates/pm-cpu_intel_mpich.yaml index 7cbe8432..578dbc3d 100644 --- a/mache/spack/templates/pm-cpu_intel_mpich.yaml +++ b/mache/spack/templates/pm-cpu_intel_mpich.yaml @@ -1,5 +1,5 @@ -{%- set compiler = "intel@2023.2.0" %} -{%- set mpi = "cray-mpich@8.1.28" %} +{%- set compiler = "intel@2024.1.0" %} +{%- set mpi = "cray-mpich@8.1.30" %} spack: specs: - {{ compiler }} @@ -35,8 +35,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.24.3 - prefix: /global/common/software/nersc9/cmake/3.24.3 + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 buildable: false findutils: externals: @@ -110,17 +110,17 @@ spack: - spec: {{ compiler }} modules: - PrgEnv-intel/8.5.0 - - intel/2023.2.0 + - intel/2024.1.0 - cray-libsci/23.12.5 - craype-accel-host - - craype/2.7.30 + - craype/2.7.32 - libfabric/1.22.0 buildable: false cray-mpich: externals: - spec: {{ mpi }}%{{ compiler }} modules: - - cray-mpich/8.1.28 + - cray-mpich/8.1.30 - libfabric/1.22.0 buildable: false cray-libsci: @@ -137,8 +137,8 @@ spack: buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/intel/2023.2 + - spec: parallel-netcdf@1.12.3.13+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.13/intel/2023.2 buildable: false netcdf-c: externals: @@ -164,10 +164,10 @@ spack: target: x86_64 modules: - PrgEnv-intel/8.5.0 - - intel/2023.2.0 + - intel/2024.1.0 - cray-libsci/23.12.5 - craype-accel-host - - craype/2.7.30 + - craype/2.7.32 - libfabric/1.22.0 environment: prepend_path: diff --git a/mache/spack/templates/pm-cpu_nvidia_mpich.csh b/mache/spack/templates/pm-cpu_nvidia_mpich.csh deleted file mode 100644 index 5c311a05..00000000 --- a/mache/spack/templates/pm-cpu_nvidia_mpich.csh +++ /dev/null @@ -1,58 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-nvidia \ - nvidia/24.5 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -setenv MPICH_MPIIO_DVS_MAXNODES 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -## Not needed -# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv MPICH_GPU_SUPPORT_ENABLED 1 -setenv FI_MR_CACHE_MONITOR kdreg2 -setenv MPICH_COLL_SYNC MPI_Bcast -setenv GATOR_INITIAL_MB 4000MB -setenv BLA_VENDOR NVHPC diff --git a/mache/spack/templates/pm-cpu_nvidia_mpich.sh b/mache/spack/templates/pm-cpu_nvidia_mpich.sh deleted file mode 100644 index 85ddbee0..00000000 --- a/mache/spack/templates/pm-cpu_nvidia_mpich.sh +++ /dev/null @@ -1,63 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-nvidia \ - nvidia/24.5 \ - cray-libsci/23.12.5 \ - craype-accel-host \ - craype/2.7.30 \ - libfabric/1.22.0 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -export MPICH_MPIIO_DVS_MAXNODES=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -## Not needed -# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export MPICH_GPU_SUPPORT_ENABLED=1 - -if [ -z "${NERSC_HOST:-}" ]; then - # happens when building spack environment - export NERSC_HOST="perlmutter" -fi -export FI_MR_CACHE_MONITOR=kdreg2 -export MPICH_COLL_SYNC=MPI_Bcast -export GATOR_INITIAL_MB=4000MB -export BLA_VENDOR=NVHPC diff --git a/mache/spack/templates/pm-cpu_nvidia_mpich.yaml b/mache/spack/templates/pm-cpu_nvidia_mpich.yaml index e1f38bcb..266e65d0 100644 --- a/mache/spack/templates/pm-cpu_nvidia_mpich.yaml +++ b/mache/spack/templates/pm-cpu_nvidia_mpich.yaml @@ -1,5 +1,5 @@ -{%- set compiler = "nvhpc@24.5" %} -{%- set mpi = "cray-mpich@8.1.28" %} +{%- set compiler = "nvhpc@25.5" %} +{%- set mpi = "cray-mpich@8.1.30" %} spack: specs: - {{ mpi }}%{{ compiler }} @@ -34,8 +34,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.24.3 - prefix: /global/common/software/nersc9/cmake/3.24.3 + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 buildable: false findutils: externals: @@ -109,7 +109,7 @@ spack: - spec: {{ mpi }}%{{ compiler }} modules: - libfabric/1.22.0 - - cray-mpich/8.1.28 + - cray-mpich/8.1.30 buildable: false cray-libsci: externals: @@ -125,8 +125,8 @@ spack: buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3 + - spec: parallel-netcdf@1.12.3.13+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.13/nvidia/23.3 buildable: false netcdf-c: externals: @@ -143,19 +143,19 @@ spack: - compiler: spec: {{ compiler }} paths: - cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc - cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++ - f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran - fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran + cc: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvc++ + f77: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvfortran + fc: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvfortran flags: {} operating_system: sles15 target: any modules: - PrgEnv-nvidia - - nvidia/24.5 + - nvidia/25.5 - cray-libsci/23.12.5 - craype-accel-host - - craype/2.7.30 + - craype/2.7.32 - libfabric/1.20 environment: prepend_path: diff --git a/mache/spack/templates/pm-gpu.sh b/mache/spack/templates/pm-gpu.sh new file mode 100644 index 00000000..5a9f8432 --- /dev/null +++ b/mache/spack/templates/pm-gpu.sh @@ -0,0 +1,4 @@ +if [ -z "${NERSC_HOST:-}" ]; then + # happens when building spack environment + export NERSC_HOST="perlmutter" +fi diff --git a/mache/spack/templates/pm-gpu_gnu_mpich.csh b/mache/spack/templates/pm-gpu_gnu_mpich.csh deleted file mode 120000 index fe10cab4..00000000 --- a/mache/spack/templates/pm-gpu_gnu_mpich.csh +++ /dev/null @@ -1 +0,0 @@ -pm-cpu_gnu_mpich.csh \ No newline at end of file diff --git a/mache/spack/templates/pm-gpu_gnu_mpich.sh b/mache/spack/templates/pm-gpu_gnu_mpich.sh deleted file mode 120000 index ab1eb811..00000000 --- a/mache/spack/templates/pm-gpu_gnu_mpich.sh +++ /dev/null @@ -1 +0,0 @@ -pm-cpu_gnu_mpich.sh \ No newline at end of file diff --git a/mache/spack/templates/pm-gpu_gnugpu_mpich.csh b/mache/spack/templates/pm-gpu_gnugpu_mpich.csh deleted file mode 100644 index f969d4e1..00000000 --- a/mache/spack/templates/pm-gpu_gnugpu_mpich.csh +++ /dev/null @@ -1,57 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cudatoolkit/12.4 \ - craype-accel-nvidia80 \ - cray-libsci/23.12.5 \ - libfabric/1.22.0 \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -setenv MPICH_MPIIO_DVS_MAXNODES 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -## Not needed -# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv MPICH_COLL_SYNC MPI_Bcast -setenv FI_MR_CACHE_MONITOR kdreg2 -setenv MPICH_GPU_SUPPORT_ENABLED 1 diff --git a/mache/spack/templates/pm-gpu_gnugpu_mpich.sh b/mache/spack/templates/pm-gpu_gnugpu_mpich.sh deleted file mode 100644 index 5258f262..00000000 --- a/mache/spack/templates/pm-gpu_gnugpu_mpich.sh +++ /dev/null @@ -1,62 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-gnu/8.5.0 \ - gcc-native/12.3 \ - cudatoolkit/12.4 \ - craype-accel-nvidia80 \ - cray-libsci/23.12.5 \ - libfabric/1.22.0 \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -export MPICH_ENV_DISPLAY=1 -export MPICH_MPIIO_DVS_MAXNODES=1 -export MPICH_VERSION_DISPLAY=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -## Not needed -# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export MPICH_COLL_SYNC=MPI_Bcast -export FI_MR_CACHE_MONITOR=kdreg2 -export MPICH_GPU_SUPPORT_ENABLED=1 - -if [ -z "${NERSC_HOST:-}" ]; then - # happens when building spack environment - export NERSC_HOST="perlmutter" -fi diff --git a/mache/spack/templates/pm-gpu_gnugpu_mpich.yaml b/mache/spack/templates/pm-gpu_gnugpu_mpich.yaml index 09e01d19..f2cd97ea 100644 --- a/mache/spack/templates/pm-gpu_gnugpu_mpich.yaml +++ b/mache/spack/templates/pm-gpu_gnugpu_mpich.yaml @@ -1,5 +1,5 @@ -{%- set compiler = "gcc@12.3" %} -{%- set mpi = "cray-mpich@8.1.28" %} +{%- set compiler = "gcc@13.2" %} +{%- set mpi = "cray-mpich@8.1.30" %} spack: specs: - {{ compiler }} @@ -34,8 +34,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.24.3 - prefix: /global/common/software/nersc9/cmake/3.24.3 + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 buildable: false findutils: externals: @@ -109,11 +109,11 @@ spack: - spec: {{ compiler }} modules: - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 + - gcc-native/13.2 - craype-accel-nvidia80 - - cray-libsci/23.12.5 + - cray-libsci/24.07.0 - libfabric/1.22.0 - - craype/2.7.30 + - craype/2.7.32 - cudatoolkit/12.4 extra_attributes: compilers: @@ -126,44 +126,44 @@ spack: - spec: cuda@12.4 modules: - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 + - gcc-native/13.2 - craype-accel-nvidia80 - - cray-libsci/23.12.5 + - cray-libsci/24.07.0 - libfabric/1.22.0 - - craype/2.7.30 + - craype/2.7.32 - cudatoolkit/12.4 buildable: false cray-mpich: externals: - spec: {{ mpi }}%{{ compiler }} - prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3 + prefix: /opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3 buildable: false cray-libsci: externals: - spec: cray-libsci@23.12.5 modules: - - cray-libsci/23.12.5 + - cray-libsci/24.07.0 buildable: false {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3 + - spec: hdf5@1.14.3.1~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.14.3.1/gnu/12.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3 + - spec: parallel-netcdf@1.12.3.13+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.13/gnu/12.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 + - spec: netcdf-c@4.9.0.13+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.13/gnu/12.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.13/gnu/12.3 buildable: false {%- endif %} compilers: @@ -179,11 +179,11 @@ spack: target: any modules: - PrgEnv-gnu/8.5.0 - - gcc-native/12.3 + - gcc-native/13.2 - craype-accel-nvidia80 - - cray-libsci/23.12.5 + - cray-libsci/24.07.0 - libfabric/1.22.0 - - craype/2.7.30 + - craype/2.7.32 - cudatoolkit/12.4 environment: prepend_path: diff --git a/mache/spack/templates/pm-gpu_nvidia_mpich.csh b/mache/spack/templates/pm-gpu_nvidia_mpich.csh deleted file mode 120000 index 289fc529..00000000 --- a/mache/spack/templates/pm-gpu_nvidia_mpich.csh +++ /dev/null @@ -1 +0,0 @@ -pm-cpu_nvidia_mpich.csh \ No newline at end of file diff --git a/mache/spack/templates/pm-gpu_nvidia_mpich.sh b/mache/spack/templates/pm-gpu_nvidia_mpich.sh deleted file mode 120000 index e6edc27b..00000000 --- a/mache/spack/templates/pm-gpu_nvidia_mpich.sh +++ /dev/null @@ -1 +0,0 @@ -pm-cpu_nvidia_mpich.sh \ No newline at end of file diff --git a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.csh b/mache/spack/templates/pm-gpu_nvidiagpu_mpich.csh deleted file mode 100644 index f27a5154..00000000 --- a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.csh +++ /dev/null @@ -1,58 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-nvidia \ - nvidia/24.5 \ - cudatoolkit/12.4 \ - craype-accel-nvidia80 \ - gcc-native-mixed/12.3 \ - cray-libsci/23.12.5 \ - libfabric/1.22.0 \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -setenv NETCDF_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX -setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -setenv MPICH_ENV_DISPLAY 1 -setenv MPICH_VERSION_DISPLAY 1 -setenv MPICH_MPIIO_DVS_MAXNODES 1 -## purposefully omitting OMP variables that cause trouble in ESMF -# setenv OMP_STACKSIZE 128M -# setenv OMP_PROC_BIND spread -# setenv OMP_PLACES threads -setenv HDF5_USE_FILE_LOCKING FALSE -setenv MPICH_COLL_SYNC MPI_Bcast -setenv FI_MR_CACHE_MONITOR kdreg2 -## Not needed -# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv MPICH_GPU_SUPPORT_ENABLED 1 diff --git a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.sh b/mache/spack/templates/pm-gpu_nvidiagpu_mpich.sh deleted file mode 100644 index bac965cd..00000000 --- a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.sh +++ /dev/null @@ -1,63 +0,0 @@ -module rm cpe \ - cray-hdf5-parallel \ - cray-netcdf-hdf5parallel \ - cray-parallel-netcdf \ - PrgEnv-gnu \ - PrgEnv-intel \ - PrgEnv-nvidia \ - PrgEnv-cray \ - PrgEnv-aocc \ - gcc-native \ - intel \ - intel-oneapi \ - cudatoolkit \ - climate-utils \ - cray-libsci \ - matlab \ - craype-accel-nvidia80 \ - craype-accel-host \ - perftools-base \ - perftools \ - darshan \ - cray-mpich &> /dev/null - -module load PrgEnv-nvidia \ - nvidia/24.5 \ - cudatoolkit/12.4 \ - craype-accel-nvidia80 \ - gcc-native-mixed/12.3 \ - cray-libsci/23.12.5 \ - libfabric/1.22.0 \ - craype/2.7.30 \ - cray-mpich/8.1.28 \ - cmake/3.24.3 -{%- if e3sm_hdf5_netcdf %} -module load cray-hdf5-parallel/1.12.2.9 \ - cray-netcdf-hdf5parallel/4.9.0.9 \ - cray-parallel-netcdf/1.12.3.9 -{%- endif %} - -{%- if e3sm_hdf5_netcdf %} -export NETCDF_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX -export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX -{%- endif %} -export MPICH_ENV_DISPLAY=1 -export MPICH_VERSION_DISPLAY=1 -export MPICH_MPIIO_DVS_MAXNODES=1 -## purposefully omitting OMP variables that cause trouble in ESMF -# export OMP_STACKSIZE=128M -# export OMP_PROC_BIND=spread -# export OMP_PLACES=threads -export HDF5_USE_FILE_LOCKING=FALSE -## Not needed -# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export MPICH_COLL_SYNC=MPI_Bcast -export FI_MR_CACHE_MONITOR=kdreg2 -export MPICH_GPU_SUPPORT_ENABLED=1 - -if [ -z "${NERSC_HOST:-}" ]; then - # happens when building spack environment - export NERSC_HOST="perlmutter" -fi diff --git a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.yaml b/mache/spack/templates/pm-gpu_nvidiagpu_mpich.yaml index 5b4d2948..2a547c16 100644 --- a/mache/spack/templates/pm-gpu_nvidiagpu_mpich.yaml +++ b/mache/spack/templates/pm-gpu_nvidiagpu_mpich.yaml @@ -1,5 +1,5 @@ -{%- set compiler = "nvhpc@24.5" %} -{%- set mpi = "cray-mpich@8.1.28" %} +{%- set compiler = "nvhpc@25.5" %} +{%- set mpi = "cray-mpich@8.1.30" %} spack: specs: - {{ mpi }}%{{ compiler }} @@ -33,8 +33,8 @@ spack: buildable: false cmake: externals: - - spec: cmake@3.24.3 - prefix: /global/common/software/nersc9/cmake/3.24.3 + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 buildable: false findutils: externals: @@ -108,11 +108,11 @@ spack: - spec: cuda@12.4 modules: - PrgEnv-nvidia - - nvidia/24.5 + - nvidia/25.5 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.22.0 - - craype/2.7.30 + - craype/2.7.32 - cudatoolkit/12.4 buildable: false cray-mpich: @@ -120,7 +120,7 @@ spack: - spec: {{ mpi }}%{{ compiler }} modules: - libfabric/1.22.0 - - cray-mpich/8.1.28 + - cray-mpich/8.1.30 buildable: false cray-libsci: externals: @@ -131,43 +131,43 @@ spack: {%- if e3sm_hdf5_netcdf %} hdf5: externals: - - spec: hdf5@1.12.2.9~cxx+fortran+hl~java+mpi+shared - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/nvidia/23.3 + - spec: hdf5@1.14.3.1~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.14.3.1/nvidia/23.3 buildable: false parallel-netcdf: externals: - - spec: parallel-netcdf@1.12.3.9+cxx+fortran+pic+shared - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/nvidia/23.3 + - spec: parallel-netcdf@1.12.3.13+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.13/nvidia/23.3 buildable: false netcdf-c: externals: - - spec: netcdf-c@4.9.0.9+mpi~parallel-netcdf - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 + - spec: netcdf-c@4.9.0.13+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.13/nvidia/23.3 buildable: false netcdf-fortran: externals: - spec: netcdf-fortran@4.5.3 - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/nvidia/23.3 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.13/nvidia/23.3 buildable: false {%- endif %} compilers: - compiler: spec: {{ compiler }} paths: - cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc - cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++ - f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran - fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran + cc: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvc++ + f77: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvfortran + fc: /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/bin/nvfortran flags: {} operating_system: sles15 target: any modules: - PrgEnv-nvidia - - nvidia/24.5 + - nvidia/25.5 - craype-accel-nvidia80 - cray-libsci/23.12.5 - libfabric/1.22.0 - - craype/2.7.30 + - craype/2.7.32 - cudatoolkit/12.4 environment: prepend_path: