diff --git a/CIME/Tools/pelayout b/CIME/Tools/pelayout index 48a13a4cc91..494db7254f7 100755 --- a/CIME/Tools/pelayout +++ b/CIME/Tools/pelayout @@ -47,6 +47,7 @@ import re logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -70,13 +71,13 @@ def parse_command_line(args, description): parser.add_argument( "--format", - default="%4C: %6T/%6H; %6R %6P", + default="%4C: %6T/%6H; %6R %6P %6X", help="Format the PE layout items for each component (see below)", ) parser.add_argument( "--header", - default="Comp NTASKS NTHRDS ROOTPE PSTRIDE", + default="Comp NTASKS NTHRDS ROOTPE PSTRIDE XSTRIDE", help="Custom header for PE layout display", ) @@ -116,26 +117,35 @@ def get_value_as_string(case, var, attribute=None, resolved=False, subgroup=None ############################################################################### -def format_pelayout(comp, ntasks, nthreads, rootpe, pstride, arg_format): +def format_pelayout(comp, ntasks, nthreads, rootpe, pstride, xstride, arg_format): ############################################################################### """ Format the PE layout information for each component, using a default format, or using the arg_format input, if it exists. """ - subs = {"C": comp, "T": ntasks, "H": nthreads, "R": rootpe, "P": pstride} + subs = { + "C": comp, + "T": ntasks, + "H": nthreads, + "R": rootpe, + "P": pstride, + "X": xstride, + } layout_str = re.sub(r"%([0-9]*)C", r"{C:\1}", arg_format) layout_str = re.sub(r"%([-+0-9]*)T", r"{T:\1}", layout_str) layout_str = re.sub(r"%([-+0-9]*)H", r"{H:\1}", layout_str) layout_str = re.sub(r"%([-+0-9]*)R", r"{R:\1}", layout_str) layout_str = re.sub(r"%([-+0-9]*)P", r"{P:\1}", layout_str) + layout_str = re.sub(r"%([-+0-9]*)X", r"{X:\1}", layout_str) layout_str = layout_str.format(**subs) return layout_str # End def format_pelayout + ############################################################################### -def print_pelayout(case, ntasks, nthreads, rootpes, pstrid, arg_format, header): +def print_pelayout(case, ntasks, nthreads, rootpes, pstrid, xstrid, arg_format, header): ############################################################################### """ Print the PE layout information for each component, using the format, @@ -155,6 +165,7 @@ def print_pelayout(case, ntasks, nthreads, rootpes, pstrid, arg_format, header): nthreads[comp], rootpes[comp], pstrid[comp], + xstrid[comp], arg_format, ) ) @@ -175,6 +186,7 @@ def print_pelayout(case, ntasks, nthreads, rootpes, pstrid, arg_format, header): # End def print_pelayout + ############################################################################### def gather_pelayout(case): ############################################################################### @@ -185,6 +197,7 @@ def gather_pelayout(case): nthreads = {} rootpes = {} pstride = {} + xstride = {} comp_classes = case.get_values("COMP_CLASSES") for comp in comp_classes: @@ -192,12 +205,14 @@ def gather_pelayout(case): nthreads[comp] = int(case.get_value("NTHRDS_" + comp)) rootpes[comp] = int(case.get_value("ROOTPE_" + comp)) pstride[comp] = int(case.get_value("PSTRID_" + comp)) + xstride[comp] = int(case.get_value("EXCL_STRIDE_" + comp)) # End for - return ntasks, nthreads, rootpes, pstride + return ntasks, nthreads, rootpes, pstride, xstride # End def gather_pelayout + ############################################################################### def set_nthreads(case, nthreads): ############################################################################### @@ -210,6 +225,7 @@ def set_nthreads(case, nthreads): # End def set_nthreads + ############################################################################### def modify_ntasks(case, new_tot_tasks): ############################################################################### @@ -258,6 +274,7 @@ def modify_ntasks(case, new_tot_tasks): # End def modify_ntasks + ############################################################################### def _main_func(description): ############################################################################### @@ -274,8 +291,10 @@ def _main_func(description): if set_ntasks is not None: modify_ntasks(case, int(set_ntasks)) # End if - ntasks, nthreads, rootpes, pstrid = gather_pelayout(case) - print_pelayout(case, ntasks, nthreads, rootpes, pstrid, arg_format, header) + ntasks, nthreads, rootpes, pstrid, xstrid = gather_pelayout(case) + print_pelayout( + case, ntasks, nthreads, rootpes, pstrid, xstrid, arg_format, header + ) # End with diff --git a/CIME/XML/pes.py b/CIME/XML/pes.py index 3254751c794..855c428fbc1 100644 --- a/CIME/XML/pes.py +++ b/CIME/XML/pes.py @@ -1,6 +1,7 @@ """ Interface to the config_pes.xml file. This class inherits from GenericXML.py """ + from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files @@ -25,6 +26,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): opes_nthrds = {} opes_rootpe = {} opes_pstrid = {} + opes_excl_stride = {} oother_settings = {} other_settings = {} append = {} @@ -41,6 +43,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): opes_nthrds, opes_rootpe, opes_pstrid, + opes_excl_stride, oother_settings, append, ocomments, @@ -60,6 +63,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): pes_nthrds, pes_rootpe, pes_pstrid, + pes_excl_stride, other_settings, os_append, comments, @@ -68,6 +72,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): pes_nthrds.update(opes_nthrds) pes_rootpe.update(opes_rootpe) pes_pstrid.update(opes_pstrid) + pes_excl_stride.update(opes_excl_stride) other_settings.update(oother_settings) os_append.update(append) if ocomments is not None: @@ -80,6 +85,8 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): pes_rootpe[i] = 0 for i in iter(pes_pstrid): pes_pstrid[i] = 0 + for i in iter(pes_excl_stride): + pes_pstrid[i] = 0 logger.info("Pes setting: grid is {} ".format(grid)) logger.info("Pes setting: compset is {} ".format(compset)) @@ -87,6 +94,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): logger.info("Pes setting: threads is {} ".format(pes_nthrds)) logger.info("Pes setting: rootpe is {} ".format(pes_rootpe)) logger.info("Pes setting: pstrid is {} ".format(pes_pstrid)) + logger.info("Pes setting: excl_stride is {} ".format(pes_excl_stride)) logger.info("Pes other settings: {}".format(other_settings)) logger.info("Pes other settings append: {}".format(os_append)) if comments is not None: @@ -97,6 +105,7 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts="M", mpilib=None): pes_nthrds, pes_rootpe, pes_pstrid, + pes_excl_stride, other_settings, os_append, comments, @@ -110,7 +119,16 @@ def _find_matches( compset_choice = None pesize_choice = None max_points = -1 - pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings, append = ( + ( + pes_ntasks, + pes_nthrds, + pes_rootpe, + pes_pstrid, + pes_excl_stride, + other_settings, + append, + ) = ( + {}, {}, {}, {}, @@ -155,22 +173,27 @@ def _find_matches( comment = self.text(node) elif "ntasks" in vid: for child in self.get_children(root=node): - pes_ntasks[ - self.name(child).upper() - ] = int(self.text(child)) + pes_ntasks[self.name(child).upper()] = ( + int(self.text(child)) + ) elif "nthrds" in vid: for child in self.get_children(root=node): - pes_nthrds[ - self.name(child).upper() - ] = int(self.text(child)) + pes_nthrds[self.name(child).upper()] = ( + int(self.text(child)) + ) elif "rootpe" in vid: for child in self.get_children(root=node): - pes_rootpe[ - self.name(child).upper() - ] = int(self.text(child)) + pes_rootpe[self.name(child).upper()] = ( + int(self.text(child)) + ) elif "pstrid" in vid: for child in self.get_children(root=node): - pes_pstrid[ + pes_pstrid[self.name(child).upper()] = ( + int(self.text(child)) + ) + elif "excl_stride" in vid: + for child in self.get_children(root=node): + pes_excl_stride[ self.name(child).upper() ] = int(self.text(child)) # if the value is already upper case its something else we are trying to set @@ -231,6 +254,11 @@ def _find_matches( elif "pstrid" in vid: for child in self.get_children(root=node): pes_pstrid[self.name(child).upper()] = int(self.text(child)) + elif "excl_stride" in vid: + for child in self.get_children(root=node): + pes_excl_stride[self.name(child).upper()] = int( + self.text(child) + ) # if the value is already upper case its something else we are trying to set elif vid == self.name(node): text = self.text(node).strip() @@ -250,6 +278,7 @@ def _find_matches( pes_nthrds, pes_rootpe, pes_pstrid, + pes_excl_stride, other_settings, append, comment, diff --git a/CIME/case/case.py b/CIME/case/case.py index ae7fae9b94e..46e4ac8edc4 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -1094,9 +1094,9 @@ def _get_component_config_data(self, files): ) drv_comp_model_specific = Component(drv_config_file_model_specific, "CPL") - self._component_description[ - "forcing" - ] = drv_comp_model_specific.get_forcing_description(self._compsetname) + self._component_description["forcing"] = ( + drv_comp_model_specific.get_forcing_description(self._compsetname) + ) logger.info( "Compset forcing is {}".format(self._component_description["forcing"]) ) @@ -1204,6 +1204,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): pes_nthrds = {} pes_rootpe = {} pes_pstrid = {} + pes_excl_stride = {} other = {} append = {} comment = None @@ -1233,6 +1234,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): pes_nthrds, pes_rootpe, pes_pstrid, + pes_excl_stride, other, append, comment, @@ -1287,17 +1289,24 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): nthrds_str = "NTHRDS_{}".format(comp_class) rootpe_str = "ROOTPE_{}".format(comp_class) pstrid_str = "PSTRID_{}".format(comp_class) + excl_stride_str = "EXCL_STRIDE_{}".format(comp_class) ntasks = pes_ntasks[ntasks_str] if ntasks_str in pes_ntasks else 1 nthrds = pes_nthrds[nthrds_str] if nthrds_str in pes_nthrds else 1 rootpe = pes_rootpe[rootpe_str] if rootpe_str in pes_rootpe else 0 pstrid = pes_pstrid[pstrid_str] if pstrid_str in pes_pstrid else 1 + excl_stride = ( + pes_excl_stride[excl_stride_str] + if excl_stride_str in pes_excl_stride + else 0 + ) totaltasks.append((ntasks + rootpe) * nthrds) mach_pes_obj.set_value(ntasks_str, ntasks) mach_pes_obj.set_value(nthrds_str, nthrds) mach_pes_obj.set_value(rootpe_str, rootpe) mach_pes_obj.set_value(pstrid_str, pstrid) + mach_pes_obj.set_value(excl_stride_str, excl_stride) # Make sure that every component has been accounted for # set, nthrds and ntasks to 1 otherwise. Also set the ninst values here. diff --git a/CIME/data/config/config_headers.xml b/CIME/data/config/config_headers.xml index 10f59f2c770..748520cc93b 100644 --- a/CIME/data/config/config_headers.xml +++ b/CIME/data/config/config_headers.xml @@ -69,6 +69,7 @@ ROOTPE: the global mpi task of the component root task, if negative, indicates nodes rather than tasks. PSTRID: the stride of MPI tasks across the global set of pes (for now set to 1) NINST : the number of component instances (will be spread evenly across NTASKS) + EXCL_STRIDE: the stride of MPI tasks owned exclusively by a component. If 0, exclusivity is disabled. for example, for NTASKS = 8, NTHRDS = 2, ROOTPE = 32, NINST = 2 the MPI tasks would be placed starting on global pe 32 and each pe would be threaded 2-ways diff --git a/doc/source/ccs/model-configuration/variables/pes.rst b/doc/source/ccs/model-configuration/variables/pes.rst index 4d158c89571..e8be806b939 100644 --- a/doc/source/ccs/model-configuration/variables/pes.rst +++ b/doc/source/ccs/model-configuration/variables/pes.rst @@ -326,6 +326,8 @@ following meanings: - The global MPI task of the component root task; if negative, indicates nodes rather than tasks. The root processor for each component is set relative to the MPI global communicator. * - PSTRID - The stride of MPI tasks across the global set of pes (for now set to 1). This variable is currently not used and is a placeholder for future development. + * - EXCL_STRIDE + - Stride of MPI tasks owned exclusively by a component. If 0, exclusivity is disabled. * - NINST - The number of component instances, which are spread evenly across NTASKS. * - COST_PER_NODE