diff --git a/dsl/pace/dsl/dace/utils.py b/dsl/pace/dsl/dace/utils.py index aadc7a31f..e86fb6de0 100644 --- a/dsl/pace/dsl/dace/utils.py +++ b/dsl/pace/dsl/dace/utils.py @@ -1,7 +1,7 @@ import logging import time from dataclasses import dataclass, field -from typing import Dict, List +from typing import Dict, List, Tuple import dace from dace.transformation.helpers import get_parent_map @@ -37,6 +37,100 @@ def __exit__(self, _type, _val, _traceback): DaCeProgress.log(self.prefix, f"{self.label}...{elapsed}s.") +def sdfg_nan_checker(sdfg: dace.SDFG): + """ + Insert a check on array after each computational map to check for NaN + in the domain. + + In current pipeline, it is to be inserter after sdfg.simplify(...). + """ + import copy + + import sympy as sp + from dace import data as dt + from dace import symbolic + from dace.sdfg import graph as gr + from dace.sdfg import utils as sdutil + + # Adds a NaN checker after every mapexit->access node + checks: List[ + Tuple[dace.SDFGState, dace.nodes.AccessNode, gr.MultiConnectorEdge[dace.Memlet]] + ] = [] + allmaps = [ + (me, state) + for me, state in sdfg.all_nodes_recursive() + if isinstance(me, dace.nodes.MapEntry) + ] + topmaps = [ + (me, state) for me, state in allmaps if get_parent_map(state, me) is None + ] + for me, state in topmaps: + mx = state.exit_node(me) + for e in state.out_edges(mx): + if isinstance(e.dst, dace.nodes.AccessNode): + if isinstance(e.dst.desc(state.parent), dt.View): # Skip views for now + continue + node = sdutil.get_last_view_node(state, e.dst) + if "diss_estd" in node.data: + continue + if state.memlet_path(e)[ + 0 + ].data.dynamic: # Skip dynamic (region) outputs + continue + + checks.append((state, node, e)) + for state, node, e in checks: + # Append node that will go after the map + newnode: dace.nodes.AccessNode = copy.deepcopy(node) + # Move all outgoing edges to new node + for oe in list(state.out_edges(node)): + state.remove_edge(oe) + state.add_edge(newnode, oe.src_conn, oe.dst, oe.dst_conn, oe.data) + + # Add map in between node and newnode + sdfg = state.parent + inparr = sdfg.arrays[newnode.data] + index_expr = ", ".join(["__i%d" % i for i in range(len(inparr.shape))]) + index_printf = ", ".join(["%d"] * len(inparr.shape)) + + # Get range from memlet (which may not be the entire array size) + def evaluate(expr): + return expr.subs({sp.Function("int_floor"): symbolic.int_floor}) + + # Infer scheduly + schedule_type = dace.ScheduleType.Default + if ( + inparr.storage == dace.StorageType.GPU_Global + or inparr.storage == dace.StorageType.GPU_Shared + ): + schedule_type = dace.ScheduleType.GPU_Device + + ranges = [] + for i, (begin, end, step) in enumerate(e.data.subset): + ranges.append( + (f"__i{i}", (evaluate(begin), evaluate(end), evaluate(step))) + ) # evaluate to resolve views & actively read/write domains + state.add_mapped_tasklet( + name="nancheck", + map_ranges=ranges, + inputs={"__inp": dace.Memlet.simple(newnode.data, index_expr)}, + code=f""" + if (__inp != __inp) {{ + printf("NaN value found at {newnode.data}, line %d, index {index_printf}\\n", __LINE__, {index_expr}); + }} + """, # noqa: E501 + schedule=schedule_type, + language=dace.Language.CPP, + outputs={ + "__out": dace.Memlet.simple(newnode.data, index_expr, num_accesses=-1) + }, + input_nodes={node.data: node}, + output_nodes={newnode.data: newnode}, + external_edges=True, + ) + logger.info(f"Added {len(checks)} NaN checks") + + def _is_ref(sd: dace.sdfg.SDFG, aname: str): found = False for node, state in sd.all_nodes_recursive(): diff --git a/fv3core/pace/fv3core/stencils/dyn_core.py b/fv3core/pace/fv3core/stencils/dyn_core.py index b2b5c5913..d1b8fd3db 100644 --- a/fv3core/pace/fv3core/stencils/dyn_core.py +++ b/fv3core/pace/fv3core/stencils/dyn_core.py @@ -607,6 +607,11 @@ def _get_da_min(self) -> float: def _checkpoint_csw(self, state: DycoreState, tag: str): if self.call_checkpointer: + if tag == "In": + # need this to compare with Fortran + self._ut[:] = 0.0 + self._vt[:] = 0.0 + self._divgd.data[:] = 0.0 self.checkpointer( f"C_SW-{tag}", delpd=state.delp, @@ -625,6 +630,8 @@ def _checkpoint_csw(self, state: DycoreState, tag: str): def _checkpoint_dsw_in(self, state: DycoreState): if self.call_checkpointer: + self._xfx[:] = 0.0 + self._yfx[:] = 0.0 self.checkpointer( "D_SW-In", ucd=state.uc, @@ -638,8 +645,8 @@ def _checkpoint_dsw_in(self, state: DycoreState): ptd=state.pt, uad=state.ua, vad=state.va, - zhd=self._zh, - divgdd=self._divgd, + # zhd=self._zh, + # divgdd=self._divgd, xfxd=self._xfx, yfxd=self._yfx, mfxd=state.mfxd, diff --git a/fv3core/pace/fv3core/stencils/remapping.py b/fv3core/pace/fv3core/stencils/remapping.py index 01c5d7579..611a872df 100644 --- a/fv3core/pace/fv3core/stencils/remapping.py +++ b/fv3core/pace/fv3core/stencils/remapping.py @@ -550,6 +550,36 @@ def __call__( mdt (in) : Remap time step bdt (in): Timestep """ + + if self._call_checkpointer: + self._checkpointer( + "Remapping-In", + pt=pt, + delp=delp, + delz=delz, + peln=peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data + u=u, + v=v, + w=w, + ua=ua, + va=va, + cappa=cappa, + pkz=pkz, + pk=pk, + pe=pe.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data + phis=hs, + te_2d=te0_2d, + ps=ps, + wsd=wsd, + omga=omga, + ak=ak, + bk=bk, + dp1=dp1, + ) # TODO: remove unused arguments (and commented code that references them) # TODO: can we trim ps or make it a temporary # TODO: pe is copied into pe1 and pe2 for vectorization reasons in the Fortran, @@ -693,3 +723,26 @@ def __call__( else: # converts virtual temperature back to virtual potential temperature self._basic_adjust_divide_stencil(pkz, pt) + + if self._call_checkpointer: + self._checkpointer( + "Remapping-Out", + pt=pt, + delp=delp, + delz=delz, + peln=peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data + u=u, + v=v, + w=w, + cappa=cappa, + pkz=pkz, + pk=pk, + pe=pe.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data + te_2d=te0_2d, + omga=omga, + dp1=dp1, + ) diff --git a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py index c5d7f69fa..f4539d786 100644 --- a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py +++ b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py @@ -1,5 +1,5 @@ import math -from typing import Dict +from typing import Dict, Optional import gt4py.cartesian.gtscript as gtscript from gt4py.cartesian.gtscript import PARALLEL, computation, horizontal, interval, region @@ -189,6 +189,10 @@ def __init__( config=stencil_factory.config.dace_config, dace_compiletime_args=["tracers"], ) + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None grid_indexing = stencil_factory.grid_indexing self.grid_indexing = grid_indexing # needed for selective validation self._tracer_count = len(tracers) @@ -285,6 +289,7 @@ def __call__( x_courant (inout): accumulated courant number in x-direction y_courant (inout): accumulated courant number in y-direction """ + self._checkpoint_input(tracers, dp1, mfxd, mfyd, cxd, cyd) # DaCe parsing issue # if len(tracers) != self._tracer_count: # raise ValueError( @@ -390,3 +395,4 @@ def __call__( # we can't use variable assignment to avoid a data copy # because of current dace limitations self._swap_dp(dp1, dp2) + self._checkpoint_output(tracers, dp1, mfxd, mfyd, cxd, cyd) diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index a4dfdb2f7..2174989ad 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -1,3 +1,4 @@ +import typing from typing import List import gt4py.cartesian.gtscript as gtscript @@ -12,6 +13,7 @@ ) from typing_extensions import Literal +import pace.dsl.gt4py_utils as utils import pace.util import pace.util.constants as constants from pace.dsl.dace.orchestration import orchestrate @@ -21,6 +23,7 @@ from pace.physics.stencils.get_phi_fv3 import get_phi_fv3 from pace.physics.stencils.get_prs_fv3 import get_prs_fv3 from pace.physics.stencils.microphysics import Microphysics +from pace.stencils.testing.translate import reshape_pace_variable_to_fortran_format from pace.util import X_DIM, Y_DIM, Z_DIM from pace.util.grid import GridData @@ -209,13 +212,18 @@ def __init__( grid_data: GridData, namelist: PhysicsConfig, active_packages: List[Literal[PHYSICS_PACKAGES]], + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, dace_compiletime_args=["physics_state"], ) - + self.grid_indexing = stencil_factory.grid_indexing grid_indexing = stencil_factory.grid_indexing self._setup_statein() self._ptop = grid_data.ptop @@ -276,7 +284,31 @@ def _setup_statein(self): self._p00 = 1.0e5 def __call__(self, physics_state: PhysicsState, timestep: float): - + if self._call_checkpointer: + self._checkpointer( + "GFSPhysicsDriver-In", + qvapor=physics_state.qvapor, + qliquid=physics_state.qliquid, + qrain=physics_state.qrain, + qsnow=physics_state.qsnow, + qice=physics_state.qice, + qgraupel=physics_state.qgraupel, + qo3mr=physics_state.qo3mr, + qsgs_tke=physics_state.qsgs_tke, + qcld=physics_state.qcld, + pt=physics_state.pt, + delp=physics_state.delp, + delz=physics_state.delz, + ua=physics_state.ua, + va=physics_state.va, + w=physics_state.w, + omga=physics_state.omga, + ) + if self._call_checkpointer: + self._checkpointer( + "AtmosPhysDriverStatein-In", + pt=physics_state.pt, + ) self._atmos_phys_driver_statein( self._prsik, physics_state.phii, @@ -295,6 +327,13 @@ def __call__(self, physics_state: PhysicsState, timestep: float): physics_state.pt, self._dm3d, ) + if self._call_checkpointer: + self._checkpointer( + "AtmosPhysDriverStatein-Out", + IPD_tgrs=reshape_pace_variable_to_fortran_format( + physics_state.pt, self.grid_indexing + ), + ) self._get_prs_fv3( physics_state.phii, physics_state.prsi, diff --git a/stencils/pace/stencils/fv_update_phys.py b/stencils/pace/stencils/fv_update_phys.py index 751d985a7..3c6fe93a9 100644 --- a/stencils/pace/stencils/fv_update_phys.py +++ b/stencils/pace/stencils/fv_update_phys.py @@ -1,3 +1,5 @@ +import typing + import gt4py.cartesian.gtscript as gtscript from gt4py.cartesian.gtscript import FORWARD, PARALLEL, computation, exp, interval, log @@ -10,7 +12,7 @@ from pace.dsl.typing import Float, FloatField, FloatFieldIJ from pace.stencils.c2l_ord import CubedToLatLon from pace.stencils.update_dwind_phys import AGrid2DGridPhysics -from pace.util import X_DIM, Y_DIM +from pace.util import X_DIM, Y_DIM, Z_INTERFACE_DIM from pace.util.grid import DriverGridData, GridData @@ -92,7 +94,12 @@ def __init__( state: fv3core.DycoreState, u_dt: pace.util.Quantity, v_dt: pace.util.Quantity, + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, @@ -154,6 +161,31 @@ def __call__( t_dt, dt: float, ): + if self._call_checkpointer: + self._checkpointer( + "FVUpdatePhys-In", + u_dt=u_dt, + v_dt=v_dt, + t_dt=t_dt, + ua=state.ua, + va=state.va, + u=state.u, + v=state.v, + qvapor=state.qvapor, + qliquid=state.qliquid, + qice=state.qice, + qrain=state.qrain, + qsnow=state.qsnow, + qgraupel=state.qgraupel, + peln=state.peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data, + delp=state.delp, + pt=state.pt, + ps=state.ps, + pe=state.pe.transpose([X_DIM, Z_INTERFACE_DIM, Y_DIM]), + pk=state.pk, + ) self._moist_cv( state.qvapor, state.qliquid, diff --git a/stencils/pace/stencils/testing/translate.py b/stencils/pace/stencils/testing/translate.py index ac807813b..4d1358264 100644 --- a/stencils/pace/stencils/testing/translate.py +++ b/stencils/pace/stencils/testing/translate.py @@ -418,3 +418,16 @@ def python_grid(self): self.make_grid_storage(pygrid) pygrid.add_data(self.data) return pygrid + + +def reshape_pace_variable_to_fortran_format(data, grid_indexing): + cn2 = grid_indexing.domain[0] * grid_indexing.domain[1] + return np.reshape( + data[ + grid_indexing.isc : grid_indexing.iec + 1, + grid_indexing.jsc : grid_indexing.jec + 1, + 0 : grid_indexing.domain[2], + ].data, + (cn2, grid_indexing.domain[2]), + order="F", + )[:, ::-1] diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index cb97fabb5..23002729e 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -1,3 +1,4 @@ +import typing from typing import Optional from gt4py.cartesian.gtscript import BACKWARD, FORWARD, PARALLEL, computation, interval @@ -9,6 +10,7 @@ from pace.dsl.stencil import StencilFactory from pace.dsl.typing import Float, FloatField from pace.stencils.fv_update_phys import ApplyPhysicsToDycore +from pace.stencils.testing.translate import reshape_pace_variable_to_fortran_format from pace.util.grid import DriverGridData, GridData @@ -159,15 +161,10 @@ def __init__( config=stencil_factory.config.dace_config, dace_compiletime_args=["dycore_state", "physics_state", "tendency_state"], ) - - self._copy_dycore_to_physics = stencil_factory.from_dims_halo( + self._copy_dycore_to_physics = stencil_factory.from_origin_domain( copy_dycore_to_physics, - compute_dims=[ - pace.util.X_INTERFACE_DIM, - pace.util.Y_INTERFACE_DIM, - pace.util.Z_INTERFACE_DIM, - ], - compute_halos=(0, 0), + origin=stencil_factory.grid_indexing.origin_full(), + domain=stencil_factory.grid_indexing.domain_full(add=(0, 0, 1)), ) self._do_dry_convective_adjustment = do_dry_convective_adjust self._dycore_only = dycore_only @@ -249,7 +246,12 @@ def __init__( dycore_only: bool, apply_tendencies: bool, tendency_state, + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, @@ -258,7 +260,7 @@ def __init__( "phy_state", ], ) - + self.grid_indexing = stencil_factory.grid_indexing grid_indexing = stencil_factory.grid_indexing self.namelist = namelist self._rdt = 1.0 / Float(self.namelist.dt_atmos) @@ -287,6 +289,7 @@ def __init__( state, tendency_state.u_dt, tendency_state.v_dt, + checkpointer=checkpointer, ) self._dycore_only = dycore_only # apply_tendencies when we have run physics or fv_subgridz @@ -313,6 +316,19 @@ def __call__( self._fill_GFS_delp( dycore_state.delp, phy_state.physics_updated_specific_humidity, 1.0e-9 ) + if self._call_checkpointer: + self._checkpointer( + "PhysUpdateTracers-In", + u_dt=u_dt, + v_dt=v_dt, + t_dt=pt_dt, + pt_t1=reshape_pace_variable_to_fortran_format( + phy_state.physics_updated_pt, self.grid_indexing + ), + pt_t0=reshape_pace_variable_to_fortran_format( + phy_state.pt, self.grid_indexing + ), + ) self._prepare_tendencies_and_update_tracers( u_dt, v_dt, @@ -339,6 +355,13 @@ def __call__( dycore_state.delp, self._rdt, ) + if self._call_checkpointer: + self._checkpointer( + "PhysUpdateTracers-Out", + u_dt=u_dt, + v_dt=v_dt, + t_dt=pt_dt, + ) if self._apply_tendencies: self._apply_physics_to_dycore( dycore_state, @@ -347,3 +370,18 @@ def __call__( pt_dt, dt=dt, ) + if self._call_checkpointer: + self._checkpointer( + "FVUpdatePhys-Out", + qvapor=dycore_state.qvapor, + qliquid=dycore_state.qliquid, + qrain=dycore_state.qrain, + qsnow=dycore_state.qsnow, + qice=dycore_state.qice, + qgraupel=dycore_state.qgraupel, + pt=dycore_state.pt, + ua=dycore_state.ua, + va=dycore_state.va, + u=dycore_state.u, + v=dycore_state.v, + ) diff --git a/tests/savepoint/thresholds/driver.yaml b/tests/savepoint/thresholds/driver.yaml new file mode 100644 index 000000000..a51181c46 --- /dev/null +++ b/tests/savepoint/thresholds/driver.yaml @@ -0,0 +1,1056 @@ +savepoints: + AtmosPhysDriverStatein-In: + - pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + - pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + AtmosPhysDriverStatein-Out: + - IPD_tgrs: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + - IPD_tgrs: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + C_SW-In: + - delpd: + absolute: 2.7284841053187847e-12 + relative: 2.6632276486803376e-15 + divgdd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.4024873582185304e-14 + uad: + absolute: 0.0 + relative: 0.0 + ucd: + absolute: 0.0 + relative: 0.0 + ud: + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 + utd: + absolute: 0.0 + relative: 0.0 + vad: + absolute: 0.0 + relative: 0.0 + vcd: + absolute: 0.0 + relative: 0.0 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + vtd: + absolute: 0.0 + relative: 0.0 + wd: + absolute: 0.0 + relative: 0.0 + - delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + divgdd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 384.0 + relative: 1.924686893998195e-09 + ucd: + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 + ud: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + utd: + absolute: 0.0 + relative: 0.0 + vad: + absolute: 384.0 + relative: 1.10022939861047e-07 + vcd: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + vd: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + vtd: + absolute: 0.0 + relative: 0.0 + wd: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + C_SW-Out: + - delpd: + absolute: 2.7284841053187847e-12 + relative: 2.6632276486803376e-15 + divgdd: + absolute: 4.942437247228842e-19 + relative: 5.152618796428155e-08 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.4024873582185304e-14 + uad: + absolute: 384.0 + relative: 2.8634008989163294e-11 + ucd: + absolute: 3.410605131648481e-13 + relative: 2.813991384236331e-10 + ud: + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 + utd: + absolute: 2.86102294921875e-05 + relative: 2.26433843291111e-10 + vad: + absolute: 384.0 + relative: 8.422447914909142e-11 + vcd: + absolute: 3.410605131648481e-13 + relative: 1.51267761412336e-10 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + vtd: + absolute: 2.86102294921875e-05 + relative: 1.5363504124939987e-10 + wd: + absolute: 0.0 + relative: 0.0 + - delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + divgdd: + absolute: 5.409006817378153e-18 + relative: 3.7456372136851146e-07 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 9.698908343125368e-13 + relative: 4.31030506277985e-08 + ud: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + utd: + absolute: 7.152557373046875e-05 + relative: 3.916045067374942e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 + vcd: + absolute: 9.592326932761353e-13 + relative: 1.535682918794925e-07 + vd: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + vtd: + absolute: 7.152557373046875e-05 + relative: 4.161008969260313e-08 + wd: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + D_SW-In: + - delpcd: + absolute: 2.86102294921875e-05 + relative: 1.5363504124939987e-10 + delpd: + absolute: 2.7284841053187847e-12 + relative: 2.6632276486803376e-15 + mfxd: + absolute: 0.0 + relative: 0.0 + mfyd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.4024873582185304e-14 + uad: + absolute: 384.0 + relative: 2.8634008989163294e-11 + ucd: + absolute: 6.288303211476887e-13 + relative: 2.008118044889811e-08 + ud: + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 + vad: + absolute: 384.0 + relative: 8.422447914909142e-11 + vcd: + absolute: 6.288303211476887e-13 + relative: 1.535682918794925e-07 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + wd: + absolute: 0.0 + relative: 0.0 + xfxd: + absolute: 0.0 + relative: 0.0 + yfxd: + absolute: 0.0 + relative: 0.0 + - delpcd: + absolute: 7.152557373046875e-05 + relative: 4.161008969260313e-08 + delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + mfxd: + absolute: 0.0 + relative: 0.0 + mfyd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 1.262989712813578e-12 + relative: 2.8125989221256106e-08 + ud: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 + vcd: + absolute: 1.262989712813578e-12 + relative: 1.0487218599584194e-07 + vd: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + wd: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + xfxd: + absolute: 0.0 + relative: 0.0 + yfxd: + absolute: 0.0 + relative: 0.0 + D_SW-Out: + - delpcd: + absolute: 2.86102294921875e-05 + relative: 5.152618796428155e-08 + delpd: + absolute: 5.4569682106375694e-12 + relative: 5.320929041303097e-15 + divgdd: + absolute: 4.942437247228842e-19 + relative: 5.152618796428155e-08 + mfxd: + absolute: 1604124672.0 + relative: 120.0 + mfyd: + absolute: 1621082112.0 + relative: 120.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.4024873582185304e-14 + uad: + absolute: 384.0 + relative: 2.8634008989163294e-11 + ucd: + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 + ud: + absolute: 1.341104507446289e-07 + relative: 1.6737607801318062e-10 + vad: + absolute: 384.0 + relative: 8.422447914909142e-11 + vcd: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + vd: + absolute: 1.7881393432617188e-07 + relative: 7.469099658749364e-11 + wd: + absolute: 0.0 + relative: 0.0 + xfxd: + absolute: 0.00010514259338378906 + relative: 2.0081181638579702e-08 + yfxd: + absolute: 0.00010532140731811523 + relative: 1.5356829033557975e-07 + - delpcd: + absolute: 6.651878356933594e-05 + relative: 3.7456372136851146e-07 + delpd: + absolute: 2.6193447411060333e-10 + relative: 5.022709085608076e-13 + divgdd: + absolute: 5.409006817378153e-18 + relative: 2.3153394949370874e-06 + mfxd: + absolute: 64332103680.0 + relative: 120.0 + mfyd: + absolute: 62617780224.0 + relative: 120.0 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 1.262989712813578e-12 + relative: 1.2492728689511802e-05 + ud: + absolute: 7.264316082000732e-07 + relative: 8.742370556648009e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 + vcd: + absolute: 1.262989712813578e-12 + relative: 3.6260027791287995e-06 + vd: + absolute: 6.370246410369873e-07 + relative: 1.0448120616784678e-08 + wd: + absolute: 1.859949694260621e-12 + relative: 3.944770141167069e-05 + xfxd: + absolute: 0.00018596649169921875 + relative: 2.8125989468113277e-08 + yfxd: + absolute: 0.00018525123596191406 + relative: 1.0487218572228159e-07 + FVDynamics-In: + - delz: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 0.0 + relative: 0.0 + u: + absolute: 0.0 + relative: 0.0 + ua: + absolute: 0.0 + relative: 0.0 + uc: + absolute: 0.0 + relative: 0.0 + v: + absolute: 0.0 + relative: 0.0 + va: + absolute: 0.0 + relative: 0.0 + vc: + absolute: 0.0 + relative: 0.0 + w: + absolute: 0.0 + relative: 0.0 + - delz: + absolute: 25769803776.0 + relative: 4.425706291051527e-12 + qvapor: + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 384.0 + relative: 1.924686893998195e-09 + uc: + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 + v: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + va: + absolute: 384.0 + relative: 1.10022939861047e-07 + vc: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + FVDynamics-Out: + - delz: + absolute: 25769803776.0 + relative: 4.425706291051527e-12 + qvapor: + absolute: 3.289035710452026e-14 + relative: 7.505716682286047e-11 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 384.0 + relative: 1.924686893998194e-09 + uc: + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 + v: + absolute: 1.0444978215673473e-12 + relative: 7.151036709605271e-08 + va: + absolute: 384.0 + relative: 1.1002293986118271e-07 + vc: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + qvapor: + absolute: 3.778227730677486e-14 + relative: 2.8773515355509475e-11 + u: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + uc: + absolute: 1.262989712813578e-12 + relative: 1.2492728689511802e-05 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + va: + absolute: 3093.0 + relative: 1.001009536835503e-07 + vc: + absolute: 1.262989712813578e-12 + relative: 3.6260027791287995e-06 + w: + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 + FVUpdatePhys-In: + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + pe: + absolute: 5.238689482212067e-10 + relative: 7.0048868034443836e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2542546489769687e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2545561792452565e-14 + ps: + absolute: .nan + relative: 5.243476108433941e-15 + pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.4986438636356084e-15 + relative: 138.0 + qliquid: + absolute: 3.3320351686128014e-14 + relative: 6.967637350538825e-09 + qrain: + absolute: 2.8374926106661258e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 + t_dt: + absolute: 3.637979403402908e-14 + relative: 1.4767081826878412e-08 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + ua: + absolute: 384.0 + relative: 1.924686893998194e-09 + v: + absolute: 1.0444978215673473e-12 + relative: 7.151036709605271e-08 + v_dt: + absolute: 2.368475785867001e-17 + relative: 120.0 + va: + absolute: 384.0 + relative: 1.1002293986118271e-07 + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + pe: + absolute: 6.984919309616089e-10 + relative: 8.74747886746669e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2749142151097106e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2798456145608727e-14 + ps: + absolute: .nan + relative: 6.990040335621623e-15 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.484916212417695e-15 + relative: .inf + qliquid: + absolute: 3.800939055685626e-14 + relative: 1.6059488926374826e-07 + qrain: + absolute: 2.8293610943724845e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.778227730677486e-14 + relative: 6.869188271838486e-12 + t_dt: + absolute: 1.5158244126046994e-14 + relative: 3.2931956762020915e-08 + u: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + v_dt: + absolute: 2.3684758432181898e-17 + relative: 120.0 + va: + absolute: 3093.0 + relative: 1.001009536835503e-07 + FVUpdatePhys-Out: + - pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.4986438636356084e-15 + relative: 138.0 + qliquid: + absolute: 3.3320351686128014e-14 + relative: 6.967637350538825e-09 + qrain: + absolute: 2.8374926106661258e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 384.0 + relative: 1.924686893998195e-09 + v: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + va: + absolute: 384.0 + relative: 1.10022939861047e-07 + - pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.484916212417695e-15 + relative: .inf + qliquid: + absolute: 3.800939055685626e-14 + relative: 1.6059488926374826e-07 + qrain: + absolute: 2.8293610943724845e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.778227730677486e-14 + relative: 6.869188271838486e-12 + u: + absolute: 1.4788170688007085e-12 + relative: 3.846326345494972e-08 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + v: + absolute: 1.4788170688007085e-12 + relative: 3.846326345494972e-08 + va: + absolute: 3093.0 + relative: 1.0012980122080339e-07 + GFSPhysicsDriver-In: + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + delz: + absolute: 25769803776.0 + relative: 4.425706291051527e-12 + omga: + absolute: 9.321803745576673e-12 + relative: 0.0001252012141681156 + pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + qcld: + absolute: 8.192313494248538e-11 + relative: 7.636900744509854e-07 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 1.1277840081491642e-14 + relative: 120.0 + qliquid: + absolute: 3.284547113457936e-14 + relative: 6.485006027709165e-09 + qo3mr: + absolute: 0.0 + relative: 0.0 + qrain: + absolute: 7.623296525288703e-21 + relative: 1.9715033202384307e-05 + qsgs_tke: + absolute: 0.0 + relative: 0.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.289035710452026e-14 + relative: 7.505716682286047e-11 + ua: + absolute: 384.0 + relative: 1.924686893998194e-09 + va: + absolute: 384.0 + relative: 1.1002293986118271e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + omga: + absolute: 1.762632487883886e-11 + relative: 0.00013614641074803746 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qcld: + absolute: 8.595690825785596e-11 + relative: 3.228782851361542e-07 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.484916212417695e-15 + relative: .inf + qliquid: + absolute: 3.7826553412993735e-14 + relative: 1.7425773968564856e-07 + qo3mr: + absolute: 0.0 + relative: 0.0 + qrain: + absolute: 2.8293610943724845e-15 + relative: 120.0 + qsgs_tke: + absolute: 0.0 + relative: 0.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.778227730677486e-14 + relative: 2.8773515355509475e-11 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + va: + absolute: 3093.0 + relative: 1.001009536835503e-07 + w: + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 + PhysUpdateTracers-In: + - pt_t0: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + pt_t1: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + t_dt: + absolute: 0.0 + relative: 0.0 + u_dt: + absolute: 0.0 + relative: 0.0 + v_dt: + absolute: 0.0 + relative: 0.0 + - pt_t0: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + pt_t1: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + t_dt: + absolute: 0.0 + relative: 0.0 + u_dt: + absolute: 0.0 + relative: 0.0 + v_dt: + absolute: 0.0 + relative: 0.0 + PhysUpdateTracers-Out: + - t_dt: + absolute: 3.637979403402908e-14 + relative: 1.4767081826878412e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + v_dt: + absolute: 2.368475785867001e-17 + relative: 120.0 + - t_dt: + absolute: 1.5158244126046994e-14 + relative: 3.2931956762020915e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + v_dt: + absolute: 2.3684758432181898e-17 + relative: 120.0 + Remapping-In: + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 1.3322676295501878e-15 + relative: 4.684744757427358e-15 + delp: + absolute: 5.4569682106375694e-12 + relative: 5.320929041303097e-15 + delz: + absolute: 25769803776.0 + relative: 4.424309879776438e-12 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.981060438552416e-15 + omga: + absolute: 0.0 + relative: 0.0 + pe: + absolute: 5.238689482212067e-10 + relative: 8.133300801432759e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.6420226645538662e-15 + pfull: + absolute: 0.0 + relative: 0.0 + phis: + absolute: 0.0 + relative: 0.0 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2798830510581378e-14 + pkz: + absolute: 4.7279513637477066e-11 + relative: 1.7652850589111428e-12 + ps: + absolute: .nan + relative: 1.7503446900789073e-15 + pt: + absolute: 3.410605131648481e-13 + relative: 2.4024873582185304e-14 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.0444978215673473e-12 + relative: 4.920683557677236e-08 + ua: + absolute: 384.0 + relative: 2.8634008989163294e-11 + v: + absolute: 1.0125233984581428e-12 + relative: 5.378610226381813e-08 + va: + absolute: 384.0 + relative: 8.422447914909142e-11 + w: + absolute: 1.859949694260621e-12 + relative: 5.4023923350536e-05 + wsd: + absolute: 0.0 + relative: 0.0 + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 3.064215547965432e-14 + relative: 1.0751531623036565e-13 + delp: + absolute: 2.6193447411060333e-10 + relative: 5.022709085608076e-13 + delz: + absolute: 25769803776.0 + relative: 4.4284261446010145e-12 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 + omga: + absolute: 9.321803745576673e-12 + relative: 6.434410522391023e-05 + pe: + absolute: 6.984919309616089e-10 + relative: 1.0616077611929113e-14 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.6420212048418394e-15 + pfull: + absolute: 0.0 + relative: 0.0 + phis: + absolute: 0.0 + relative: 0.0 + pk: + absolute: 2.984279490192421e-13 + relative: 1.3065571699261469e-14 + pkz: + absolute: 4.7407411329913884e-11 + relative: 1.7669600579817248e-12 + ps: + absolute: .nan + relative: 5.243476108433941e-15 + pt: + absolute: 6.480149750132114e-12 + relative: 5.904546306310534e-13 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.3962164757685969e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 3093.0 + relative: 9.980260518341024e-09 + v: + absolute: 1.4788170688007085e-12 + relative: 3.755502628095818e-08 + va: + absolute: 3093.0 + relative: 9.98355958190459e-09 + w: + absolute: 2.369833496107532e-12 + relative: 0.00013648630486358366 + wsd: + absolute: 0.0 + relative: 0.0 + Remapping-Out: + - cappa: + absolute: 2.9976021664879227e-14 + relative: 1.0517782687951213e-13 + delp: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + delz: + absolute: 25769803776.0 + relative: 4.425706291051527e-12 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.981060438552416e-15 + omga: + absolute: 0.0 + relative: 0.0 + pe: + absolute: 5.238689482212067e-10 + relative: 7.0048868034443836e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2542546489769687e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2545561792452565e-14 + pkz: + absolute: 4.732214620162267e-11 + relative: 1.76983385674747e-12 + pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + v: + absolute: 1.0231815394945443e-12 + relative: 7.151036709605271e-08 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - cappa: + absolute: 3.4638958368304884e-14 + relative: 1.213988092423256e-13 + delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 + omga: + absolute: 9.321803745576673e-12 + relative: 6.434410522391023e-05 + pe: + absolute: 6.984919309616089e-10 + relative: 8.74747886746669e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2749142151097106e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2798456145608727e-14 + pkz: + absolute: 4.736477876576828e-11 + relative: 1.7694778604229885e-12 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.4175327578413999e-12 + relative: 4.7436825986745986e-08 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + w: + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 + Tracer2D1L-In: + - cxd: + absolute: 2.2638141361497333e-16 + relative: 2.0081178953558838e-08 + cyd: + absolute: 2.2768245622195593e-16 + relative: 1.53568292783154e-07 + dp1: + absolute: 2.7284841053187847e-12 + relative: 2.6632276486803376e-15 + mfxd: + absolute: 1604124672.0 + relative: 120.0 + mfyd: + absolute: 1621082112.0 + relative: 120.0 + - cxd: + absolute: 4.683753385137379e-16 + relative: 2.8125989101612067e-08 + cyd: + absolute: 4.501607420159814e-16 + relative: 1.0487218492835087e-07 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + mfxd: + absolute: 64332103680.0 + relative: 120.0 + mfyd: + absolute: 62617780224.0 + relative: 120.0 + Tracer2D1L-Out: + - cxd: + absolute: 7.546047120499111e-17 + relative: 2.0081177721573792e-08 + cyd: + absolute: 7.611099250848241e-17 + relative: 1.535682939641504e-07 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.981060438552416e-15 + mfxd: + absolute: 534724608.0 + relative: 120.0 + mfyd: + absolute: 540352512.0 + relative: 120.0 + - cxd: + absolute: 1.5612511283791264e-16 + relative: 2.812598910161206e-08 + cyd: + absolute: 1.496198998029996e-16 + relative: 1.0487218492835087e-07 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 + mfxd: + absolute: 21444034560.0 + relative: 120.0 + mfyd: + absolute: 20872593408.0 + relative: 120.0 diff --git a/util/pace/util/testing/perturbation.py b/util/pace/util/testing/perturbation.py index 25e423028..e28a5f0f9 100644 --- a/util/pace/util/testing/perturbation.py +++ b/util/pace/util/testing/perturbation.py @@ -10,10 +10,20 @@ def perturb(input: Mapping[str, np.ndarray]): Will only make changes to float64 or float32 arrays. """ roundoff = 1e-16 - for data in input.values(): + for name, data in input.items(): if isinstance(data, np.ndarray) and data.dtype in (np.float64, np.float32): - not_fill_value = data < 1e30 + valid_points = data < 1e30 + if name in [ + "q_con", + "qcld", + "qice", + "qliquid", + "qrain", + "qsnow", + "qgraupel", + ]: + valid_points = np.where((data < 1e30) & (data > 0)) # multiply data by roundoff-level error - data[not_fill_value] *= 1.0 + np.random.uniform( - low=-roundoff, high=roundoff, size=data[not_fill_value].shape + data[valid_points] *= 1.0 + np.random.uniform( + low=-roundoff, high=roundoff, size=data[valid_points].shape )