more cleanup

Author: jedwards4b

cmip7_prep/cmor_utils.py

@@ -57,6 +57,8 @@ def filled_for_cmor(
     # keep attrs helpful for downstream
     da2.attrs["_FillValue"] = f
     da2.attrs["missing_value"] = f
+    if da2.dtype != np.float32:
+        da2 = da2.astype(np.float32)
     return da2, f
 
 

cmip7_prep/cmor_writer.py

@@ -292,13 +292,15 @@ def _get_1d_with_bounds(dsi: xr.Dataset, name: str, units_default: str):
         lon_id = None
         sdepth_id = None
 
-        print(f"[CMOR axis debug] var_dims: {var_dims}")
+        logger.debug("[CMOR axis debug] var_dims: %s", var_dims)
         if "xh" in var_dims and "yh" in var_dims:
             # MOM6/curvilinear grid: register xh/yh as generic axes (i/j), not as lat/lon
             # Define the native grid using the coordinate arrays
-            print(
-                f"[CMOR axis debug] Defining unstructured grid for variable {var_name}."
+            logger.debug(
+                "[CMOR axis debug] Defining unstructured grid for variable %s.",
+                var_name,
             )
+
             i_id = cmor.axis(
                 table_entry="i",
                 units="1",
@@ -309,7 +311,7 @@ def _get_1d_with_bounds(dsi: xr.Dataset, name: str, units_default: str):
                 units="1",
                 length=ds["yh"].size,
             )
-            print("[CMOR axis debug] Defining unstructured grid_id.")
+            logger.debug("[CMOR axis debug] Defining unstructured grid_id.")
             grid_id = cmor.grid(
                 axis_ids=[j_id, i_id],  # note CMOR wants fastest varying last
                 longitude=ds["geolon"].values,
@@ -331,9 +333,9 @@ def _get_1d_with_bounds(dsi: xr.Dataset, name: str, units_default: str):
                         f" in variable '{var_name}' (curvilinear)"
                     )
                 axes_ids.append(axis_id)
-            print(f"[CMOR axis debug] Appending grid_id: {grid_id}")
+            logger.debug("[CMOR axis debug] Appending grid_id: %s", grid_id)
             axes_ids.append(grid_id)
-            print(f"[CMOR axis debug] axes_ids: {axes_ids}")
+            logger.debug("[CMOR axis debug] axes_ids: %s", axes_ids)
             return axes_ids
 
         # --- horizontal axes (use CMOR names) ----
@@ -482,6 +484,8 @@ def _get_1d_with_bounds(dsi: xr.Dataset, name: str, units_default: str):
             values = ds["sdepth"].values
             logger.info("write sdepth axis")
             bnds = bounds_from_centers_1d(values, "sdepth")
+            if bnds[0, 0] < 0:
+                bnds[0, 0] = 0.0  # no negative soil depth bounds
             sdepth_id = cmor.axis(
                 table_entry="sdepth",
                 units="m",
@@ -641,8 +645,9 @@ def write_variable(
         cmor.load_table(table_filename)
 
         data = ds[vdef.name]
+        logger.info("Prepare data for CMOR %s", data.dtype)  # debug
         data_filled, fillv = filled_for_cmor(data)
-        logger.info("Define axes")
+        logger.info("Define axes data_filled dtype: %s", data_filled.dtype)  # debug
         axes_ids = self._define_axes(ds, vdef)
         units = getattr(vdef, "units", "") or ""
         # Debug logging for axis mapping
@@ -651,7 +656,7 @@ def write_variable(
         try:
             for i, aid in enumerate(axes_ids):
                 entry = cmor.axis_entry(aid) if hasattr(cmor, "axis_entry") else None
-                logger.info(
+                logger.debug(
                     "[CMOR DEBUG] axis %d: id=%s, table_entry=%s", i, aid, entry
                 )
         # pylint: disable=broad-exception-caught
@@ -664,7 +669,6 @@ def write_variable(
             positive=getattr(vdef, "positive", None),
             missing_value=fillv,
         )
-        data = ds[varname]
 
         # ---- Prepare time info for this write (local, not cached) ----
         time_da = ds.coords.get("time")

cmip7_prep/mapping_compat.py

@@ -217,7 +217,7 @@ def realize(self, ds: xr.Dataset, cmip_name: str) -> xr.DataArray:
         vc = self._vars[cmip_name]
 
         da = _realize_core(ds, vc)
-        print(f"da is {da}")
+
         if da is not None:
             if vc.unit_conversion is not None:
                 da = _apply_unit_conversion(da, vc.unit_conversion)

cmip7_prep/pipeline.py

@@ -278,7 +278,18 @@ def realize_regrid_prepare(
         names_to_regrid.append("PS")
 
     # 7) Rename levgrnd if present to sdepth
-    if "levgrnd" in ds_native.dims and "levgrnd" in ds_vert.coords:
+
+    # Check if 'levgrnd' is a dimension of any variable in names_to_regrid
+    needs_levgrnd_rename = any(
+        (v in ds_vert and "levgrnd" in getattr(ds_vert[v], "dims", []))
+        for v in names_to_regrid
+    )
+    if (
+        needs_levgrnd_rename
+        and "levgrnd" in ds_native.dims
+        and "levgrnd" in ds_native.coords
+    ):
+        logger.info("Renaming 'levgrnd' dimension to 'sdepth'")
         ds_vert = ds_vert.rename_dims({"levgrnd": "sdepth"})
         # Ensure the coordinate variable is also copied
         ds_vert = ds_vert.assign_coords(sdepth=ds_native["levgrnd"].values)

cmip7_prep/regrid.py

@@ -498,14 +498,13 @@ def regrid_to_1deg(
                 *non_spatial, "lat", "lon"
             )  # ensure last two dims are ('lat','lon')
         )
-        logger.info("here %s, %s", hdim, da2_2d.dims)
     else:
         da2_2d = da2.rename({"xh": "lon", "yh": "lat"}).transpose(
             *non_spatial, "lat", "lon"
         )
 
         da2_2d = da2_2d.assign_coords(lon=((da2_2d.lon % 360)))
-    logger.info(
+    logger.debug(
         "da2_2d range: %f to %f lat, %f to %f lon",
         da2_2d["lat"].min().item(),
         da2_2d["lat"].max().item(),
@@ -666,7 +665,6 @@ def _build_fx_native(ds_native: xr.Dataset) -> xr.Dataset:
         pieces["sftlf"] = sftlf
     # Also extract sftof (sea fraction) if present
     sftof = None
-    print("ds_native contains:", list(ds_native.data_vars))
     for name in ["sftof", "ocnfrac", "wet"]:
         if name in ds_native:
             logger.info("Extracting sftof from native variable %s", name)

scripts/fullLmon.sh

@@ -9,5 +9,5 @@
 module load conda
 conda activate CMORDEV
 NCPUS=$(cat $PBS_NODEFILE | wc -l)
-poetry run python ./scripts/monthly_cmor.py --realm lnd --test --workers $NCPUS --skip-timeseries --cmip-vars tsl
+poetry run python ./scripts/monthly_cmor.py --realm lnd --test --workers $NCPUS --skip-timeseries --cmip-vars cProduct
 #/glade/u/home/cmip7/cases/b.e30_beta06.B1850C_LTso.ne30_t232_wgx3.192.wrkflw.1 /glade/work/hannay/cesm_tags/cesm3_0_beta06/cime

scripts/monthly_cmor.py

@@ -182,11 +182,15 @@ def process_one_var(
                 parallel=True,
                 open_kwargs=open_kwargs,
             )
+
             # Append ocn_fx_fields to ds_native if available
             if realm == "ocn" and ocn_fx_fields is not None:
                 ds_native = ds_native.merge(ocn_fx_fields)
-            logger.info(
-                f"ds_native keys: {list(ds_native.keys())} for var {varname} with dims {dims}"
+            logger.debug(
+                "ds_native keys: %s for var %s with dims %s",
+                list(ds_native.keys()),
+                varname,
+                dims,
             )
             if var is None:
                 logger.warning(f"Source variable(s) not found for {varname}")
@@ -215,8 +219,9 @@ def _extract_array(val):
                     )
             else:
                 # For lnd/atm or any other dims, use existing logic
-                logger.info(f"Processing {varname} for dims {dims} (atm/lnd or other)")
-                sftlf_path = next(Path(outdir).rglob("sftlf_fx_*.nc"), None)
+                logger.debug(
+                    "Processing %s for dims %s (atm/lnd or other)", varname, dims
+                )
                 ds_cmor = realize_regrid_prepare(
                     mapping,
                     ds_native,
@@ -232,12 +237,14 @@ def _extract_array(val):
                 )
         except Exception as e:
             logger.error(
-                f"Exception during regridding of {varname} with dims {dims}: {e!r}"
+                "Exception during regridding of %s with dims %s: %r",
+                varname,
+                dims,
+                e,
             )
             continue
         try:
             # CMORize
-            logger.info(f"CMOR writing for {varname} with dims {dims}")
             log_dir = outdir + "/logs"
             with CmorSession(
                 tables_path=tables_path,
@@ -262,7 +269,9 @@ def _extract_array(val):
                         "levels": cfg.get("levels", None),
                     },
                 )()
-                logger.info(f"Writing variable {varname} with dims {dims}")
+                logger.info(
+                    f"Writing variable {varname} with dims {dims} and type {ds_cmor[varname].dtype}"
+                )
                 cm.write_variable(ds_cmor, varname, vdef)
             logger.info(f"Finished processing for {varname} with dims {dims}")
             results.append((varname, "ok"))
@@ -428,7 +437,7 @@ def main():
             sys.exit(0)
         hf_collection = HFCollection(input_head_dir, dask_client=client)
         for include_pattern in include_patterns:
-            logger.info(f"Processing files with pattern: {include_pattern}")
+            logger.info("Processing files with pattern: %s", include_pattern)
             hfp_collection = hf_collection.include_patterns([include_pattern])
             hfp_collection.pull_metadata()
             ts_collection = TSCollection(
@@ -491,13 +500,26 @@ def main():
             results = []
             for _, result in as_completed(futures, with_results=True):
                 try:
-                    results.append(result)  # (v, status)
+                    # Handle result types: list of tuples, tuple, or other
+                    if isinstance(result, list):
+                        # If it's a list, check if it's a list of tuples
+                        if all(isinstance(x, tuple) and len(x) == 2 for x in result):
+                            results.extend(result)
+                        else:
+                            # Not a list of tuples, wrap as unknown
+                            results.append((str(result), "unknown"))
+                    elif isinstance(result, tuple) and len(result) == 2:
+                        results.append(result)
+                    else:
+                        # Not a tuple/list, wrap as unknown
+                        results.append((str(result), "unknown"))
                 except Exception as e:
                     logger.error("Task error:", e)
                     raise
 
         for v, status in results:
             logger.info(f"Variable {v} processed with status: {status}")
+
     else:
         logger.info("No results to process.")
     if client: