Add --on-var-failure flag to handle failure behaviors

docs/source/usage.rst

@@ -71,6 +71,10 @@ Please be aware that some arguments are required or optional based on how ``e3sm
                            (WARNING: NOT WORKING AS OF 1.8.2)
    -s, --serial          Run in serial mode (by default parallel). Useful for
                            debugging purposes.
+   --on-var-failure {ignore,fail,stop} Behavior when a variable fails:
+                           ignore - continue and exit 0 (default)
+                           fail   - process all variables, exit 1 if any failed
+                           stop   - exit immediately on first failure, useful for debugging
    optional arguments (run settings):
    --realm <realm>       The realm to process. Must be atm, lnd, mpaso or
                            mpassi. Default is atm.
@@ -162,8 +166,11 @@ when the output is intended for analysis, but is not suited for publication.
 
 Serial
 ^^^^^^
-For debugging purposes, or when running in a resource constrained environment, the "--serial" or "-s" boolean flag can be used to cause the conversion process
-to be run in serial, using the main process.
+For debugging purposes, or when running in a resource constrained environment, the "--serial" or "-s" boolean flag can be used to cause the conversion process to be run in serial, using the main process.
+
+On-var-failure
+^^^^^^^^^^^^^^^
+This optional flag controls the behavior of the tool when a variable fails to process. The default behavior is to ignore the failure and continue processing the remaining variables, exiting with a return code of 0. The "fail" option will cause the tool to continue processing all variables, but exit with a return code of 1 if any variable failed. The "stop" option will cause the tool to exit immediately on the first variable failure, which is useful for debugging.
 
 Optional arguments (run settings)
 ---------------------------------

e3sm_to_cmip/argparser.py

@@ -11,6 +11,7 @@ def setup_argparser() -> argparse.ArgumentParser:
         prog="e3sm_to_cmip",
         usage="%(prog)s [-h]",
         add_help=False,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
 
     # Argument groups to organize the numerous arguments printed by --help.
@@ -38,19 +39,16 @@ def setup_argparser() -> argparse.ArgumentParser:
         "--info",
         action="store_true",
         help=(
-            "Produce information about the CMIP6 variables passed in the --var-list "
-            "argument and exit without doing any processing. There are three modes "
-            "for getting the info. (Mode 1) If you just pass the --info flag with the "
-            "--var-list then it will print out the handler information as yaml data for "
-            "the requested variable to your default output path (or to a file designated "
-            "by the --info-out path). (Mode 2) If the --freq <frequency> is passed "
-            "along with the --tables-path, then the variable handler information will "
-            "only be output if the requested variables are present in the CMIP6 table matching the freq. "
-            "NOTE: For MPAS data, one must also include --realm mpaso (or mpassi) and --map no_map. "
-            "(Mode 3) For non-MPAS data, if the --freq <freq> is passed with the --tables-path, and the "
-            "--input-path, and the input-path points to raw unprocessed E3SM files, "
-            "then an additional check will me made for if the required raw "
-            "variables are present in the E3SM native output. "
+            "Produce information about the CMIP6 variables passed in the "
+            "--var-list argument and exit without processing. Modes:\n"
+            "  1) Default: Print handler info as YAML for requested variables "
+            "to the default output path, or to a file specified by --info-out.\n"
+            "  2) With --freq <frequency> and --tables-path: Output handler info "
+            "only if variables are in the CMIP6 table matching the frequency. "
+            "For MPAS data, include --realm mpaso/mpassi and --map no_map.\n"
+            "  3) For non-MPAS data: With --freq <frequency>, --tables-path, "
+            "and --input-path pointing to raw E3SM files, check if required raw "
+            "variables are present in the E3SM native output."
         ),
     )
     optional_mode.add_argument(
@@ -68,6 +66,18 @@ def setup_argparser() -> argparse.ArgumentParser:
         action="store_true",
     )
 
+    optional_mode.add_argument(
+        "--on-var-failure",
+        choices=["ignore", "fail", "stop"],
+        default="ignore",
+        help=(
+            "Behavior when a variable fails:\n"
+            "  1) 'ignore' - continue and exit 0 (default)\n"
+            "  2) 'fail'   - process all variables, exit 1 if any failed\n"
+            "  3) 'stop'   - exit immediately on first failure, useful for debugging\n"
+        ),
+    )
+
     # ======================================================================
     # Run settings.
     # ======================================================================

e3sm_to_cmip/cmor_handlers/utils.py

@@ -33,7 +33,7 @@
 
 def load_all_handlers(
     realm: Realm | MPASRealm, cmip_vars: list[str]
-) -> list[dict[str, Any]]:
+) -> tuple[list[dict[str, Any]], list[str]]:
     """Loads variable handlers based on a list of variable names.
 
     This function is used specifically for printing out the handler information
@@ -48,13 +48,9 @@ def load_all_handlers(
 
     Returns
     -------
-    list[dict[str, Any]]:
-        A list of the dictionary representation of VarHandler objects.
-
-    Raises
-    ------
-    KeyError
-        If no handlers are defined for a CMIP6 variable in `handlers.yaml`.
+    tuple[list[dict[str, Any]], list[str]]:
+        A list of the dictionary representation of VarHandler objects
+        and a list of variable names that are missing handlers if any.
     """
     handlers_by_var: dict[str, list[dict[str, Any]]] = _get_handlers_by_var()
 
@@ -72,19 +68,13 @@ def load_all_handlers(
 
             handlers = handlers + var_handler
 
-        if len(missing_handlers) > 0:
-            logger.warning(
-                f"No handlers are defined for the variables: {missing_handlers}. "
-                "Make sure at least one variable handler is defined for each of these "
-                f"variables in `{HANDLER_DEFINITIONS_PATH}`."
-            )
     else:
         handlers = _get_mpas_handlers(cmip_vars)
 
-    return handlers
+    return handlers, missing_handlers
 
 
-def _get_mpas_handlers(cmip_vars: list[str]):
+def _get_mpas_handlers(cmip_vars: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
     """Get MPAS variable handlers using the list of CMIP variables.
 
     All current MPAS variable handlers are defined as modules and there is only
@@ -97,8 +87,9 @@ def _get_mpas_handlers(cmip_vars: list[str]):
 
     Returns
     -------
-    KeyError
-        If no handlers are defined for the MPAS CMIP6 variable.
+    tuple[list[dict[str, Any]], list[str]]:
+        A list of the dictionary representation of VarHandler objects and
+        a list of variable names that are missing handlers if any.
     """
     handlers = _get_handlers_from_modules(MPAS_HANDLER_DIR_PATH)
 
@@ -121,7 +112,7 @@ def _get_mpas_handlers(cmip_vars: list[str]):
             f"`{MPAS_HANDLER_DIR_PATH}`."
         )
 
-    return derived_handlers
+    return derived_handlers, missing_handlers
 
 
 def derive_handlers(
@@ -130,7 +121,7 @@ def derive_handlers(
     e3sm_vars: list[str],
     freq: Frequency,
     realm: Realm | MPASRealm,
-) -> list[dict[str, Any]]:
+) -> tuple[list[dict[str, Any]], list[str], list[str]]:
     """Derives the appropriate handler for each CMIP variable.
 
     For each CMIP variable the user wants to CMORize (`cmip_vars`), a variable
@@ -162,16 +153,12 @@ def derive_handlers(
 
     Returns
     -------
-    list[dict[str, Any]]:
-        A list of the dictionary representation of VarHandler objects.
+    tuple[list[dict[str, Any]], list[str]], list[str]]:
+        A list of the dictionary representation of VarHandler objects, a list of
+        variable names that are missing handlers (if any), and a list of
+        variable names that could not be derived using the input E3SM variables
+        if (any).
 
-    Raises
-    ------
-    KeyError
-        If no handlers are defined for a CMIP6 variable in `handlers.yaml`.
-    KeyError
-        If a handler could not be derived for a CMIP6 variable using the existing
-        E3SM variables.
     """
     # TODO: Refactor the function parameters.
     handlers_by_var: dict[str, list[dict[str, Any]]] = _get_handlers_by_var()
@@ -180,7 +167,7 @@ def derive_handlers(
     # Stores variable names that are missing handlers or the handler cannot
     # be derived using the input E3SM variables.
     missing_handlers: list[str] = []
-    cannot_derive: list[str] = []
+    non_derivable_handlers: list[str] = []
 
     for var in cmip_vars:
         var_handlers = handlers_by_var.get(var)
@@ -197,30 +184,17 @@ def derive_handlers(
             var_handlers, freq, realm, cmip_tables_path, e3sm_vars
         )
 
-        # If no handler could be derived, add it to the cannot_derive list.
-        # This can happen if the handler has no matching CMIP table for the
-        # requested frequency, or if the handler's raw E3SM variables do not
-        # match the input E3SM variables.
+        # If no handler is defined variable itself could be derived, add it to
+        # the cannot_derive list. This can happen if the handler has no matching
+        # CMIP table for the requested frequency, or if the handler's raw E3SM
+        # variables do not match the input E3SM variables.
         if derived_handler is None:
-            cannot_derive.append(var)
+            non_derivable_handlers.append(var)
             continue
 
         derived_handlers.append(derived_handler)
 
-    if len(missing_handlers) > 0:
-        logger.warning(
-            f"No handlers are defined for the variables: {missing_handlers}. "
-            "Make sure handlers are defined for these variables in `handlers.yaml`."
-        )
-
-    if len(cannot_derive) > 0:
-        logger.warning(
-            f"No handlers could be derived for the variables: {cannot_derive}. "
-            "Make sure the input E3SM datasets have the variables needed for "
-            "derivation."
-        )
-
-    return derived_handlers
+    return derived_handlers, missing_handlers, non_derivable_handlers
 
 
 def _get_handlers_by_var() -> dict[str, list[dict[str, Any]]]: