Add log_and_raise helper to merge fre_logger.error and raise steps

fre/__init__.py

@@ -4,6 +4,7 @@
 
 import logging
 import os
+from typing import NoReturn
 version = os.getenv("GIT_DESCRIBE_TAG", "2026.01.alpha2")
 __version__ = version
 
@@ -14,3 +15,49 @@
                     format = FORMAT,
                     filename = None,
                     encoding = 'utf-8' )
+
+
+def log_and_raise(msg, exc_type=ValueError, exc=None) -> NoReturn:
+    """
+    Log an error message via fre_logger and raise an exception with the same message.
+    Avoids the need to duplicate error text in both fre_logger.error() and raise calls.
+
+    Per Python logging best practices (see :pep:`282` and the
+    `logging HOWTO <https://docs.python.org/3/howto/logging.html>`_):
+
+    * ``exc_info=True`` is passed when *exc* is given so the caught exception's
+      traceback is written to every configured handler (including any log file
+      set up via ``fre -l``).
+    * ``stack_info=True`` is always passed so the call-site stack trace appears
+      in every handler, making it easy to locate the origin of the error in a
+      log file.
+    * ``stacklevel=2`` attributes the log record to the **caller** of
+      ``log_and_raise``, not this helper itself.
+
+    :param msg: Error message to log and include in the exception.
+    :type msg: str
+    :param exc_type: Exception class to raise. Defaults to ValueError.
+    :type exc_type: type
+    :param exc: Optional original exception to chain from (uses ``raise ... from exc``).
+                When provided, ``exc_info=True`` is set so the caught exception's
+                traceback is included in the log output.
+    :type exc: Exception, optional
+    :raises exc_type: Always raised with the given message.
+
+    Examples::
+
+        # raises ValueError (default) and logs the message at ERROR level
+        log_and_raise("something went wrong")
+
+        # raises a specific exception type
+        log_and_raise("file not found", OSError)
+
+        # chains from an original exception (raise ... from exc)
+        except KeyError as e:
+            log_and_raise("update failed", KeyError, exc=e)
+    """
+    if exc is not None:
+        fre_logger.error(msg, exc_info=True, stack_info=True, stacklevel=2)
+        raise exc_type(msg) from exc
+    fre_logger.error(msg, stack_info=True, stacklevel=2)
+    raise exc_type(msg)

fre/app/generate_time_averages/cdoTimeAverager.py

@@ -8,6 +8,7 @@
 import cdo
 from cdo import Cdo
 
+from fre import log_and_raise
 from .timeAverager import timeAverager
 
 fre_logger = logging.getLogger(__name__)
@@ -32,8 +33,7 @@ def generate_timavg(self, infile = None, outfile = None):
         """
 
         if self.avg_type not in ['all', 'seas', 'month']:
-            fre_logger.error('requested unknown avg_type %s.', self.avg_type)
-            raise ValueError(f'requested unknown avg_type {self.avg_type}')
+            log_and_raise(f'requested unknown avg_type {self.avg_type}')
 
         if self.var is not None:
             fre_logger.warning('WARNING: variable specification not twr supported for cdo time averaging. ignoring!')

fre/app/generate_time_averages/combine.py

@@ -11,6 +11,7 @@
 import xarray as xr
 
 from ..helpers import change_directory
+from fre import log_and_raise
 
 fre_logger = logging.getLogger(__name__)
 duration_parser = metomi.isodatetime.parsers.DurationParser()
@@ -35,7 +36,7 @@ def form_bronx_directory_name(frequency: str,
     elif frequency == "yr":
         frequency_label = "annual"
     else:
-        raise ValueError(f"Frequency '{frequency}' not recognized or supported")
+        log_and_raise(f"Frequency '{frequency}' not recognized or supported", ValueError)
     interval_object = duration_parser.parse(interval)
     return frequency_label + '_' + str(interval_object.years) + 'yr'
 
@@ -57,9 +58,9 @@ def merge_netcdfs(input_file_glob: str, output_file: str) -> None:
     if len(input_files) >= 1:
         fre_logger.debug(f"Input file search string '{input_file_glob}' matched {len(input_files)} files")
     else:
-        raise FileNotFoundError(f"'{input_file_glob}' resolves to no files")
+        log_and_raise(f"'{input_file_glob}' resolves to no files", FileNotFoundError)
     if Path(output_file).exists():
-        raise FileExistsError(f"Output file '{output_file}' already exists")
+        log_and_raise(f"Output file '{output_file}' already exists", FileExistsError)
 
     ds = xr.open_mfdataset(input_files, compat='override', coords='minimal')
     ds.to_netcdf(output_file, unlimited_dims=['time'])
@@ -93,7 +94,7 @@ def combine( root_in_dir: str,
     :rtype: None
     """
     if frequency not in ["yr", "mon"]:
-        raise ValueError(f"Frequency '{frequency}' not recognized or supported")
+        log_and_raise(f"Frequency '{frequency}' not recognized or supported", ValueError)
 
     if frequency == "yr":
         frequency_iso = "P1Y"

fre/app/generate_time_averages/frenctoolsTimeAverager.py

@@ -7,6 +7,7 @@
 
 from cdo import Cdo
 from .timeAverager import timeAverager
+from fre import log_and_raise
 
 fre_logger = logging.getLogger(__name__)
 
@@ -35,7 +36,7 @@ def generate_timavg(self, infile=None, outfile=None):
         """
         exitstatus=1
         if self.avg_type not in ['month','all']:
-            raise ValueError(f'avg_type= {self.avg_type} not supported by this class at this time.')
+            log_and_raise(f'avg_type= {self.avg_type} not supported by this class at this time.', ValueError)
 
         if self.unwgt:
             fre_logger.warning('unwgt=True unsupported by frenctoolsAverager. Ignoring!!!')
@@ -45,15 +46,15 @@ def generate_timavg(self, infile=None, outfile=None):
                    ' not currently supported for frenctols time averaging. ignoring!', self.var)
 
         if infile is None:
-            raise ValueError('Need an input file, specify a value for the infile argument')
+            log_and_raise('Need an input file, specify a value for the infile argument', ValueError)
 
         if outfile is None:
             outfile='frenctoolsTimeAverage_'+infile
             fre_logger.warning('No output filename given, setting outfile= %s', outfile)
 
         #check for existence of timavg.csh. If not found, issue might be that user is not in env with frenctools.
         if shutil.which('timavg.csh') is None:
-            raise ValueError('did not find timavg.csh')
+            log_and_raise('did not find timavg.csh', ValueError)
         fre_logger.info('timeaverager using: %s', shutil.which("timavg.csh"))
 
 
@@ -106,7 +107,7 @@ def generate_timavg(self, infile=None, outfile=None):
 
                     if subp.returncode != 0:
                         fre_logger.error('stderror = %s', stderror)
-                        raise ValueError(f'error: timavg.csh had a problem, subp.returncode = {subp.returncode}')
+                        log_and_raise(f'error: timavg.csh had a problem, subp.returncode = {subp.returncode}', ValueError)
 
                     fre_logger.info('%s climatology successfully ran',nc_monthly_file)
                     exitstatus=0
@@ -130,7 +131,7 @@ def generate_timavg(self, infile=None, outfile=None):
 
             if subp.returncode != 0:
                 fre_logger.error('stderror  = %s', stderror)
-                raise ValueError(f'error: timavgcsh command not properly executed, subp.returncode = {subp.returncode}')
+                log_and_raise(f'error: timavgcsh command not properly executed, subp.returncode = {subp.returncode}', ValueError)
 
             fre_logger.info('climatology successfully ran')
             exitstatus = 0

fre/app/generate_time_averages/generate_time_averages.py

@@ -11,6 +11,8 @@
 from .frenctoolsTimeAverager import frenctoolsTimeAverager
 from .frepytoolsTimeAverager import frepytoolsTimeAverager
 
+from fre import log_and_raise
+
 fre_logger = logging.getLogger(__name__)
 
 def generate_time_average(infile: Union[str, List[str]] = None,
@@ -40,9 +42,9 @@ def generate_time_average(infile: Union[str, List[str]] = None,
     start_time = time.perf_counter()
     fre_logger.debug('called generate_time_average')
     if None in [infile, outfile, pkg]:
-        raise ValueError('infile, outfile, and pkg are required inputs')
+        log_and_raise('infile, outfile, and pkg are required inputs', ValueError)
     if pkg not in ['cdo', 'fre-nctools', 'fre-python-tools']:
-        raise ValueError(f'argument pkg = {pkg} not known, must be one of: cdo, fre-nctools, fre-python-tools')
+        log_and_raise(f'argument pkg = {pkg} not known, must be one of: cdo, fre-nctools, fre-python-tools', ValueError)
     exitstatus = 1
     myavger = None
 
@@ -98,8 +100,7 @@ def generate_time_average(infile: Union[str, List[str]] = None,
         exitstatus = myavger.generate_timavg( infile = infile,
                                               outfile = outfile)
     else:
-        fre_logger.error('averager is None, check generate_time_average in generate_time_averages.py!')
-        raise ValueError
+        log_and_raise('averager is None, check generate_time_average in generate_time_averages.py!')
 
     # remove the new merged file if we created it.
     if merged:

fre/app/generate_time_averages/wrapper.py

@@ -8,6 +8,7 @@
 from metomi.isodatetime.dumpers import TimePointDumper
 
 from . import generate_time_averages
+from fre import log_and_raise
 
 fre_logger = logging.getLogger(__name__)
 one_year = DurationParser().parse('P1Y')
@@ -77,7 +78,7 @@ def generate_wrapper(cycle_point: str,
     input_interval = DurationParser().parse(input_interval)
 
     if frequency not in ["yr", "mon"]:
-        raise ValueError(f"Frequency '{frequency}' not recognized or supported")
+        log_and_raise(f"Frequency '{frequency}' not recognized or supported", ValueError)
 
     # convert frequency 'yr' or 'mon' to ISO8601
     if frequency == 'mon':
@@ -109,7 +110,7 @@ def generate_wrapper(cycle_point: str,
                         fre_logger.debug("Annual ts to annual climo from source %s:%s variables",
                                          source, len(variables))
                     else:
-                        raise FileNotFoundError(f"Expected files not found in {subdir_yr}")
+                        log_and_raise(f"Expected files not found in {subdir_yr}", FileNotFoundError)
                 elif subdir_mon.exists():
                     results = glob.glob(str(subdir_mon / f"{source}.{yyyy}01-{zzzz}12.*.nc"))
                     if results:
@@ -118,7 +119,7 @@ def generate_wrapper(cycle_point: str,
                         fre_logger.debug("monthly ts to annual climo from source %s:%s variables",
                                          source, len(variables))
                     else:
-                        raise FileNotFoundError(f"Expected files not found in {subdir_mon}")
+                        log_and_raise(f"Expected files not found in {subdir_mon}", FileNotFoundError)
                 else:
                     fre_logger.debug('Skipping %s as it does not appear to be monthly or annual frequency', source)
                     fre_logger.debug('neither %s nor %s  exists', subdir_mon, subdir_yr)
@@ -132,7 +133,7 @@ def generate_wrapper(cycle_point: str,
                         fre_logger.debug("monthly ts to monthly climo from source %s:%s variables",
                                          source, len(variables))
                     else:
-                        raise FileNotFoundError(f"Expected files not found in {subdir}")
+                        log_and_raise(f"Expected files not found in {subdir}", FileNotFoundError)
                 else:
                     fre_logger.debug("Skipping %s as it does not appear to be monthly frequency", source)
                     fre_logger.debug(" %s does not exist", subdir)

fre/app/helpers.py

@@ -6,6 +6,8 @@
 
 import yaml
 
+from fre import log_and_raise
+
 
 fre_logger = logging.getLogger(__name__)
 
@@ -25,7 +27,7 @@ def get_variables(yml: dict, pp_comp: str) -> dict:
     fre_logger.debug(f"Yaml file information: {yml}")
     fre_logger.debug(f"PP component: {pp_comp}")
     if not isinstance(yml, dict):
-        raise TypeError("yml should be of type dict, but was of type " + str(type(yml)))
+        log_and_raise("yml should be of type dict, but was of type " + str(type(yml)), TypeError)
 
     src_vars={}
 
@@ -61,7 +63,7 @@ def get_variables(yml: dict, pp_comp: str) -> dict:
     # If the dictionary is empty (no overlap of pp components and components
     # in pp yaml) --> error
     if not src_vars:
-        raise ValueError(f"PP component, {pp_comp}, not found in pp yaml configuration!")
+        log_and_raise(f"PP component, {pp_comp}, not found in pp yaml configuration!", ValueError)
 
     return src_vars
 

fre/app/mask_atmos_plevel/mask_atmos_plevel.py

@@ -8,6 +8,8 @@
 
 import xarray as xr
 
+from fre import log_and_raise
+
 fre_logger = logging.getLogger(__name__)
 
 
@@ -42,7 +44,7 @@ def mask_atmos_plevel_subtool(infile: str = None,
     """
     # check if input file exists, raise an error if not
     if not os.path.exists(infile):
-        raise FileNotFoundError(f"ERROR: Input file {infile} does not exist")
+        log_and_raise(f"ERROR: Input file {infile} does not exist", FileNotFoundError)
 
     # Warn if outfile exists, but continue and recreate
     if os.path.exists(outfile):
@@ -62,7 +64,7 @@ def mask_atmos_plevel_subtool(infile: str = None,
     if "ps" not in list(ds_ps.variables):
         fre_logger.warning('pressure variable ps not found in target pressure file')
         if not warn_no_ps:
-            raise ValueError(f"Surface pressure file {psfile} does not contain surface pressure.")
+            log_and_raise(f"Surface pressure file {psfile} does not contain surface pressure.", ValueError)
         fre_logger.warning('warn_no_ps is True! this means I\'m going to no-op gracefully instead of raising an error')
         return
 
@@ -143,7 +145,7 @@ def mask_field_above_surface_pressure(ds: xr.Dataset,
     try:
         missing_value = ds[var].encoding['missing_value']
     except Exception as exc:
-        raise KeyError("input file does not contain missing_value, a required variable attribute") from exc
+        log_and_raise("input file does not contain missing_value, a required variable attribute", KeyError, exc=exc)
 
     fre_logger.info('masking do not need looping')
     masked = xr.where(plev_extended > ps_extended, missing_value, ds[var])

fre/app/regrid_xy/regrid_xy.py

@@ -7,6 +7,7 @@
 import yaml
 
 from fre.app import helpers
+from fre import log_and_raise
 
 fre_logger = logging.getLogger(__name__)
 
@@ -83,7 +84,7 @@ def get_grid_spec(datadict: dict) -> str:
     elif Path("grid_spec.nc").exists():
         grid_spec = "grid_spec.nc"
     else:
-        raise IOError(f"Cannot find mosaic.nc or grid_spec.nc in tar file {pp_grid_spec_tar}")
+        log_and_raise(f"Cannot find mosaic.nc or grid_spec.nc in tar file {pp_grid_spec_tar}", IOError)
 
     fre_logger.debug(f"Current directory: {Path.cwd()}")
 
@@ -122,7 +123,7 @@ def get_input_mosaic(datadict: dict) -> str:
 
     #check if the mosaic file exists in the current directory
     if not Path(mosaic_file).exists():
-        raise IOError(f"Cannot find mosaic file {mosaic_file} in current work directory {work_dir}")
+        log_and_raise(f"Cannot find mosaic file {mosaic_file} in current work directory {work_dir}", IOError)
 
     return mosaic_file
 
@@ -299,16 +300,15 @@ def regrid_xy(yamlfile: str,
 
     #check if input_dir exists
     if not Path(input_dir).exists():
-        raise RuntimeError(f"Input directory {input_dir} containing the input data files does not exist")
+        log_and_raise(f"Input directory {input_dir} containing the input data files does not exist", RuntimeError)
 
     #check if output_dir exists
     if not Path(output_dir).exists():
-        raise RuntimeError(f"Output directory {output_dir} where regridded data" \
-                            "will be outputted does not exist")
+        log_and_raise(f"Output directory {output_dir} where regridded data will be outputted does not exist", RuntimeError)
 
     #check if work_dir exists
     if not Path(work_dir).exists():
-        raise RuntimeError(f"Specified working directory {work_dir} does not exist")
+        log_and_raise(f"Specified working directory {work_dir} does not exist", RuntimeError)
 
     #work in working directory
     with helpers.change_directory(work_dir):
@@ -399,4 +399,4 @@ def regrid_xy(yamlfile: str,
             if fregrid_job.returncode == 0:
                 fre_logger.info(fregrid_job.stdout.split("\n")[-3:])
             else:
-                raise RuntimeError(fregrid_job.stderr)
+                log_and_raise(fregrid_job.stderr, RuntimeError)