Merge pull request #155 from arbennett/simplify_config

Simplify config

Author: arbennett

docs/configuration.rst

@@ -32,27 +32,13 @@ __domain_vars__ for more details.
 ``out_dir :: path``: The location to write output to.  If this path doesn't
 exist, it will be created.
 
-``out_state :: path/filename.nc``: The location to write state file to.
-
 ``forcing_fmt :: str``: A string representing the type of input files specified in
 the ``forcing`` entry.  Can be one of the following: ``ascii``, ``binary``,
 ``netcdf``, or ``data``.
 
-``state_fmt :: str``: A string representing the type of state file specified in
-the ``state`` entry.  Can be either ``netcdf`` or ``data``.
-
-``domain_fmt :: str``: A string representing the type of state file specified in
-the ``domain`` entry.  Can be either ``netcdf`` or ``data``.
-
-``out_fmt:: str``: A string representing the type of output to write to
-``out_dir``.  Can be either ``netcdf``, ``data``, or ``ascii``.
-
-``method :: str``: A string representing the simulation methods to use.  The
-current implementation only supports ``mtclim``.
-
 **Optional Variables**
 
-``out_prefix :: str``: The output file base name. Defaults to ``forcing``.
+``output_prefix :: str``: The output file base name. Defaults to ``forcing``.
 
 ``out_precision :: str``: Precision to use when writing output.  Defaults to
 ``f8``.  Can be either ``f4`` or ``f8``.
@@ -63,19 +49,13 @@ memory.  Each chunk of output is written as ``{out_prefix}_{date_range}`` when
 active. Any valid ``pandas.TimeGrouper`` string may be used (e.g. use '10AS'
 for 10 year chunks).
 
-``iter_dims :: list``: The dimensions of input data to iterate over to
-accumulate sites.  Defaults to ``['lat', 'lon']``.
-
 ``verbose :: bool``: Whether to print output to ``stdout``.  Should be set using
 the ``-v`` flag for command line usage.  This can be set for scripting purposes,
 if desired. Set to ``1`` to print output; defaults to ``0``.
 
 ``sw_prec_thresh :: float``: Minimum precipitation threshold to take into
 account when simulating incoming shortwave radiation.  Defaults to ``0``.
 
-``mtclim_swe_corr :: bool``: Whether to activate MtClim's SWE correction
-algorithm. Default to ``False``.
-
 ``utc_offset :: bool``: Whether to use UTC timecode offsets for shifting
 timeseries. Without this option all times should be considered local to
 the gridcell being processed. Large domain runs probably want to set this
@@ -96,12 +76,6 @@ dewpoint temperature in MtClim.  Defaults to ``1e-6``.
 ``tmax_daylength_fraction :: float`` : Weight for calculation of time of maximum
 daily temperature.  Must be between ``0`` and ``1``.  Defaults to ``0.67``.
 
-``snow_crit_temp :: float``: Critical temperature for snow to melt.  Defaults to
-``-6.0 C``.
-
-``snow_melt_rate :: float``: Melt rate when temperature is less than
-``snow_crit_temp``.  Defaults to ``0.042 cm/K``.
-
 ``rain_scalar :: float``: Scale factor for calculation of cloudy sky
 transmittance.  Defaults to ``0.75``, range should be between ``0`` and
 ``1``.

examples/example_ascii.conf

@@ -1,5 +1,6 @@
 # This is an example of an input file for MetSim
 [MetSim]
+out_vars = ['temp', 'prec', 'shortwave', 'longwave', 'vapor_pressure', 'rel_humid', 'air_pressure', 'wind']
 
 # Time step in minutes
 time_step = 60
@@ -15,15 +16,10 @@ forcing = ./metsim/data/ascii
 domain  = ./metsim/data/stehekin.nc
 state = ./metsim/data/state_vic.nc
 forcing_fmt = ascii
-domain_fmt = netcdf
-state_fmt = netcdf
-out_fmt = netcdf
 out_dir = ./results
-out_state = ./results/state.nc
 
 # How to disaggregate
 method = mtclim
-prec_type = uniform
 
 [chunks]
 lat = 10
@@ -40,7 +36,6 @@ wind = wind
 prec = prec
 t_max = t_max
 t_min = t_min
-swe = swe
 
 [domain_vars]
 lat = lat

examples/example_bin.conf

@@ -1,5 +1,6 @@
 # This is an example of an input file for MetSim
 [MetSim]
+out_vars = ['temp', 'prec', 'shortwave', 'longwave', 'vapor_pressure', 'rel_humid', 'air_pressure', 'wind']
 
 # Time step in minutes
 time_step = 30
@@ -14,15 +15,7 @@ forcing = ./metsim/data/binary
 domain  = ./metsim/data/stehekin.nc
 state = ./metsim/data/state_vic.nc
 forcing_fmt = binary
-domain_fmt = netcdf
-state_fmt = netcdf
-
 out_dir = ./results
-out_fmt = netcdf
-
-# How to disaggregate
-method = mtclim
-prec_type = uniform
 
 [chunks]
 lat = 10
@@ -39,7 +32,6 @@ wind = 100.0 signed
 prec = prec
 t_max = t_max
 t_min = t_min
-swe = swe
 
 [domain_vars]
 lat = lat

examples/example_nc.conf

@@ -1,5 +1,6 @@
 # This is an example of an input file for MetSim
 [MetSim]
+out_vars = ['temp', 'prec', 'shortwave', 'longwave', 'vapor_pressure', 'rel_humid', 'air_pressure', 'wind']
 
 # Time step in minutes
 time_step = 30
@@ -14,17 +15,13 @@ stop = 1950/1/31
 forcing = ./metsim/data/test.nc
 domain  = ./metsim/data/domain.nc
 state = ./metsim/data/state_nc.nc
+
 forcing_fmt = netcdf
-domain_fmt = netcdf
-state_fmt = netcdf
+in_format = netcdf
+
 out_dir = ./results
 out_prefix = forcing
-in_format = netcdf
-out_fmt = netcdf
-out_state = ./results/state.nc
 
-# How to disaggregate
-method = mtclim
 prec_type = triangle
 utc_offset = True
 
@@ -33,21 +30,20 @@ lat = 3
 lon = 3
 
 [forcing_vars]
-Prec = prec
-Tmax = t_max
-Tmin = t_min
-wind = wind
+prec  = Prec
+t_max = Tmax
+t_min = Tmin
+wind  = wind
 
 [state_vars]
-prec = prec
+prec  = prec
 t_max = t_max
 t_min = t_min
-swe = swe
 
 [domain_vars]
-lat = lat
-lon = lon
+lat  = lat
+lon  = lon
 mask = mask
 elev = elev
 t_pk = t_pk
-dur = dur
+dur  = dur

metsim/cli/ms.py

@@ -56,20 +56,19 @@ def init(opts):
     config.optionxform = str
     config.read(opts.config)
     conf = OrderedDict(config['MetSim'])
-    conf['forcing_vars'] = OrderedDict(config['forcing_vars'])
-    conf['domain_vars'] = OrderedDict(config['domain_vars'])
-    conf['state_vars'] = OrderedDict(config['state_vars'])
-    conf['chunks'] = OrderedDict(config['chunks'])
-    out_dir = os.path.abspath(conf['out_dir'])
-    out_state = conf.get('out_state', None)
-    if out_state is None:
-        out_state = os.path.join(out_dir, 'state.nc')
 
-    method = conf['method']
+    def invert_dict(d):
+        return OrderedDict({v: k for k, v in d.items()})
 
-    prec_type = conf.get('prec_type', None)
-    if prec_type is None:
-        prec_type = 'uniform'
+    def to_list(s):
+        return json.loads(s.replace("'", '"'))
+
+    conf['forcing_vars'] = OrderedDict(config['forcing_vars'])
+    if conf['forcing_fmt'] != 'binary':
+        conf['forcing_vars'] = invert_dict(conf['forcing_vars'])
+    conf['domain_vars'] = invert_dict(OrderedDict(config['domain_vars']))
+    conf['state_vars'] = invert_dict(OrderedDict(config['state_vars']))
+    conf['chunks'] = OrderedDict(config['chunks'])
 
     # If the forcing variable is a directory, scan it for files
     if os.path.isdir(conf['forcing']):
@@ -78,26 +77,14 @@ def init(opts):
     else:
         forcing_files = conf['forcing']
 
-    # We assume there is only one domain file and one state file
-    domain_file = conf['domain']
-    state_file = conf['state']
-    chunks = conf['chunks']
-
-    def to_list(s):
-        return json.loads(s.replace("'", '"'))
-
+    # Update the full configuration
     conf.update({"calendar": conf.get('calendar', 'standard'),
                  "scheduler": opts.scheduler,
                  "num_workers": opts.num_workers,
-                 "method": method,
-                 "out_dir": out_dir,
-                 "out_state": out_state,
-                 "state": state_file,
-                 "domain": domain_file,
-                 "forcing": forcing_files,
-                 "chunks": chunks,
                  "verbose": logging.DEBUG if opts.verbose else logging.INFO,
-                 "prec_type": prec_type})
+                 "forcing": forcing_files,
+                 "out_dir": os.path.abspath(conf['out_dir']),
+                 "prec_type": conf.get('prec_type', 'uniform')})
     conf['out_vars'] = to_list(conf.get('out_vars', '[]'))
     conf['iter_dims'] = to_list(conf.get('iter_dims', '["lat", "lon"]'))
     conf = {k: v for k, v in conf.items() if v != []}

metsim/io.py

@@ -48,23 +48,14 @@ def read_met_data(params: dict, domain: xr.Dataset) -> xr.Dataset:
 
 def read_domain(params: dict) -> xr.Dataset:
     """Load in a domain file"""
-    read_funcs = {
-        "netcdf": read_netcdf,
-        "data": read_data
-    }
-    return read_funcs[params['domain_fmt']](
+    return read_netcdf(
         params['domain'], calendar=params['calendar'],
         var_dict=params.get('domain_vars', None))
 
 
 def read_state(params: dict, domain: xr.Dataset) -> xr.Dataset:
     """Load in a state file"""
-    read_funcs = {
-        "netcdf": read_netcdf,
-        "data": read_data
-    }
-
-    return read_funcs[params['state_fmt']](
+    return read_netcdf(
         params['state'], domain=domain,
         start=params['state_start'], stop=params['state_stop'],
         calendar=params['calendar'],

metsim/metsim.py

@@ -112,17 +112,16 @@ class MetSim(object):
     # Class variables
     methods = {'mtclim': mtclim}
     params = {
-        "method": '',
+        "method": 'mtclim',
         "domain": '',
         "state": '',
         "out_dir": '',
-        "out_prefix": 'forcing',
+        "output_prefix": 'forcing',
         "start": 'forcing',
         "stop": 'forcing',
         "time_step": -1,
         "calendar": 'standard',
         "prec_type": 'uniform',
-        "out_fmt": '',
         "out_precision": 'f4',
         "verbose": 0,
         "sw_prec_thresh": 0.0,
@@ -445,7 +444,7 @@ def _get_output_times(self, freq=None):
 
     def _get_output_filename(self, times):
         suffix = self.get_nc_output_suffix(times)
-        fname = '{}_{}.nc'.format(self.params['out_prefix'], suffix)
+        fname = '{}_{}.nc'.format(self.params['output_prefix'], suffix)
         output_filename = os.path.join(
             os.path.abspath(self.params['out_dir']), fname)
         return output_filename
@@ -531,8 +530,7 @@ def _validate_setup(self):
             errs.append("Requires input forcings to be specified")
 
         # Parameters that can't be empty strings or None
-        non_empty = ['method', 'out_dir', 'time_step',
-                     'forcing_fmt', 'domain_fmt', 'state_fmt']
+        non_empty = ['out_dir', 'time_step', 'forcing_fmt']
         for each in non_empty:
             if self.params.get(each, None) is None or self.params[each] == '':
                 errs.append("Cannot have empty value for {}".format(each))