Merge pull request #111 from arbennett/feature/update_summa

Feature/update summa

Author: arbennett

README.md

@@ -1,3 +1,5 @@
+# pysumma
+
 pysumma is a Python wrapper for manipulating, running, managing, and analyzing
 of SUMMA (Structure for Unifying Multiple Modeling Alternatives)
 * [SUMMA web site at UCAR ](https://www.rap.ucar.edu/projects/summa)
@@ -6,6 +8,7 @@ pysumma provides methods for:
  - Running SUMMA
  - Modifying SUMMA input files
  - Automatically parallelizing distributed and sensitivity analysis type experiments
+ - Calibration via OSTRICH
  - Visualizing output
 
 # Installation

docs/configuration.rst

@@ -44,52 +44,38 @@ Then, you can see what is in it simply by printing it out:
 
     print(fm)
 
-    > 'SUMMA_FILE_MANAGER_V1.0'    ! filemanager_version
-    > '/home/user/summa_setup_template/settings/'    ! settings_path
-    > '/home/user/summa_setup_template/forcing/'    ! input_path
-    > '/home/user/summa_setup_template/output/'    ! output_path
-    > 'decisions.txt'    ! decisions_path
-    > '[notUsed]'    ! meta_time
-    > '[notUsed]'    ! meta_attr
-    > '[notUsed]'    ! meta_type
-    > '[notUsed]'    ! meta_force
-    > '[notUsed]'    ! meta_localparam
-    > 'output_control.txt'    ! output_control
-    > '[notUsed]'    ! meta_localindex
-    > '[notUsed]'    ! meta_basinparam
-    > '[notUsed]'    ! meta_basinmvar
-    > 'local_attributes.nc'    ! local_attributes
-    > 'local_param_info.txt'    ! local_param_info
-    > 'basin_param_info.txt'    ! basin_param_info
-    > 'forcing_file_list.txt'    ! forcing_file_list
-    > 'initial_conditions.nc'    ! model_init_cond
-    > 'parameter_trial.nc'    ! parameter_trial
-    > 'test'    ! output_prefix
+    > controlVersion          'SUMMA_FILE_MANAGER_V3.0.0'
+    > simStartTime            '2002-10-01 00:00'
+    > simEndTime              '2003-05-31 00:00'
+    > tmZoneInfo              'localTime'
+    > settingsPath            '/pool0/data/andrbenn/dana_3_test/.pysumma/_test/settings/'
+    > forcingPath             './forcings/'
+    > outputPath              './output/'
+    > decisionsFile           'decisions.txt'
+    > outputControlFile       'output_control.txt'
+    > globalHruParamFile      '../params/local_param_info.txt'
+    > globalGruParamFile      '../params/basin_param_info.txt'
+    > attributeFile           '../params/local_attributes.nc'
+    > trialParamFile          '../params/parameter_trial.nc'
+    > forcingListFile         '../forcings/forcing_file_list.txt'
+    > initConditionFile       '../params/initial_conditions.nc'
+    > outFilePrefix           'template_output'
+    > vegTableFile            'VEGPARM.TBL'
+    > soilTableFile           'SOILPARM.TBL'
+    > generalTableFile        'GENPARM.TBL'
+    > noahmpTableFile         'MPTABLE.TBL'
 
 To see how to access each of these specific options you can use the ``list_options`` method.
-
-::
-
-    print(fm.list_options)
-
-    > ['filemanager_version', 'settings_path', 'input_path',
-    >  'output_path', 'decisions_path', 'meta_time',
-    >  'meta_attr', 'meta_type', 'meta_force', 'meta_localparam',
-    >  'output_control', 'meta_localindex', 'meta_basinparam',
-    >  'meta_basinmvar', 'local_attributes', 'local_param_info',
-    >  'basin_param_info', 'forcing_file_list', 'model_init_cond',
-    >  'parameter_trial', 'output_prefix']
-
 Then, each of these keys can be accessed directly similarly to how is done with python dictionaries.
 This can be used to inspect the values of each option as well as modify their values.
 
 ::
 
-    print(fm['output_prefix'])
+    print(fm['outputPrefix'])
 
     > 'test'    ! output_prefix
 
-    fm['output_prefix'] = 'tutorial'
+    fm['outputPrefix'] = 'tutorial'
 
     print(fm['output_prefix'])
 
@@ -114,14 +100,13 @@ Once instantiated you can inspect the available decisions and the options availa
 
     print(dec.list_options())
 
-    > ['simulStart', 'simulFinsh', 'tmZoneInfo', 'soilCatTbl',
-    >  'vegeParTbl', 'soilStress', 'stomResist', 'num_method',
+    > ['soilCatTbl', 'vegeParTbl', 'soilStress', 'stomResist',
     >  'fDerivMeth', 'LAI_method', 'f_Richards', 'groundwatr',
     >  'hc_profile', 'bcUpprTdyn', 'bcLowrTdyn', 'bcUpprSoiH',
     >  'bcLowrSoiH', 'veg_traits', 'canopyEmis', 'snowIncept',
     >  'windPrfile', 'astability', 'canopySrad', 'alb_method',
     >  'compaction', 'snowLayers', 'thCondSnow', 'thCondSoil',
-    >  'spatial_gw', 'subRouting']
+    >  'spatial_gw', 'subRouting', 'num_method']
 
     print(dec['snowLayers'])
 
@@ -136,12 +121,12 @@ Once instantiated you can inspect the available decisions and the options availa
 Forcing file list
 -----------------
 The forcing file list contains a listing of each of the forcing files available for use as SUMMA input.
-To instantiate the `ForceFileList` you will have to specify the path that is set as the ``input_path`` in your ``FileManager``. Below we show using the ``FileManager`` (``fm``) to do so.
-Once instantiated you can also use the `ForceFileList` object to inspect the forcing files themselves.
+To instantiate the `ForcingList` you will have to specify the path that is set as the ``input_path`` in your ``FileManager``. Below we show using the ``FileManager`` (``fm``) to do so.
+Once instantiated you can also use the `ForcingList` object to inspect the forcing files themselves.
 
 ::
 
-    ff = ps.ForceFileList('.', 'forcingFileList.1hr.txt', fm['input_path'])
+    ff = ps.ForcingList('.', 'forcingFileList.1hr.txt', fm['input_path'])
     print(ff)
 
     >> 'forcing_file.nc'
@@ -204,9 +189,9 @@ The format of the output control file mirrors the way that it is described in th
     >> sum
 
 
-Local parameter info
+GlobalParams
 --------------------
-The local parameter info file contains a listing of global parameters. Spatially dependent parameters are specified
+The GlobalParams object listing of global parameters. Spatially dependent parameters are specified
 in the parameter trial NetCDF file. Values which are specified in the local parameter info file will be overwritten
 by those specified in the parameter trial file.
 As with the output control file, there are many parameters which can be specified, so we omit them for brevity.
@@ -215,7 +200,7 @@ this out currently is by looking at the SUMMA source code directly.
 
 ::
 
-    lpi = ps.LocalParamInfo('.', 'local_param_info.txt')
+    lpi = ps.GlobalParams('.', 'global_param_info.txt')
     print(lpi.list_options())
 
     >> ['upperBoundHead', 'lowerBoundHead', 'upperBoundTheta', 'lowerBoundTheta',
@@ -230,7 +215,7 @@ NetCDF based files
 ==================
 The following input files are NetCDF-based and therefore, should be interacted with via ``xarray`` when using pysumma:
 
- - Parameter trial
+ - Parameter trial (Spatially distributed parameters)
  - Basin parameters
  - Local attributes
  - Initial conditions

environment.yml

@@ -19,5 +19,6 @@ dependencies:
   - geopandas
   - jupyter
   - pip
+  - summa=3.0.0
   - pip:
     - hs_restclient

pysumma/__init__.py

@@ -4,7 +4,7 @@
 from .file_manager import FileManager
 from .decisions import Decisions
 from .output_control import OutputControl
-from .local_param_info import LocalParamInfo
-from .force_file_list import ForceFileList
+from .global_params import GlobalParams
+from .force_file_list import ForcingList
 from . import utils
 from .calibration import Ostrich, OstrichParam

pysumma/decisions.py

@@ -22,11 +22,7 @@ def __init__(self, name, value):
         self.set_value(value)
 
     def set_value(self, new_value):
-        datestring = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}"
-        if (self.name in ['simulStart', 'simulFinsh'] and
-                re.match(datestring, new_value) is not None):
-            self.value = new_value
-        elif new_value in self.available_options:
+        if new_value in self.available_options:
             self.value = new_value
         else:
             raise ValueError(os.linesep.join([

pysumma/file_manager.py

@@ -7,8 +7,8 @@
 from .option import BaseOption, OptionContainer
 from .decisions import Decisions
 from .output_control import OutputControl
-from .local_param_info import LocalParamInfo
-from .force_file_list import ForceFileList
+from .global_params import GlobalParams
+from .force_file_list import ForcingList
 
 # Option names for the file manager, this is just a list,
 # as the order of these values matters. They may also not be
@@ -31,7 +31,7 @@ def set_value(self, new_value):
         self.value = new_value
 
     def __str__(self):
-        return "'{}'    ! {}".format(self.value, self.name)
+        return "{} '{}'".format(self.name.ljust(36), self.value)
 
 
 class FileManager(OptionContainer):
@@ -42,96 +42,104 @@ class FileManager(OptionContainer):
 
     def __init__(self, path, name):
         super().__init__(FileManagerOption, path, name)
+        assert self.get_value('controlVersion') == 'SUMMA_FILE_MANAGER_V3.0.0'
 
     def set_option(self, key, value):
         o = self.get_option(key)
         o.set_value(value)
 
     def get_constructor_args(self, line):
-        return (OPTION_NAMES[self.opt_count],
-                line.split('!')[0].replace("'", "").strip())
+        name, *value = line.split('!')[0].strip().split()
+        if isinstance(value, list):
+            value = " ".join(value).replace("'", "")
+        return (name.strip(), value.strip().replace("'", "").strip())
 
     @property
     def decisions(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('decisions_path')
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('decisionsFile')
         self._decisions = Decisions(p1, p2)
         return self._decisions
 
     @property
     def output_control(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('output_control')
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('outputControlFile')
         self._output_control = OutputControl(p1, p2)
         return self._output_control
 
     @property
-    def local_param_info(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('local_param_info')
-        self._local_param_info = LocalParamInfo(p1, p2)
-        return self._local_param_info
+    def global_hru_params(self):
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('globalHruParamFile')
+        self._hru_params = GlobalParams(p1, p2)
+        return self._hru_params
 
     @property
-    def basin_param_info(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('basin_param_info')
-        self._basin_param_info = LocalParamInfo(p1, p2)
-        return self._basin_param_info
+    def global_gru_params(self):
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('globalGruParamFile')
+        self._gru_params = GlobalParams(p1, p2)
+        return self._gru_params
 
     @property
     def force_file_list(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('forcing_file_list')
-        p3 = self.get_value('input_path')
-        self._force_file_list = ForceFileList(p1, p2, p3)
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('forcingListFile')
+        p3 = self.get_value('forcingPath')
+        self._force_file_list = ForcingList(p1, p2, p3)
         return self._force_file_list
 
     @property
     def local_attributes(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('local_attributes')
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('attributeFile')
         self._local_attrs = xr.open_dataset(p1 + p2)
         return self._local_attrs
 
     @property
-    def parameter_trial(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('parameter_trial')
-        self._param_trial = xr.open_dataset(p1 + p2)
-        return self._param_trial
+    def trial_params(self):
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('trialParamFile')
+        self._trial_params = xr.open_dataset(p1 + p2)
+        return self._trial_params
 
     @property
     def initial_conditions(self):
-        p1 = self.get_value('settings_path')
-        p2 = self.get_value('model_init_cond')
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('initConditionFile')
         self._init_cond = xr.open_dataset(p1 + p2)
         return self._init_cond
 
     @property
     def genparm(self):
-        p1, p2 = self.get_value('settings_path'), 'GENPARM.TBL'
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('generalTableFile')
+        print(p1, p2)
         with open(p1 + p2, 'r') as f:
             self._genparm = f.readlines()
         return self._genparm
 
     @property
     def mptable(self):
-        p1, p2 = self.get_value('settings_path'), 'MPTABLE.TBL'
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('noahmpTableFile')
         with open(p1 + p2, 'r') as f:
             self._mptable = f.readlines()
         return self._mptable
 
     @property
     def soilparm(self):
-        p1, p2 = self.get_value('settings_path'), 'SOILPARM.TBL'
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('soilTableFile')
         with open(p1 + p2, 'r') as f:
             self._soilparm = f.readlines()
         return self._soilparm
 
     @property
     def vegparm(self):
-        p1, p2 = self.get_value('settings_path'), 'VEGPARM.TBL'
+        p1 = self.get_value('settingsPath')
+        p2 = self.get_value('vegTableFile')
         with open(p1 + p2, 'r') as f:
             self._vegparm = f.readlines()
         return self._vegparm

pysumma/force_file_list.py

@@ -5,7 +5,7 @@
 from .option import OptionContainer
 
 
-class ForceFileListOption(BaseOption):
+class ForcingOption(BaseOption):
 
     def __init__(self, name):
         super().__init__(name)
@@ -26,13 +26,13 @@ def __str__(self):
         return "'{}'".format(self.name.split('/')[-1])
 
 
-class ForceFileList(OptionContainer):
+class ForcingList(OptionContainer):
 
     prefix: str = ''
 
     def __init__(self, dirpath, filepath, force_file_prefix_path):
         self.prefix = str(force_file_prefix_path)
-        super().__init__(ForceFileListOption, dirpath, filepath)
+        super().__init__(ForcingOption, dirpath, filepath)
 
     def set_option(self, key, value):
         o = self.get_option(key)