Merge pull request #39 from IBM/v0.5.4.1b

V0.5.4.1
IBM · Jan 24, 2022 · f831294 · f831294
2 parents 42a9078 + 77eaea0
commit f831294
Show file tree

Hide file tree

Showing 36 changed files with 1,342 additions and 220 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]## [0.5.4.1b]
+## [Unreleased]## [0.5.4.2b]
+
+## [0.5.4.1] - 2022-01-24
+### Changed
+- Fixed bug in ScenarioDbManager._read_scenario_tables_from_db
+- Fixed bug in ScenarioDbManager.update_scenario_output_tables_in_db
+- Fixed bug in ScenarioManager.add_file_as_data_asset when using CPDaaS
+- Fixed bug in ScenarioManager.load_data_from_excel when using CPDaaS
+- DOModelDeployer working in CPD4.0.3
+### Added
+- Added DataManager.set_parameters()
+- Added print of num rows and columns inserting in ScenarioDbManager.update_scenario_output_tables_in_db
+- ScenarioDbManager.insert_scenarios_from_zip
+- ScenarioManager.insert_scenarios_from_zip
+- AutoScenarioDbTable reflection to get SQLAlchemy table metadata from DB. Fixes bug using AutoScenarioDbTable.
+- ScenarioManager.load_data_from_excel now accepts excel_file_name with and without `.xlsx` extension
+- ScenarioManager.write_data_to_excel now accepts excel_file_name with and without `.xlsx` extension
 
 ## [0.5.4.0] - 2022-01-11
 ### Changed

diff --git a/docs/doc_build/doctrees/dse_do_utils.doctree b/docs/doc_build/doctrees/dse_do_utils.doctree
diff --git a/docs/doc_build/doctrees/environment.pickle b/docs/doc_build/doctrees/environment.pickle
diff --git a/docs/doc_build/html/.buildinfo b/docs/doc_build/html/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: bd2b9a92071f1c40e294c4bb1fb76baa
+config: 11af026b252a6ffaf304856232bb7c3f
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/doc_build/html/_modules/dse_do_utils.html b/docs/doc_build/html/_modules/dse_do_utils.html
@@ -6,7 +6,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>dse_do_utils &#8212; DSE DO Utils 0.5.4.0 documentation</title>
+    <title>dse_do_utils &#8212; DSE DO Utils 0.5.4.1 documentation</title>
     <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../_static/bizstyle.css" />
 
@@ -31,7 +31,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="index.html" accesskey="U">Module code</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils</a></li> 
       </ul>
@@ -169,7 +169,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="index.html" >Module code</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils</a></li> 
       </ul>

diff --git a/docs/doc_build/html/_modules/dse_do_utils/cpd25utilities.html b/docs/doc_build/html/_modules/dse_do_utils/cpd25utilities.html
@@ -6,7 +6,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>dse_do_utils.cpd25utilities &#8212; DSE DO Utils 0.5.4.0 documentation</title>
+    <title>dse_do_utils.cpd25utilities &#8212; DSE DO Utils 0.5.4.1 documentation</title>
     <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/bizstyle.css" />
 
@@ -31,7 +31,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" accesskey="U">dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.cpd25utilities</a></li> 
@@ -211,7 +211,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" >dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.cpd25utilities</a></li> 

diff --git a/docs/doc_build/html/_modules/dse_do_utils/datamanager.html b/docs/doc_build/html/_modules/dse_do_utils/datamanager.html
@@ -6,7 +6,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>dse_do_utils.datamanager &#8212; DSE DO Utils 0.5.4.0 documentation</title>
+    <title>dse_do_utils.datamanager &#8212; DSE DO Utils 0.5.4.1 documentation</title>
     <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/bizstyle.css" />
 
@@ -31,7 +31,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" accesskey="U">dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.datamanager</a></li> 
@@ -52,6 +52,8 @@ <h1>Source code for dse_do_utils.datamanager</h1><div class="highlight"><pre>
 <span class="c1"># DataManager</span>
 <span class="c1"># -----------------------------------------------------------------------------------</span>
 <span class="c1"># -----------------------------------------------------------------------------------</span>
+<span class="kn">import</span> <span class="nn">types</span>
+
 <span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Optional</span>
 
@@ -102,6 +104,7 @@ <h1>Source code for dse_do_utils.datamanager</h1><div class="highlight"><pre>
 
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">pass</span></div>
+        <span class="c1"># self.set_parameters()  # TODO: do we always want to call set_parameter here?</span>
 
 <div class="viewcode-block" id="DataManager.prepare_output_data_frames"><a class="viewcode-back" href="../../dse_do_utils.html#dse_do_utils.datamanager.DataManager.prepare_output_data_frames">[docs]</a>    <span class="k">def</span> <span class="nf">prepare_output_data_frames</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Placeholder to process output data frames.</span>
@@ -141,6 +144,21 @@ <h1>Source code for dse_do_utils.datamanager</h1><div class="highlight"><pre>
     <span class="c1">#     &quot;&quot;&quot;</span>
     <span class="c1">#     return self.prep_parameters()</span>
 
+<div class="viewcode-block" id="DataManager.set_parameters"><a class="viewcode-back" href="../../dse_do_utils.html#dse_do_utils.datamanager.DataManager.set_parameters">[docs]</a>    <span class="k">def</span> <span class="nf">set_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Set the parameters as properties of the self.param object.</span>
+<span class="sd">        This allows for easy access to the parameters, e.g. `dm.param.time_limit`</span>
+<span class="sd">        To be overridden. Make sure to call the `super().set_parameters()`</span>
+
+<span class="sd">        Creates the self.param SimpleNamespace to be able to add the individual parameter properties.</span>
+<span class="sd">        Creates the self.params pd.DataFrame to be able to easily extract the parameter values.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prep_parameters</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">param</span> <span class="o">=</span> <span class="n">types</span><span class="o">.</span><span class="n">SimpleNamespace</span><span class="p">()</span></div>
+
+        <span class="c1">#Examples of extraction of parameters:</span>
+        <span class="c1"># self.param.time_limit = self.get_parameter_value(self.params, &#39;solveTimeLimit&#39;, param_type=&#39;int&#39;, default_value=600)</span>
+        <span class="c1"># Supported param_type values are int, float, str, bool, datetime</span>
+
 <div class="viewcode-block" id="DataManager.prep_parameters"><a class="viewcode-back" href="../../dse_do_utils.html#dse_do_utils.datamanager.DataManager.prep_parameters">[docs]</a>    <span class="k">def</span> <span class="nf">prep_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
         <span class="sd">&quot;&quot;&quot;Pre-process the Parameter(s) input table.</span>
 <span class="sd">        Assumes the inputs contains a table named `Parameter` or `Parameters` with key `param` and column `value`.</span>
@@ -155,6 +173,7 @@ <h1>Source code for dse_do_utils.datamanager</h1><div class="highlight"><pre>
         <span class="c1"># self.params = params</span>
         <span class="k">return</span> <span class="n">params</span></div>
 
+
 <div class="viewcode-block" id="DataManager.get_parameter_value"><a class="viewcode-back" href="../../dse_do_utils.html#dse_do_utils.datamanager.DataManager.get_parameter_value">[docs]</a>    <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">get_parameter_value</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">param_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">param_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">default_value</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                             <span class="n">value_format</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S&#39;</span><span class="p">):</span>
@@ -437,7 +456,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" >dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.datamanager</a></li> 

diff --git a/docs/doc_build/html/_modules/dse_do_utils/deployeddomodel.html b/docs/doc_build/html/_modules/dse_do_utils/deployeddomodel.html
@@ -6,7 +6,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>dse_do_utils.deployeddomodel &#8212; DSE DO Utils 0.5.4.0 documentation</title>
+    <title>dse_do_utils.deployeddomodel &#8212; DSE DO Utils 0.5.4.1 documentation</title>
     <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/bizstyle.css" />
 
@@ -31,7 +31,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" accesskey="U">dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.deployeddomodel</a></li> 
@@ -325,7 +325,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" >dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.deployeddomodel</a></li> 

diff --git a/docs/doc_build/html/_modules/dse_do_utils/deployeddomodelcpd21.html b/docs/doc_build/html/_modules/dse_do_utils/deployeddomodelcpd21.html
@@ -6,7 +6,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>dse_do_utils.deployeddomodelcpd21 &#8212; DSE DO Utils 0.5.4.0 documentation</title>
+    <title>dse_do_utils.deployeddomodelcpd21 &#8212; DSE DO Utils 0.5.4.1 documentation</title>
     <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/bizstyle.css" />
 
@@ -31,7 +31,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" accesskey="U">dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.deployeddomodelcpd21</a></li> 
@@ -737,7 +737,7 @@ <h3>Navigation</h3>
         <li class="right" >
           <a href="../../py-modindex.html" title="Python Module Index"
              >modules</a> |</li>
-        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.0 documentation</a> &#187;</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">DSE DO Utils 0.5.4.1 documentation</a> &#187;</li>
           <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li>
           <li class="nav-item nav-item-2"><a href="../dse_do_utils.html" >dse_do_utils</a> &#187;</li>
         <li class="nav-item nav-item-this"><a href="">dse_do_utils.deployeddomodelcpd21</a></li>