Implement LIVVkit 3 for EVV (#8)

* Update resources and script to match LIVV3 interface

* Update PGN module for LIVV3

* Update ks extension (MVK test) for LIVVkit 3.0
Update to new elements, adds variable description and groups variables
into accept/reject groups for figures

* Clean up unused code, fix summary table

* Fix details table creation, cleanup code in main

* Re-order tables, rejected first...should be shorter

* Update to use LIVVkit 3 API

* Revert to un-grouped validations

* Remove Python 2

* Update contact and version info

* Fix commas

* Add web resources for LIVV3

* Remove unused code from tsc extn

* Make LIVVkit 3.0.1 lower bound requirement

Author: mkstratos

README.md

@@ -3,9 +3,9 @@
 
 EVV is a python-based toolkit for extended verification and validation of Earth
 system models (ESMs). Currently, it provides a number tests to determine if
-modifications to an ESM is *climate changing.*  
-  
-  
+modifications to an ESM is *climate changing.*
+
+
   Contact
 ===========
 
@@ -15,6 +15,9 @@ report bugs, ask questions, or contact us for any reason, use the
 
 Want to send us a private message?
 
+**Michael Kelleher**
+* github: @mkstratos
+* email: <a href="mailto:[email protected]">kelleherme [at] ornl.gov</a>
 **Joseph H. Kennedy** 
 * github: @jhkennedy
 * email: <a href="mailto:[email protected]">kennedyjh [at] ornl.gov</a>

evv4esm/__init__.py

@@ -27,9 +27,8 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import, division, print_function, unicode_literals
 
-__version_info__ = (0, 2, 5)
+__version_info__ = (0, 3, 0)
 __version__ = '.'.join(str(vi) for vi in __version_info__)
 
 PASS_COLOR = '#389933'

evv4esm/__main__.py

@@ -28,8 +28,6 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import os
 import sys
 import time
@@ -75,8 +73,8 @@ def parse_args(args=None):
     from evv4esm import resources
     args.livv_resource_dir = livvkit.resource_dir
     livvkit.resource_dir = os.sep.join(resources.__path__)
-    
-    return args 
+
+    return args
 
 
 def main(cl_args=None):
@@ -106,11 +104,10 @@ def main(cl_args=None):
     from livvkit.components import validation
     from livvkit import scheduler
     from livvkit.util import functions
-    from livvkit.util import elements
+    from livvkit import elements
 
     if args.extensions:
         functions.setup_output()
-    
         summary_elements = []
         validation_config = {}
         print(" -----------------------------------------------------------------")
@@ -120,31 +117,24 @@ def main(cl_args=None):
         for conf in livvkit.validation_model_configs:
             validation_config = functions.merge_dicts(validation_config,
                                                       functions.read_json(conf))
-        summary_elements.extend(scheduler.run_quiet(validation, validation_config,
-                                                    group=False))
+            summary_elements.extend(scheduler.run_quiet("validation", validation, validation_config,
+                                                        group=False))
         print(" -----------------------------------------------------------------")
         print("   Extensions test suite complete ")
         print(" -----------------------------------------------------------------")
         print("")
 
-        result = elements.page("Summary", "", element_list=summary_elements)
-        functions.write_json(result, livvkit.output_dir, "index.json")
+        result = elements.Page("Summary", "", elements=summary_elements)
+        with open(os.path.join(livvkit.output_dir, "index.json"), "w") as index_data:
+            index_data.write(result._repr_json())
         print("-------------------------------------------------------------------")
         print(" Done!  Results can be seen in a web browser at:")
         print("   " + os.path.join(livvkit.output_dir, 'index.html'))
         print("-------------------------------------------------------------------")
 
     if args.serve:
-        try:
-            # Python 3
-            import http.server as server
-            import socketserver as socket
-        except ImportError:
-            # Python 2
-            # noinspection PyPep8Naming
-            import SimpleHTTPServer as server
-            # noinspection PyPep8Naming
-            import SocketServer as socket
+        import http.server as server
+        import socketserver as socket
 
         httpd = socket.TCPServer(('', args.serve), server.SimpleHTTPRequestHandler)
 

evv4esm/ensembles/e3sm.py

@@ -30,7 +30,6 @@
 
 """E3SM specific ensemble functions."""
 
-from __future__ import absolute_import, division, print_function, unicode_literals
 import six
 
 import os
@@ -102,8 +101,16 @@ def gather_monthly_averages(ensemble_files, variable_set=None):
                         continue
                     else:
                         m = np.mean(data.variables[var][0, ...])
-
-                    monthly_avgs.append((case, var, '{:04}'.format(inst), date_str, m))
-
-    monthly_avgs = pd.DataFrame(monthly_avgs, columns=('case', 'variable', 'instance', 'date', 'monthly_mean'))
+                    try:
+                        _name = f": {data.variables[var].getncattr('long_name')}"
+                    except AttributeError:
+                        _name = ""
+                    try:
+                        _units = f" [{data.variables[var].getncattr('units')}]"
+                    except AttributeError:
+                        _units = ""
+                    desc = f"{_name}{_units}"
+                    monthly_avgs.append((case, var, '{:04}'.format(inst), date_str, m, desc))
+
+    monthly_avgs = pd.DataFrame(monthly_avgs, columns=('case', 'variable', 'instance', 'date', 'monthly_mean', 'desc'))
     return monthly_avgs

evv4esm/extensions/ks.py

@@ -44,27 +44,25 @@
 techniques.
 """
 
-from __future__ import absolute_import, division, print_function, unicode_literals
-import six
-
-import os
 import argparse
-
-from pprint import pprint
+import os
 from collections import OrderedDict
-    
-import numpy as np
-from scipy import stats
+from pathlib import Path
+from pprint import pprint
 
 import livvkit
-from livvkit.util import elements as el
+import numpy as np
+import pandas as pd
+import six
+from livvkit import elements as el
 from livvkit.util import functions as fn
 from livvkit.util.LIVVDict import LIVVDict
+from scipy import stats
 
+from evv4esm import EVVException, human_color_names
 from evv4esm.ensembles import e3sm
 from evv4esm.ensembles.tools import monthly_to_annual_avg, prob_plot
 from evv4esm.utils import bib2html
-from evv4esm import human_color_names, EVVException
 
 
 def variable_set(name):
@@ -118,7 +116,7 @@ def parse_args(args=None):
                         default=13, type=float,
                         help='The critical value (desired significance level) for rejecting the ' +
                         'null hypothesis.')
-   
+
     parser.add_argument('--img-dir',
                         default=os.getcwd(),
                         help='Image output location.')
@@ -138,13 +136,26 @@ def parse_args(args=None):
                 args.config['ks'][key] = val
 
         config_arg_list = []
-        [config_arg_list.extend(['--'+key, str(val)]) for key, val in args.config['ks'].items()
-         if key != 'config']
+        _ = [
+            config_arg_list.extend(['--'+key, str(val)])
+            for key, val in args.config['ks'].items() if key != 'config'
+        ]
         args, _ = parser.parse_known_args(config_arg_list)
 
     return args
 
 
+def col_fmt(dat):
+    """Format results for table output."""
+    if dat is not None:
+        try:
+            _out = "{:.3e}, {:.3e}".format(*dat)
+        except TypeError:
+            _out = dat
+    else:
+        _out = "-"
+    return _out
+
 def run(name, config):
     """
     Runs the analysis.
@@ -156,7 +167,7 @@ def run(name, config):
     Returns:
        The result of elements.page with the list of elements to display
     """
-   
+
     config_arg_list = []
     [config_arg_list.extend(['--'+key, str(val)]) for key, val in config.items()]
 
@@ -167,31 +178,52 @@ def run(name, config):
 
     details, img_gal = main(args)
 
-    tbl_data = OrderedDict(sorted(details.items()))
-    tbl_el = {'Type': 'V-H Table',
-              'Title': 'Validation',
-              'TableTitle': 'Analyzed variables',
-              'Headers': ['h0', 'K-S test (D, p)', 'T test (t, p)'],
-              'Data': {'': tbl_data}
-              }
+    table_data = pd.DataFrame(details).T
+    _hdrs = [
+        "h0",
+        "K-S test (D, p)",
+        "T test (t, p)",
+        "mean (test case, ref. case)",
+        "std (test case, ref. case)",
+    ]
+    table_data = table_data[_hdrs]
+    for _hdr in _hdrs[1:]:
+        table_data[_hdr] = table_data[_hdr].apply(col_fmt)
+
+    tables = [
+        el.Table("Rejected", data=table_data[table_data["h0"] == "reject"]),
+        el.Table("Accepted", data=table_data[table_data["h0"] == "accept"]),
+        el.Table("Null", data=table_data[~table_data["h0"].isin(["accept", "reject"])])
+    ]
+
     bib_html = bib2html(os.path.join(os.path.dirname(__file__), 'ks.bib'))
-    tl = [el.tab('Figures', element_list=[img_gal]),
-          el.tab('Details', element_list=[tbl_el]),
-          el.tab('References', element_list=[el.html(bib_html)])]
-
-    rejects = [var for var, dat in tbl_data.items() if dat['h0'] == 'reject']
-    results = {'Type': 'Table',
-               'Title': 'Results',
-               'Headers': ['Test status', 'Variables analyzed', 'Rejecting', 'Critical value', 'Ensembles'],
-               'Data': {'Test status': 'pass' if len(rejects) < args.critical else 'fail',
-                        'Variables analyzed': len(tbl_data.keys()),
-                        'Rejecting': len(rejects),
-                        'Critical value': args.critical,
-                        'Ensembles': 'statistically identical' if len(rejects) < args.critical else 'statistically different'}
-               }
+
+    tabs = el.Tabs(
+        {
+            "Figures": img_gal,
+            "Details": tables,
+            "References": [el.RawHTML(bib_html)]
+        }
+    )
+    rejects = [var for var, dat in details.items() if dat["h0"] == "reject"]
+
+    results = el.Table(
+        title="Results",
+        data=OrderedDict(
+            {
+                'Test status': ['pass' if len(rejects) < args.critical else 'fail'],
+                'Variables analyzed': [len(details.keys())],
+                'Rejecting': [len(rejects)],
+                'Critical value': [int(args.critical)],
+                'Ensembles': [
+                    'statistically identical' if len(rejects) < args.critical else 'statistically different'
+                ]
+            }
+        )
+    )
 
     # FIXME: Put into a ___ function
-    page = el.page(name, __doc__.replace('\n\n', '<br><br>'), element_list=[results], tab_list=tl)
+    page = el.Page(name, __doc__.replace('\n\n', '<br><br>'), elements=[results, tabs])
     return page
 
 
@@ -232,17 +264,17 @@ def print_details(details):
 
 
 def summarize_result(results_page):
-    summary = {'Case': results_page['Title']}
-    for elem in results_page['Data']['Elements']:
-        if elem['Type'] == 'Table' and elem['Title'] == 'Results':
-            summary['Test status'] = elem['Data']['Test status']
-            summary['Variables analyzed'] = elem['Data']['Variables analyzed']
-            summary['Rejecting'] = elem['Data']['Rejecting']
-            summary['Critical value'] = elem['Data']['Critical value']
-            summary['Ensembles'] = elem['Data']['Ensembles']
+    summary = {'Case': results_page.title}
+
+    for elem in results_page.elements:
+        if isinstance(elem, el.Table) and elem.title == "Results":
+            summary['Test status'] = elem.data['Test status'][0]
+            summary['Variables analyzed'] = elem.data['Variables analyzed'][0]
+            summary['Rejecting'] = elem.data['Rejecting'][0]
+            summary['Critical value'] = elem.data['Critical value'][0]
+            summary['Ensembles'] = elem.data['Ensembles'][0]
             break
-        else:
-            continue
+
     return {'': summary}
 
 
@@ -251,13 +283,13 @@ def populate_metadata():
     Generates the metadata responsible for telling the summary what
     is done by this module's run method
     """
-    
+
     metadata = {'Type': 'ValSummary',
                 'Title': 'Validation',
                 'TableTitle': 'Kolmogorov-Smirnov test',
                 'Headers': ['Test status', 'Variables analyzed', 'Rejecting', 'Critical value', 'Ensembles']}
     return metadata
-    
+
 
 def main(args):
     ens_files, key1, key2 = case_files(args)
@@ -276,7 +308,7 @@ def main(args):
     if not common_vars:
         raise EVVException('No common variables between {} and {} to analyze!'.format(args.test_case, args.ref_case))
 
-    img_list = []
+    images = {"accept": [], "reject": [], "-": []}
     details = LIVVDict()
     for var in sorted(common_vars):
         annuals_1 = annual_avgs.query('case == @args.test_case & variable == @var').monthly_mean.values
@@ -307,24 +339,32 @@ def main(args):
         img_file = os.path.relpath(os.path.join(args.img_dir, var + '.png'), os.getcwd())
         prob_plot(annuals_1, annuals_2, 20, img_file, test_name=args.test_case, ref_name=args.ref_case,
                   pf=details[var]['h0'])
-        
-        img_desc = 'Mean annual global average of {var} for <em>{testcase}</em> ' \
+        _desc = monthly_avgs.query('case == @args.test_case & variable == @var').desc.values[0]
+        img_desc = 'Mean annual global average of {var}{desc} for <em>{testcase}</em> ' \
                    'is {testmean:.3e} and for <em>{refcase}</em> is {refmean:.3e}. ' \
                    'Pass (fail) is indicated by {cpass} ({cfail}) coloring of the ' \
                    'plot markers and bars.'.format(var=var,
+                                                   desc=_desc,
                                                    testcase=args.test_case,
                                                    testmean=details[var]['mean (test case, ref. case)'][0],
                                                    refcase=args.ref_case,
                                                    refmean=details[var]['mean (test case, ref. case)'][1],
                                                    cfail=human_color_names['fail'][0],
                                                    cpass=human_color_names['pass'][0])
 
-        img_link = os.path.join(os.path.basename(args.img_dir), os.path.basename(img_file))
-        img_list.append(el.image(var, img_desc, img_link))
-        
-    img_gal = el.gallery('Analyzed variables', img_list)
+        img_link = Path(*Path(args.img_dir).parts[-2:], Path(img_file).name)
+        _img = el.Image(var, img_desc, img_link, relative_to="", group=details[var]['h0'])
+        images[details[var]['h0']].append(_img)
+
+    gals = []
+    for group in ["reject", "accept", "-"]:
+        _group_name = {
+            "reject": "Failed variables", "accept": "Passed variables", "-": "Null variables"
+        }
+        if images[group]:
+            gals.append(el.Gallery(_group_name[group], images[group]))
 
-    return details, img_gal
+    return details, gals
 
 
 if __name__ == '__main__':