Merge pull request #20 from jhamman/feature/pycompat

add pycompat module and included some minor python3 fixes

Author: Joe Hamman

tests/test_pycompat.py

@@ -0,0 +1,24 @@
+from tonic.pycompat import pyrange, pyzip, iteritems
+
+
+def test_pyrange():
+    x = pyrange(5)
+    assert list(x) == [0, 1, 2, 3, 4]
+
+
+def test_pyzip():
+    a = [1, 2, 3]
+    b = ['a', 'b', 'c']
+
+    z = pyzip(a, b)
+    assert hasattr(z, '__iter__')
+    lz = list(pyzip(a, b))
+    assert lz[0] == (1, 'a')
+    assert len(lz) == len(a)  # == len(b)
+
+
+def test_iteritems():
+    d = {'a': 0, 'b': 1, 'c': 2}
+    for k, v in iteritems(d):
+        assert type(k) == str
+        assert type(v) == int

tests/test_vic2netcdf.py

@@ -1,4 +1,3 @@
-#!/usr/local/env python
 """Set to run with pytest
 
 Usage: py.test

tonic/io.py

@@ -1,15 +1,8 @@
 #!/usr/bin/env python
 """Input/Output functions"""
 from netCDF4 import Dataset
-try:
-    from cyordereddict import OrderedDict
-except:
-    from collections import OrderedDict
-try:
-    from configparser import SafeConfigParser
-except:
-    from ConfigParser import SafeConfigParser
 import configobj
+from .pycompat import OrderedDict, SafeConfigParser
 
 
 # -------------------------------------------------------------------- #

tonic/models/vic/compare_soil_params.py

@@ -14,6 +14,7 @@
 from matplotlib import rcParams
 from tonic.io import read_netcdf
 from tonic.plot_utils import sub_plot_pcolor, cmap_discretize
+from tonic.pycompat import pyrange
 
 description = 'Create plots comparing two sets of VIC soil parameters'
 help = 'Create plots comparing two sets of VIC soil parameters'
@@ -275,7 +276,7 @@ def my_plot9(lons, lats, d1, d2, units=None,
     f, axarr = plt.subplots(3, 3, figsize=(13.5, 9), dpi=150)
     f.tight_layout()
 
-    for layer in xrange(3):
+    for layer in pyrange(3):
         plt.sca(axarr[layer, 0])
         sub_plot_pcolor(lons, lats, np.ma.masked_where(mask, d1[layer]),
                         vmin=vmin, vmax=vmax, units=units,

tonic/models/vic/grid_params.py

@@ -15,7 +15,10 @@
 import socket
 from getpass import getuser
 from collections import OrderedDict
+from warnings import warn
 from tonic.io import read_netcdf
+from tonic.pycompat import pyrange, pyzip
+
 
 # -------------------------------------------------------------------- #
 description = 'Converter for VIC ASCII style parameters to gridded netCDF'
@@ -123,12 +126,11 @@ def __init__(self, nlayers=3, snow_bands=5):
                                    ('lib_albedo', np.arange(16, 28)),
                                    ('lib_veg_rough', np.arange(28, 40)),
                                    ('lib_displacement', np.arange(40, 52)),
-                                   ('lib_wind_h', np.array(52)),
-                                   ('lib_RGL', np.array(53)),
-                                   ('lib_rad_atten', np.array(54)),
-                                   ('lib_wind_atten', np.array(55)),
-                                   ('lib_trunk_ratio', np.array(56)),
-                                   ('lib_snow_albedo', np.array(57))])
+                                   ('lib_wind_h', np.array([52])),
+                                   ('lib_RGL', np.array([53])),
+                                   ('lib_rad_atten', np.array([54])),
+                                   ('lib_wind_atten', np.array([55])),
+                                   ('lib_trunk_ratio', np.array([56]))])
 # -------------------------------------------------------------------- #
 
 
@@ -507,14 +509,14 @@ def calc_grid(lats, lons, decimals=4):
 
     # check that counts and steps make sense
     if lon_step != lat_step:
-        print('WARNING: lon_step ({0}) and lat_step ({1}) do not '
-              'match'.format(lon_step, lat_step))
+        warn('lon_step ({0}) and lat_step ({1}) do not '
+             'match'.format(lon_step, lat_step))
     if lat_count / len(ulats) < 0.95:
-        print('WARNING: lat_count of mode is less than 95% ({0}%) of'
-              ' len(lats)'.format(lat_count / len(ulats)))
+        warn('lat_count of mode is less than 95% ({0}%) of'
+             ' len(lats)'.format(lat_count / len(ulats)))
     if lon_count / len(ulons) < 0.95:
-        print('WARNING: lon_count of mode is less than 95% ({0}%) of'
-              ' len(lons)'.format(lon_count / len(ulons)))
+        warn('lon_count of mode is less than 95% ({0}%) of'
+             ' len(lons)'.format(lon_count / len(ulons)))
 
     if lats.min() < -55 and lats.max() > 70:
         # assume global grid
@@ -644,7 +646,7 @@ def grid_params(soil_dict, target_grid, snow_dict, veg_dict, veglib_dict,
                 steps = mydict[var].shape[1]
                 out_dict[var] = np.ma.zeros((steps, ysize, xsize),
                                             dtype=dtype)
-                for i in xrange(steps):
+                for i in pyrange(steps):
                     out_dict[var][i, yi, xi] = mydict[var][:, i]
                 out_dict[var][:, ymask, xmask] = fill_val
 
@@ -653,10 +655,10 @@ def grid_params(soil_dict, target_grid, snow_dict, veg_dict, veglib_dict,
                 k = mydict[var].shape[2]
                 out_dict[var] = np.ma.zeros((j, k, ysize, xsize),
                                             dtype=dtype)
-                for jj in xrange(j):
-                    for kk in xrange(k):
+                for jj in pyrange(j):
+                    for kk in pyrange(k):
                         out_dict[var][jj, kk, yi, xi] = mydict[var][:, jj, kk]
-                for y, x in zip(ymask, xmask):
+                for y, x in pyzip(ymask, xmask):
                     out_dict[var][:, :, y, x] = fill_val
 
             out_dict[var] = np.ma.masked_values(out_dict[var], fill_val)
@@ -710,7 +712,7 @@ def grid_params(soil_dict, target_grid, snow_dict, veg_dict, veglib_dict,
             new = np.zeros(shape) + FILLVALUE_F
             new[:-1, :, yi, xi] = veglib_dict[lib_var][:, :, np.newaxis]
             new[-1, :, yi, xi] = 0
-            for y, x in zip(ymask, xmask):
+            for y, x in pyzip(ymask, xmask):
                 new[:, :, y, x] = fill_val
             out_dicts['veg_dict'][var] = np.ma.masked_values(new, FILLVALUE_F)
 
@@ -1021,7 +1023,6 @@ def veg(veg_file, soil_dict, max_roots=3, veg_classes=11,
 
     row = 0
     cell = 0
-
     while row < len(lines):
         line = lines[row].strip('\n').split(' ')
         gridcel[cell], nveg[cell] = np.array(line).astype(int)
@@ -1078,7 +1079,7 @@ def veg(veg_file, soil_dict, max_roots=3, veg_classes=11,
 
 
 # -------------------------------------------------------------------- #
-def veg_class(veg_file, maxcols=58, skiprows=3, c=Cols()):
+def veg_class(veg_file, maxcols=57, skiprows=3, c=Cols()):
     """
     Load the entire vegetation library file into a dictionary of numpy arrays.
     Also reorders data to match gridcell order of soil file.
@@ -1087,7 +1088,6 @@ def veg_class(veg_file, maxcols=58, skiprows=3, c=Cols()):
     print('reading {0}'.format(veg_file))
 
     usecols = np.arange(maxcols)
-    print(usecols, maxcols, skiprows)
 
     data = np.loadtxt(veg_file, usecols=usecols, skiprows=skiprows)
 

tonic/models/vic/ncparam2ascii.py

@@ -10,6 +10,7 @@
 from scipy.spatial import cKDTree
 from . import grid_params
 from tonic.io import read_netcdf
+from tonic.pycompat import pyzip, iteritems
 
 FILL_VALUE = -9999
 
@@ -111,7 +112,7 @@ def rasm_soil(data, soil_file):
     numcells = data['mask'].size
 
     arrayshape = (numcells, 1 + np.max([np.max(cols) for v, cols in
-                                        c.soil_param.iteritems()]))
+                                        iteritems(c.soil_param)]))
     soil_params = np.empty(arrayshape)
     dtypes = ['%1i'] * arrayshape[1]
     # ---------------------------------------------------------------- #
@@ -190,7 +191,7 @@ def rasm_soil(data, soil_file):
     # For rasm, all cols are shifted one to right to make room for nveg in
     # col 0
     i = -1
-    for var, cols in c.soil_param.iteritems():
+    for var, cols in iteritems(c.soil_param):
         for col in cols:
             dtypes[col] = f.soil_param[var]
 
@@ -260,7 +261,7 @@ def rasm_soil(data, soil_file):
 
     # ---------------------------------------------------------------- #
     # Print summary of variables
-    for var, cols in c.soil_param.iteritems():
+    for var, cols in iteritems(c.soil_param):
         print('{0: <12}--> min: {1:<09.3f}, max: {2:<09.3f}, mean:'
               ' {3:<09.3f}'.format(var,
                                    soil_params[:, cols].min(),
@@ -361,7 +362,7 @@ def veg(data, xinds, yinds, veg_file, rootzones=3, global_lai=True):
 
     f = open(veg_file, 'w')
 
-    for y, x in zip(yinds, xinds):
+    for y, x in pyzip(yinds, xinds):
         gridcell = int(data['gridcell'][y, x])
         n_veg = int(data['Nveg'][y, x])
         cv = data['Cv'][:, y, x]

tonic/models/vic/netcdf2vic.py

@@ -6,6 +6,7 @@
 import struct
 import os
 from tonic.io import read_netcdf, read_config
+from tonic.pycompat import pyzip
 
 description = 'Convert netCDF meteorological forcings to VIC sytle format'
 help = 'Convert netCDF meteorological forcings to VIC sytle format'
@@ -47,7 +48,7 @@ def _run(args):
             xs[posinds] -= 360
             print('adjusted xs lon minimum')
 
-            for y, x in zip(yi, xi):
+            for y, x in pyzip(yi, xi):
                 active_flag = False
                 for key in var_keys:
                     if (d[key][:, y, x].all() is np.ma.masked) \
@@ -62,7 +63,7 @@ def _run(args):
         else:
             append = True
 
-        for y, x, point in zip(ylist, xlist, pointlist):
+        for y, x, point in pyzip(ylist, xlist, pointlist):
 
             data = np.empty((d[var_keys[0]].shape[0], len(var_keys)))
 

tonic/models/vic/plot_params.py

@@ -8,7 +8,8 @@
 import matplotlib.pyplot as plt
 import matplotlib.gridspec as gridspec
 from mpl_toolkits.basemap import Basemap
-from share import read_netcdf
+from tonic.io import read_netcdf
+from tonic.pycompat import pyrange
 description = ''
 help = ''
 
@@ -43,7 +44,7 @@ def plot_veg_types(yc, xc, cv, baresoil):
 
     gs1 = gridspec.GridSpec(4, 3)
 
-    for loc in xrange(11):
+    for loc in pyrange(11):
         ax = fig.add_subplot(gs1[loc])
         plot_map(ax, yc, xc, cv[loc], projection_parameters, vmin=0,
                  cmap='Jet')

tonic/models/vic/vic2netcdf.py

@@ -33,6 +33,7 @@
 import time as tm
 from tonic.io import read_config, SafeConfigParser
 from tonic.tonic import calc_grid, get_grid_inds, NcVar
+from tonic.pycompat import pyzip, pyrange
 
 description = 'Convert a set of VIC ascii outputs to gridded netCDF'
 help = 'Convert a set of VIC ascii outputs to gridded netCDF'
@@ -155,12 +156,12 @@ def get_lats(self):
         return np.array([p.lat for p in self])
 
     def add_xs(self, xinds):
-        for i in range(len(self)):
+        for i in pyrange(len(self)):
             self[i].x = xinds[i]
         return
 
     def add_ys(self, yinds):
-        for i in range(len(self)):
+        for i in pyrange(len(self)):
             self[i].y = yinds[i]
         return
 
@@ -190,7 +191,7 @@ def set_fileformat(self, fileformat):
             elif fileformat == 'binary':
                 p.open = p._open_binary
                 p.read = p._read_binary
-                p.dt = np.dtype(list(zip(p.names, p.bin_dtypes)))
+                p.dt = np.dtype(list(pyzip(p.names, p.bin_dtypes)))
             elif fileformat == 'netcdf':
                 p.open = p._open_netcdf
                 p.read = p._read_netcdf
@@ -438,7 +439,7 @@ def nc_add_data_to_array(self, point):
                 point.df[name].values[self.slice]
         for name in self.four_dim_vars:
             varshape = self.f.variables[name].shape[1]
-            for i in range(varshape):
+            for i in pyrange(varshape):
                 subname = name + str(i)
                 self.data[name][:, i, point.y,
                                 point.x] = point.df[subname].values[self.slice]
@@ -452,7 +453,7 @@ def nc_add_data_standard(self, points):
                 self.f.variables[name][:, ys, xs] = data
             for name in self.four_dim_vars:
                 varshape = self.f.variables[name].shape[1]
-                for i in range(varshape):
+                for i in pyrange(varshape):
                     sn = name + str(i)
                     self.f.variables[name][:, i, ys,
                                            xs] = p.df[sn].values[self.slice]
@@ -640,7 +641,7 @@ def vic2nc(options, global_atts, domain_dict, fields):
             + end_date.month - start_date.month + 1
         month = start_date.month
         year = start_date.year
-        for i in range(num_segments + 1):
+        for i in pyrange(num_segments + 1):
             segment_dates.append(datetime(year, month, 1))
             month += 1
             if month == 13:
@@ -649,13 +650,13 @@ def vic2nc(options, global_atts, domain_dict, fields):
     elif options['time_segment'] == 'year':
         num_segments = end_date.year - start_date.year + 1
         year = start_date.year
-        for i in range(num_segments + 1):
+        for i in pyrange(num_segments + 1):
             segment_dates.append(datetime(year, 1, 1))
             year += 1
     elif options['time_segment'] == 'decade':
         num_segments = (end_date.year - start_date.year) / 10 + 1
         year = start_date.year
-        for i in range(num_segments + 1):
+        for i in pyrange(num_segments + 1):
             segment_dates.append(datetime(year, 1, 1))
             year += 10
     elif options['time_segment'] == 'all':
@@ -676,7 +677,7 @@ def vic2nc(options, global_atts, domain_dict, fields):
     # Setup Segments
     segments = deque()
 
-    for num in range(num_segments):
+    for num in pyrange(num_segments):
         # Segment time bounds
         t0 = segment_dates[num]
         t1 = segment_dates[num + 1]
@@ -801,7 +802,7 @@ def vic2nc(options, global_atts, domain_dict, fields):
     # order. Note that sorted(usecols) is not strictly necessary, since
     # apparently that is done in read_table, but it keeps the names and columns
     # in the same order
-    names = [x for (y, x) in sorted(zip(usecols, names))]
+    names = [x for (y, x) in sorted(pyzip(usecols, names))]
     usecols = sorted(usecols)
     points.set_names(names)
     points.set_usecols(usecols)

tonic/pycompat.py

@@ -0,0 +1,45 @@
+import sys
+
+PY3 = sys.version_info[0] >= 3
+
+if PY3:  # pragma: no cover
+    basestring = str
+    unicode_type = str
+    bytes_type = bytes
+
+    def iteritems(d):
+        return iter(d.items())
+
+    def itervalues(d):
+        return iter(d.values())
+
+    pyrange = range
+    pyzip = zip
+    from functools import reduce as pyreduce
+    import builtins
+    from configparser import SafeConfigParser
+else:  # pragma: no cover
+    # Python 2
+    basestring = basestring
+    unicode_type = unicode
+    bytes_type = str
+
+    def iteritems(d):
+        return d.iteritems()
+
+    def itervalues(d):
+        return d.itervalues()
+
+    pyrange = xrange
+    from itertools import izip as pyzip
+    from itertools import imap as pymap
+    pyreduce = reduce
+    import __builtin__ as builtins
+    from ConfigParser import SafeConfigParser
+try:
+    from cyordereddict import OrderedDict
+except ImportError:  # pragma: no cover
+    try:
+        from collections import OrderedDict
+    except ImportError:
+        from ordereddict import OrderedDict