diff --git a/dimarray/__init__.py b/dimarray/__init__.py index efe9cda..d69bfab 100644 --- a/dimarray/__init__.py +++ b/dimarray/__init__.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,5 +7,5 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -from dimarray import DimArray,Dim -from attrarray import AttrArray +from .dimarray import DimArray, Dim +from .attrarray import AttrArray diff --git a/dimarray/attrarray.py b/dimarray/attrarray.py index c94e7b6..8002bfb 100644 --- a/dimarray/attrarray.py +++ b/dimarray/attrarray.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -16,7 +16,7 @@ HAS_H5PY = True except ImportError: HAS_H5PY = False - + ################################# # New array class with attributes @@ -38,7 +38,7 @@ class AttrArray(np.ndarray): None when no attributes are required (as is the case for instances of AttrArray) or a dictionary that specifies required attributes (for child classes of AttrArray, such as Dim and DimArray). - + Examples -------- >>> import numpy as np @@ -55,7 +55,7 @@ class AttrArray(np.ndarray): These custom attributes are maintained when copying or manipulating the data in an AttrArray: - + >>> data2 = data.mean() >>> data2.hello good bye @@ -67,7 +67,7 @@ class AttrArray(np.ndarray): # type is required, "object" should be specified. E.g., # {'name':str} or {'misc':object} _required_attrs = None - + def __new__(cls, data, dtype=None, copy=False, hdf5_group=None, **kwargs): # see if linking to hdf5 file @@ -79,7 +79,7 @@ def __new__(cls, data, dtype=None, copy=False, # else: # cls._hdf5_file = None # #cls.hdf5_group = hdf5_group - + # get the data in the proper format, copied if desired # PBS: Does this clobber the attrs? result = np.array(data, dtype=dtype, copy=copy) @@ -92,7 +92,7 @@ def __new__(cls, data, dtype=None, copy=False, # get the new attrs, kwargs has priority # see if do deep copy of attrs newattrs = {} - if hasattr(data,'_attrs'): + if hasattr(data, '_attrs'): # add those to the list of attributes if copy: newattrs = copylib.deepcopy(data._attrs) @@ -109,18 +109,16 @@ def __new__(cls, data, dtype=None, copy=False, result._chk_req_attr() return result - - - def __array_finalize__(self,obj): + + def __array_finalize__(self, obj): if not hasattr(self, '_attrs'): self._attrs = copylib.deepcopy(getattr(obj, '_attrs', {})) # Set all attributes: self._set_all_attr() # Ensure that the required attributes are present: # PBS: I don't think we need to call this here - #self._chk_req_attr() + # self._chk_req_attr() - def __setattr__(self, name, value): # Do not allow changes to _required_attrs if name == '_required_attrs': @@ -128,16 +126,16 @@ def __setattr__(self, name, value): "The attribute '_required_attrs' is read-only!") # set the value in the attribute list if self._required_attrs: - if (self._required_attrs.has_key(name) and - (not isinstance(value,self._required_attrs[name]))): - raise AttributeError("Attribute '"+name +"' must be "+ - str(self._required_attrs[name])+ - "\nSupplied value and type:\n"+ - str(value)+"\n"+str(type(value))) + if (name in self._required_attrs and + (not isinstance(value, self._required_attrs[name]))): + raise AttributeError("Attribute '" + name + "' must be " + + str(self._required_attrs[name]) + + "\nSupplied value and type:\n" + + str(value) + "\n" + str(type(value))) # save whether it already existed # must do this before the call to ndarray.__setattr__ - if hasattr(self,name): + if hasattr(self, name): attr_existed = True else: attr_existed = False @@ -147,7 +145,7 @@ def __setattr__(self, name, value): # update the attrs if necessary # CTW: shouln't _attr be always updated? - if self._attrs.has_key(name) or \ + if name in self._attrs or \ (name != '_attrs' and not attr_existed): self._attrs[name] = value @@ -158,11 +156,11 @@ def __delattr__(self, name): if name == '_required_attrs': raise AttributeError( "The attribute '_required_attrs' is read-only!") - if name in self._required_attrs.keys(): - raise AttributeError("Attribute '"+name +"' is required, and cannot "+ + if name in list(self._required_attrs.keys()): + raise AttributeError("Attribute '" + name + "' is required, and cannot " + "be deleted!") ret = np.ndarray.__delattr__(self, name) - if self._attrs.has_key(name): + if name in self._attrs: del self._attrs[name] return ret @@ -179,14 +177,15 @@ def _chk_req_attr(self): Make sure the required attributes are set """ # if there are no required attributes, no check is required: - if self._required_attrs is None: return - - for name in self._required_attrs.keys(): - if ((not self._attrs.has_key(name)) or - (not isinstance(self._attrs[name], self._required_attrs[name]))): - raise AttributeError("Attribute '"+name+"' is required, and "+ - "must be "+str(self._required_attrs[name])) - + if self._required_attrs is None: + return + + for name in list(self._required_attrs.keys()): + if ((name not in self._attrs) or + (not isinstance(self._attrs[name], self._required_attrs[name]))): + raise AttributeError("Attribute '" + name + "' is required, and " + + "must be " + str(self._required_attrs[name])) + # def __repr__(self): # # make the attribute kwargs list @@ -199,7 +198,7 @@ def _chk_req_attr(self): # else: # retrepr = "AttrArray(%s)" % \ # (np.ndarray.__repr__(self)) - + # return retrepr def __reduce__(self): @@ -208,17 +207,17 @@ def __reduce__(self): # append the custom object attributes subclass_state = (self._attrs,) - object_state[2] = (object_state[2],subclass_state) + object_state[2] = (object_state[2], subclass_state) # convert back to tuple and return return tuple(object_state) - - def __setstate__(self,state): + + def __setstate__(self, state): # get the ndarray state and the subclass state nd_state, own_state = state # refresh the ndarray state - np.ndarray.__setstate__(self,nd_state) + np.ndarray.__setstate__(self, nd_state) # get the subclass attributes attrs, = own_state @@ -249,7 +248,7 @@ def h5save(self, filename, group=None, mode='w', **kwargs): # see if already exists grp_name = '' for name in os.path.split(group): - grp_name = '/'.join([grp_name,name]) + grp_name = '/'.join([grp_name, name]) if grp_name in f: grp = f[grp_name] else: @@ -257,7 +256,7 @@ def h5save(self, filename, group=None, mode='w', **kwargs): # grp now has the group where we're going to put the new group # for this AttrArray - + pass def nanvar(a, axis=None, ddof=0): @@ -268,7 +267,7 @@ def nanvar(a, axis=None, ddof=0): spread of a distribution, treating nans as missing values. The variance is computed for the flattened array by default, otherwise over the specified axis. - + Parameters ---------- a : array_like @@ -281,7 +280,7 @@ def nanvar(a, axis=None, ddof=0): calculations is ``N - ddof``, where ``N`` represents the number of elements. By default `ddof` is zero (biased estimate). - + Returns ------- variance : {ndarray, scalar} @@ -301,7 +300,7 @@ def nanvar(a, axis=None, ddof=0): If no nan values are present, returns the same value as numpy.var, otherwise, the variance is calculated as if the nan values were not present. - + The variance is the average of the squared deviations from the mean, i.e., var = mean(abs(x - x.mean())**2). The mean is normally calculated as ``x.sum() / N``, where ``N = len(x)``. @@ -328,7 +327,7 @@ def nanvar(a, axis=None, ddof=0): >>> a.nanvar(1) AttrArray([ 0.0, 0.25, 0.25]) """ - + if axis is None: return a[~np.isnan(a)].var(ddof=ddof) @@ -339,30 +338,30 @@ def nanvar(a, axis=None, ddof=0): n_orig = a.shape[axis] # number of nans: - n_nan = np.float64(np.sum(np.isnan(a),axis)) + n_nan = np.float64(np.sum(np.isnan(a), axis)) # number of non-nan values: n = n_orig - n_nan # compute the mean for all non-nan values: a[np.isnan(a)] = 0. - m1 = np.sum(a,axis)/n + m1 = np.sum(a, axis) / n # Kludge to subtract m1 from the correct axis - if axis!=0: + if axis != 0: shape = np.arange(a.ndim).tolist() shape.remove(axis) - shape.insert(0,axis) + shape.insert(0, axis) a = a.transpose(tuple(shape)) - d = (a-m1)**2.0 + d = (a - m1)**2.0 shape = tuple(np.array(shape).argsort()) d = d.transpose(shape) else: - d = (a-m1)**2.0 + d = (a - m1)**2.0 # calculate numerator for variance: - m2 = np.float64(np.sum(d,axis)-(m1*m1)*n_nan) - + m2 = np.float64(np.sum(d, axis) - (m1 * m1) * n_nan) + # devide by appropriate denominator: m2c = m2 / (n - ddof) return(m2c) @@ -376,7 +375,7 @@ def nanstd(a, axis=None, ddof=0): distribution, of the array elements, treating nans as missing values. The standard deviation is computed for the flattened array by default, otherwise over the specified axis. - + Parameters ---------- a : array_like @@ -390,7 +389,7 @@ def nanstd(a, axis=None, ddof=0): calculations is ``N - ddof``, where ``N`` represents the number of elements. By default `ddof` is zero (biased estimate). - + Returns ------- standard_deviation : {ndarray, scalar} @@ -410,7 +409,7 @@ def nanstd(a, axis=None, ddof=0): If no nan values are present, returns the same value as numpy.std, otherwise, the standard deviation is calculated as if the nan values were not present. - + The standard deviation is the square root of the average of the squared deviations from the mean, i.e., ``var = sqrt(mean(abs(x - x.mean())**2))``. @@ -438,8 +437,8 @@ def nanstd(a, axis=None, ddof=0): AttrArray([ 1., 1.6329931618554521]) >>> a.nanstd(1) AttrArray([ 0.0, 0.5, 0.5]) - """ - return np.sqrt(a.nanvar(axis,ddof)) + """ + return np.sqrt(a.nanvar(axis, ddof)) def nanmean(a, axis=None): """ @@ -458,7 +457,7 @@ def nanmean(a, axis=None): axis : int, optional Axis along which the means are computed. The default is to compute the mean of the flattened array. - + Returns ------- mean : {ndarray, scalar} @@ -492,7 +491,7 @@ def nanmean(a, axis=None): >>> a.nanmean(1) AttrArray([ 2.0, 3.5, 5.5]) """ - + if axis is None: return a[~np.isnan(a)].mean() @@ -502,8 +501,7 @@ def nanmean(a, axis=None): # number of all observations n_orig = a.shape[axis] - factor = 1.0-np.sum(np.isnan(a),axis)*1.0/n_orig + factor = 1.0 - np.sum(np.isnan(a), axis) * 1.0 / n_orig a[np.isnan(a)] = 0 - - return(np.mean(a,axis)/factor) + return(np.mean(a, axis) / factor) diff --git a/dimarray/dimarray.py b/dimarray/dimarray.py index 6ee3f6f..2a883eb 100644 --- a/dimarray/dimarray.py +++ b/dimarray/dimarray.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,12 +11,13 @@ import copy as copylib import re -from attrarray import AttrArray +from .attrarray import AttrArray ############################### # New dimensioned array class ############################### + class Dim(AttrArray): """ Dim(data, name, dtype=None, copy=False, **kwargs) @@ -53,20 +54,20 @@ class Dim(AttrArray): >>> print test [1 2 3] """ - _required_attrs = {'name':str} - + _required_attrs = {'name': str} + def __new__(cls, data, name=None, dtype=None, copy=False, **kwargs): if name is None: # if 'name' is not specified see if data already has a # name attribute: - name = getattr(data,'name',None) + name = getattr(data, 'name', None) if name is None: raise AttributeError("A 'name' attribute must be specified!") # set the kwargs to have name kwargs['name'] = name # make new AttrArray: - dim = AttrArray(data,dtype=dtype,copy=copy,**kwargs) + dim = AttrArray(data, dtype=dtype, copy=copy, **kwargs) # check the dim if dim.ndim > 1: @@ -79,18 +80,19 @@ def __new__(cls, data, name=None, dtype=None, copy=False, **kwargs): elif ndim == 0: dim.shape = (1,) else: - raise ValueError("Dim instances must be 1-dimensional!\ndim:\n"+ - str(dim)+"\nnewshape:",newshape) + raise ValueError("Dim instances must be 1-dimensional!\ndim:\n" + + str(dim) + "\nnewshape:", newshape) # if the array is 0-D, make it 1-D: elif dim.ndim == 0: dim.shape = (1,) - - #if dim.shape[0] != np.unique(np.asarray(dim)).shape[0]: + + # if dim.shape[0] != np.unique(np.asarray(dim)).shape[0]: # raise ValueError("Data for Dim objects must be unique!") # convert to Dim and return: return dim.view(cls) + class DimIndex(tuple): """ Tuple representing a fancy index of a Dim along with its siblings. @@ -99,43 +101,44 @@ def __new__(typ, ind, bool_ind): res = tuple.__new__(typ, ind) res._bool_ind = bool_ind return res - + def __and__(self, other): # compare each bool # check other is DimIndex ind = [] - for l,r in zip(self._bool_ind,other._bool_ind): - ind.append(l&r) - return DimIndex(np.ix_(*ind),ind) + for l, r in zip(self._bool_ind, other._bool_ind): + ind.append(l & r) + return DimIndex(np.ix_(*ind), ind) def __or__(self, other): # compare each bool # check other is DimIndex ind = [] - for l,r in zip(self._bool_ind,other._bool_ind): - ind.append(l|r) - return DimIndex(np.ix_(*ind),ind) + for l, r in zip(self._bool_ind, other._bool_ind): + ind.append(l | r) + return DimIndex(np.ix_(*ind), ind) def __xor__(self, other): # compare each bool # check other is DimIndex ind = [] - for l,r in zip(self._bool_ind,other._bool_ind): - ind.append(l^r) - return DimIndex(np.ix_(*ind),ind) + for l, r in zip(self._bool_ind, other._bool_ind): + ind.append(l ^ r) + return DimIndex(np.ix_(*ind), ind) + class DimSelect(Dim): """ Dim that supports boolean comparisons for fancy indexing. """ - _required_attrs = {'name':str, - '_parent_dim_shapes':list, - '_parent_dim_index':int} - + _required_attrs = {'name': str, + '_parent_dim_shapes': list, + '_parent_dim_index': int} + def __new__(cls, dim, parent_dim_shapes, parent_dim_index): # verify that dim is a Dim instance - + # set the kwargs to have what we need kwargs = {} kwargs['name'] = dim.name @@ -150,73 +153,81 @@ def __new__(cls, dim, parent_dim_shapes, parent_dim_index): def __lt__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) < other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def __le__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) <= other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def __gt__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) > other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def __ge__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) >= other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def __eq__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) == other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def __ne__(self, other): # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension ind[self._parent_dim_index] = np.asarray(self) != other # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def is_in(self, vals): """ Elementwise boolean check for membership in a list. """ # get starting indicies - ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] + ind = [np.ones(shape, dtype=np.bool) + for shape in self._parent_dim_shapes] # do the comparison along the desired dimension - ind[self._parent_dim_index] = np.lib.arraysetops.in1d(np.asarray(self),vals) + ind[self._parent_dim_index] = np.lib.arraysetops.in1d( + np.asarray(self), vals) # i = self._parent_dim_index # self_array = np.asarray(self) # ind[i] = False @@ -224,14 +235,14 @@ def is_in(self, vals): # ind[i] = ind[i] | (self_array == val) # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) def index(self, index): """ Index into elements along the dimension. """ # get starting indicies - ind = [np.ones(shape, dtype=np.bool) + ind = [np.ones(shape, dtype=np.bool) for shape in self._parent_dim_shapes] # make sure 1d @@ -246,13 +257,13 @@ def index(self, index): ind[self._parent_dim_index][index] = True # create the final master index from the list of filtered indices - return DimIndex(np.ix_(*ind),ind) + return DimIndex(np.ix_(*ind), ind) class DimArray(AttrArray): """ DimArray(data, dims=None, dtype=None, copy=False, **kwargs) - + A DimArray (short for Dimensioned Array) is a child class of AttrArray with the constraints that each instance and have a dims attribute which specifies the dimensions as an array of Dim @@ -265,7 +276,7 @@ class DimArray(AttrArray): beyond normal ndarrays. These include the ability to refer to dimensions by name and to select subsets of the data based on complex queries using the dimension names. - + Parameters ---------- data : array_like @@ -288,10 +299,10 @@ class DimArray(AttrArray): >>> data = da.DimArray(np.random.rand(5,3),[dim1,dim2]) >>> data DimArray([[ 0.59645979, 0.92462876, 0.76882167], - [ 0.3581822 , 0.57873905, 0.76889117], - [ 0.40272846, 0.69372032, 0.59006832], - [ 0.69862889, 0.68334188, 0.10891802], - [ 0.14111733, 0.97454223, 0.73193147]]) + [ 0.3581822 , 0.57873905, 0.76889117], + [ 0.40272846, 0.69372032, 0.59006832], + [ 0.69862889, 0.68334188, 0.10891802], + [ 0.14111733, 0.97454223, 0.73193147]]) >>> data.dims array([[0 1 2 3 4], ['a' 'b' 'c']], dtype=object) >>> data['Dim1'] @@ -335,10 +346,10 @@ class DimArray(AttrArray): >>> data.dims array([[0 1 2 3], [0 1 2 3 4]], dtype=object) """ - _required_attrs = {'dims':np.ndarray} + _required_attrs = {'dims': np.ndarray} dim_names = property(lambda self: - [dim.name for dim in self.dims], - doc=""" + [dim.name for dim in self.dims], + doc=""" List of dimension names This is a property that is created from the dims @@ -356,43 +367,44 @@ class DimArray(AttrArray): ['First dimension', 'Second dimension'] """) _dim_namesRE = property(lambda self: - re.compile('(?=i]+=1 + tokeep[tokeep >= i] += 1 tokeep = tokeep.tolist() - tokeep.insert(i+1,i) + tokeep.insert(i + 1, i) tokeep = np.array(tokeep) - elif not isinstance(ind, slice) and len(ind)==0: + elif not isinstance(ind, slice) and len(ind) == 0: # handle where we selected nothing ret.dims[i] = ret.dims[i][[]] else: # slice the dims based on the index - #if isinstance(ind,np.ndarray) and ind.dtype==bool: + # if isinstance(ind,np.ndarray) and ind.dtype==bool: # if len(ind.shape)>1: # ret = ret.view(AttrArray) if not isinstance(ind, slice): # squeeze it to maintain dimensionality ind = np.asanyarray(ind) - tosqueeze = [0]*len(ind.shape) + tosqueeze = [0] * len(ind.shape) tosqueeze[i] = slice(None) ind = ind[tuple(tosqueeze)] # if a boolean array is given as an index the # dimensions get lost, so we need to cast to # an AttrArray if there's more than 1 dim: - if isinstance(ind,np.ndarray) and ind.dtype==bool: - if len(self.shape)>1: + if isinstance(ind, np.ndarray) and ind.dtype == bool: + if len(self.shape) > 1: ret = ret.view(AttrArray) ret.dims[i] = ret.dims[i][ind] @@ -658,27 +671,27 @@ def __getitem__(self, index): # remove the specified dims from the main array if np.any(remove_dim): - ind = np.asarray([slice(None)]*len(ret.shape)) + ind = np.asarray([slice(None)] * len(ret.shape)) ind[remove_dim] = 0 ind = tuple(ind) ret = ret[ind] return ret - - def __getslice__(self,i,j): - try: + + def __getslice__(self, i, j): + try: # skip the dim check b/c we're going to fiddle with them self._skip_dim_check = True - ret = AttrArray.__getslice__(self,i,j) + ret = AttrArray.__getslice__(self, i, j) finally: # reset the _skip_dim_check flag: self._skip_dim_check = False - ret.dims[0] = ret.dims[0].__getslice__(i,j) + ret.dims[0] = ret.dims[0].__getslice__(i, j) return ret - - def find(self,*args,**kwargs): + + def find(self, *args, **kwargs): """ - Returns a tuple of index arrays for the selected conditions. + Returns a tuple of index arrays for the selected conditions. There are three different ways to specify a filterstring illustrated in the examples. @@ -721,10 +734,10 @@ def find(self,*args,**kwargs): DimArray([[ True], [ True]], dtype=bool) """ - m_ind,ind,remove_dim = self._select_ind(*args,**kwargs) + m_ind, ind, remove_dim = self._select_ind(*args, **kwargs) return m_ind - def select(self,*args,**kwargs): + def select(self, *args, **kwargs): """ Returns a slice of the data filtered with the select conditions. @@ -775,7 +788,7 @@ def select(self,*args,**kwargs): DimArray([[ True], [ True]], dtype=bool) """ - m_ind,ind,remove_dim = self._select_ind(*args,**kwargs) + m_ind, ind, remove_dim = self._select_ind(*args, **kwargs) return self[m_ind] def _split_bins(self, dim, bins, function, bin_labels, @@ -792,10 +805,10 @@ def _split_bins(self, dim, bins, function, bin_labels, split = np.array_split # Create the new dimension: - split_dim = split(self.dims[dim],bins) + split_dim = split(self.dims[dim], bins) if bin_labels == 'function': - new_dim_dat = np.array([function(x,**kwargs) for x in split_dim]) - new_dim = Dim(new_dim_dat,self.dim_names[dim]) + new_dim_dat = np.array([function(x, **kwargs) for x in split_dim]) + new_dim = Dim(new_dim_dat, self.dim_names[dim]) elif bin_labels == 'sequential': new_dim = Dim(np.arange(len(split_dim)), self.dim_names[dim]) @@ -811,24 +824,25 @@ def _split_bins(self, dim, bins, function, bin_labels, str(bin_labels)) # Create the new data: - split_dat = split(self.view(AttrArray),bins,axis=dim) - new_dat = np.array([function(x,axis=dim,**kwargs) for x in split_dat]) - + split_dat = split(self.view(AttrArray), bins, axis=dim) + new_dat = np.array([function(x, axis=dim, **kwargs) + for x in split_dat]) + # Now the dimensions of the array need be re-arranged in the correct # order: dim_order = np.arange(len(new_dat.shape)) dim_order[dim] = 0 - dim_order[0:dim] = np.arange(1,dim+1) - dim_order[dim+1:len(new_dat.shape)] = np.arange(dim+1, - len(new_dat.shape)) + dim_order[0:dim] = np.arange(1, dim + 1) + dim_order[dim + 1:len(new_dat.shape)] = np.arange(dim + 1, + len(new_dat.shape)) new_dat = new_dat.transpose(dim_order) - + # Create and return new DimArray object: new_dims = copylib.deepcopy(self.dims) new_dims[dim] = new_dim new_attrs = self._attrs.copy() new_attrs['dims'] = new_dims - return self.__class__(new_dat,**new_attrs) + return self.__class__(new_dat, **new_attrs) def _select_bins(self, dim, bins, function, bin_labels, error_on_nonexact, **kwargs): @@ -837,12 +851,12 @@ def _select_bins(self, dim, bins, function, bin_labels, a list of intervals. See make_bins method. """ # Create the new dimension: - dimbin_indx = np.array([((self.dims[dim]>=x[0]) & - (self.dims[dim]= x[0]) & + (self.dims[dim] < x[1])) for x in bins]) if np.shape(bins[-1])[-1] == 3: new_dim_dat = np.array([x[2] for x in bins]) elif bin_labels == 'function': - new_dim_dat = np.array([function(x,**kwargs) for x in + new_dim_dat = np.array([function(x, **kwargs) for x in [self.dims[dim][indx] for indx in dimbin_indx]]) elif bin_labels == 'sequential': @@ -856,47 +870,46 @@ def _select_bins(self, dim, bins, function, bin_labels, " list/array of labels of the same length as " + "bins.\n bins: " + str(bins) + "\n bin_labels: " + str(bin_labels)) - - new_dim = Dim(data=new_dim_dat,name=self.dim_names[dim]) - + + new_dim = Dim(data=new_dim_dat, name=self.dim_names[dim]) + # Create the new data: # We need to transpose the data array so that dim is the first # dimension. We store the new order of dimensions in totrans: - totrans = range(len(self.shape)) + totrans = list(range(len(self.shape))) totrans[0] = dim totrans[dim] = 0 - + # Now we are ready to do the transpose: tmpdata = self.copy() - tmpdata = np.transpose(tmpdata.view(np.ndarray),totrans) - + tmpdata = np.transpose(tmpdata.view(np.ndarray), totrans) + # Now loop through each bin applying the function and concatenate the # data: new_dat = None - for b,bin in enumerate(bins): - bindata = function(tmpdata[dimbin_indx[b]],axis=0,**kwargs) + for b, bin in enumerate(bins): + bindata = function(tmpdata[dimbin_indx[b]], axis=0, **kwargs) if new_dat is None: - new_dat = bindata[np.newaxis,:] + new_dat = bindata[np.newaxis, :] else: - new_dat = np.r_[new_dat,bindata[np.newaxis,:]] - + new_dat = np.r_[new_dat, bindata[np.newaxis, :]] + # transpose back: new_dat = new_dat.transpose(totrans) - + # Create and return new DimArray object: new_dims = copylib.deepcopy(self.dims) new_dims[dim] = new_dim new_attrs = self._attrs.copy() new_attrs['dims'] = new_dims - return self.__class__(new_dat,**new_attrs) + return self.__class__(new_dat, **new_attrs) - - def make_bins(self,axis,bins,function,bin_labels='function', - error_on_nonexact=True,**kwargs): + def make_bins(self, axis, bins, function, bin_labels='function', + error_on_nonexact=True, **kwargs): """ Return a copy of the data with dimension (specified by axis) binned as specified. - + Parameters ---------- axis : int @@ -932,16 +945,16 @@ def make_bins(self,axis,bins,function,bin_labels='function', number of bins (this parameter is only applicable when bins is an integer specifying the number of bins). When True, the function numpy.split is used, when False the - function numpy.array_split is used. + function numpy.array_split is used. kwargs : keyword arguments, optional Optional key word arguments to be passed on to function. - + Returns ------- binned : DimArray A new DimArray instance in which one of the dimensions is binned as specified. - + Examples -------- >>> import numpy as np @@ -966,38 +979,38 @@ def make_bins(self,axis,bins,function,bin_labels='function', # Makes sure dim is index (convert dim name if necessary): dim = self.get_axis(axis) tmp_bins = np.atleast_2d(bins) - if len(tmp_bins.shape)>2: + if len(tmp_bins.shape) > 2: raise ValueError("Invalid bins! Acceptable values are: number of" + " bins, 1-D container of index values, 2-D " + "container of min and max values and (optionally)" + - " a label for each bin. Provided bins: "+str(bins)) + " a label for each bin. Provided bins: " + str(bins)) if np.atleast_2d(bins).shape[1] == 1: - return self._split_bins(dim,bins,function,bin_labels, - error_on_nonexact,**kwargs) + return self._split_bins(dim, bins, function, bin_labels, + error_on_nonexact, **kwargs) elif np.atleast_2d(bins).shape[1] == 2: if not error_on_nonexact: raise ValueError("When bins are explicitly specified, " + "error_on_nonexact must be True. Provided " + "value: " + str(error_on_nonexact)) - return self._select_bins(dim,bins,function,bin_labels, - error_on_nonexact,**kwargs) + return self._select_bins(dim, bins, function, bin_labels, + error_on_nonexact, **kwargs) elif np.atleast_2d(bins).shape[1] == 3: if bin_labels != 'function': raise ValueError("Simultaneously specification of bin labels " + "in bins and bin_labels is not allowed. " + - "Provided bins: " + str(bins)+" Provided " + + "Provided bins: " + str(bins) + " Provided " + "bin_labels: " + str(bin_labels)) if not error_on_nonexact: raise ValueError("When bins are explicitly specified, " + - "error_on_nonexact must be True. Provided " + - "value: " + str(error_on_nonexact)) - return self._select_bins(dim,bins,function,bin_labels, - error_on_nonexact,**kwargs) + "error_on_nonexact must be True. Provided " + + "value: " + str(error_on_nonexact)) + return self._select_bins(dim, bins, function, bin_labels, + error_on_nonexact, **kwargs) else: raise ValueError("Invalid bins! Acceptable values are: number of" + " bins, 1-D container of index values, 2-D " + "container of min and max values and (optionally)" + - " a label for each bin. Provided bins: "+str(bins)) + " a label for each bin. Provided bins: " + str(bins)) def extend(self, data, axis=0): """ @@ -1024,7 +1037,7 @@ def extend(self, data, axis=0): the exception of the updated dimension). """ # make sure we have a list - if isinstance(data,DimArray): + if isinstance(data, DimArray): data = [data] else: data = list(data) @@ -1033,17 +1046,18 @@ def extend(self, data, axis=0): axis = self.get_axis(axis) # make sure all dim_names match: - dim_names_deviations = [np.sum(d.dim_names!=self.dim_names) for d in data] - if np.sum(dim_names_deviations)>0: + dim_names_deviations = [ + np.sum(d.dim_names != self.dim_names) for d in data] + if np.sum(dim_names_deviations) > 0: raise ValueError('Names of the dimensions do not match!') - + # make sure all dims except for the extended one match: - dim_deviations = [np.sum(d.dims!=self.dims) for d in data] - if np.sum(dim_deviations)>1: + dim_deviations = [np.sum(d.dims != self.dims) for d in data] + if np.sum(dim_deviations) > 1: raise ValueError('Dimensions do not match!') - + # add the current DimArray to the beginning of list: - data.insert(0,self) + data.insert(0, self) # list of dims to be concatenated: conc_dims = [d.dims[axis] for d in data] @@ -1053,13 +1067,13 @@ def extend(self, data, axis=0): # functions rather than the DimArray functions): data = [d.view(AttrArray) for d in data] #dim_names = [d.name for dat in data for d in dat.dims] - - new_dat = np.concatenate(data,axis=axis) + + new_dat = np.concatenate(data, axis=axis) new_dims = copylib.deepcopy(self.dims) - new_dims[axis] = Dim(np.concatenate(conc_dims),self.dim_names[axis]) + new_dims[axis] = Dim(np.concatenate(conc_dims), self.dim_names[axis]) new_attrs = self._attrs.copy() new_attrs['dims'] = new_dims - return self.__class__(new_dat,**new_attrs) + return self.__class__(new_dat, **new_attrs) # # create new array: # result = np.concatenate(data,axis=axis).view(AttrArray) @@ -1067,7 +1081,7 @@ def extend(self, data, axis=0): # result.__array_finalize__(self) # # update dims & return: # result.dims[axis] = Dim(np.concatenate(conc_dims),self.dim_names[axis]) - # return result.view(self.__class__) + # return result.view(self.__class__) def add_dim(self, dim): """ @@ -1075,13 +1089,13 @@ def add_dim(self, dim): each value of the new dim. """ # add the axis and repeat the data - ret = self[np.newaxis].repeat(len(dim),axis=0) + ret = self[np.newaxis].repeat(len(dim), axis=0) # now replace the dim ret.dims[0] = dim # return as desired class return ret.view(self.__class__) - def get_axis(self,axis): + def get_axis(self, axis): """ Get the axis number for a dimension name. @@ -1096,13 +1110,13 @@ def get_axis(self,axis): ---------- axis : str The name of a dimension. - + Returns ------- The axis number corresponding to the dimension name. - If axis is not a string, it is returned unchanged. + If axis is not a string, it is returned unchanged. """ - if isinstance(axis,str): + if isinstance(axis, str): # must convert to index dim axis = self.dim_names.index(axis) return axis @@ -1129,7 +1143,7 @@ def get_dim_name(self, axis): else: dim_name = self.dim_names[axis] return dim_name - + def _ret_func(self, ret, axis): """ Return function output for functions that take an axis @@ -1140,34 +1154,33 @@ def _ret_func(self, ret, axis): return ret.view(AttrArray) else: # pop the dim - ret.dims = ret.dims[np.arange(len(ret.dims))!=axis] + ret.dims = ret.dims[np.arange(len(ret.dims)) != axis] return ret.view(self.__class__) - - + def all(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).all(axis=axis, out=out) - return self._ret_func(ret,axis) - + return self._ret_func(ret, axis) + def any(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).any(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def argmax(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).argmax(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def argmin(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).argmin(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def argsort(self, axis=-1, kind='quicksort', order=None): if axis is None: return self.view(AttrArray).argsort(axis=axis, kind=kind, - order=order) + order=order) else: axis = self.get_axis(axis) ret = self.view(AttrArray).argsort(axis=axis, kind=kind, @@ -1187,7 +1200,7 @@ def compress(self, condition, axis=None, out=None): def cumprod(self, axis=None, dtype=None, out=None): if axis is None: return self.view(AttrArray).cumprod(axis=axis, dtype=dtype, - out=out) + out=out) else: axis = self.get_axis(axis) ret = self.view(AttrArray).cumprod(axis=axis, dtype=dtype, out=out) @@ -1196,7 +1209,7 @@ def cumprod(self, axis=None, dtype=None, out=None): def cumsum(self, axis=None, dtype=None, out=None): if axis is None: return self.view(AttrArray).cumsum(axis=axis, dtype=dtype, - out=out) + out=out) else: axis = self.get_axis(axis) ret = self.view(AttrArray).cumsum(axis=axis, dtype=dtype, out=out) @@ -1211,32 +1224,32 @@ def flatten(self, *args, **kwargs): def max(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).max(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def mean(self, axis=None, dtype=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).mean(axis=axis, dtype=dtype, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def min(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).min(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def nanmean(self, axis=None): axis = self.get_axis(axis) ret = self.view(AttrArray).nanmean(axis=axis) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def nanstd(self, axis=None, ddof=0): axis = self.get_axis(axis) ret = self.view(AttrArray).nanstd(axis=axis, ddof=0) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def nanvar(self, axis=None, ddof=0): axis = self.get_axis(axis) ret = self.view(AttrArray).nanvar(axis=axis, ddof=0) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def nonzero(self, *args, **kwargs): return self.view(AttrArray).nonzero(*args, **kwargs) @@ -1244,12 +1257,12 @@ def nonzero(self, *args, **kwargs): def prod(self, axis=None, dtype=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).prod(axis=axis, dtype=dtype, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def ptp(self, axis=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).ptp(axis=axis, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def ravel(self, *args, **kwargs): return self.view(AttrArray).ravel(*args, **kwargs) @@ -1259,7 +1272,7 @@ def repeat(self, repeats, axis=None): return self.view(AttrArray).repeat(repeats, axis=axis) def reshape(self, shape, order='C'): - return np.reshape(self.view(AttrArray),shape,order) + return np.reshape(self.view(AttrArray), shape, order) def resize(self, *args, **kwargs): """Resizing is not possible for dimensioned arrays. Calling @@ -1271,7 +1284,7 @@ def resize(self, *args, **kwargs): def sort(self, axis=-1, kind='quicksort', order=None): if axis is None: - raise ValueError("Please specify an axis! To sort a flattened "+ + raise ValueError("Please specify an axis! To sort a flattened " + "array convert to (e.g.) numpy.ndarray.") axis = self.get_axis(axis) self.view(AttrArray).sort(axis=axis, kind=kind, order=order) @@ -1283,8 +1296,8 @@ def squeeze(self): d = 0 for dms in ret.dims: if len(ret.dims[d]) == 1: - #ret.dims.pop(d) - ret.dims = ret.dims[np.arange(len(ret.dims))!=d] + # ret.dims.pop(d) + ret.dims = ret.dims[np.arange(len(ret.dims)) != d] else: d += 1 return ret.view(self.__class__) @@ -1292,17 +1305,17 @@ def squeeze(self): def std(self, axis=None, dtype=None, out=None, ddof=0): axis = self.get_axis(axis) ret = self.view(AttrArray).std(axis=axis, dtype=dtype, out=out, ddof=0) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def sum(self, axis=None, dtype=None, out=None): axis = self.get_axis(axis) ret = self.view(AttrArray).sum(axis=axis, dtype=dtype, out=out) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) def swapaxes(self, axis1, axis2): axis1 = self.get_axis(axis1) axis2 = self.get_axis(axis2) - ret = self.view(AttrArray).swapaxes(axis1,axis2) + ret = self.view(AttrArray).swapaxes(axis1, axis2) tmp = ret.dims[axis1] ret.dims[axis1] = ret.dims[axis2] ret.dims[axis2] = tmp @@ -1319,21 +1332,21 @@ def take(self, indices, axis=None, out=None, mode='raise'): ret.dims[axis] = ret.dims[axis].take(indices, axis=0, out=out, mode=mode) return ret.view(self.__class__) - + def trace(self, *args, **kwargs): return self.view(AttrArray).trace(*args, **kwargs) def transpose(self, *axes): axes = np.squeeze(axes) # PBS (I think this was wrong):if len(axes.shape)==len(self): - if(len(np.shape(axes))>0): # needs to be evaluated separately + if(len(np.shape(axes)) > 0): # needs to be evaluated separately # b/c len(axes) won't work on None if(len(axes) == self.ndim): axes = [self.get_axis(a) for a in axes] ret = self.view(AttrArray).transpose(*axes) # ret.dims = [ret.dims[a] for a in axes] ret.dims = ret.dims[axes] - return ret.view(self.__class__) + return ret.view(self.__class__) ret = self.view(AttrArray).transpose() # ret.dims.reverse() ret.dims = ret.dims[-1::-1] @@ -1343,73 +1356,74 @@ def var(self, axis=None, dtype=None, out=None, ddof=0): axis = self.get_axis(axis) ret = self.view(AttrArray).var(axis=axis, dtype=dtype, out=out, ddof=ddof) - return self._ret_func(ret,axis) + return self._ret_func(ret, axis) # set the doc strings + # Methods that return DimArrays and take an axis argument: axis_msg =\ -""" + """ **Below is the docstring from numpy.ndarray.** **For DimArray instances, the axis may be specified as string (dimension name).** """ axis_msg_aa =\ -""" + """ **Below is the docstring from AttrArray.** **For DimArray instances, the axis may be specified as string (dimension name).** """ axes_msg =\ -""" + """ **Below is the docstring from numpy.ndarray.** **The axes may be specified as strings (dimension names).** """ -DimArray.all.im_func.func_doc = axis_msg+np.ndarray.all.__doc__ -DimArray.any.im_func.func_doc = axis_msg+np.ndarray.any.__doc__ -DimArray.argmax.im_func.func_doc = axis_msg+np.ndarray.argmax.__doc__ -DimArray.argmin.im_func.func_doc = axis_msg+np.ndarray.argmin.__doc__ -DimArray.argsort.im_func.func_doc = axis_msg+np.ndarray.argsort.__doc__ -DimArray.compress.im_func.func_doc = axis_msg+np.ndarray.compress.__doc__ -DimArray.cumprod.im_func.func_doc = axis_msg+np.ndarray.cumprod.__doc__ -DimArray.cumsum.im_func.func_doc = axis_msg+np.ndarray.cumsum.__doc__ -DimArray.max.im_func.func_doc = axis_msg+np.ndarray.max.__doc__ -DimArray.mean.im_func.func_doc = axis_msg+np.ndarray.mean.__doc__ -DimArray.min.im_func.func_doc = axis_msg+np.ndarray.min.__doc__ -DimArray.nanmean.im_func.func_doc = axis_msg_aa+AttrArray.nanmean.__doc__ -DimArray.nanstd.im_func.func_doc = axis_msg_aa+AttrArray.nanstd.__doc__ -DimArray.nanvar.im_func.func_doc = axis_msg_aa+AttrArray.nanvar.__doc__ -DimArray.prod.im_func.func_doc = axis_msg+np.ndarray.prod.__doc__ -DimArray.ptp.im_func.func_doc = axis_msg+np.ndarray.ptp.__doc__ -DimArray.sort.im_func.func_doc = axis_msg+np.ndarray.sort.__doc__ -DimArray.std.im_func.func_doc = axis_msg+np.ndarray.std.__doc__ -DimArray.sum.im_func.func_doc = axis_msg+np.ndarray.sum.__doc__ -DimArray.swapaxes.im_func.func_doc = axes_msg+np.ndarray.swapaxes.__doc__ -DimArray.take.im_func.func_doc = axis_msg+np.ndarray.take.__doc__ -DimArray.transpose.im_func.func_doc = axes_msg+np.ndarray.transpose.__doc__ -DimArray.var.im_func.func_doc = axis_msg+np.ndarray.var.__doc__ +DimArray.all.__doc__ = axis_msg + np.ndarray.all.__doc__ +DimArray.any.__doc__ = axis_msg + np.ndarray.any.__doc__ +DimArray.argmax.__doc__ = axis_msg + np.ndarray.argmax.__doc__ +DimArray.argmin.__doc__ = axis_msg + np.ndarray.argmin.__doc__ +DimArray.argsort.__doc__ = axis_msg + np.ndarray.argsort.__doc__ +DimArray.compress.__doc__ = axis_msg + np.ndarray.compress.__doc__ +DimArray.cumprod.__doc__ = axis_msg + np.ndarray.cumprod.__doc__ +DimArray.cumsum.__doc__ = axis_msg + np.ndarray.cumsum.__doc__ +DimArray.max.__doc__ = axis_msg + np.ndarray.max.__doc__ +DimArray.mean.__doc__ = axis_msg + np.ndarray.mean.__doc__ +DimArray.min.__doc__ = axis_msg + np.ndarray.min.__doc__ +DimArray.nanmean.__doc__ = axis_msg_aa + AttrArray.nanmean.__doc__ +DimArray.nanstd.__doc__ = axis_msg_aa + AttrArray.nanstd.__doc__ +DimArray.nanvar.__doc__ = axis_msg_aa + AttrArray.nanvar.__doc__ +DimArray.prod.__doc__ = axis_msg + np.ndarray.prod.__doc__ +DimArray.ptp.__doc__ = axis_msg + np.ndarray.ptp.__doc__ +DimArray.sort.__doc__ = axis_msg + np.ndarray.sort.__doc__ +DimArray.std.__doc__ = axis_msg + np.ndarray.std.__doc__ +DimArray.sum.__doc__ = axis_msg + np.ndarray.sum.__doc__ +DimArray.swapaxes.__doc__ = axes_msg + np.ndarray.swapaxes.__doc__ +DimArray.take.__doc__ = axis_msg + np.ndarray.take.__doc__ +DimArray.transpose.__doc__ = axes_msg + np.ndarray.transpose.__doc__ +DimArray.var.__doc__ = axis_msg + np.ndarray.var.__doc__ # Methods that return DimArrays and do not take an axis argument: -DimArray.nonzero.im_func.func_doc = np.ndarray.nonzero.__doc__ -DimArray.squeeze.im_func.func_doc = np.ndarray.squeeze.__doc__ +DimArray.nonzero.__doc__ = np.ndarray.nonzero.__doc__ +DimArray.squeeze.__doc__ = np.ndarray.squeeze.__doc__ # Methods that return AttrArrays: Prefic docstring with warning! cast_msg =\ -""" + """ **CAUTION: the output of this method is cast to an AttrArray instance.** **Some attributes may no longer be valid after this Method is applied!** """ -DimArray.diagonal.im_func.func_doc = cast_msg+np.ndarray.diagonal.__doc__ -DimArray.flatten.im_func.func_doc = cast_msg+np.ndarray.flatten.__doc__ -DimArray.ravel.im_func.func_doc = cast_msg+np.ndarray.ravel.__doc__ -DimArray.repeat.im_func.func_doc = cast_msg+np.ndarray.repeat.__doc__ -DimArray.reshape.im_func.func_doc = cast_msg+np.ndarray.reshape.__doc__ -#DimArray.resize.im_func.func_doc = cast_msg+np.ndarray.resize.__doc__ -DimArray.trace.im_func.func_doc = cast_msg+np.ndarray.trace.__doc__ +DimArray.diagonal.__doc__ = cast_msg + np.ndarray.diagonal.__doc__ +DimArray.flatten.__doc__ = cast_msg + np.ndarray.flatten.__doc__ +DimArray.ravel.__doc__ = cast_msg + np.ndarray.ravel.__doc__ +DimArray.repeat.__doc__ = cast_msg + np.ndarray.repeat.__doc__ +DimArray.reshape.__doc__ = cast_msg + np.ndarray.reshape.__doc__ +#DimArray.resize.im_func.func_doc = cast_msg+np.ndarray.resize.__doc__ +DimArray.trace.__doc__ = cast_msg + np.ndarray.trace.__doc__ diff --git a/dimarray/tests/test_attrarray.py b/dimarray/tests/test_attrarray.py index 316c9d8..fce9e41 100644 --- a/dimarray/tests/test_attrarray.py +++ b/dimarray/tests/test_attrarray.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,11 +9,12 @@ import numpy as np from numpy.testing import TestCase,\ - assert_array_equal, assert_array_almost_equal + assert_array_equal, assert_array_almost_equal from dimarray import AttrArray -import cPickle as pickle +import pickle as pickle + class test_AttrArray(TestCase): def setUp(self): @@ -21,92 +22,93 @@ def setUp(self): def test_new(self): # test new instantiation (attributes are defined by kwargs) - + # instatiation with a numpy ndarray: shape = (10,) arr = np.random.random_sample(shape) - dat_array = AttrArray(arr,name='randvals') + dat_array = AttrArray(arr, name='randvals') self.assertTrue(dat_array.name == 'randvals') - self.assertEquals(shape,dat_array.shape) - self.assertTrue((dat_array==arr).all()) + self.assertEqual(shape, dat_array.shape) + self.assertTrue((dat_array == arr).all()) # another instatioation with an ndarray, but this time with # dtype set: - shape = (1,2,3,4) + shape = (1, 2, 3, 4) arr = np.random.random_sample(shape) - dat_array = AttrArray(arr,name='randvals',dtype=np.float32) + dat_array = AttrArray(arr, name='randvals', dtype=np.float32) self.assertTrue(dat_array.name == 'randvals') - self.assertEquals(shape,dat_array.shape) + self.assertEqual(shape, dat_array.shape) # "almost" equal because of the casting to np.float32: - assert_array_almost_equal(dat_array,arr) - self.assertTrue(dat_array.dtype==np.float32) + assert_array_almost_equal(dat_array, arr) + self.assertTrue(dat_array.dtype == np.float32) # # another ndarray, with copy = True vs. copy = False - shape = (10,9,8,7,6,1,8) + shape = (10, 9, 8, 7, 6, 1, 8) arr = np.random.random_sample(shape) - dat_array = AttrArray(arr,name='randvals', test1=33, - test2='test', copy = True) + dat_array = AttrArray(arr, name='randvals', test1=33, + test2='test', copy=True) self.assertTrue(dat_array.name == 'randvals') self.assertTrue(dat_array.test1 == 33) self.assertTrue(dat_array.test2 == 'test') - self.assertEquals(shape,dat_array.shape) - assert_array_equal(dat_array,arr) + self.assertEqual(shape, dat_array.shape) + assert_array_equal(dat_array, arr) dat_array[0] += 5 # # "almost" equal because of slight inaccuracies in the the # # representation of floats: - assert_array_almost_equal((dat_array[0]-5), arr[0]) - dat_array = AttrArray(arr,name='randvals', test1=33, - test2='test', copy = False) + assert_array_almost_equal((dat_array[0] - 5), arr[0]) + dat_array = AttrArray(arr, name='randvals', test1=33, + test2='test', copy=False) self.assertTrue(dat_array.name == 'randvals') self.assertTrue(dat_array.test1 == 33) self.assertTrue(dat_array.test2 == 'test') - self.assertEquals(shape,dat_array.shape) - self.assertTrue((dat_array==arr).all()) + self.assertEqual(shape, dat_array.shape) + self.assertTrue((dat_array == arr).all()) dat_array[0] += 5 - assert_array_equal(dat_array[0],arr[0]) - + assert_array_equal(dat_array[0], arr[0]) + # instantiation with a list: - lst = range(10) - dat_list = AttrArray(lst,name='range') + lst = list(range(10)) + dat_list = AttrArray(lst, name='range') self.assertTrue(dat_list.name == 'range') - self.assertTrue((lst==dat_list).all()) - lst = [['a','b','c']] - dat_list = AttrArray(lst,name='list') + self.assertTrue((lst == dat_list).all()) + lst = [['a', 'b', 'c']] + dat_list = AttrArray(lst, name='list') self.assertTrue(dat_list.name == 'list') - self.assertTrue((lst==dat_list).all()) - lst = [[1,2,3],[4.5,6,7]] - dat_list = AttrArray(lst,name='list') + self.assertTrue((lst == dat_list).all()) + lst = [[1, 2, 3], [4.5, 6, 7]] + dat_list = AttrArray(lst, name='list') self.assertTrue(dat_list.name == 'list') - self.assertTrue((lst==dat_list).all()) + self.assertTrue((lst == dat_list).all()) # instantiation with a AttrArray: - dat_attrarray = AttrArray(dat_array,name='attrarray') + dat_attrarray = AttrArray(dat_array, name='attrarray') self.assertTrue(dat_attrarray.name == 'attrarray') - dat_attrarray = AttrArray(dat_list,newname='attrarray',test=44) + dat_attrarray = AttrArray(dat_list, newname='attrarray', test=44) self.assertTrue(dat_attrarray.newname == 'attrarray') - self.assertTrue(dat_attrarray.test == 44) + self.assertTrue(dat_attrarray.test == 44) def test_setattr(self): - dat = AttrArray(np.random.rand(10),name='randvals') + dat = AttrArray(np.random.rand(10), name='randvals') # add a custom attribute: dat.custom = 'attribute' - self.assertEquals(dat.custom,'attribute') + self.assertEqual(dat.custom, 'attribute') # _required_attrs is read only: - self.assertRaises(AttributeError,dat.__setattr__,'_required_attrs','test') + self.assertRaises(AttributeError, dat.__setattr__, + '_required_attrs', 'test') def test_getattr(self): - dat = AttrArray(np.random.rand(10),name='randvals') - self.assertEquals(dat.name,'randvals') + dat = AttrArray(np.random.rand(10), name='randvals') + self.assertEqual(dat.name, 'randvals') def test_method(self): # make sure ndarray methods work and return a new AttrArray # instance with the attributes intact - dat = AttrArray(np.random.rand(10),name='randvals') + dat = AttrArray(np.random.rand(10), name='randvals') sdat = np.sqrt(dat) - self.assertEquals(sdat.name,'randvals') + self.assertEqual(sdat.name, 'randvals') def test_pickle(self): # make sure we can pickle this thing - dat = AttrArray(np.random.rand(10),name='randvals') + dat = AttrArray(np.random.rand(10), name='randvals') # dump to string pstr = pickle.dumps(dat) @@ -114,239 +116,247 @@ def test_pickle(self): dat2 = pickle.loads(pstr) # make sure data same - assert_array_equal(dat,dat2) + assert_array_equal(dat, dat2) # make sure has attr and it's correct - self.assertTrue(hasattr(dat2,'_attrs')) - self.assertTrue(hasattr(dat2,'name')) - self.assertEquals(dat2.name, 'randvals') + self.assertTrue(hasattr(dat2, '_attrs')) + self.assertTrue(hasattr(dat2, 'name')) + self.assertEqual(dat2.name, 'randvals') # make sure has required attr - self.assertTrue(hasattr(dat2,'_required_attrs')) + self.assertTrue(hasattr(dat2, '_required_attrs')) def test_nanstd(self): - arr = np.random.rand(10,10,10) - dat = AttrArray(arr,name='randvals') + arr = np.random.rand(10, 10, 10) + dat = AttrArray(arr, name='randvals') # if there are no NaN's, std and nanstd should give equal # results: - self.assertEquals(dat.std(),dat.nanstd()) - assert_array_almost_equal(dat.std(0),dat.nanstd(0)) - self.assertEquals(dat.nanstd(0).name, 'randvals') - assert_array_almost_equal(dat.std(1),dat.nanstd(1)) - self.assertEquals(dat.nanstd(1).name, 'randvals') - assert_array_almost_equal(dat.std(2),dat.nanstd(2)) - self.assertEquals(dat.nanstd(2).name, 'randvals') + self.assertEqual(dat.std(), dat.nanstd()) + assert_array_almost_equal(dat.std(0), dat.nanstd(0)) + self.assertEqual(dat.nanstd(0).name, 'randvals') + assert_array_almost_equal(dat.std(1), dat.nanstd(1)) + self.assertEqual(dat.nanstd(1).name, 'randvals') + assert_array_almost_equal(dat.std(2), dat.nanstd(2)) + self.assertEqual(dat.nanstd(2).name, 'randvals') # test ddof: for d in range(3): - self.assertEquals(dat.std(ddof=d),dat.nanstd(ddof=d)) - assert_array_almost_equal(dat.std(0,ddof=d),dat.nanstd(0,ddof=d)) - self.assertEquals(dat.nanstd(0,ddof=d).name, 'randvals') - assert_array_almost_equal(dat.std(1,ddof=d),dat.nanstd(1,ddof=d)) - self.assertEquals(dat.nanstd(1,ddof=d).name, 'randvals') - assert_array_almost_equal(dat.std(2,ddof=d),dat.nanstd(2,ddof=d)) - self.assertEquals(dat.nanstd(2,ddof=d).name, 'randvals') + self.assertEqual(dat.std(ddof=d), dat.nanstd(ddof=d)) + assert_array_almost_equal( + dat.std(0, ddof=d), dat.nanstd(0, ddof=d)) + self.assertEqual(dat.nanstd(0, ddof=d).name, 'randvals') + assert_array_almost_equal( + dat.std(1, ddof=d), dat.nanstd(1, ddof=d)) + self.assertEqual(dat.nanstd(1, ddof=d).name, 'randvals') + assert_array_almost_equal( + dat.std(2, ddof=d), dat.nanstd(2, ddof=d)) + self.assertEqual(dat.nanstd(2, ddof=d).name, 'randvals') # Now, make sure results are as expected with NaN present: - arr[0,0,0] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].std(),dat.nanstd()) + arr[0, 0, 0] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].std(), dat.nanstd()) for i in range(len(arr.shape)): tmp1 = dat.std(i) - tmp1[0,0] = 0 + tmp1[0, 0] = 0 tmp2 = dat.nanstd(i) - tmp2[0,0] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - arr[3,6,2] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].std(),dat.nanstd()) + tmp2[0, 0] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + arr[3, 6, 2] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].std(), dat.nanstd()) tmp1 = dat.std(0) - tmp1[0,0] = 0 - tmp1[6,2] = 0 + tmp1[0, 0] = 0 + tmp1[6, 2] = 0 tmp2 = dat.nanstd(0) - tmp2[0,0] = 0 - tmp2[6,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[6, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.std(1) - tmp1[0,0] = 0 - tmp1[3,2] = 0 + tmp1[0, 0] = 0 + tmp1[3, 2] = 0 tmp2 = dat.nanstd(1) - tmp2[0,0] = 0 - tmp2[3,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.std(2) - tmp1[0,0] = 0 - tmp1[3,6] = 0 + tmp1[0, 0] = 0 + tmp1[3, 6] = 0 tmp2 = dat.nanstd(2) - tmp2[0,0] = 0 - tmp2[3,6] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 6] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') # Test ddof: for d in range(3): - self.assertEquals(dat[~np.isnan(dat)].std(ddof=d),dat.nanstd(ddof=d)) - tmp1 = dat.std(0,ddof=d) - tmp1[0,0] = 0 - tmp1[6,2] = 0 - tmp2 = dat.nanstd(0,ddof=d) - tmp2[0,0] = 0 - tmp2[6,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - tmp1 = dat.std(1,ddof=d) - tmp1[0,0] = 0 - tmp1[3,2] = 0 - tmp2 = dat.nanstd(1,ddof=d) - tmp2[0,0] = 0 - tmp2[3,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - tmp1 = dat.std(2,ddof=d) - tmp1[0,0] = 0 - tmp1[3,6] = 0 - tmp2 = dat.nanstd(2,ddof=d) - tmp2[0,0] = 0 - tmp2[3,6] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + self.assertEqual(dat[~np.isnan(dat)].std( + ddof=d), dat.nanstd(ddof=d)) + tmp1 = dat.std(0, ddof=d) + tmp1[0, 0] = 0 + tmp1[6, 2] = 0 + tmp2 = dat.nanstd(0, ddof=d) + tmp2[0, 0] = 0 + tmp2[6, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + tmp1 = dat.std(1, ddof=d) + tmp1[0, 0] = 0 + tmp1[3, 2] = 0 + tmp2 = dat.nanstd(1, ddof=d) + tmp2[0, 0] = 0 + tmp2[3, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + tmp1 = dat.std(2, ddof=d) + tmp1[0, 0] = 0 + tmp1[3, 6] = 0 + tmp2 = dat.nanstd(2, ddof=d) + tmp2[0, 0] = 0 + tmp2[3, 6] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') def test_nanvar(self): - arr = np.random.rand(10,10,10) - dat = AttrArray(arr,name='randvals') + arr = np.random.rand(10, 10, 10) + dat = AttrArray(arr, name='randvals') # if there are no NaN's, var and nanvar should give equal # results: - self.assertEquals(dat.var(),dat.nanvar()) - assert_array_almost_equal(dat.var(0),dat.nanvar(0)) - self.assertEquals(dat.nanvar(0).name, 'randvals') - assert_array_almost_equal(dat.var(1),dat.nanvar(1)) - self.assertEquals(dat.nanvar(1).name, 'randvals') - assert_array_almost_equal(dat.var(2),dat.nanvar(2)) - self.assertEquals(dat.nanvar(2).name, 'randvals') + self.assertEqual(dat.var(), dat.nanvar()) + assert_array_almost_equal(dat.var(0), dat.nanvar(0)) + self.assertEqual(dat.nanvar(0).name, 'randvals') + assert_array_almost_equal(dat.var(1), dat.nanvar(1)) + self.assertEqual(dat.nanvar(1).name, 'randvals') + assert_array_almost_equal(dat.var(2), dat.nanvar(2)) + self.assertEqual(dat.nanvar(2).name, 'randvals') # test ddof: for d in range(3): - self.assertEquals(dat.var(ddof=d),dat.nanvar(ddof=d)) - assert_array_almost_equal(dat.var(0,ddof=d),dat.nanvar(0,ddof=d)) - self.assertEquals(dat.nanvar(0,ddof=d).name, 'randvals') - assert_array_almost_equal(dat.var(1,ddof=d),dat.nanvar(1,ddof=d)) - self.assertEquals(dat.nanvar(1,ddof=d).name, 'randvals') - assert_array_almost_equal(dat.var(2,ddof=d),dat.nanvar(2,ddof=d)) - self.assertEquals(dat.nanvar(2,ddof=d).name, 'randvals') + self.assertEqual(dat.var(ddof=d), dat.nanvar(ddof=d)) + assert_array_almost_equal( + dat.var(0, ddof=d), dat.nanvar(0, ddof=d)) + self.assertEqual(dat.nanvar(0, ddof=d).name, 'randvals') + assert_array_almost_equal( + dat.var(1, ddof=d), dat.nanvar(1, ddof=d)) + self.assertEqual(dat.nanvar(1, ddof=d).name, 'randvals') + assert_array_almost_equal( + dat.var(2, ddof=d), dat.nanvar(2, ddof=d)) + self.assertEqual(dat.nanvar(2, ddof=d).name, 'randvals') # Now, make sure results are as expected with NaN present: - arr[0,0,0] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].var(),dat.nanvar()) + arr[0, 0, 0] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].var(), dat.nanvar()) for i in range(len(arr.shape)): tmp1 = dat.var(i) - tmp1[0,0] = 0 + tmp1[0, 0] = 0 tmp2 = dat.nanvar(i) - tmp2[0,0] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - arr[3,6,2] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].var(),dat.nanvar()) + tmp2[0, 0] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + arr[3, 6, 2] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].var(), dat.nanvar()) tmp1 = dat.var(0) - tmp1[0,0] = 0 - tmp1[6,2] = 0 + tmp1[0, 0] = 0 + tmp1[6, 2] = 0 tmp2 = dat.nanvar(0) - tmp2[0,0] = 0 - tmp2[6,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[6, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.var(1) - tmp1[0,0] = 0 - tmp1[3,2] = 0 + tmp1[0, 0] = 0 + tmp1[3, 2] = 0 tmp2 = dat.nanvar(1) - tmp2[0,0] = 0 - tmp2[3,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.var(2) - tmp1[0,0] = 0 - tmp1[3,6] = 0 + tmp1[0, 0] = 0 + tmp1[3, 6] = 0 tmp2 = dat.nanvar(2) - tmp2[0,0] = 0 - tmp2[3,6] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 6] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') # Test ddof: for d in range(3): - self.assertEquals(dat[~np.isnan(dat)].var(ddof=d),dat.nanvar(ddof=d)) - tmp1 = dat.var(0,ddof=d) - tmp1[0,0] = 0 - tmp1[6,2] = 0 - tmp2 = dat.nanvar(0,ddof=d) - tmp2[0,0] = 0 - tmp2[6,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - tmp1 = dat.var(1,ddof=d) - tmp1[0,0] = 0 - tmp1[3,2] = 0 - tmp2 = dat.nanvar(1,ddof=d) - tmp2[0,0] = 0 - tmp2[3,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - tmp1 = dat.var(2,ddof=d) - tmp1[0,0] = 0 - tmp1[3,6] = 0 - tmp2 = dat.nanvar(2,ddof=d) - tmp2[0,0] = 0 - tmp2[3,6] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + self.assertEqual(dat[~np.isnan(dat)].var( + ddof=d), dat.nanvar(ddof=d)) + tmp1 = dat.var(0, ddof=d) + tmp1[0, 0] = 0 + tmp1[6, 2] = 0 + tmp2 = dat.nanvar(0, ddof=d) + tmp2[0, 0] = 0 + tmp2[6, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + tmp1 = dat.var(1, ddof=d) + tmp1[0, 0] = 0 + tmp1[3, 2] = 0 + tmp2 = dat.nanvar(1, ddof=d) + tmp2[0, 0] = 0 + tmp2[3, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + tmp1 = dat.var(2, ddof=d) + tmp1[0, 0] = 0 + tmp1[3, 6] = 0 + tmp2 = dat.nanvar(2, ddof=d) + tmp2[0, 0] = 0 + tmp2[3, 6] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') def test_nanmean(self): - arr = np.random.rand(10,10,10) - dat = AttrArray(arr,name='randvals') + arr = np.random.rand(10, 10, 10) + dat = AttrArray(arr, name='randvals') # if there are no NaN's, mean and nanmean should give equal # results: - self.assertEquals(dat.mean(),dat.nanmean()) - assert_array_almost_equal(dat.mean(0),dat.nanmean(0)) - self.assertEquals(dat.nanmean(0).name, 'randvals') - assert_array_almost_equal(dat.mean(1),dat.nanmean(1)) - self.assertEquals(dat.nanmean(1).name, 'randvals') - assert_array_almost_equal(dat.mean(2),dat.nanmean(2)) - self.assertEquals(dat.nanmean(2).name, 'randvals') + self.assertEqual(dat.mean(), dat.nanmean()) + assert_array_almost_equal(dat.mean(0), dat.nanmean(0)) + self.assertEqual(dat.nanmean(0).name, 'randvals') + assert_array_almost_equal(dat.mean(1), dat.nanmean(1)) + self.assertEqual(dat.nanmean(1).name, 'randvals') + assert_array_almost_equal(dat.mean(2), dat.nanmean(2)) + self.assertEqual(dat.nanmean(2).name, 'randvals') # Now, make sure results are as expected with NaN present: - arr[0,0,0] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].mean(),dat.nanmean()) + arr[0, 0, 0] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].mean(), dat.nanmean()) for i in range(len(arr.shape)): tmp1 = dat.mean(i) - tmp1[0,0] = 0 + tmp1[0, 0] = 0 tmp2 = dat.nanmean(i) - tmp2[0,0] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') - arr[3,6,2] = np.nan - dat = AttrArray(arr,name='randvals') - self.assertEquals(dat[~np.isnan(dat)].mean(),dat.nanmean()) + tmp2[0, 0] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') + arr[3, 6, 2] = np.nan + dat = AttrArray(arr, name='randvals') + self.assertEqual(dat[~np.isnan(dat)].mean(), dat.nanmean()) tmp1 = dat.mean(0) - tmp1[0,0] = 0 - tmp1[6,2] = 0 + tmp1[0, 0] = 0 + tmp1[6, 2] = 0 tmp2 = dat.nanmean(0) - tmp2[0,0] = 0 - tmp2[6,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[6, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.mean(1) - tmp1[0,0] = 0 - tmp1[3,2] = 0 + tmp1[0, 0] = 0 + tmp1[3, 2] = 0 tmp2 = dat.nanmean(1) - tmp2[0,0] = 0 - tmp2[3,2] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 2] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') tmp1 = dat.mean(2) - tmp1[0,0] = 0 - tmp1[3,6] = 0 + tmp1[0, 0] = 0 + tmp1[3, 6] = 0 tmp2 = dat.nanmean(2) - tmp2[0,0] = 0 - tmp2[3,6] = 0 - assert_array_almost_equal(tmp1,tmp2) - self.assertEquals(tmp2.name, 'randvals') + tmp2[0, 0] = 0 + tmp2[3, 6] = 0 + assert_array_almost_equal(tmp1, tmp2) + self.assertEqual(tmp2.name, 'randvals') diff --git a/dimarray/tests/test_dimarray.py b/dimarray/tests/test_dimarray.py index 0c0d0fc..c842fcd 100644 --- a/dimarray/tests/test_dimarray.py +++ b/dimarray/tests/test_dimarray.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,42 +9,44 @@ import numpy as np from numpy.testing import TestCase,\ - assert_array_equal, assert_array_almost_equal + assert_array_equal, assert_array_almost_equal from numpy.random import random_sample as rnd from dimarray import DimArray, Dim from dimarray import AttrArray -import cPickle as pickle +import pickle as pickle # Dim class + + class test_Dim(TestCase): def setUp(self): pass - + def test_new(self): # should raise AttributeError if no name is specified: - self.assertRaises(AttributeError,Dim,range(3)) + self.assertRaises(AttributeError, Dim, list(range(3))) # should raise ValueError if not 1-D: - self.assertRaises(ValueError,Dim,rnd((2,3)),name='test') + self.assertRaises(ValueError, Dim, rnd((2, 3)), name='test') # should raise ValueError if data is not unique - self.assertRaises(ValueError,Dim,[1,2,2,3],name='test') + self.assertRaises(ValueError, Dim, [1, 2, 2, 3], name='test') # should work fine with any number of dimensions as long as it # is squeezable or expandable to 1-D: - tst = Dim(rnd((3,1,1,1,1)),name='test') - self.assertEquals(tst.name,'test') - tst = Dim(np.array(5),name='test2') - self.assertEquals(tst.name,'test2') + tst = Dim(rnd((3, 1, 1, 1, 1)), name='test') + self.assertEqual(tst.name, 'test') + tst = Dim(np.array(5), name='test2') + self.assertEqual(tst.name, 'test2') # custom attributes should work, too: - tst = Dim(range(2),name='test3',custom='attribute') - self.assertEquals(tst.name,'test3') - self.assertEquals(tst.custom,'attribute') + tst = Dim(list(range(2)), name='test3', custom='attribute') + self.assertEqual(tst.name, 'test3') + self.assertEqual(tst.custom, 'attribute') # should raise Attribute Error if name is removed: - self.assertRaises(AttributeError,tst.__setattr__,'name',None) + self.assertRaises(AttributeError, tst.__setattr__, 'name', None) def test_pickle(self): # make sure we can pickle this thing - dat = Dim(np.random.rand(10),name='randvals') + dat = Dim(np.random.rand(10), name='randvals') # dump to string pstr = pickle.dumps(dat) @@ -53,135 +55,135 @@ def test_pickle(self): dat2 = pickle.loads(pstr) # make sure data same - assert_array_equal(dat,dat2) + assert_array_equal(dat, dat2) # make sure has attr and it's correct - self.assertTrue(hasattr(dat2,'_attrs')) - self.assertTrue(hasattr(dat2,'name')) - self.assertEquals(dat2.name, 'randvals') + self.assertTrue(hasattr(dat2, '_attrs')) + self.assertTrue(hasattr(dat2, 'name')) + self.assertEqual(dat2.name, 'randvals') # make sure has required attr - self.assertTrue(hasattr(dat2,'_required_attrs')) - self.assertEquals(dat._required_attrs,dat2._required_attrs) + self.assertTrue(hasattr(dat2, '_required_attrs')) + self.assertEqual(dat._required_attrs, dat2._required_attrs) # DimArray class class test_DimArray(TestCase): def setUp(self): pass - + def test_new(self): # should raise Error if dims contains non-Dim instances: - self.assertRaises(AttributeError,DimArray,np.random.rand(5,10), - dims = np.arange(4)) - self.assertRaises(AttributeError,DimArray,np.random.rand(5,10), - dims=[Dim(range(5),name='freqs',unit='Hz'), - AttrArray(range(10),name='time',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(5,10), - dims=[AttrArray(range(5),name='freqs',unit='Hz'), - Dim(range(10),name='time',unit='sec')]) - + self.assertRaises(AttributeError, DimArray, np.random.rand(5, 10), + dims=np.arange(4)) + self.assertRaises(AttributeError, DimArray, np.random.rand(5, 10), + dims=[Dim(list(range(5)), name='freqs', unit='Hz'), + AttrArray(list(range(10)), name='time', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(5, 10), + dims=[AttrArray(list(range(5)), name='freqs', unit='Hz'), + Dim(list(range(10)), name='time', unit='sec')]) + # should throw Error if dims do not match data shape: - self.assertRaises(AttributeError,DimArray,np.random.rand(5,10), - dims=[Dim(range(10),name='freqs',unit='Hz'), - Dim(range(5),name='time',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(5,10), - dims=[Dim(range(5),name='freqs',unit='Hz')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(5, 10), + dims=[Dim(list(range(10)), name='freqs', unit='Hz'), + Dim(list(range(5)), name='time', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(5, 10), + dims=[Dim(list(range(5)), name='freqs', unit='Hz')]) # should throw Error if 2 dims have the same name: - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(5),name='dim1',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,3,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(3),name='dim2',unit='Hz'), - Dim(range(5),name='dim1',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,3,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(3),name='dim1',unit='Hz'), - Dim(range(5),name='dim1',unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(5)), name='dim1', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 3, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(3)), name='dim2', unit='Hz'), + Dim(list(range(5)), name='dim1', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 3, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(3)), name='dim1', unit='Hz'), + Dim(list(range(5)), name='dim1', unit='sec')]) # should throw Error if a dim name is not a valid identifier: - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(5),name='dim 2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim 1',unit='Hz'), - Dim(range(5),name='dim2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim 1',unit='Hz'), - Dim(range(5),name='dim 2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(5),name='dim$2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='$dim1',unit='Hz'), - Dim(range(5),name='dim2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='1dim1',unit='Hz'), - Dim(range(5),name='dim:2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='dim1',unit='Hz'), - Dim(range(5),name='',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='',unit='Hz'), - Dim(range(5),name='dim2',unit='sec')]) - self.assertRaises(AttributeError,DimArray,np.random.rand(10,5), - dims=[Dim(range(10),name='',unit='Hz'), - Dim(range(5),name='',unit='sec')]) - + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(5)), name='dim 2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim 1', unit='Hz'), + Dim(list(range(5)), name='dim2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim 1', unit='Hz'), + Dim(list(range(5)), name='dim 2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(5)), name='dim$2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='$dim1', unit='Hz'), + Dim(list(range(5)), name='dim2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='1dim1', unit='Hz'), + Dim(list(range(5)), name='dim:2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='dim1', unit='Hz'), + Dim(list(range(5)), name='', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='', unit='Hz'), + Dim(list(range(5)), name='dim2', unit='sec')]) + self.assertRaises(AttributeError, DimArray, np.random.rand(10, 5), + dims=[Dim(list(range(10)), name='', unit='Hz'), + Dim(list(range(5)), name='', unit='sec')]) + # this is a proper initialization: - dat = DimArray(np.random.rand(5,10), - dims=[Dim(range(5),name='freqs',unit='Hz'), - Dim(range(10),name='time',unit='sec')]) + dat = DimArray(np.random.rand(5, 10), + dims=[Dim(list(range(5)), name='freqs', unit='Hz'), + Dim(list(range(10)), name='time', unit='sec')]) # should raise Attribute Error if dims is removed: - self.assertRaises(AttributeError,dat.__setattr__,'dims',None) + self.assertRaises(AttributeError, dat.__setattr__, 'dims', None) # ensure dim_names attribute is set properly: - self.assertEquals(dat.dim_names,['freqs','time']) + self.assertEqual(dat.dim_names, ['freqs', 'time']) # ensure proper shape - self.assertEquals(dat.shape,(5,10)) + self.assertEqual(dat.shape, (5, 10)) # ensure dims have proper lengths: - self.assertEquals(len(dat.dims[0]),5) - self.assertEquals(len(dat.dims[1]),10) + self.assertEqual(len(dat.dims[0]), 5) + self.assertEqual(len(dat.dims[1]), 10) # ensure that dims attributes are copied properly: - self.assertEquals(dat.dims[0].unit,'Hz') - self.assertEquals(dat.dims[1].unit,'sec') + self.assertEqual(dat.dims[0].unit, 'Hz') + self.assertEqual(dat.dims[1].unit, 'sec') # check that dims values are preserved: - self.assertEquals(dat.dims[0][-1],4) - self.assertEquals(dat.dims[1][-1],9) - - dat = DimArray(np.random.rand(2,4,5), - dims=[Dim(range(2),name='dim1',unit='Hz'), - Dim(range(4),name='dim2',bla='bla'), - Dim(range(5),name='dim3',attr1='attr1', + self.assertEqual(dat.dims[0][-1], 4) + self.assertEqual(dat.dims[1][-1], 9) + + dat = DimArray(np.random.rand(2, 4, 5), + dims=[Dim(list(range(2)), name='dim1', unit='Hz'), + Dim(list(range(4)), name='dim2', bla='bla'), + Dim(list(range(5)), name='dim3', attr1='attr1', attr2='attr2')]) # ensure dim_names attribute is set properly: - self.assertEquals(dat.dim_names,['dim1','dim2','dim3']) + self.assertEqual(dat.dim_names, ['dim1', 'dim2', 'dim3']) # ensure proper shape - self.assertEquals(dat.shape,(2,4,5)) + self.assertEqual(dat.shape, (2, 4, 5)) # ensure dims have proper lengths: - self.assertEquals(len(dat.dims[0]),2) - self.assertEquals(len(dat.dims[1]),4) - self.assertEquals(len(dat.dims[2]),5) + self.assertEqual(len(dat.dims[0]), 2) + self.assertEqual(len(dat.dims[1]), 4) + self.assertEqual(len(dat.dims[2]), 5) # ensure that dims attributes are copied properly: - self.assertEquals(dat.dims[0].unit,'Hz') - self.assertEquals(dat.dims[1].bla,'bla') - self.assertEquals(dat.dims[2].attr1,'attr1') - self.assertEquals(dat.dims[2].attr2,'attr2') + self.assertEqual(dat.dims[0].unit, 'Hz') + self.assertEqual(dat.dims[1].bla, 'bla') + self.assertEqual(dat.dims[2].attr1, 'attr1') + self.assertEqual(dat.dims[2].attr2, 'attr2') # check that dims values are preserved: - self.assertEquals(dat.dims[0][-1],1) - self.assertEquals(dat.dims[1][-1],3) - self.assertEquals(dat.dims[2][-1],4) + self.assertEqual(dat.dims[0][-1], 1) + self.assertEqual(dat.dims[1][-1], 3) + self.assertEqual(dat.dims[2][-1], 4) # check filling in of default dims if left out - dat = DimArray(np.random.rand(4,3)) - self.assertEquals(dat.dim_names, ['dim1','dim2']) - assert_array_equal(dat['dim1'],np.arange(dat.shape[0])) - assert_array_equal(dat['dim2'],np.arange(dat.shape[1])) + dat = DimArray(np.random.rand(4, 3)) + self.assertEqual(dat.dim_names, ['dim1', 'dim2']) + assert_array_equal(dat['dim1'], np.arange(dat.shape[0])) + assert_array_equal(dat['dim2'], np.arange(dat.shape[1])) def test_pickle(self): # make sure we can pickle this thing - dat = DimArray(np.random.rand(4,3)) + dat = DimArray(np.random.rand(4, 3)) # dump to string pstr = pickle.dumps(dat) @@ -190,326 +192,332 @@ def test_pickle(self): dat2 = pickle.loads(pstr) # make sure data same - assert_array_equal(dat,dat2) + assert_array_equal(dat, dat2) # make sure has attr and it's correct - self.assertTrue(hasattr(dat2,'_attrs')) - self.assertTrue(hasattr(dat2,'dims')) - assert_array_equal(dat.dims[0],dat2.dims[0]) - assert_array_equal(dat.dims[1],dat2.dims[1]) + self.assertTrue(hasattr(dat2, '_attrs')) + self.assertTrue(hasattr(dat2, 'dims')) + assert_array_equal(dat.dims[0], dat2.dims[0]) + assert_array_equal(dat.dims[1], dat2.dims[1]) # make sure has required attr - self.assertTrue(hasattr(dat2,'_required_attrs')) - self.assertEquals(dat._required_attrs,dat2._required_attrs) + self.assertTrue(hasattr(dat2, '_required_attrs')) + self.assertEqual(dat._required_attrs, dat2._required_attrs) def test_getitem(self): # make ndarray an Dimaray with identical data arr = np.random.rand(3) - dat = DimArray(arr,dims=[Dim(range(3),name='dim1')]) - self.assertEquals(dat[0],dat['dim1==0']) - self.assertEquals(dat[1],dat['dim1==1']) - self.assertEquals(dat[2],dat['dim1==2']) - self.assertEquals(arr[0],dat['dim1==0']) - self.assertEquals(arr[1],dat['dim1==1']) - self.assertEquals(arr[2],dat['dim1==2']) - - arr = np.random.rand(3,2) - dat = DimArray(arr,dims=[Dim(range(3),name='dim1'), - Dim(range(2),name='dim2')]) - assert_array_equal(dat[:,0],dat['dim2==0']) - assert_array_equal(dat[1],dat['dim1==1']) - assert_array_equal(dat[2],dat['dim1==2']) - assert_array_equal(arr[0],dat['dim1==0']) - assert_array_equal(arr[1],dat['dim1==1']) - assert_array_equal(arr[2],dat['dim1==2']) - - assert_array_equal(dat[0,0],dat['dim1==0','dim2==0']) - assert_array_equal(dat[0,1],dat['dim1==0','dim2==1']) - assert_array_equal(dat[1,0],dat['dim1==1','dim2==0']) - assert_array_equal(dat[1,1],dat['dim1==1','dim2==1']) - assert_array_equal(dat[2,0],dat['dim1==2','dim2==0']) - assert_array_equal(dat[2,1],dat['dim1==2','dim2==1']) - - bool_indx = np.zeros(arr.shape,np.bool) - bool_indx[2,1] = True - assert_array_equal(dat[2,1],dat[bool_indx]) - bool_indx[1,1] = True - assert_array_equal(dat[1:3,1],dat[bool_indx]) + dat = DimArray(arr, dims=[Dim(list(range(3)), name='dim1')]) + self.assertEqual(dat[0], dat['dim1==0']) + self.assertEqual(dat[1], dat['dim1==1']) + self.assertEqual(dat[2], dat['dim1==2']) + self.assertEqual(arr[0], dat['dim1==0']) + self.assertEqual(arr[1], dat['dim1==1']) + self.assertEqual(arr[2], dat['dim1==2']) + + arr = np.random.rand(3, 2) + dat = DimArray(arr, dims=[Dim(list(range(3)), name='dim1'), + Dim(list(range(2)), name='dim2')]) + assert_array_equal(dat[:, 0], dat['dim2==0']) + assert_array_equal(dat[1], dat['dim1==1']) + assert_array_equal(dat[2], dat['dim1==2']) + assert_array_equal(arr[0], dat['dim1==0']) + assert_array_equal(arr[1], dat['dim1==1']) + assert_array_equal(arr[2], dat['dim1==2']) + + assert_array_equal(dat[0, 0], dat['dim1==0', 'dim2==0']) + assert_array_equal(dat[0, 1], dat['dim1==0', 'dim2==1']) + assert_array_equal(dat[1, 0], dat['dim1==1', 'dim2==0']) + assert_array_equal(dat[1, 1], dat['dim1==1', 'dim2==1']) + assert_array_equal(dat[2, 0], dat['dim1==2', 'dim2==0']) + assert_array_equal(dat[2, 1], dat['dim1==2', 'dim2==1']) + + bool_indx = np.zeros(arr.shape, np.bool) + bool_indx[2, 1] = True + assert_array_equal(dat[2, 1], dat[bool_indx]) + bool_indx[1, 1] = True + assert_array_equal(dat[1:3, 1], dat[bool_indx]) # The below test makes sure that one can work with the results # of a Boolean slice into a DimArray object. Because the # dimensions get lost with Boolean indices we need to test # that there are no complaints from dimension checking (the # result should be upcast as an AttrArray): test1 = dat[bool_indx] + 1 - test2 = dat[1:3,1] + 1 - assert_array_equal(test1,test2) + test2 = dat[1:3, 1] + 1 + assert_array_equal(test1, test2) arr = np.random.rand(3) dat = DimArray(arr) - bool_indx = np.array([True,False,True]) - assert_array_equal(dat[bool_indx],arr[bool_indx]) - assert_array_equal(dat[bool_indx].dims[0],dat.dims[0][bool_indx]) + bool_indx = np.array([True, False, True]) + assert_array_equal(dat[bool_indx], arr[bool_indx]) + assert_array_equal(dat[bool_indx].dims[0], dat.dims[0][bool_indx]) - dat_array = np.random.rand(2,4,5) + dat_array = np.random.rand(2, 4, 5) dat = DimArray(dat_array, - dims=[Dim(range(2),name='dim1',unit='Hz'), - Dim(range(4),name='dim2',bla='bla'), - Dim(range(5),name='dim3',attr1='attr1', + dims=[Dim(list(range(2)), name='dim1', unit='Hz'), + Dim(list(range(4)), name='dim2', bla='bla'), + Dim(list(range(5)), name='dim3', attr1='attr1', attr2='attr2')]) # check that the correct elements are returned: - self.assertEquals(dat[0,0,0],dat_array[0,0,0]) - self.assertEquals(dat[0,1,2],dat_array[0,1,2]) - self.assertEquals(dat[1,0,3],dat_array[1,0,3]) - + self.assertEqual(dat[0, 0, 0], dat_array[0, 0, 0]) + self.assertEqual(dat[0, 1, 2], dat_array[0, 1, 2]) + self.assertEqual(dat[1, 0, 3], dat_array[1, 0, 3]) + # check that the correct elements are returned: - self.assertEquals(dat['dim1==0','dim2==0','dim3==0'],dat_array[0,0,0]) - self.assertEquals(dat['dim1==0','dim2==1','dim3==2'],dat_array[0,1,2]) - self.assertEquals(dat['dim1==1','dim2==0','dim3==3'],dat_array[1,0,3]) - + self.assertEqual( + dat['dim1==0', 'dim2==0', 'dim3==0'], dat_array[0, 0, 0]) + self.assertEqual( + dat['dim1==0', 'dim2==1', 'dim3==2'], dat_array[0, 1, 2]) + self.assertEqual( + dat['dim1==1', 'dim2==0', 'dim3==3'], dat_array[1, 0, 3]) + # check that the returned DimArray and its dims have proper shapes: - self.assertEquals(dat[0].shape,dat_array[0].shape) - self.assertEquals(len(dat[0].dims[0]),dat_array[0].shape[0]) - self.assertEquals(len(dat[0].dims[1]),dat_array[0].shape[1]) - self.assertEquals(dat[0].dim_names,['dim2','dim3']) - - self.assertEquals(dat[1].shape,dat_array[1].shape) - self.assertEquals(len(dat[1].dims[0]),dat_array[1].shape[0]) - self.assertEquals(len(dat[1].dims[1]),dat_array[1].shape[1]) - self.assertEquals(dat[1].dim_names,['dim2','dim3']) - - self.assertEquals(dat[0,0].shape,dat_array[0,0].shape) - self.assertEquals(len(dat[0,0].dims[0]),dat_array[0,0].shape[0]) - self.assertEquals(dat[0,0].dim_names,['dim3']) - - self.assertEquals(dat[:,:,0].shape,dat_array[:,:,0].shape) - self.assertEquals(len(dat[:,:,0].dims[0]),dat_array[:,:,0].shape[0]) - self.assertEquals(len(dat[:,:,0].dims[1]),dat_array[:,:,0].shape[1]) - self.assertEquals(dat[:,:,0].dim_names,['dim1','dim2']) - - self.assertEquals(dat[0:1,2,0:3].shape,dat_array[0:1,2,0:3].shape) - self.assertEquals(len(dat[0:1,2,0:3].dims[0]), - dat_array[0:1,2,0:3].shape[0]) - self.assertEquals(len(dat[0:1,2,0:3].dims[1]), - dat_array[0:1,2,0:3].shape[1]) - self.assertEquals(dat[0:1,2,0:3].dim_names,['dim1','dim3']) - - self.assertEquals(dat[0:1].shape,dat_array[0:1].shape) - self.assertEquals(len(dat[0:1].dims[0]), - dat_array[0:1].shape[0]) - self.assertEquals(len(dat[0:1].dims[1]), - dat_array[0:1].shape[1]) - self.assertEquals(dat[0:1].dim_names,['dim1','dim2','dim3']) - - self.assertEquals(dat[1].shape,dat_array[1].shape) - self.assertEquals(len(dat[1].dims[0]),dat_array[1].shape[0]) - self.assertEquals(len(dat[1].dims[1]),dat_array[1].shape[1]) - self.assertEquals(dat[1].dim_names,['dim2','dim3']) - - self.assertEquals(dat[0,0].shape,dat_array[0,0].shape) - self.assertEquals(len(dat[0,0].dims[0]),dat_array[0,0].shape[0]) - self.assertEquals(dat[0,0].dim_names,['dim3']) - - self.assertEquals(dat[:,:,0].shape,dat_array[:,:,0].shape) - self.assertEquals(len(dat[:,:,0].dims[0]),dat_array[:,:,0].shape[0]) - self.assertEquals(len(dat[:,:,0].dims[1]),dat_array[:,:,0].shape[1]) - self.assertEquals(dat[:,:,0].dim_names,['dim1','dim2']) - - self.assertEquals(dat[0:1,2,0:3].shape,dat_array[0:1,2,0:3].shape) - self.assertEquals(len(dat[0:1,2,0:3].dims[0]), - dat_array[0:1,2,0:3].shape[0]) - self.assertEquals(len(dat[0:1,2,0:3].dims[1]), - dat_array[0:1,2,0:3].shape[1]) - self.assertEquals(dat[0:1,2,0:3].dim_names,['dim1','dim3']) - print dat.dims - print dat['dim2>0'].dims - assert_array_equal(dat['dim2>0'].dims[1],dat.dims[1][1:]) - - assert_array_equal(dat[1:,1:],dat['dim1>0','dim2>0']) - - + self.assertEqual(dat[0].shape, dat_array[0].shape) + self.assertEqual(len(dat[0].dims[0]), dat_array[0].shape[0]) + self.assertEqual(len(dat[0].dims[1]), dat_array[0].shape[1]) + self.assertEqual(dat[0].dim_names, ['dim2', 'dim3']) + + self.assertEqual(dat[1].shape, dat_array[1].shape) + self.assertEqual(len(dat[1].dims[0]), dat_array[1].shape[0]) + self.assertEqual(len(dat[1].dims[1]), dat_array[1].shape[1]) + self.assertEqual(dat[1].dim_names, ['dim2', 'dim3']) + + self.assertEqual(dat[0, 0].shape, dat_array[0, 0].shape) + self.assertEqual(len(dat[0, 0].dims[0]), dat_array[0, 0].shape[0]) + self.assertEqual(dat[0, 0].dim_names, ['dim3']) + + self.assertEqual(dat[:, :, 0].shape, dat_array[:, :, 0].shape) + self.assertEqual(len(dat[:, :, 0].dims[0]), + dat_array[:, :, 0].shape[0]) + self.assertEqual(len(dat[:, :, 0].dims[1]), + dat_array[:, :, 0].shape[1]) + self.assertEqual(dat[:, :, 0].dim_names, ['dim1', 'dim2']) + + self.assertEqual(dat[0:1, 2, 0:3].shape, dat_array[0:1, 2, 0:3].shape) + self.assertEqual(len(dat[0:1, 2, 0:3].dims[0]), + dat_array[0:1, 2, 0:3].shape[0]) + self.assertEqual(len(dat[0:1, 2, 0:3].dims[1]), + dat_array[0:1, 2, 0:3].shape[1]) + self.assertEqual(dat[0:1, 2, 0:3].dim_names, ['dim1', 'dim3']) + + self.assertEqual(dat[0:1].shape, dat_array[0:1].shape) + self.assertEqual(len(dat[0:1].dims[0]), + dat_array[0:1].shape[0]) + self.assertEqual(len(dat[0:1].dims[1]), + dat_array[0:1].shape[1]) + self.assertEqual(dat[0:1].dim_names, ['dim1', 'dim2', 'dim3']) + + self.assertEqual(dat[1].shape, dat_array[1].shape) + self.assertEqual(len(dat[1].dims[0]), dat_array[1].shape[0]) + self.assertEqual(len(dat[1].dims[1]), dat_array[1].shape[1]) + self.assertEqual(dat[1].dim_names, ['dim2', 'dim3']) + + self.assertEqual(dat[0, 0].shape, dat_array[0, 0].shape) + self.assertEqual(len(dat[0, 0].dims[0]), dat_array[0, 0].shape[0]) + self.assertEqual(dat[0, 0].dim_names, ['dim3']) + + self.assertEqual(dat[:, :, 0].shape, dat_array[:, :, 0].shape) + self.assertEqual(len(dat[:, :, 0].dims[0]), + dat_array[:, :, 0].shape[0]) + self.assertEqual(len(dat[:, :, 0].dims[1]), + dat_array[:, :, 0].shape[1]) + self.assertEqual(dat[:, :, 0].dim_names, ['dim1', 'dim2']) + + self.assertEqual(dat[0:1, 2, 0:3].shape, dat_array[0:1, 2, 0:3].shape) + self.assertEqual(len(dat[0:1, 2, 0:3].dims[0]), + dat_array[0:1, 2, 0:3].shape[0]) + self.assertEqual(len(dat[0:1, 2, 0:3].dims[1]), + dat_array[0:1, 2, 0:3].shape[1]) + self.assertEqual(dat[0:1, 2, 0:3].dim_names, ['dim1', 'dim3']) + print(dat.dims) + print(dat['dim2>0'].dims) + assert_array_equal(dat['dim2>0'].dims[1], dat.dims[1][1:]) + + assert_array_equal(dat[1:, 1:], dat['dim1>0', 'dim2>0']) + # when the name of a Dim instance is given, that dim should be # returned: - self.assertTrue(isinstance(dat['dim1'],Dim)) - self.assertTrue(isinstance(dat['dim2'],Dim)) - self.assertTrue(isinstance(dat['dim3'],Dim)) - - self.assertEquals(dat['dim1'].name,'dim1') - self.assertEquals(dat['dim1'].unit,'Hz') - self.assertEquals(dat['dim1'][-1],1) - self.assertEquals(len(dat['dim1']),2) - self.assertEquals(dat['dim2'].name,'dim2') - self.assertEquals(dat['dim2'].bla,'bla') - self.assertEquals(dat['dim2'][-1],3) - self.assertEquals(len(dat['dim2']),4) - self.assertEquals(dat['dim3'].name,'dim3') - self.assertEquals(dat['dim3'].attr1,'attr1') - self.assertEquals(dat['dim3'].attr2,'attr2') - self.assertEquals(dat['dim3'][-1],4) - self.assertEquals(len(dat['dim3']),5) + self.assertTrue(isinstance(dat['dim1'], Dim)) + self.assertTrue(isinstance(dat['dim2'], Dim)) + self.assertTrue(isinstance(dat['dim3'], Dim)) + + self.assertEqual(dat['dim1'].name, 'dim1') + self.assertEqual(dat['dim1'].unit, 'Hz') + self.assertEqual(dat['dim1'][-1], 1) + self.assertEqual(len(dat['dim1']), 2) + self.assertEqual(dat['dim2'].name, 'dim2') + self.assertEqual(dat['dim2'].bla, 'bla') + self.assertEqual(dat['dim2'][-1], 3) + self.assertEqual(len(dat['dim2']), 4) + self.assertEqual(dat['dim3'].name, 'dim3') + self.assertEqual(dat['dim3'].attr1, 'attr1') + self.assertEqual(dat['dim3'].attr2, 'attr2') + self.assertEqual(dat['dim3'][-1], 4) + self.assertEqual(len(dat['dim3']), 5) # when another string is given, it should be evaluated: - self.assertEquals(dat['dim1==0'].shape,(4,5)) - self.assertEquals(len(dat['dim1==0'].dims[0]),4) - self.assertEquals(len(dat['dim1==0'].dims[1]),5) - self.assertEquals(dat['dim1==0'].dim_names,['dim2','dim3']) - - self.assertEquals(dat['dim2==1'].shape,(2,5)) - self.assertEquals(len(dat['dim2==1'].dims[0]),2) - self.assertEquals(len(dat['dim2==1'].dims[1]),5) - self.assertEquals(dat['dim2==1'].dim_names,['dim1','dim3']) - - self.assertEquals(dat['dim2<2'].shape,(2,2,5)) - self.assertEquals(len(dat['dim2<2'].dims[0]),2) - self.assertEquals(len(dat['dim2<2'].dims[1]),2) - self.assertEquals(len(dat['dim2<2'].dims[2]),5) - self.assertEquals(dat['dim2<2'].dim_names,['dim1','dim2','dim3']) - - self.assertEquals(dat['dim3!=2'].shape,(2,4,4)) - self.assertEquals(len(dat['dim3!=2'].dims[0]),2) - self.assertEquals(len(dat['dim3!=2'].dims[1]),4) - self.assertEquals(len(dat['dim3!=2'].dims[2]),4) - self.assertEquals(dat['dim3!=2'].dim_names,['dim1','dim2','dim3']) + self.assertEqual(dat['dim1==0'].shape, (4, 5)) + self.assertEqual(len(dat['dim1==0'].dims[0]), 4) + self.assertEqual(len(dat['dim1==0'].dims[1]), 5) + self.assertEqual(dat['dim1==0'].dim_names, ['dim2', 'dim3']) + + self.assertEqual(dat['dim2==1'].shape, (2, 5)) + self.assertEqual(len(dat['dim2==1'].dims[0]), 2) + self.assertEqual(len(dat['dim2==1'].dims[1]), 5) + self.assertEqual(dat['dim2==1'].dim_names, ['dim1', 'dim3']) + + self.assertEqual(dat['dim2<2'].shape, (2, 2, 5)) + self.assertEqual(len(dat['dim2<2'].dims[0]), 2) + self.assertEqual(len(dat['dim2<2'].dims[1]), 2) + self.assertEqual(len(dat['dim2<2'].dims[2]), 5) + self.assertEqual(dat['dim2<2'].dim_names, ['dim1', 'dim2', 'dim3']) + + self.assertEqual(dat['dim3!=2'].shape, (2, 4, 4)) + self.assertEqual(len(dat['dim3!=2'].dims[0]), 2) + self.assertEqual(len(dat['dim3!=2'].dims[1]), 4) + self.assertEqual(len(dat['dim3!=2'].dims[2]), 4) + self.assertEqual(dat['dim3!=2'].dim_names, ['dim1', 'dim2', 'dim3']) # check that the right values are returned: - self.assertEquals(dat['dim3!=2'][0,0,0],dat_array[0,0,0]) - self.assertEquals(dat['dim3!=2'][1,2,1],dat_array[1,2,1]) - self.assertEquals(dat['dim3!=2'][1,2,3],dat_array[1,2,4]) + self.assertEqual(dat['dim3!=2'][0, 0, 0], dat_array[0, 0, 0]) + self.assertEqual(dat['dim3!=2'][1, 2, 1], dat_array[1, 2, 1]) + self.assertEqual(dat['dim3!=2'][1, 2, 3], dat_array[1, 2, 4]) # check indexing with a tuple of arrays and with 1-level dimensions: - dim1=Dim(['dim'],'dim1') - dim2=Dim([1,2],'dim2') - dim3=Dim([3,4,5],'dim3') - dat=DimArray([[[6,7,8],[9,10,11]]],[dim1,dim2,dim3]) - self.assertEquals(dat[np.ix_([0],[0,1],[0,1])].shape,(1,2,2)) + dim1 = Dim(['dim'], 'dim1') + dim2 = Dim([1, 2], 'dim2') + dim3 = Dim([3, 4, 5], 'dim3') + dat = DimArray([[[6, 7, 8], [9, 10, 11]]], [dim1, dim2, dim3]) + self.assertEqual(dat[np.ix_([0], [0, 1], [0, 1])].shape, (1, 2, 2)) # test string index returning nothing # test list index - dim1=Dim(['dim'],'dim1') - dim2=Dim([1,2],'dim2') - dim3=Dim([3,4,5],'dim3') - dat=DimArray([[[6,7,8],[9,10,11]]],[dim1,dim2,dim3]) - assert_array_equal(dat[[0]],dat[np.array([0])]) + dim1 = Dim(['dim'], 'dim1') + dim2 = Dim([1, 2], 'dim2') + dim3 = Dim([3, 4, 5], 'dim3') + dat = DimArray([[[6, 7, 8], [9, 10, 11]]], [dim1, dim2, dim3]) + assert_array_equal(dat[[0]], dat[np.array([0])]) def test_select(self): # check indexing with a tuple of arrays and with 1-level dimensions: - dim1=Dim(['dim'],'dim1') - dim2=Dim([1,2],'dim2') - dim3=Dim([3,4,5],'dim3') - dat=DimArray([[[6,7,8],[9,10,11]]],[dim1,dim2,dim3]) - self.assertEquals(dat.select(dim2=dat['dim2']>1, - dim3=dat['dim3']>3).shape,(1,1,2)) + dim1 = Dim(['dim'], 'dim1') + dim2 = Dim([1, 2], 'dim2') + dim3 = Dim([3, 4, 5], 'dim3') + dat = DimArray([[[6, 7, 8], [9, 10, 11]]], [dim1, dim2, dim3]) + self.assertEqual(dat.select(dim2=dat['dim2'] > 1, + dim3=dat['dim3'] > 3).shape, (1, 1, 2)) def test_find(self): # check indexing with a tuple of arrays and with 1-level dimensions: - dim1=Dim(['dim'],'dim1') - dim2=Dim([1,2],'dim2') - dim3=Dim([3,4,5],'dim3') - dat=DimArray([[[6,7,8],[9,10,11]]],[dim1,dim2,dim3]) - indx = dat.find(dim2=dat['dim2']>1,dim3=dat['dim3']>3) - assert_array_equal(dat.select(dim2=dat['dim2']>1,dim3=dat['dim3']>3), - dat[indx]) + dim1 = Dim(['dim'], 'dim1') + dim2 = Dim([1, 2], 'dim2') + dim3 = Dim([3, 4, 5], 'dim3') + dat = DimArray([[[6, 7, 8], [9, 10, 11]]], [dim1, dim2, dim3]) + indx = dat.find(dim2=dat['dim2'] > 1, dim3=dat['dim3'] > 3) + assert_array_equal(dat.select(dim2=dat['dim2'] > 1, dim3=dat['dim3'] > 3), + dat[indx]) def test_get_axis(self): - dat = DimArray(np.random.rand(5,10,3), - dims=[Dim(range(5),name='one'), - Dim(range(10),name='two'), - Dim(range(3),name='three')],test='tst') - self.assertEqual(dat.get_axis(0),0) - self.assertEqual(dat.get_axis(1),1) - self.assertEqual(dat.get_axis(2),2) - self.assertEqual(dat.get_axis('one'),0) - self.assertEqual(dat.get_axis('two'),1) - self.assertEqual(dat.get_axis('three'),2) + dat = DimArray(np.random.rand(5, 10, 3), + dims=[Dim(list(range(5)), name='one'), + Dim(list(range(10)), name='two'), + Dim(list(range(3)), name='three')], test='tst') + self.assertEqual(dat.get_axis(0), 0) + self.assertEqual(dat.get_axis(1), 1) + self.assertEqual(dat.get_axis(2), 2) + self.assertEqual(dat.get_axis('one'), 0) + self.assertEqual(dat.get_axis('two'), 1) + self.assertEqual(dat.get_axis('three'), 2) def test_reshape(self): # make ndarray an Dimaray with identical data - arr = np.random.rand(5,12,3,1) - dat = DimArray(arr,dims=[Dim(range(5),name='one'), - Dim(range(12),name='two'), - Dim(range(3),name='three'), - Dim(range(1),name='four')],test='tst') - newshapes = [(5,2,2,3,3),(2,3,5,3,2),(15,12),(6,2,15,1,1,1,1,1,1,1), - 180,(1,1,1,180,1,1,1)] + arr = np.random.rand(5, 12, 3, 1) + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one'), + Dim(list(range(12)), name='two'), + Dim(list(range(3)), name='three'), + Dim(list(range(1)), name='four')], test='tst') + newshapes = [(5, 2, 2, 3, 3), (2, 3, 5, 3, 2), (15, 12), (6, 2, 15, 1, 1, 1, 1, 1, 1, 1), + 180, (1, 1, 1, 180, 1, 1, 1)] for newshape in newshapes: - assert_array_equal(arr.reshape(newshape),dat.reshape(newshape)) - assert_array_equal(np.reshape(arr,newshape), - np.reshape(dat,newshape)) - + assert_array_equal(arr.reshape(newshape), dat.reshape(newshape)) + assert_array_equal(np.reshape(arr, newshape), + np.reshape(dat, newshape)) + def test_resize(self): # make ndarray an Dimaray with identical data - arr = np.random.rand(5,12,3,1) - dat = DimArray(arr,dims=[Dim(range(5),name='one'), - Dim(range(12),name='two'), - Dim(range(3),name='three'), - Dim(range(1),name='four')],test='tst') - self.assertRaises(NotImplementedError,dat.resize,(5,2,2,3,3)) + arr = np.random.rand(5, 12, 3, 1) + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one'), + Dim(list(range(12)), name='two'), + Dim(list(range(3)), name='three'), + Dim(list(range(1)), name='four')], test='tst') + self.assertRaises(NotImplementedError, dat.resize, (5, 2, 2, 3, 3)) def test_newaxis(self): - # make ndarray an Dimaray with identical data - arr = np.random.rand(5,12,3,1) - dat = DimArray(arr,dims=[Dim(range(5),name='one'), - Dim(range(12),name='two'), - Dim(range(3),name='three'), - Dim(range(1),name='four')],test='tst') + # make ndarray an Dimaray with identical data + arr = np.random.rand(5, 12, 3, 1) + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one'), + Dim(list(range(12)), name='two'), + Dim(list(range(3)), name='three'), + Dim(list(range(1)), name='four')], test='tst') # add a new axis at beginning - d0 = dat[np.newaxis,:] - self.assertEquals(d0.dim_names[0],'newaxis_0') - self.assertEquals(d0.dim_names[-1],'four') - self.assertEquals(len(d0.shape),len(arr.shape)+1) + d0 = dat[np.newaxis, :] + self.assertEqual(d0.dim_names[0], 'newaxis_0') + self.assertEqual(d0.dim_names[-1], 'four') + self.assertEqual(len(d0.shape), len(arr.shape) + 1) # add a new axis at end - d0 = dat[:,:,:,:,np.newaxis] - self.assertEquals(d0.dim_names[-1],'newaxis_4') - self.assertEquals(d0.dim_names[0],'one') - self.assertEquals(len(d0.shape),len(arr.shape)+1) + d0 = dat[:, :, :, :, np.newaxis] + self.assertEqual(d0.dim_names[-1], 'newaxis_4') + self.assertEqual(d0.dim_names[0], 'one') + self.assertEqual(len(d0.shape), len(arr.shape) + 1) # add two axes at once - d0 = dat[np.newaxis,:,:,:,:,np.newaxis] - self.assertEquals(d0.dim_names[-1],'newaxis_5') - self.assertEquals(d0.dim_names[0],'newaxis_0') - self.assertEquals(len(d0.shape),len(arr.shape)+2) + d0 = dat[np.newaxis, :, :, :, :, np.newaxis] + self.assertEqual(d0.dim_names[-1], 'newaxis_5') + self.assertEqual(d0.dim_names[0], 'newaxis_0') + self.assertEqual(len(d0.shape), len(arr.shape) + 2) # make sure the attribute is still there d0.test = 'tst' def test_add_dim(self): - # make ndarray an Dimaray with identical data + # make ndarray an Dimaray with identical data arr = np.random.rand(5) - dat = DimArray(arr,dims=[Dim(range(5),name='one')]) + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one')]) # make new dim to add - d = Dim(range(10),name='replicate') + d = Dim(list(range(10)), name='replicate') # add it to the dat ndat = dat.add_dim(d) # test that it worked # verify shape - self.assertEquals(len(ndat.shape),len(dat.shape)+1) - self.assertEquals(ndat.shape[0],10) - self.assertEquals(ndat.shape[1],5) + self.assertEqual(len(ndat.shape), len(dat.shape) + 1) + self.assertEqual(ndat.shape[0], 10) + self.assertEqual(ndat.shape[1], 5) # verify contents (a couple random spots) - assert_array_equal(ndat[4],dat) - assert_array_equal(ndat[7],dat) - assert_array_equal(ndat.dims[0],d) - assert_array_equal(ndat.dims[1],dat.dims[0]) + assert_array_equal(ndat[4], dat) + assert_array_equal(ndat[7], dat) + assert_array_equal(ndat.dims[0], d) + assert_array_equal(ndat.dims[1], dat.dims[0]) def test_extend(self): """Test the extend method""" # make ndarrays and DimArrays with identical data - arr1 = np.arange(256).reshape((4,4,4,4)) - dat1 = DimArray(arr1,dims=[Dim(np.arange(100,500,100),name='one'), - Dim(np.arange(30,70,10),name='two'), - Dim(np.arange(4),name='three'), - Dim(np.arange(1000,1200,50),name='four')], + arr1 = np.arange(256).reshape((4, 4, 4, 4)) + dat1 = DimArray(arr1, dims=[Dim(np.arange(100, 500, 100), name='one'), + Dim(np.arange(30, 70, 10), name='two'), + Dim(np.arange(4), name='three'), + Dim(np.arange(1000, 1200, 50), name='four')], test='tst') - arr2 = np.arange(256,512).reshape((4,4,4,4)) - dat2 = DimArray(arr2,dims=[Dim(np.arange(100,500,100),name='one'), - Dim(np.arange(30,70,10),name='two'), - Dim(np.arange(4,8),name='three'), - Dim(np.arange(1000,1200,50),name='four')], + arr2 = np.arange(256, 512).reshape((4, 4, 4, 4)) + dat2 = DimArray(arr2, dims=[Dim(np.arange(100, 500, 100), name='one'), + Dim(np.arange(30, 70, 10), name='two'), + Dim(np.arange(4, 8), name='three'), + Dim(np.arange(1000, 1200, 50), name='four')], test='tst') # extend - dat1dat2 = dat1.extend(dat2,'three') - arr1arr2 = np.concatenate([arr1,arr2],2) - assert_array_equal(dat1dat2,arr1arr2) - + dat1dat2 = dat1.extend(dat2, 'three') + arr1arr2 = np.concatenate([arr1, arr2], 2) + assert_array_equal(dat1dat2, arr1arr2) + # # test making bins on all dimensions: # test1a = dat.make_bins('one',2,np.mean) # assert_array_equal(test1a.dims[0],np.array([150,350])) @@ -612,7 +620,7 @@ def test_extend(self): # dat = DimArray(arr,dims=[Dim(np.arange(4),name='one'), # Dim(np.arange(16),name='two'), # Dim(np.arange(4),name='three')],test='tst') - + # self.assertRaises(ValueError,dat.make_bins,'two',3,np.mean) # test5a = dat.make_bins('two',3,np.mean,bin_labels=['1st','2nd','3rd'], # error_on_nonexact=False) @@ -631,75 +639,78 @@ def test_extend(self): # for d,dn in enumerate(dat.dim_names): # self.assertEquals(test5a.dim_names[d],dn) # self.assertEquals(test5b.dim_names[d],dn) - - + def test_make_bins(self): """Test the make_bins method""" # make ndarray and DimArray with identical data - arr = np.arange(256).reshape((4,4,4,4)) - dat = DimArray(arr,dims=[Dim(np.arange(100,500,100),name='one'), - Dim(np.arange(30,70,10),name='two'), - Dim(np.arange(4),name='three'), - Dim(np.arange(1000,1200,50),name='four')], + arr = np.arange(256).reshape((4, 4, 4, 4)) + dat = DimArray(arr, dims=[Dim(np.arange(100, 500, 100), name='one'), + Dim(np.arange(30, 70, 10), name='two'), + Dim(np.arange(4), name='three'), + Dim(np.arange(1000, 1200, 50), name='four')], test='tst') - + # test making bins on all dimensions: - test1a = dat.make_bins('one',2,np.mean) - assert_array_equal(test1a.dims[0],np.array([150,350])) - test1b = dat.make_bins(0,2,np.mean,bin_labels='sequential') - assert_array_equal(test1b.dims[0],np.array([0,1])) - assert_array_equal(test1a,test1b) - test2a = dat.make_bins('two',2,np.mean) - assert_array_equal(test2a.dims[1],np.array([35,55])) - test2b = dat.make_bins(1,2,np.mean,bin_labels=['a','b']) - assert_array_equal(test2b.dims[1],np.array(['a','b'])) - assert_array_equal(test2a,test2b) - test3a = dat.make_bins('three',2,np.mean,bin_labels='function') - assert_array_equal(test3a.dims[2],np.array([0.5,2.5])) - test3b = dat.make_bins(2,2,np.mean) - assert_array_equal(test3b.dims[2],np.array([0.5,2.5])) - assert_array_equal(test3a,test3b) - test4a = dat.make_bins('four',2,np.mean) - assert_array_equal(test4a.dims[3],np.array([1025,1125])) - test4b = dat.make_bins(3,2,np.mean) - assert_array_equal(test4b.dims[3],np.array([1025,1125])) - assert_array_equal(test4a,test4b) + test1a = dat.make_bins('one', 2, np.mean) + assert_array_equal(test1a.dims[0], np.array([150, 350])) + test1b = dat.make_bins(0, 2, np.mean, bin_labels='sequential') + assert_array_equal(test1b.dims[0], np.array([0, 1])) + assert_array_equal(test1a, test1b) + test2a = dat.make_bins('two', 2, np.mean) + assert_array_equal(test2a.dims[1], np.array([35, 55])) + test2b = dat.make_bins(1, 2, np.mean, bin_labels=['a', 'b']) + assert_array_equal(test2b.dims[1], np.array(['a', 'b'])) + assert_array_equal(test2a, test2b) + test3a = dat.make_bins('three', 2, np.mean, bin_labels='function') + assert_array_equal(test3a.dims[2], np.array([0.5, 2.5])) + test3b = dat.make_bins(2, 2, np.mean) + assert_array_equal(test3b.dims[2], np.array([0.5, 2.5])) + assert_array_equal(test3a, test3b) + test4a = dat.make_bins('four', 2, np.mean) + assert_array_equal(test4a.dims[3], np.array([1025, 1125])) + test4b = dat.make_bins(3, 2, np.mean) + assert_array_equal(test4b.dims[3], np.array([1025, 1125])) + assert_array_equal(test4a, test4b) # test specifiying bins: - test4c = dat.make_bins('four',[[1000,1100],[1100,2000]],np.mean) - test4d = dat.make_bins(3,[[1000,1100],[1100,2000]],np.mean) - assert_array_equal(test4c,test4d) - assert_array_equal(test4a,test4d) + test4c = dat.make_bins('four', [[1000, 1100], [1100, 2000]], np.mean) + test4d = dat.make_bins(3, [[1000, 1100], [1100, 2000]], np.mean) + assert_array_equal(test4c, test4d) + assert_array_equal(test4a, test4d) split = np.split # compare output to reproduced output for ndarray: - test1c = np.array(split(arr,2,axis=0)).mean(1) - assert_array_equal(test1a,test1c) - test2c = np.array(split(arr,2,axis=1)).mean(2).transpose([1,0,2,3]) - assert_array_equal(test2a,test2c) - test3c = np.array(split(arr,2,axis=2)).mean(3).transpose([1,2,0,3]) - assert_array_equal(test3a,test3c) - test4e = np.array(split(arr,2,axis=3)).mean(4).transpose([1,2,3,0]) - assert_array_equal(test4a,test4e) + test1c = np.array(split(arr, 2, axis=0)).mean(1) + assert_array_equal(test1a, test1c) + test2c = np.array(split(arr, 2, axis=1)).mean( + 2).transpose([1, 0, 2, 3]) + assert_array_equal(test2a, test2c) + test3c = np.array(split(arr, 2, axis=2)).mean( + 3).transpose([1, 2, 0, 3]) + assert_array_equal(test3a, test3c) + test4e = np.array(split(arr, 2, axis=3)).mean( + 4).transpose([1, 2, 3, 0]) + assert_array_equal(test4a, test4e) # compare sequential applications of make_bins to desired output: - test12a = test1a.make_bins('two',2,np.mean) - assert_array_equal(test1a.dims[0],test12a.dims[0]) - assert_array_equal(test2a.dims[1],test12a.dims[1]) - test21a = test2a.make_bins('one',2,np.mean) - assert_array_equal(test1a.dims[0],test21a.dims[0]) - assert_array_equal(test2a.dims[1],test21a.dims[1]) - assert_array_equal(test12a,test21a) - test12b = test1a.make_bins(1,2,np.mean) - assert_array_equal(test1a.dims[0],test12b.dims[0]) - assert_array_equal(test2a.dims[1],test12b.dims[1]) - test21b = test2a.make_bins(0,2,np.mean) - assert_array_equal(test1a.dims[0],test21b.dims[0]) - assert_array_equal(test2a.dims[1],test21b.dims[1]) - assert_array_equal(test12b,test21b) + test12a = test1a.make_bins('two', 2, np.mean) + assert_array_equal(test1a.dims[0], test12a.dims[0]) + assert_array_equal(test2a.dims[1], test12a.dims[1]) + test21a = test2a.make_bins('one', 2, np.mean) + assert_array_equal(test1a.dims[0], test21a.dims[0]) + assert_array_equal(test2a.dims[1], test21a.dims[1]) + assert_array_equal(test12a, test21a) + test12b = test1a.make_bins(1, 2, np.mean) + assert_array_equal(test1a.dims[0], test12b.dims[0]) + assert_array_equal(test2a.dims[1], test12b.dims[1]) + test21b = test2a.make_bins(0, 2, np.mean) + assert_array_equal(test1a.dims[0], test21b.dims[0]) + assert_array_equal(test2a.dims[1], test21b.dims[1]) + assert_array_equal(test12b, test21b) # check that attributes are preserved: for a in dat._attrs: - if a == 'dims': continue + if a == 'dims': + continue self.assertEqual(dat.__getattribute__(a), test1a.__getattribute__(a)) self.assertEqual(dat.__getattribute__(a), @@ -726,80 +737,80 @@ def test_make_bins(self): test21a.__getattribute__(a)) self.assertEqual(dat.__getattribute__(a), test21b.__getattribute__(a)) - for d,dn in enumerate(dat.dim_names): - self.assertEquals(test1a.dim_names[d],dn) - self.assertEquals(test1b.dim_names[d],dn) - self.assertEquals(test2a.dim_names[d],dn) - self.assertEquals(test2b.dim_names[d],dn) - self.assertEquals(test3a.dim_names[d],dn) - self.assertEquals(test3b.dim_names[d],dn) - self.assertEquals(test4a.dim_names[d],dn) - self.assertEquals(test4b.dim_names[d],dn) - self.assertEquals(test12a.dim_names[d],dn) - self.assertEquals(test12b.dim_names[d],dn) - self.assertEquals(test21a.dim_names[d],dn) - self.assertEquals(test21b.dim_names[d],dn) + for d, dn in enumerate(dat.dim_names): + self.assertEqual(test1a.dim_names[d], dn) + self.assertEqual(test1b.dim_names[d], dn) + self.assertEqual(test2a.dim_names[d], dn) + self.assertEqual(test2b.dim_names[d], dn) + self.assertEqual(test3a.dim_names[d], dn) + self.assertEqual(test3b.dim_names[d], dn) + self.assertEqual(test4a.dim_names[d], dn) + self.assertEqual(test4b.dim_names[d], dn) + self.assertEqual(test12a.dim_names[d], dn) + self.assertEqual(test12b.dim_names[d], dn) + self.assertEqual(test21a.dim_names[d], dn) + self.assertEqual(test21b.dim_names[d], dn) # test unequal bins: - arr = np.arange(256).reshape((4,16,4)) - dat = DimArray(arr,dims=[Dim(np.arange(4),name='one'), - Dim(np.arange(16),name='two'), - Dim(np.arange(4),name='three')],test='tst') - - self.assertRaises(ValueError,dat.make_bins,'two',3,np.mean) - test5a = dat.make_bins('two',3,np.mean,bin_labels=['1st','2nd','3rd'], + arr = np.arange(256).reshape((4, 16, 4)) + dat = DimArray(arr, dims=[Dim(np.arange(4), name='one'), + Dim(np.arange(16), name='two'), + Dim(np.arange(4), name='three')], test='tst') + + self.assertRaises(ValueError, dat.make_bins, 'two', 3, np.mean) + test5a = dat.make_bins('two', 3, np.mean, bin_labels=['1st', '2nd', '3rd'], error_on_nonexact=False) - test5b = dat.make_bins(1,[[0,6,'1st'],[6,11,'2nd'],[11,16,'3rd']], + test5b = dat.make_bins(1, [[0, 6, '1st'], [6, 11, '2nd'], [11, 16, '3rd']], np.mean) - assert_array_equal(test5a,test5b) - assert_array_equal(test5a.dims[1],np.array(['1st','2nd','3rd'])) - assert_array_equal(test5a.dims[1],test5b.dims[1]) + assert_array_equal(test5a, test5b) + assert_array_equal(test5a.dims[1], np.array(['1st', '2nd', '3rd'])) + assert_array_equal(test5a.dims[1], test5b.dims[1]) # check that attributes are preserved: for a in dat._attrs: - if a == 'dims': continue + if a == 'dims': + continue self.assertEqual(dat.__getattribute__(a), test5a.__getattribute__(a)) self.assertEqual(dat.__getattribute__(a), test5b.__getattribute__(a)) - for d,dn in enumerate(dat.dim_names): - self.assertEquals(test5a.dim_names[d],dn) - self.assertEquals(test5b.dim_names[d],dn) - - + for d, dn in enumerate(dat.dim_names): + self.assertEqual(test5a.dim_names[d], dn) + self.assertEqual(test5b.dim_names[d], dn) + def test_funcs(self): """Test the numpy functions""" # make ndarray an Dimaray with identical data - arr = np.random.rand(5,12,3,1) - dat = DimArray(arr,dims=[Dim(range(5),name='one'), - Dim(range(12),name='two'), - Dim(range(3),name='three'), - Dim(range(1),name='four')],test='tst') - + arr = np.random.rand(5, 12, 3, 1) + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one'), + Dim(list(range(12)), name='two'), + Dim(list(range(3)), name='three'), + Dim(list(range(1)), name='four')], test='tst') + # these are functions that take an axis argument: - funcs = [np.mean,np.all,np.any,np.argmax,np.argmin,np.argsort, - np.cumprod,np.cumsum,np.max,np.mean,np.min,np.prod, - np.ptp,np.std,np.sum,np.var] - + funcs = [np.mean, np.all, np.any, np.argmax, np.argmin, np.argsort, + np.cumprod, np.cumsum, np.max, np.mean, np.min, np.prod, + np.ptp, np.std, np.sum, np.var] + # The axes for the ndarray: - axes_arr = [None,0,1,2,3,0,1,2,3] + axes_arr = [None, 0, 1, 2, 3, 0, 1, 2, 3] # The axes for the DimArray (we want to test indexing them by # number and name): - axes_dat = [None,0,1,2,3,'one','two','three','four'] + axes_dat = [None, 0, 1, 2, 3, 'one', 'two', 'three', 'four'] # loop through the functions and axes: for func in funcs: for a in range(len(axes_arr)): # apply the function to the ndarray and the DimArray - arr_func = func(arr,axis=axes_arr[a]) - dat_func = func(dat,axis=axes_dat[a]) + arr_func = func(arr, axis=axes_arr[a]) + dat_func = func(dat, axis=axes_dat[a]) # make sure they are the same: - assert_array_equal(arr_func,dat_func) + assert_array_equal(arr_func, dat_func) if not(axes_dat[a] is None): # ensure we still have a DimArray - self.assertTrue(isinstance(dat_func,DimArray)) + self.assertTrue(isinstance(dat_func, DimArray)) # ensure that the attributes are preserved - self.assertEquals(dat_func.test,'tst') - + self.assertEqual(dat_func.test, 'tst') + # same tests as above but this time calling the DimArray # methods directly (this test is necessary because it is in # principle possible for the numpy function to work and the @@ -836,79 +847,79 @@ def test_funcs(self): assert_array_equal(arr.var(axes_arr[a]), dat.var(axes_dat[a])) if not(axes_dat[a] is None): - self.assertTrue(isinstance(dat.all(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.any(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.argmax(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.argmin(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.argsort(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.cumprod(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.cumsum(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.max(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.mean(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.min(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.prod(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.ptp(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.std(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.sum(axes_arr[a]),DimArray)) - self.assertTrue(isinstance(dat.var(axes_arr[a]),DimArray)) - - self.assertEquals(dat.all(axes_arr[a]).test,'tst') - self.assertEquals(dat.any(axes_arr[a]).test,'tst') - self.assertEquals(dat.argmax(axes_arr[a]).test,'tst') - self.assertEquals(dat.argmin(axes_arr[a]).test,'tst') - self.assertEquals(dat.argsort(axes_arr[a]).test,'tst') - self.assertEquals(dat.cumprod(axes_arr[a]).test,'tst') - self.assertEquals(dat.cumsum(axes_arr[a]).test,'tst') - self.assertEquals(dat.max(axes_arr[a]).test,'tst') - self.assertEquals(dat.mean(axes_arr[a]).test,'tst') - self.assertEquals(dat.min(axes_arr[a]).test,'tst') - self.assertEquals(dat.prod(axes_arr[a]).test,'tst') - self.assertEquals(dat.ptp(axes_arr[a]).test,'tst') - self.assertEquals(dat.std(axes_arr[a]).test,'tst') - self.assertEquals(dat.sum(axes_arr[a]).test,'tst') - self.assertEquals(dat.var(axes_arr[a]).test,'tst') - + self.assertTrue(isinstance(dat.all(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.any(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.argmax(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.argmin(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.argsort(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.cumprod(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.cumsum(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.max(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.mean(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.min(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.prod(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.ptp(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.std(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.sum(axes_arr[a]), DimArray)) + self.assertTrue(isinstance(dat.var(axes_arr[a]), DimArray)) + + self.assertEqual(dat.all(axes_arr[a]).test, 'tst') + self.assertEqual(dat.any(axes_arr[a]).test, 'tst') + self.assertEqual(dat.argmax(axes_arr[a]).test, 'tst') + self.assertEqual(dat.argmin(axes_arr[a]).test, 'tst') + self.assertEqual(dat.argsort(axes_arr[a]).test, 'tst') + self.assertEqual(dat.cumprod(axes_arr[a]).test, 'tst') + self.assertEqual(dat.cumsum(axes_arr[a]).test, 'tst') + self.assertEqual(dat.max(axes_arr[a]).test, 'tst') + self.assertEqual(dat.mean(axes_arr[a]).test, 'tst') + self.assertEqual(dat.min(axes_arr[a]).test, 'tst') + self.assertEqual(dat.prod(axes_arr[a]).test, 'tst') + self.assertEqual(dat.ptp(axes_arr[a]).test, 'tst') + self.assertEqual(dat.std(axes_arr[a]).test, 'tst') + self.assertEqual(dat.sum(axes_arr[a]).test, 'tst') + self.assertEqual(dat.var(axes_arr[a]).test, 'tst') + # test functions that require function specific input: for a in range(len(axes_arr)): if axes_arr[a] is None: length = len(arr) else: length = np.shape(arr)[axes_arr[a]] - cond = np.random.random(length)>0.5 + cond = np.random.random(length) > 0.5 # calling the compress method directly: - arr_func = arr.compress(cond,axis=axes_arr[a]) - dat_func = dat.compress(cond,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = arr.compress(cond, axis=axes_arr[a]) + dat_func = dat.compress(cond, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a] is not None: - self.assertTrue(isinstance(dat_func,DimArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, DimArray)) + self.assertEqual(dat_func.test, 'tst') # calling the numpy compress function: - arr_func = np.compress(cond,arr,axis=axes_arr[a]) - dat_func = np.compress(cond,dat,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = np.compress(cond, arr, axis=axes_arr[a]) + dat_func = np.compress(cond, dat, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a] is not None: - self.assertTrue(isinstance(dat_func,DimArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, DimArray)) + self.assertEqual(dat_func.test, 'tst') # the below tests should not run with axis==None: if axes_arr[a] is None: continue - + reps = np.random.random_integers(low=1, high=10, size=length) # calling the repeat method directly: - arr_func = arr.repeat(reps,axis=axes_arr[a]) - dat_func = dat.repeat(reps,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = arr.repeat(reps, axis=axes_arr[a]) + dat_func = dat.repeat(reps, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a] is not None: - self.assertTrue(isinstance(dat_func,AttrArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, AttrArray)) + self.assertEqual(dat_func.test, 'tst') # calling the numpy repeat function: - arr_func = np.repeat(arr,reps,axis=axes_arr[a]) - dat_func = np.repeat(dat,reps,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = np.repeat(arr, reps, axis=axes_arr[a]) + dat_func = np.repeat(dat, reps, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a] is not None: - self.assertTrue(isinstance(dat_func,AttrArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, AttrArray)) + self.assertEqual(dat_func.test, 'tst') # skip the last dimension for this test for # convenience (the last dimension only has 1 level): @@ -917,83 +928,81 @@ def test_funcs(self): indcs = np.arange(len(arr.shape)) # calling the take method directly (squeeze, to get rid of # the last dimension): - arr_func = arr.squeeze().take(indcs,axis=axes_arr[a]) - dat_func = dat.squeeze().take(indcs,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = arr.squeeze().take(indcs, axis=axes_arr[a]) + dat_func = dat.squeeze().take(indcs, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a]: - self.assertTrue(isinstance(dat_func,AttrArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, AttrArray)) + self.assertEqual(dat_func.test, 'tst') # calling the numpy take function directly (squeeze, to get rid of # the last dimension): - arr_func = np.take(arr.squeeze(),indcs,axis=axes_arr[a]) - dat_func = np.take(dat.squeeze(),indcs,axis=axes_dat[a]) - assert_array_equal(arr_func,dat_func) + arr_func = np.take(arr.squeeze(), indcs, axis=axes_arr[a]) + dat_func = np.take(dat.squeeze(), indcs, axis=axes_dat[a]) + assert_array_equal(arr_func, dat_func) if axes_dat[a]: - self.assertTrue(isinstance(dat_func,AttrArray)) - self.assertEquals(dat_func.test,'tst') + self.assertTrue(isinstance(dat_func, AttrArray)) + self.assertEqual(dat_func.test, 'tst') # This should work with numpy 1.2 but doesn't # with 1.1.1 or below (therfore commented out for now): # arr_func = arr.clip(0.4,0.6) # dat_func = dat.clip(0.4,0.6) # assert_array_equal(arr_func,dat_func) - #self.assertTrue(isinstance(dat_func,DimArray)) - #self.assertEquals(dat_func.test,'tst') + # self.assertTrue(isinstance(dat_func,DimArray)) + # self.assertEquals(dat_func.test,'tst') #arr_func = np.clip(arr,0.4,0.6) #dat_func = np.clip(dat,0.4,0.6) - #assert_array_equal(arr_func,dat_func) - #self.assertTrue(isinstance(dat_func,DimArray)) - #self.assertEquals(dat_func.test,'tst') + # assert_array_equal(arr_func,dat_func) + # self.assertTrue(isinstance(dat_func,DimArray)) + # self.assertEquals(dat_func.test,'tst') # other functions that don't necessarily take return a # DimArray: - funcs = [np.diagonal,np.nonzero,np.ravel,np.squeeze, - np.sort,np.trace,np.transpose] + funcs = [np.diagonal, np.nonzero, np.ravel, np.squeeze, + np.sort, np.trace, np.transpose] for func in funcs: arr_func = func(arr) dat_func = func(dat) - assert_array_equal(arr_func,dat_func) + assert_array_equal(arr_func, dat_func) # same tests as above, but calling the methods directly: - assert_array_equal(arr.diagonal(),dat.diagonal()) - assert_array_equal(arr.nonzero(),dat.nonzero()) - assert_array_equal(arr.ravel(),dat.ravel()) - assert_array_equal(arr.squeeze(),dat.squeeze()) - assert_array_equal(arr.sort(),dat.sort()) - assert_array_equal(arr.trace(),dat.trace()) - assert_array_equal(arr.transpose(),dat.transpose()) + assert_array_equal(arr.diagonal(), dat.diagonal()) + assert_array_equal(arr.nonzero(), dat.nonzero()) + assert_array_equal(arr.ravel(), dat.ravel()) + assert_array_equal(arr.squeeze(), dat.squeeze()) + assert_array_equal(arr.sort(), dat.sort()) + assert_array_equal(arr.trace(), dat.trace()) + assert_array_equal(arr.transpose(), dat.transpose()) # there is no numpy.flatten() function, so we only call the # method directly: - assert_array_equal(arr.flatten(),dat.flatten()) - - assert_array_equal(arr.swapaxes(0,1),dat.swapaxes(0,1)) - self.assertTrue(isinstance(dat.swapaxes(0,1),DimArray)) - self.assertEquals(dat.swapaxes(0,1).test,'tst') - assert_array_equal(arr.swapaxes(0,1),dat.swapaxes('one','two')) - self.assertTrue(isinstance(dat.swapaxes('one','two'),DimArray)) - self.assertEquals(dat.swapaxes('one','two').test,'tst') - assert_array_equal(arr.swapaxes(1,3),dat.swapaxes(1,3)) - self.assertTrue(isinstance(dat.swapaxes(1,3),DimArray)) - self.assertEquals(dat.swapaxes(1,3).test,'tst') - assert_array_equal(arr.swapaxes(1,3),dat.swapaxes('two','four')) - self.assertTrue(isinstance(dat.swapaxes('two','four'),DimArray)) - self.assertEquals(dat.swapaxes('two','four').test,'tst') - + assert_array_equal(arr.flatten(), dat.flatten()) + + assert_array_equal(arr.swapaxes(0, 1), dat.swapaxes(0, 1)) + self.assertTrue(isinstance(dat.swapaxes(0, 1), DimArray)) + self.assertEqual(dat.swapaxes(0, 1).test, 'tst') + assert_array_equal(arr.swapaxes(0, 1), dat.swapaxes('one', 'two')) + self.assertTrue(isinstance(dat.swapaxes('one', 'two'), DimArray)) + self.assertEqual(dat.swapaxes('one', 'two').test, 'tst') + assert_array_equal(arr.swapaxes(1, 3), dat.swapaxes(1, 3)) + self.assertTrue(isinstance(dat.swapaxes(1, 3), DimArray)) + self.assertEqual(dat.swapaxes(1, 3).test, 'tst') + assert_array_equal(arr.swapaxes(1, 3), dat.swapaxes('two', 'four')) + self.assertTrue(isinstance(dat.swapaxes('two', 'four'), DimArray)) + self.assertEqual(dat.swapaxes('two', 'four').test, 'tst') + def test_ufuncs(self): """Test the numpy u-functions""" # make ndarray an Dimaray with identical data arr = np.random.rand(5) - dat = DimArray(arr,dims=[Dim(range(5),name='one')], + dat = DimArray(arr, dims=[Dim(list(range(5)), name='one')], test='tst') x = np.ones(10) * dat[[0]] - self.assertTrue(isinstance(x,np.ndarray)) + self.assertTrue(isinstance(x, np.ndarray)) x = np.ones(1) * dat[[0]] - self.assertTrue(isinstance(x,DimArray)) + self.assertTrue(isinstance(x, DimArray)) x = 22 * dat - self.assertTrue(isinstance(x,DimArray)) - assert_array_equal(x,arr*22) - - + self.assertTrue(isinstance(x, DimArray)) + assert_array_equal(x, arr * 22) diff --git a/docs/conf.py b/docs/conf.py index 0e1238a..4f87e20 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os #import sphinx.ext.autodoc # If your extensions are in another directory, add it here. If the directory @@ -29,7 +30,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc','numpydoc'] +extensions = ['sphinx.ext.autodoc', 'numpydoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -44,8 +45,8 @@ master_doc = 'index' # General information about the project. -project = u'PTSA' -copyright = u'2009-2010, Per B. Sederberg & Christoph T. Weidemann' +project = 'PTSA' +copyright = '2009-2010, Per B. Sederberg & Christoph T. Weidemann' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -108,7 +109,7 @@ # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = None #'_static/logo.png' +html_logo = None # '_static/logo.png' # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 @@ -171,8 +172,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'PTSA.tex', ur'PTSA Documentation', - ur'Per B. Sederberg and Christoph T. Weidemann', 'manual'), + ('index', 'PTSA.tex', r'PTSA Documentation', + r'Per B. Sederberg and Christoph T. Weidemann', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/docs/sphinxexts/docscrape.py b/docs/sphinxexts/docscrape.py index f374b3d..e060f9b 100644 --- a/docs/sphinxexts/docscrape.py +++ b/docs/sphinxexts/docscrape.py @@ -6,13 +6,16 @@ import textwrap import re import pydoc -from StringIO import StringIO +from io import StringIO from warnings import warn 4 + + class Reader(object): """A line-based string reader. """ + def __init__(self, data): """ Parameters @@ -21,10 +24,10 @@ def __init__(self, data): String with lines separated by '\n'. """ - if isinstance(data,list): + if isinstance(data, list): self._str = data else: - self._str = data.split('\n') # store string as list of lines + self._str = data.split('\n') # store string as list of lines self.reset() @@ -32,7 +35,7 @@ def __getitem__(self, n): return self._str[n] def reset(self): - self._l = 0 # current line nr + self._l = 0 # current line nr def read(self): if not self.eof(): @@ -59,11 +62,12 @@ def read_to_condition(self, condition_func): return self[start:self._l] self._l += 1 if self.eof(): - return self[start:self._l+1] + return self[start:self._l + 1] return [] def read_to_next_empty_line(self): self.seek_next_non_empty_line() + def is_empty(line): return not line.strip() return self.read_to_condition(is_empty) @@ -73,7 +77,7 @@ def is_unindented(line): return (line.strip() and (len(line.lstrip()) == len(line))) return self.read_to_condition(is_unindented) - def peek(self,n=0): + def peek(self, n=0): if self._l + n < len(self._str): return self[self._l + n] else: @@ -84,7 +88,7 @@ def is_empty(self): class NumpyDocString(object): - def __init__(self,docstring): + def __init__(self, docstring): docstring = textwrap.dedent(docstring).split('\n') self._doc = Reader(docstring) @@ -105,15 +109,15 @@ def __init__(self,docstring): 'References': '', 'Examples': '', 'index': {} - } + } self._parse() - def __getitem__(self,key): + def __getitem__(self, key): return self._parsed_data[key] - def __setitem__(self,key,val): - if not self._parsed_data.has_key(key): + def __setitem__(self, key, val): + if key not in self._parsed_data: warn("Unknown section %s" % key) else: self._parsed_data[key] = val @@ -129,25 +133,27 @@ def _is_at_section(self): if l1.startswith('.. index::'): return True - l2 = self._doc.peek(1).strip() # ---------- or ========== - return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1)) + l2 = self._doc.peek(1).strip() # ---------- or ========== + return l2.startswith('-' * len(l1)) or l2.startswith('=' * len(l1)) - def _strip(self,doc): + def _strip(self, doc): i = 0 j = 0 - for i,line in enumerate(doc): - if line.strip(): break + for i, line in enumerate(doc): + if line.strip(): + break - for j,line in enumerate(doc[::-1]): - if line.strip(): break + for j, line in enumerate(doc[::-1]): + if line.strip(): + break - return doc[i:len(doc)-j] + return doc[i:len(doc) - j] def _read_to_next_section(self): section = self._doc.read_to_next_empty_line() while not self._is_at_section() and not self._doc.eof(): - if not self._doc.peek(-1).strip(): # previous line was empty + if not self._doc.peek(-1).strip(): # previous line was empty section += [''] section += self._doc.read_to_next_empty_line() @@ -159,14 +165,14 @@ def _read_sections(self): data = self._read_to_next_section() name = data[0].strip() - if name.startswith('..'): # index section + if name.startswith('..'): # index section yield name, data[1:] elif len(data) < 2: yield StopIteration else: yield name, self._strip(data[2:]) - def _parse_param_list(self,content): + def _parse_param_list(self, content): r = Reader(content) params = [] while not r.eof(): @@ -179,13 +185,13 @@ def _parse_param_list(self,content): desc = r.read_to_next_unindented_line() desc = dedent_lines(desc) - params.append((arg_name,arg_type,desc)) + params.append((arg_name, arg_type, desc)) return params - _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) + def _parse_see_also(self, content): """ func_name : Descriptive text @@ -216,9 +222,10 @@ def push_item(name, rest): current_func = None rest = [] - + for line in content: - if not line.strip(): continue + if not line.strip(): + continue m = self._name_rgx.match(line) if m and line[m.end():].strip().startswith(':'): @@ -258,7 +265,7 @@ def strip_each_in(lst): if len(line) > 2: out[line[1]] = strip_each_in(line[2].split(',')) return out - + def _parse_summary(self): """Grab signature (if given) and summary""" if self._is_at_section(): @@ -275,14 +282,15 @@ def _parse_summary(self): if not self._is_at_section(): self['Extended Summary'] = self._read_to_next_section() - + def _parse(self): self._doc.reset() self._parse_summary() - for (section,content) in self._read_sections(): + for (section, content) in self._read_sections(): if not section.startswith('..'): - section = ' '.join([s.capitalize() for s in section.split(' ')]) + section = ' '.join([s.capitalize() + for s in section.split(' ')]) if section in ('Parameters', 'Attributes', 'Methods', 'Returns', 'Raises', 'Warns'): self[section] = self._parse_param_list(content) @@ -296,17 +304,17 @@ def _parse(self): # string conversion routines def _str_header(self, name, symbol='-'): - return [name, len(name)*symbol] + return [name, len(name) * symbol] def _str_indent(self, doc, indent=4): out = [] for line in doc: - out += [' '*indent + line] + out += [' ' * indent + line] return out def _str_signature(self): if self['Signature']: - return [self['Signature'].replace('*','\*')] + [''] + return [self['Signature'].replace('*', '\*')] + [''] else: return [''] @@ -326,7 +334,7 @@ def _str_param_list(self, name): out = [] if self[name]: out += self._str_header(name) - for param,param_type,desc in self[name]: + for param, param_type, desc in self[name]: out += ['%s : %s' % (param, param_type)] out += self._str_indent(desc) out += [''] @@ -341,7 +349,8 @@ def _str_section(self, name): return out def _str_see_also(self, func_role): - if not self['See Also']: return [] + if not self['See Also']: + return [] out = [] out += self._str_header("See Also") last_had_desc = True @@ -368,8 +377,8 @@ def _str_see_also(self, func_role): def _str_index(self): idx = self['index'] out = [] - out += ['.. index:: %s' % idx.get('default','')] - for section, references in idx.iteritems(): + out += ['.. index:: %s' % idx.get('default', '')] + for section, references in idx.items(): if section == 'default': continue out += [' :%s: %s' % (section, ', '.join(references))] @@ -380,46 +389,48 @@ def __str__(self, func_role=''): out += self._str_signature() out += self._str_summary() out += self._str_extended_summary() - for param_list in ('Parameters','Returns','Raises'): + for param_list in ('Parameters', 'Returns', 'Raises'): out += self._str_param_list(param_list) out += self._str_section('Warnings') out += self._str_see_also(func_role) - for s in ('Notes','References','Examples'): + for s in ('Notes', 'References', 'Examples'): out += self._str_section(s) out += self._str_index() return '\n'.join(out) -def indent(str,indent=4): - indent_str = ' '*indent +def indent(str, indent=4): + indent_str = ' ' * indent if str is None: return indent_str lines = str.split('\n') return '\n'.join(indent_str + l for l in lines) + def dedent_lines(lines): """Deindent a list of lines maximally""" return textwrap.dedent("\n".join(lines)).split("\n") + def header(text, style='-'): - return text + '\n' + style*len(text) + '\n' + return text + '\n' + style * len(text) + '\n' class FunctionDoc(NumpyDocString): def __init__(self, func, role='func', doc=None): self._f = func - self._role = role # e.g. "func" or "meth" + self._role = role # e.g. "func" or "meth" if doc is None: doc = inspect.getdoc(func) or '' try: NumpyDocString.__init__(self, doc) - except ValueError, e: - print '*'*78 - print "ERROR: '%s' while parsing `%s`" % (e, self._f) - print '*'*78 - #print "Docstring follows:" - #print doclines - #print '='*78 + except ValueError as e: + print('*' * 78) + print("ERROR: '%s' while parsing `%s`" % (e, self._f)) + print('*' * 78) + # print "Docstring follows:" + # print doclines + # print '='*78 if not self['Signature']: func, func_name = self.get_func() @@ -427,9 +438,9 @@ def __init__(self, func, role='func', doc=None): # try to read signature argspec = inspect.getargspec(func) argspec = inspect.formatargspec(*argspec) - argspec = argspec.replace('*','\*') + argspec = argspec.replace('*', '\*') signature = '%s%s' % (func_name, argspec) - except TypeError, e: + except TypeError as e: signature = '%s()' % func_name self['Signature'] = signature @@ -440,7 +451,7 @@ def get_func(self): else: func = self._f return func, func_name - + def __str__(self): out = '' @@ -451,9 +462,9 @@ def __str__(self): 'meth': 'method'} if self._role: - if not roles.has_key(self._role): - print "Warning: invalid role %s" % self._role - out += '.. %s:: %s\n \n\n' % (roles.get(self._role,''), + if self._role not in roles: + print("Warning: invalid role %s" % self._role) + out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''), func_name) out += super(FunctionDoc, self).__str__(func_role=self._role) @@ -461,7 +472,7 @@ def __str__(self): class ClassDoc(NumpyDocString): - def __init__(self,cls,modulename='',func_doc=FunctionDoc,doc=None): + def __init__(self, cls, modulename='', func_doc=FunctionDoc, doc=None): if not inspect.isclass(cls): raise ValueError("Initialise using a class. Got %r" % cls) self._cls = cls @@ -479,7 +490,7 @@ def __init__(self,cls,modulename='',func_doc=FunctionDoc,doc=None): @property def methods(self): - return [name for name,func in inspect.getmembers(self._cls) + return [name for name, func in inspect.getmembers(self._cls) if not name.startswith('_') and callable(func)] def __str__(self): @@ -487,11 +498,9 @@ def __str__(self): out += super(ClassDoc, self).__str__() out += "\n\n" - #for m in self.methods: + # for m in self.methods: # print "Parsing `%s`" % m # out += str(self._func_doc(getattr(self._cls,m), 'meth')) + '\n\n' # out += '.. index::\n single: %s; %s\n\n' % (self._name, m) return out - - diff --git a/docs/sphinxexts/docscrape_sphinx.py b/docs/sphinxexts/docscrape_sphinx.py index 77ed271..8342ed6 100644 --- a/docs/sphinxexts/docscrape_sphinx.py +++ b/docs/sphinxexts/docscrape_sphinx.py @@ -1,6 +1,10 @@ -import re, inspect, textwrap, pydoc +import re +import inspect +import textwrap +import pydoc from docscrape import NumpyDocString, FunctionDoc, ClassDoc + class SphinxDocString(NumpyDocString): # string conversion routines def _str_header(self, name, symbol='`'): @@ -12,7 +16,7 @@ def _str_field_list(self, name): def _str_indent(self, doc, indent=4): out = [] for line in doc: - out += [' '*indent + line] + out += [' ' * indent + line] return out def _str_signature(self): @@ -33,11 +37,11 @@ def _str_param_list(self, name): if self[name]: out += self._str_field_list(name) out += [''] - for param,param_type,desc in self[name]: + for param, param_type, desc in self[name]: out += self._str_indent(['**%s** : %s' % (param.strip(), param_type)]) out += [''] - out += self._str_indent(desc,8) + out += self._str_indent(desc, 8) out += [''] return out @@ -72,8 +76,8 @@ def _str_index(self): if len(idx) == 0: return out - out += ['.. index:: %s' % idx.get('default','')] - for section, references in idx.iteritems(): + out += ['.. index:: %s' % idx.get('default', '')] + for section, references in idx.items(): if section == 'default': continue elif section == 'refguide': @@ -99,22 +103,25 @@ def __str__(self, indent=0, func_role="obj"): out += self._str_summary() out += self._str_extended_summary() for param_list in ('Parameters', 'Attributes', 'Methods', - 'Returns','Raises'): + 'Returns', 'Raises'): out += self._str_param_list(param_list) out += self._str_warnings() out += self._str_see_also(func_role) out += self._str_section('Notes') out += self._str_references() out += self._str_section('Examples') - out = self._str_indent(out,indent) + out = self._str_indent(out, indent) return '\n'.join(out) + class SphinxFunctionDoc(SphinxDocString, FunctionDoc): pass + class SphinxClassDoc(SphinxDocString, ClassDoc): pass + def get_doc_object(obj, what=None, doc=None): if what is None: if inspect.isclass(obj): @@ -133,4 +140,3 @@ def get_doc_object(obj, what=None, doc=None): if doc is None: doc = pydoc.getdoc(obj) return SphinxDocString(doc) - diff --git a/docs/sphinxexts/numpydoc.py b/docs/sphinxexts/numpydoc.py index 926667d..9138486 100755 --- a/docs/sphinxexts/numpydoc.py +++ b/docs/sphinxexts/numpydoc.py @@ -16,23 +16,26 @@ """ -import os, re, pydoc +import os +import re +import pydoc from docscrape_sphinx import get_doc_object, SphinxDocString import inspect + def mangle_docstrings(app, what, name, obj, options, lines, reference_offset=[0]): if what == 'module': # Strip top title title_re = re.compile(r'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*', - re.I|re.S) + re.I | re.S) lines[:] = title_re.sub('', "\n".join(lines)).split("\n") else: doc = get_doc_object(obj, what, "\n".join(lines)) lines[:] = str(doc).split("\n") if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \ - obj.__name__: + obj.__name__: if hasattr(obj, '__module__'): v = dict(full_name="%s.%s" % (obj.__module__, obj.__name__)) else: @@ -49,7 +52,7 @@ def mangle_docstrings(app, what, name, obj, options, lines, try: references.append(int(l[len('.. ['):l.index(']')])) except ValueError: - print "WARNING: invalid reference in %s docstring" % name + print("WARNING: invalid reference in %s docstring" % name) # Start renaming from the biggest number, otherwise we may # overwrite references. @@ -65,30 +68,35 @@ def mangle_docstrings(app, what, name, obj, options, lines, reference_offset[0] += len(references) + def mangle_signature(app, what, name, obj, options, sig, retann): # Do not try to inspect classes that don't define `__init__` if (inspect.isclass(obj) and - 'initializes x; see ' in pydoc.getdoc(obj.__init__)): + 'initializes x; see ' in pydoc.getdoc(obj.__init__)): return '', '' - if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')): return - if not hasattr(obj, '__doc__'): return + if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')): + return + if not hasattr(obj, '__doc__'): + return doc = SphinxDocString(pydoc.getdoc(obj)) if doc['Signature']: sig = re.sub("^[^(]*", "", doc['Signature']) return sig, '' + def initialize(app): try: app.connect('autodoc-process-signature', mangle_signature) except: monkeypatch_sphinx_ext_autodoc() + def setup(app, get_doc_object_=get_doc_object): global get_doc_object get_doc_object = get_doc_object_ - + app.connect('autodoc-process-docstring', mangle_docstrings) app.connect('builder-inited', initialize) app.add_config_value('numpydoc_edit_link', None, True) @@ -97,6 +105,7 @@ def setup(app, get_doc_object_=get_doc_object): # Monkeypatch sphinx.ext.autodoc to accept argspecless autodocs (Sphinx < 0.5) #------------------------------------------------------------------------------ + def monkeypatch_sphinx_ext_autodoc(): global _original_format_signature import sphinx.ext.autodoc @@ -104,10 +113,11 @@ def monkeypatch_sphinx_ext_autodoc(): if sphinx.ext.autodoc.format_signature is our_format_signature: return - print "[numpydoc] Monkeypatching sphinx.ext.autodoc ..." + print("[numpydoc] Monkeypatching sphinx.ext.autodoc ...") _original_format_signature = sphinx.ext.autodoc.format_signature sphinx.ext.autodoc.format_signature = our_format_signature + def our_format_signature(what, obj): r = mangle_signature(None, what, None, obj, None, None, None) if r is not None: diff --git a/examples/basic_analysis.py b/examples/basic_analysis.py index 466fdcd..9443bd0 100644 --- a/examples/basic_analysis.py +++ b/examples/basic_analysis.py @@ -8,8 +8,8 @@ # some general info nchan = 2 samplerate = 200 -nsamples = samplerate*100 -event_dur = samplerate*1 +nsamples = samplerate * 100 +event_dur = samplerate * 1 buf_dur = 1.0 # generate fake data @@ -17,60 +17,60 @@ aw = ArrayWrapper(dat, samplerate) # generate fake events -eoffset = np.arange(event_dur*2,nsamples-(2*event_dur),event_dur) -esrc = [aw]*len(eoffset) -nrec = len(eoffset)/2 -recalled = [True]*nrec + [False]*(len(eoffset)-nrec) -events = Events(np.rec.fromarrays([esrc,eoffset,recalled], +eoffset = np.arange(event_dur * 2, nsamples - (2 * event_dur), event_dur) +esrc = [aw] * len(eoffset) +nrec = len(eoffset) / 2 +recalled = [True] * int(nrec) + [False] * int((len(eoffset) - nrec)) +events = Events(np.rec.fromarrays([esrc, eoffset, recalled], names='esrc,eoffset,recalled')) # load in data with events (filter at the same time) -rdat = events[events.recalled==True].get_data(0, # channel - 1.0, # duration in sec - 0.0, # offset in sec - buf_dur, # buffer in sec - filt_freq = 20., - filt_type = 'low', - keep_buffer=True - ) -ndat = events[events.recalled==False].get_data(0, # channel - 1.0, # duration in sec - 0.0, # offset in sec - buf_dur, # buffer in sec - filt_freq = 20., - filt_type = 'low', - keep_buffer=True - ) +rdat = events[events.recalled == True].get_data(0, # channel + 1.0, # duration in sec + 0.0, # offset in sec + buf_dur, # buffer in sec + filt_freq=20., + filt_type='low', + keep_buffer=True + ) +ndat = events[events.recalled == False].get_data(0, # channel + 1.0, # duration in sec + 0.0, # offset in sec + buf_dur, # buffer in sec + filt_freq=20., + filt_type='low', + keep_buffer=True + ) # calc wavelet power -freqs = np.arange(2,50,2) -rpow = phase_pow_multi(freqs,rdat,to_return='power') -npow = phase_pow_multi(freqs,ndat,to_return='power') +freqs = np.arange(2, 50, 2) +rpow = phase_pow_multi(freqs, rdat, to_return='power') +npow = phase_pow_multi(freqs, ndat, to_return='power') # remove the buffer now that we have filtered and calculated power -#for ts in [rdat,ndat,rpow,npow]: +# for ts in [rdat,ndat,rpow,npow]: # ts = ts.remove_buffer(buf_dur) # why does the above not work? rdat = rdat.remove_buffer(buf_dur) ndat = ndat.remove_buffer(buf_dur) rpow = rpow.remove_buffer(buf_dur) npow = npow.remove_buffer(buf_dur) - + # plot ERP pl.figure(1) pl.clf() -pl.plot(rdat['time'],rdat.nanmean('events'),'r') -pl.plot(ndat['time'],ndat.nanmean('events'),'b') -pl.legend(('Recalled','Not Recalled'),loc=0) +pl.plot(rdat['time'], rdat.nanmean('events'), 'r') +pl.plot(ndat['time'], ndat.nanmean('events'), 'b') +pl.legend(('Recalled', 'Not Recalled'), loc=0) pl.xlabel('Time (s)') pl.ylabel('Voltage') # plot power spectrum pl.figure(2) pl.clf() -pl.plot(rpow['freqs'],rpow.nanmean('events').nanmean('time'),'r') -pl.plot(npow['freqs'],npow.nanmean('events').nanmean('time'),'b') -pl.legend(('Recalled','Not Recalled'),loc=0) +pl.plot(rpow['freqs'], rpow.nanmean('events').nanmean('time'), 'r') +pl.plot(npow['freqs'], npow.nanmean('events').nanmean('time'), 'b') +pl.legend(('Recalled', 'Not Recalled'), loc=0) pl.xlabel('Frequency (Hz)') pl.ylabel('Power') diff --git a/examples/da_wish.py b/examples/da_wish.py index ece96a4..8de877d 100644 --- a/examples/da_wish.py +++ b/examples/da_wish.py @@ -3,7 +3,7 @@ # import numpy as np -from dimarray import Dim,DimArray,AttrArray +from dimarray import Dim, DimArray, AttrArray if __name__ == "__main__": @@ -11,14 +11,14 @@ Dim(data=np.arange(10), name='freqs'), Dim(data=np.arange(30), name='events')] - dat = DimArray(data=np.random.rand(20,10,30), dims=dims) + dat = DimArray(data=np.random.rand(20, 10, 30), dims=dims) # select some data ind = ((dat['time'] > 10) & - ((dat['events']<10) | (dat['events']>20)) & - (dat['freqs'].is_in(range(0,10,2)))) + ((dat['events'] < 10) | (dat['events'] > 20)) & + (dat['freqs'].is_in(list(range(0, 10, 2))))) subdat = dat[ind] - print dat.shape - print subdat.shape + print(dat.shape) + print(subdat.shape) diff --git a/examples/dataWaveDemo.py b/examples/dataWaveDemo.py index 3817460..50e253d 100644 --- a/examples/dataWaveDemo.py +++ b/examples/dataWaveDemo.py @@ -12,76 +12,76 @@ # hypothetical test case # load events -print "Loading events..." +print("Loading events...") ev = createEventsFromMatFile('/home1/per/eeg/free/CH012/events/events.mat') # we leave the buffer on after getting the data, but pull it off # in the call to tsPhasePow -freqs = range(2,81,2) +freqs = list(range(2, 81, 2)) chan = 27 dur = 2.5 offset = -.500 buf = 1.000 resampledRate = 200 -filtFreq = [58.0,62.0] +filtFreq = [58.0, 62.0] # load the eeg data -print "Loading EEG data..." -rEEG = ev.select(ev['recalled']==1).get_data(chan, - dur, - offset, - buf, - resampledRate, - filtFreq=filtFreq, - keepBuffer=True) -nEEG = ev.select(ev['recalled']==0).get_data(chan, - dur, - offset, - buf, - resampledRate, - filtFreq=filtFreq, - keepBuffer=True) +print("Loading EEG data...") +rEEG = ev.select(ev['recalled'] == 1).get_data(chan, + dur, + offset, + buf, + resampledRate, + filtFreq=filtFreq, + keepBuffer=True) +nEEG = ev.select(ev['recalled'] == 0).get_data(chan, + dur, + offset, + buf, + resampledRate, + filtFreq=filtFreq, + keepBuffer=True) # power for recalled events -print "Calculating power..." +print("Calculating power...") rRes = wavelet.tsPhasePow(freqs, rEEG, - verbose=True,toReturn='pow') + verbose=True, toReturn='pow') # power for not recalled events nRes = wavelet.tsPhasePow(freqs, nEEG, - verbose=True,toReturn='pow') + verbose=True, toReturn='pow') # get mean power across events (axis=1) -print "Taking mean power..." -rPow = rRes.apply_func(N.log10).aggregate('event',N.mean) -nPow = nRes.apply_func(N.log10).aggregate('event',N.mean) +print("Taking mean power...") +rPow = rRes.apply_func(N.log10).aggregate('event', N.mean) +nPow = nRes.apply_func(N.log10).aggregate('event', N.mean) -print "Generating plots..." +print("Generating plots...") fig = 0 # erp -fig+=1 +fig += 1 pylab.figure(fig) -pylab.plot(rEEG['time'],rEEG.aggregate('event',N.mean).data,'r') -pylab.plot(nEEG['time'],nEEG.aggregate('event',N.mean).data,'b') -pylab.legend(('Recalled','Not Recalled')) +pylab.plot(rEEG['time'], rEEG.aggregate('event', N.mean).data, 'r') +pylab.plot(nEEG['time'], nEEG.aggregate('event', N.mean).data, 'b') +pylab.legend(('Recalled', 'Not Recalled')) pylab.xlabel('Time (s)') pylab.ylabel('Voltage (mV)') # power spectrum -fig+=1 +fig += 1 pylab.figure(fig) -pylab.plot(rPow['freq'],N.squeeze(rPow.aggregate('time',N.mean).data),'r') -pylab.plot(nPow['freq'],N.squeeze(nPow.data.mean(nPow.dim('time'))),'b') -pylab.legend(('Recalled','Not Recalled')) +pylab.plot(rPow['freq'], N.squeeze(rPow.aggregate('time', N.mean).data), 'r') +pylab.plot(nPow['freq'], N.squeeze(nPow.data.mean(nPow.dim('time'))), 'b') +pylab.legend(('Recalled', 'Not Recalled')) pylab.xlabel('Frequency (Hz)') pylab.ylabel(r'Power ($log_{10}(mV^2)$)') # plot the diff in mean power -fig+=1 +fig += 1 pylab.figure(fig) -pylab.contourf(rPow['time'],rPow['freq'],rPow.data-nPow.data) +pylab.contourf(rPow['time'], rPow['freq'], rPow.data - nPow.data) pylab.colorbar() pylab.xlabel('Time (s)') pylab.ylabel('Frequency (Hz)') @@ -89,5 +89,3 @@ # show the plots pylab.show() - - diff --git a/examples/process_edf.py b/examples/process_edf.py index 9230946..556f7d9 100644 --- a/examples/process_edf.py +++ b/examples/process_edf.py @@ -11,50 +11,49 @@ elif os.path.exists('example_data/sinus.bdf'): edfw = EdfWrapper('example_data/sinus.bdf') else: - raise IOError('Example data file sinus.bdf not found! '+ + raise IOError('Example data file sinus.bdf not found! ' + 'This file must be in example_data folder!') -for chan_num in [0,1]: +for chan_num in [0, 1]: samplerate = edfw.get_samplerate(chan_num) - nsamples = samplerate*100 - event_dur = samplerate*1 + nsamples = samplerate * 100 + event_dur = samplerate * 1 buf_dur = 1.0 - - + # generate fake events (one every .25 second) - eoffset = np.arange(20)*samplerate/4 - esrc = [edfw]*len(eoffset) - events = Events(np.rec.fromarrays([esrc,eoffset], + eoffset = np.arange(20) * samplerate / 4 + esrc = [edfw] * len(eoffset) + events = Events(np.rec.fromarrays([esrc, eoffset], names='esrc,eoffset')) - + # load in data with events (resample at the same time) # check out the ringing induced in the saw-tooth with the resample! - dat = events.get_data(chan_num, # channel - 1.0, # duration in sec - 0.0, # offset in sec - buf_dur, # buffer in sec + dat = events.get_data(chan_num, # channel + 1.0, # duration in sec + 0.0, # offset in sec + buf_dur, # buffer in sec keep_buffer=True, resampled_rate=500 ) # calc wavelet power - freqs = np.arange(2,50,2) - datpow = phase_pow_multi(freqs,dat,to_return='power') - + freqs = np.arange(2, 50, 2) + datpow = phase_pow_multi(freqs, dat, to_return='power') + # remove the buffer now that we have filtered and calculated power dat = dat.remove_buffer(buf_dur) datpow = datpow.remove_buffer(buf_dur) - + # plot ERP pl.figure() pl.clf() - pl.plot(dat['time'],dat.nanmean('events'),'r') + pl.plot(dat['time'], dat.nanmean('events'), 'r') pl.xlabel('Time (s)') pl.ylabel('Voltage') - + # plot power spectrum pl.figure() pl.clf() - pl.plot(datpow['freqs'],datpow.nanmean('events').nanmean('time'),'r') + pl.plot(datpow['freqs'], datpow.nanmean('events').nanmean('time'), 'r') pl.xlabel('Frequency (Hz)') pl.ylabel('Power') diff --git a/examples/topoPlotDemo.py b/examples/topoPlotDemo.py index d83062e..32794f4 100644 --- a/examples/topoPlotDemo.py +++ b/examples/topoPlotDemo.py @@ -1,5 +1,6 @@ from pylab import loadtxt, rand, figure, xlim, ylim, show -from ptsa.plotting.topoplot import topoplot +from ptsa.plotting.topo import topoplot + def getElecs(): # read in testLocs.dat that was generated in Matlab as follows: @@ -7,79 +8,88 @@ def getElecs(): # locs=locs_orig(4:end); %ignore orig locations 1-3, these are frontal ones we dont have # tmp = [locs.theta; locs.radius]; # save testLocs.dat tmp -ascii - locs=loadtxt("testLocs.dat") - theta=-locs[0]+90 - - #theta=deg2rad(theta) - radius=locs[1]#*(headRad/0.5) - #x,y=pol2cart(theta,radius,radians=False) - return theta,radius #x,y + locs = loadtxt("testLocs.dat") + theta = -locs[0] + 90 + + # theta=deg2rad(theta) + radius = locs[1] # *(headRad/0.5) + # x,y=pol2cart(theta,radius,radians=False) + return theta, radius # x,y + def getPowerVals(): # read in toPlotDiff toPlot = loadtxt("toPlotDiff.dat") return toPlot + els = getElecs() -toPlot=rand(129)#getPowerVals() +toPlot = rand(129) # getPowerVals() -fig=1 +fig = 1 figure(fig) topoplot() -#show() +# show() -fig+=1 +fig += 1 figure(fig) topoplot(sensors=els) -#show() +# show() -fig+=1 +fig += 1 figure(fig) -topoplot(sensors=els,colors=[None,'black','black']) -#show() +topoplot(sensors=els, colors=[None, 'black', 'black']) +# show() -fig+=1 +fig += 1 figure(fig) -topoplot(values=toPlot,sensors=els,colors=['black',None,'black']) +topoplot(values=toPlot, sensors=els, colors=['black', None, 'black']) -fig+=1 +fig += 1 figure(fig) -topoplot(sensors=els,values=toPlot,colors=['black','black',None]) +topoplot(sensors=els, values=toPlot, colors=['black', 'black', None]) -fig+=1 +fig += 1 figure(fig) -topoplot(sensors=els,values=toPlot,colors=[None,None,'black']) +topoplot(sensors=els, values=toPlot, colors=[None, None, 'black']) -fig+=1 +fig += 1 figure(fig) -topoplot(center=(0,0),sensors=els,values=toPlot,plot_mask='linear') -topoplot(center=(2,0),sensors=els,values=toPlot,plot_mask='circular') -topoplot(center=(4,0),sensors=els,values=toPlot,plot_mask='square') -xlim(-1,5) +topoplot(center=(0, 0), sensors=els, values=toPlot, plot_mask='linear') +topoplot(center=(2, 0), sensors=els, values=toPlot, plot_mask='circular') +topoplot(center=(4, 0), sensors=els, values=toPlot, plot_mask='square') +xlim(-1, 5) -grid=100 -fig+=1 +grid = 100 +fig += 1 figure(fig) -topoplot(center=(0,0),radius=0.2,sensors=els,values=toPlot,resolution=grid) -topoplot(center=(1.5,0),radius=0.5,sensors=els,values=toPlot,resolution=grid) -topoplot(center=(4,0),radius=1,sensors=els,values=toPlot,resolution=grid) -topoplot(center=(8.5,0),radius=2,sensors=els,values=toPlot,resolution=grid) -xlim(-0.5,12) -#axis('on') -#show() - -fig+=1 +topoplot(center=(0, 0), radius=0.2, sensors=els, + values=toPlot, resolution=grid) +topoplot(center=(1.5, 0), radius=0.5, sensors=els, + values=toPlot, resolution=grid) +topoplot(center=(4, 0), radius=1, sensors=els, values=toPlot, resolution=grid) +topoplot(center=(8.5, 0), radius=2, sensors=els, + values=toPlot, resolution=grid) +xlim(-0.5, 12) +# axis('on') +# show() + +fig += 1 figure(fig) -topoplot(center=(0,0),nose_dir=-45,sensors=els,values=toPlot,colors=['black',None,'black'],resolution=grid) -topoplot(center=(0,2),nose_dir=-135,sensors=els,values=toPlot,colors=['black',None,'black'],resolution=grid) -topoplot(center=(2,0),nose_dir=135,sensors=els,values=toPlot,colors=['black',None,'black'],resolution=grid) -topoplot(center=(2,2),nose_dir=45,sensors=els,values=toPlot,colors=['black',None,'black'],resolution=grid) -xlim(-1,3) -ylim(-1,3) -#show() - -#topoplot +topoplot(center=(0, 0), nose_dir=-45, sensors=els, values=toPlot, + colors=['black', None, 'black'], resolution=grid) +topoplot(center=(0, 2), nose_dir=-135, sensors=els, values=toPlot, + colors=['black', None, 'black'], resolution=grid) +topoplot(center=(2, 0), nose_dir=135, sensors=els, values=toPlot, + colors=['black', None, 'black'], resolution=grid) +topoplot(center=(2, 2), nose_dir=45, sensors=els, values=toPlot, + colors=['black', None, 'black'], resolution=grid) +xlim(-1, 3) +ylim(-1, 3) +# show() + +# topoplot show() diff --git a/ptsa/__init__.py b/ptsa/__init__.py index b15de88..9db1adb 100644 --- a/ptsa/__init__.py +++ b/ptsa/__init__.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -29,6 +29,7 @@ 'ptsa.plotting.tests', 'ptsa.stats') + def _test(method, level, verbosity, flags): """ Run test suite with level and verbosity. @@ -48,11 +49,16 @@ def _test(method, level, verbosity, flags): importall('ptsa') return getattr(NumpyTest(), method)(level, verbosity=2) + def test(level=1, verbosity=1, flags=[]): _test('test', level=level, verbosity=verbosity, flags=flags) -test.__doc__ = "Using NumpyTest test method.\n"+_test.__doc__ + + +test.__doc__ = "Using NumpyTest test method.\n" + _test.__doc__ + def testall(level=1, verbosity=1, flags=[]): _test('testall', level=level, verbosity=verbosity, flags=flags) -testall.__doc__ = "Using NumpyTest testall method.\n"+_test.__doc__ + +testall.__doc__ = "Using NumpyTest testall method.\n" + _test.__doc__ diff --git a/ptsa/_arraytools.py b/ptsa/_arraytools.py index 1cc3f34..80f5472 100644 --- a/ptsa/_arraytools.py +++ b/ptsa/_arraytools.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: """ Functions for acting on a axis of an array. """ @@ -80,7 +80,7 @@ def odd_ext(x, n, axis=-1): return x if n > x.shape[axis] - 1: raise ValueError(("The extension length n (%d) is too big. " + - "It must not exceed x.shape[axis]-1, which is %d.") + "It must not exceed x.shape[axis]-1, which is %d.") % (n, x.shape[axis] - 1)) left_end = axis_slice(x, start=0, stop=1, axis=axis) left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) @@ -116,7 +116,7 @@ def even_ext(x, n, axis=-1): return x if n > x.shape[axis] - 1: raise ValueError(("The extension length n (%d) is too big. " + - "It must not exceed x.shape[axis]-1, which is %d.") + "It must not exceed x.shape[axis]-1, which is %d.") % (n, x.shape[axis] - 1)) left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) right_ext = axis_slice(x, start=-2, stop=-(n + 2), step=-1, axis=axis) diff --git a/ptsa/contributed.py b/ptsa/contributed.py index 40433be..4fe9ccf 100644 --- a/ptsa/contributed.py +++ b/ptsa/contributed.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -13,16 +13,16 @@ #from filter import decimate #from helper import reshapeTo2D,reshapeFrom2D -from ptsa.data import TimeSeries,Dim,Dims,DimData +from ptsa.data import TimeSeries, Dim, Dims, DimData from ptsa import wavelet import scipy.stats as stats - import pdb -def tsZtransPow(freqs,tseries,zTrans=True,log=True,width=5,resample=None, - keepBuffer=False,verbose=False,to_return='both',freqDimName='freq'): + +def tsZtransPow(freqs, tseries, zTrans=True, log=True, width=5, resample=None, + keepBuffer=False, verbose=False, to_return='both', freqDimName='freq'): """ Calculate z-transformed power (and optionally phase) on a TimeSeries, returning new TimeSeries instances. @@ -36,28 +36,28 @@ def tsZtransPow(freqs,tseries,zTrans=True,log=True,width=5,resample=None, # Get the power (and optionally phase) for tseries: if to_return == 'both': - phaseAll,powerAll = wavelet.tsPhasePow(freqs=freqs,tseries=tseries,width=width, - resample=resample,keepBuffer=keepBuffer, - verbose=verbose,to_return=to_return, - freqDimName=freqDimName) + phaseAll, powerAll = wavelet.tsPhasePow(freqs=freqs, tseries=tseries, width=width, + resample=resample, keepBuffer=keepBuffer, + verbose=verbose, to_return=to_return, + freqDimName=freqDimName) else: - powerAll = wavelet.tsPhasePow(freqs=freqs,tseries=tseries,width=width, - resample=resample,keepBuffer=keepBuffer, - verbose=verbose,to_return=to_return, - freqDimName=freqDimName) + powerAll = wavelet.tsPhasePow(freqs=freqs, tseries=tseries, width=width, + resample=resample, keepBuffer=keepBuffer, + verbose=verbose, to_return=to_return, + freqDimName=freqDimName) - if log: # Ensure power is positive and log10 transform: - powerAll.data[powerAll.data<=0] = N.finfo(powerAll.data.dtype).eps + if log: # Ensure power is positive and log10 transform: + powerAll.data[powerAll.data <= 0] = N.finfo(powerAll.data.dtype).eps powerAll.data = N.log10(powerAll.data) # Get zmean and zstd (DimData objects with a frequency dimension each): - if isinstance(zTrans,tuple): # zmean and zstd are passed as zTrans - if ((len(zTrans) != 2) or (not isinstance(zTrans[0],DimData)) or - (not isinstance(zTrans[1],DimData)) or (zTrans[0].ndim!=1) or - (zTrans[1].ndim!=1) or (zTrans[0].dims.names[0]!=freqDimName) or - (zTrans[1].dims.names[0]!=freqDimName) or - (zTrans[0][freqDimName]!=powerAll[freqDimName]).any() or - (zTrans[1][freqDimName]!=powerAll[freqDimName]).any()): + if isinstance(zTrans, tuple): # zmean and zstd are passed as zTrans + if ((len(zTrans) != 2) or (not isinstance(zTrans[0], DimData)) or + (not isinstance(zTrans[1], DimData)) or (zTrans[0].ndim != 1) or + (zTrans[1].ndim != 1) or (zTrans[0].dims.names[0] != freqDimName) or + (zTrans[1].dims.names[0] != freqDimName) or + (zTrans[0][freqDimName] != powerAll[freqDimName]).any() or + (zTrans[1][freqDimName] != powerAll[freqDimName]).any()): raise ValueError("The ztrans tuple needs to conform to the\ following format: (zmean,zstd). Where zmean and zstd are both\ instances of DimData each with a single frequency dimension.\ @@ -69,14 +69,14 @@ def tsZtransPow(freqs,tseries,zTrans=True,log=True,width=5,resample=None, %f" % zTrans[1].data.min()) zmean = zTrans[0] zstd = zTrans[1] - else: # zmean and zstd must be calculated - if isinstance(zTrans,TimeSeries): + else: # zmean and zstd must be calculated + if isinstance(zTrans, TimeSeries): # Get the power for the provided baseline time series: - zpow = wavelet.tsPhasePow(freqs=freqs,tseries=zTrans,width=width, - resample=resample,keepBuffer=False,verbose=verbose, - to_return='pow',freqDimName=freqDimName) + zpow = wavelet.tsPhasePow(freqs=freqs, tseries=zTrans, width=width, + resample=resample, keepBuffer=False, verbose=verbose, + to_return='pow', freqDimName=freqDimName) if log: - zpow.data[zpow.data<=0] = N.finfo(zpow.data.dtype).eps + zpow.data[zpow.data <= 0] = N.finfo(zpow.data.dtype).eps zpow.data = N.log10(zpow.data) else: # Copy the power for the entire time series: @@ -85,11 +85,13 @@ def tsZtransPow(freqs,tseries,zTrans=True,log=True,width=5,resample=None, # Now calculate zmean and zstd from zpow: # (using stats.std will calculate the unbiased std) if log: - zmean = zpow.margin(freqDimName,stats.mean,unit="mean log10 power") - zstd = zpow.margin(freqDimName,stats.std,unit="std of log10 power") + zmean = zpow.margin(freqDimName, stats.mean, + unit="mean log10 power") + zstd = zpow.margin(freqDimName, stats.std, + unit="std of log10 power") else: - zmean = zpow.margin(freqDimName,stats.mean,unit="mean power") - zstd = zpow.margin(freqDimName,stats.std,unit="std of power") + zmean = zpow.margin(freqDimName, stats.mean, unit="mean power") + zstd = zpow.margin(freqDimName, stats.std, unit="std of power") # For the transformation {zmean,zstd}.data need to have a compatible shape. # Calculate the dimensions with which to reshape (all 1 except for the @@ -100,9 +102,8 @@ def tsZtransPow(freqs,tseries,zTrans=True,log=True,width=5,resample=None, # z transform using reshapedims to make the arrays compatible: powerAll.data = powerAll.data - zmean.data.reshape(reshapedims) powerAll.data = powerAll.data / zstd.data.reshape(reshapedims) - + if to_return == 'both': - return phaseAll,powerAll,(zmean,zstd) + return phaseAll, powerAll, (zmean, zstd) else: - return powerAll,(zmean,zstd) - + return powerAll, (zmean, zstd) diff --git a/ptsa/data/__init__.py b/ptsa/data/__init__.py index ab91a0f..c2a75b4 100644 --- a/ptsa/data/__init__.py +++ b/ptsa/data/__init__.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,13 +7,11 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -from dimarray import Dim,DimArray,AttrArray -from timeseries import TimeSeries +from dimarray import Dim, DimArray, AttrArray +from .timeseries import TimeSeries -from basewrapper import BaseWrapper -from arraywrapper import ArrayWrapper +from .basewrapper import BaseWrapper +from .arraywrapper import ArrayWrapper #from edfwrapper import EdfWrapper -from events import Events - - +from .events import Events diff --git a/ptsa/data/align.py b/ptsa/data/align.py index 28e7db9..1abb584 100644 --- a/ptsa/data/align.py +++ b/ptsa/data/align.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -13,7 +13,7 @@ import os import csv import numpy as np -from basewrapper import BaseWrapper +from .basewrapper import BaseWrapper def times_to_offsets(eeg_times, beh_times, ev_times, blen=10, tolerance=.0015): @@ -23,9 +23,9 @@ def times_to_offsets(eeg_times, beh_times, ev_times, blen=10, tolerance=.0015): """ start_ind = None # get starting ind for the beh_times - for i in range(len(beh_times)//2): - if np.abs(np.diff(eeg_times[:blen]) - - np.diff(beh_times[i:blen+i])).sum()<(tolerance*blen): + for i in range(len(beh_times) // 2): + if np.abs(np.diff(eeg_times[:blen]) - + np.diff(beh_times[i:blen + i])).sum() < (tolerance * blen): start_ind = i break if start_ind is None: @@ -35,31 +35,32 @@ def times_to_offsets(eeg_times, beh_times, ev_times, blen=10, tolerance=.0015): etimes = [] btimes = [] j = 0 - for i,bt in enumerate(beh_times[start_ind:]): - if (i == 0) or (np.abs((bt-btimes[-1])-(eeg_times[j]-etimes[-1]))<(tolerance)): + for i, bt in enumerate(beh_times[start_ind:]): + if (i == 0) or (np.abs((bt - btimes[-1]) - (eeg_times[j] - etimes[-1])) < (tolerance)): # looks good, so append etimes.append(eeg_times[j]) btimes.append(bt) # increment eeg times counter j += 1 - #print i, + # print i, else: # no good, so say we're skipping - print '.', #(np.abs((bt-btimes[-1])-(eeg_times[j]-etimes[-1]))), - print + # (np.abs((bt-btimes[-1])-(eeg_times[j]-etimes[-1]))), + print('.', end=' ') + print() # convert to arrays etimes = np.array(etimes) btimes = np.array(btimes) - print "Num. matching: ", len(etimes) #,len(btimes) - #plot(etimes,btimes,'o') + print("Num. matching: ", len(etimes)) # ,len(btimes) + # plot(etimes,btimes,'o') # fit a line to convert between behavioral and eeg times A = np.vstack([btimes, np.ones(len(btimes))]).T m, c = np.linalg.lstsq(A, etimes)[0] - print "Slope and Offset: ", m ,c + print("Slope and Offset: ", m, c) # convert to get eoffsets - eoffsets = ev_times*m + c + eoffsets = ev_times * m + c return eoffsets @@ -71,27 +72,29 @@ def load_pyepl_eeg_pulses(logfile, event_label='UP'): usually saved by EEG systems. """ # open and read each line - reader = csv.reader(open(logfile,'rU'),dialect=csv.excel_tab) + reader = csv.reader(open(logfile, 'rU'), dialect=csv.excel_tab) pulses = [] for row in reader: if row[2] == event_label: - pulses.append(long(row[0])) + pulses.append(int(row[0])) return np.asarray(pulses) + def find_needle_in_haystack(needle, haystack, maxdiff): """ Look for a matching subsequence in a long sequence. """ nlen = len(needle) found = False - for i in range(len(haystack)-nlen): - if (haystack[i:i+nlen] - needle).max() < maxdiff: + for i in range(len(haystack) - nlen): + if (haystack[i:i + nlen] - needle).max() < maxdiff: found = True break if not found: i = None return i + def times_to_offsets_old(eeg_times, eeg_offsets, beh_times, samplerate, window=100, thresh_ms=10): """ @@ -100,53 +103,55 @@ def times_to_offsets_old(eeg_times, eeg_offsets, beh_times, """ pulse_ms = eeg_times annot_ms = eeg_offsets - + # pick beginning and end (needle in haystack) s_ind = None e_ind = None - for i in xrange(len(annot_ms)-window): - s_ind = find_needle_in_haystack(np.diff(annot_ms[i:i+window]), - np.diff(pulse_ms),thresh_ms) + for i in range(len(annot_ms) - window): + s_ind = find_needle_in_haystack(np.diff(annot_ms[i:i + window]), + np.diff(pulse_ms), thresh_ms) if not s_ind is None: break if s_ind is None: - raise ValueError("Unable to find a start window.") # get better error here - start_annot_vals = annot_ms[i:i+window] - start_pulse_vals = pulse_ms[s_ind:s_ind+window] - - for i in xrange(len(annot_ms)-window): - e_ind = find_needle_in_haystack(np.diff(annot_ms[::-1][i:i+window]), - np.diff(pulse_ms[::-1]),thresh_ms) + # get better error here + raise ValueError("Unable to find a start window.") + start_annot_vals = annot_ms[i:i + window] + start_pulse_vals = pulse_ms[s_ind:s_ind + window] + + for i in range(len(annot_ms) - window): + e_ind = find_needle_in_haystack(np.diff(annot_ms[::-1][i:i + window]), + np.diff(pulse_ms[::-1]), thresh_ms) if not e_ind is None: break if e_ind is None: - raise ValueError("Unable to find a end window.") # get better error here + # get better error here + raise ValueError("Unable to find a end window.") - # correct the end ind + # correct the end ind e_ind = len(pulse_ms) - e_ind - window i = len(annot_ms) - i - window - end_annot_vals = annot_ms[i:i+window] - end_pulse_vals = pulse_ms[e_ind:e_ind+window] + end_annot_vals = annot_ms[i:i + window] + end_pulse_vals = pulse_ms[e_ind:e_ind + window] # fit line with regression - x = np.r_[start_pulse_vals,end_pulse_vals] - y = np.r_[start_annot_vals,end_annot_vals] - m,c = np.linalg.lstsq(np.vstack([x-x[0],np.ones(len(x))]).T, y)[0] - c = c - x[0]*m + x = np.r_[start_pulse_vals, end_pulse_vals] + y = np.r_[start_annot_vals, end_annot_vals] + m, c = np.linalg.lstsq(np.vstack([x - x[0], np.ones(len(x))]).T, y)[0] + c = c - x[0] * m # calc the event time in offsets #samplerate = w.samplerate #offsets = np.int64(np.round((m*beh_times + c)*samplerate/1000.)) # return seconds - offsets = (m*beh_times + c)/1000. + offsets = (m * beh_times + c) / 1000. return offsets - -def align_pyepl(wrappedfile, eeglog, events, annot_id='S255', - channel_for_sr=0, + +def align_pyepl(wrappedfile, eeglog, events, annot_id='S255', + channel_for_sr=0, window=100, thresh_ms=10, event_time_id='event_time', eeg_event_label='UP'): """ @@ -159,9 +164,9 @@ def align_pyepl(wrappedfile, eeglog, events, annot_id='S255', It returns the updated Events. """ - if(not isinstance(wrappedfile,BaseWrapper)): + if(not isinstance(wrappedfile, BaseWrapper)): raise ValueError('BaseWrapper instance required!') - + # point to wrapper w = wrappedfile @@ -172,16 +177,15 @@ def align_pyepl(wrappedfile, eeglog, events, annot_id='S255', annot = w.annotations # convert seconds to ms for annot_ms - annot_ms = annot[annot['annotations']==annot_id]['onsets'] * 1000 + annot_ms = annot[annot['annotations'] == annot_id]['onsets'] * 1000 # get the offsets offsets = times_to_offsets(pulse_ms, annot_ms, events[event_time_id], w.samplerate, window=window, thresh_ms=thresh_ms) # add esrc and eoffset to the Events instance - events = events.add_fields(esrc=np.repeat(w,len(events)), + events = events.add_fields(esrc=np.repeat(w, len(events)), eoffset=offsets) # return the updated events return events - diff --git a/ptsa/data/arraywrapper.py b/ptsa/data/arraywrapper.py index 0ea473f..5d45604 100644 --- a/ptsa/data/arraywrapper.py +++ b/ptsa/data/arraywrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,17 +8,19 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # local imports -from basewrapper import BaseWrapper +from .basewrapper import BaseWrapper # global imports import numpy as np + class ArrayWrapper(BaseWrapper): """ Interface to data stored in a numpy ndarray where the first dimension is the channel and the second dimension is samples. """ - def __init__(self,data,samplerate,annotations=None): + + def __init__(self, data, samplerate, annotations=None): """Initialize the interface to the data. You must specify the data and the samplerate.""" # set up the basic params of the data @@ -39,23 +41,23 @@ def _get_samplerate(self, channel=None): def _get_annotations(self): return self._annotations - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): - """ + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): + """ """ # allocate for data - eventdata = np.empty((len(channels),len(event_offsets),dur_samp), - dtype=self._data.dtype)*np.nan + eventdata = np.empty((len(channels), len(event_offsets), dur_samp), + dtype=self._data.dtype) * np.nan - # loop over events - for e,evOffset in enumerate(event_offsets): + # loop over events + for e, evOffset in enumerate(event_offsets): # set the range - ssamp = offset_samp+evOffset + ssamp = offset_samp + evOffset esamp = ssamp + dur_samp - + # check the ranges if ssamp < 0 or esamp > self._data.shape[1]: - raise IOError('Event with offset '+str(evOffset)+ + raise IOError('Event with offset ' + str(evOffset) + ' is outside the bounds of the data.') - eventdata[:,e,:] = self._data[channels,ssamp:esamp] + eventdata[:, e, :] = self._data[channels, ssamp:esamp] return eventdata diff --git a/ptsa/data/basewrapper.py b/ptsa/data/basewrapper.py index a475ea3..0635500 100644 --- a/ptsa/data/basewrapper.py +++ b/ptsa/data/basewrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,11 +9,12 @@ # local imports #from events import Events,TsEvents -from timeseries import TimeSeries,Dim +from .timeseries import TimeSeries, Dim # global imports import numpy as np + class BaseWrapper(object): """ Base class to provide interface to data. Child classes will @@ -23,7 +24,7 @@ class BaseWrapper(object): """ # required methods that the child class must define. - def _get_samplerate(self,channel=None): + def _get_samplerate(self, channel=None): """ Returns sample rate for a dataset or given channel. @@ -32,15 +33,15 @@ def _get_samplerate(self,channel=None): channel : {None,int,str} If specified, an integer or (if appropriate for a given data format) a string label specifying the channel. - + Returns ------- samplerate : {float} Samplerate for the specified channel. """ raise NotImplementedError - - def _get_nsamples(self,channel=None): + + def _get_nsamples(self, channel=None): """ Returns the number of samples for a dataset or given channel. @@ -49,25 +50,25 @@ def _get_nsamples(self,channel=None): channel : {None,int,str} If specified, an integer or (if appropriate for a given data format) a string label specifying the channel. - + Returns ------- nsamples : {float} Number of samples for the dataset or specified channel. """ raise NotImplementedError - + def _get_nchannels(self): """ Returns the number of channels in a dataset. - + Returns ------- nchannels : {int} Number of channels. """ raise NotImplementedError - + def _get_annotations(self): """ Returns the annotations associated with the dataset. @@ -78,14 +79,14 @@ def _get_annotations(self): Annotations """ raise NotImplementedError - + def _set_annotations(self, annotations): """ Set the annotations associated with the dataset. """ raise NotImplementedError - + def _get_channel_info(self): """ Returns the channel info associated with the dataset. @@ -96,19 +97,19 @@ def _get_channel_info(self): Channel information (e.g., names, locations, etc...) """ # generate recarray of channel info based on nchannels - return np.rec.fromarrays(zip(*[(i+1,'Ch%d'%(i+1)) - for i in range(self.nchannels)]), + return np.rec.fromarrays(list(zip(*[(i + 1, 'Ch%d' % (i + 1)) + for i in range(self.nchannels)])), names='number,name') #raise NotImplementedError - + def _set_channel_info(self, channel_info): """ Set the channel info associated with the dataset. """ raise NotImplementedError - - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): + + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): """ Method for loading data that each child wrapper class must implement. @@ -139,18 +140,18 @@ def append_data(self, data): """ """ raise NotImplementedError - + def set_channel_data(self, channel, data): """ """ raise NotImplementedError - - def get_event_data(self,channels,events, - start_time,end_time,buffer_time=0.0, + + def get_event_data(self, channels, events, + start_time, end_time, buffer_time=0.0, resampled_rate=None, - filt_freq=None,filt_type='stop',filt_order=4, + filt_freq=None, filt_type='stop', filt_order=4, keep_buffer=False, - loop_axis=None,num_mp_procs=0,eoffset='eoffset', + loop_axis=None, num_mp_procs=0, eoffset='eoffset', eoffset_in_time=True): """ Return an TimeSeries containing data for the specified channel @@ -183,7 +184,7 @@ def get_event_data(self,channels,events, The order of the filter. keep_buffer: {boolean},optional Whether to keep the buffer when returning the data. - eoffset_in_time: {boolean},optional + eoffset_in_time: {boolean},optional If True, the unit of the event offsets is taken to be time (unit of the data), otherwise samples. """ @@ -194,22 +195,22 @@ def get_event_data(self,channels,events, buf = buffer_time # get the event offsets - if ((not (hasattr(events,'dtype') or hasattr(events,'columns'))) or - (hasattr(events,'dtype') and events.dtype.names is None)): + if ((not (hasattr(events, 'dtype') or hasattr(events, 'columns'))) or + (hasattr(events, 'dtype') and events.dtype.names is None)): # they just passed in a list event_offsets = events elif ((hasattr(events, 'dtype') and (eoffset in events.dtype.names)) or (hasattr(events, 'columns') and (eoffset in events.columns))): event_offsets = events[eoffset] else: - raise ValueError(eoffset+' must be a valid fieldname '+ + raise ValueError(eoffset + ' must be a valid fieldname ' + 'specifying the offset for the data.') - + # Sanity checks: - if(dur<0): - raise ValueError('Duration must not be negative! '+ - 'Specified duration: '+str(dur)) - if(np.min(event_offsets)<0): + if(dur < 0): + raise ValueError('Duration must not be negative! ' + + 'Specified duration: ' + str(dur)) + if(np.min(event_offsets) < 0): raise ValueError('Event offsets must not be negative!') # make sure the events are an actual array: @@ -217,91 +218,92 @@ def get_event_data(self,channels,events, if eoffset_in_time: # convert to samples event_offsets = np.atleast_1d(np.int64( - np.round(event_offsets*self.samplerate))) - + np.round(event_offsets * self.samplerate))) + # set event durations from rate # get the samplesize - samplesize = 1./self.samplerate + samplesize = 1. / self.samplerate # get the number of buffer samples - buf_samp = int(np.ceil(buf/samplesize)) + buf_samp = int(np.ceil(buf / samplesize)) # calculate the offset samples that contains the desired offset - offset_samp = int(np.ceil((np.abs(offset)-samplesize*.5)/samplesize)* + offset_samp = int(np.ceil((np.abs(offset) - samplesize * .5) / samplesize) * np.sign(offset)) # finally get the duration necessary to cover the desired span #dur_samp = int(np.ceil((dur - samplesize*.5)/samplesize)) - dur_samp = (int(np.ceil((dur+offset - samplesize*.5)/samplesize)) - + dur_samp = (int(np.ceil((dur + offset - samplesize * .5) / samplesize)) - offset_samp + 1) - + # add in the buffer - dur_samp += 2*buf_samp + dur_samp += 2 * buf_samp offset_samp -= buf_samp # check that we have all the data we need before every event: - if(np.min(event_offsets+offset_samp)<0): - bad_evs = ((event_offsets+offset_samp)<0) - raise ValueError('The specified values for offset and buffer '+ - 'require more data than is available before '+ - str(np.sum(bad_evs))+' of all '+ - str(len(bad_evs))+' events.') + if(np.min(event_offsets + offset_samp) < 0): + bad_evs = ((event_offsets + offset_samp) < 0) + raise ValueError('The specified values for offset and buffer ' + + 'require more data than is available before ' + + str(np.sum(bad_evs)) + ' of all ' + + str(len(bad_evs)) + ' events.') # process the channels if isinstance(channels, dict): # turn into indices ch_info = self.channels - key = channels.keys()[0] - channels = [np.nonzero(ch_info[key]==c)[0][0] + key = list(channels.keys())[0] + channels = [np.nonzero(ch_info[key] == c)[0][0] for c in channels[key]] elif isinstance(channels, str): # find that channel by name - channels = np.nonzero(self.channels['name']==channels)[0][0] - if channels is None or len(np.atleast_1d(channels))==0: + channels = np.nonzero(self.channels['name'] == channels)[0][0] + if channels is None or len(np.atleast_1d(channels)) == 0: channels = np.arange(self.nchannels) channels = np.atleast_1d(channels) # if channels is a container with channel names, convert to # indices (can't directly change elements in array, because # dtype would remain str): - channels = np.array([np.nonzero(self.channels['name']==c)[0][0] + channels = np.array([np.nonzero(self.channels['name'] == c)[0][0] if isinstance(c, str) else c for c in channels]) channels.sort() # load the timeseries (this must be implemented by subclasses) - eventdata = self._load_data(channels,event_offsets,dur_samp,offset_samp) + eventdata = self._load_data( + channels, event_offsets, dur_samp, offset_samp) # calc the time range # get the samplesize - samp_start = offset_samp*samplesize - samp_end = samp_start + (dur_samp-1)*samplesize - time_range = np.linspace(samp_start,samp_end,dur_samp) + samp_start = offset_samp * samplesize + samp_end = samp_start + (dur_samp - 1) * samplesize + time_range = np.linspace(samp_start, samp_end, dur_samp) # make it a timeseries - dims = [Dim(self.channels[channels],'channels'), # can index into channels - Dim(events,'events'), - Dim(time_range,'time')] + dims = [Dim(self.channels[channels], 'channels'), # can index into channels + Dim(events, 'events'), + Dim(time_range, 'time')] eventdata = TimeSeries(np.asarray(eventdata), 'time', - self.samplerate,dims=dims) + self.samplerate, dims=dims) - # filter if desired - if not(filt_freq is None): - # filter that data + # filter if desired + if not(filt_freq is None): + # filter that data eventdata = eventdata.filtered(filt_freq, filt_type=filt_type, order=filt_order) - # resample if desired - if (not(resampled_rate is None) and - not(resampled_rate == eventdata.samplerate)): - # resample the data + # resample if desired + if (not(resampled_rate is None) and + not(resampled_rate == eventdata.samplerate)): + # resample the data eventdata = eventdata.resampled(resampled_rate, loop_axis=loop_axis, num_mp_procs=num_mp_procs) - # remove the buffer and set the time range - if buf > 0 and not(keep_buffer): - # remove the buffer + # remove the buffer and set the time range + if buf > 0 and not(keep_buffer): + # remove the buffer eventdata = eventdata.remove_buffer(buf) # return the timeseries @@ -314,36 +316,36 @@ def get_all_data(self, channels=None): if channels is None: channels = np.arange(self.nchannels) dur_samp = self.nsamples - data = self._load_data(channels,[0],dur_samp,0) + data = self._load_data(channels, [0], dur_samp, 0) # remove events dimension - data = data[:,0,:] + data = data[:, 0, :] # turn it into a TimeSeries # get the samplesize - samplesize = 1./self.samplerate + samplesize = 1. / self.samplerate # set timerange - samp_start = 0*samplesize - samp_end = samp_start + (dur_samp-1)*samplesize - time_range = np.linspace(samp_start,samp_end,dur_samp) + samp_start = 0 * samplesize + samp_end = samp_start + (dur_samp - 1) * samplesize + time_range = np.linspace(samp_start, samp_end, dur_samp) - # make it a timeseries - dims = [Dim(self.channels[channels],'channels'), - Dim(time_range,'time')] + # make it a timeseries + dims = [Dim(self.channels[channels], 'channels'), + Dim(time_range, 'time')] data = TimeSeries(np.asarray(data), 'time', - self.samplerate,dims=dims) + self.samplerate, dims=dims) return data - + # class properties samplerate = property(lambda self: self._get_samplerate()) nsamples = property(lambda self: self._get_nsamples()) nchannels = property(lambda self: self._get_nchannels()) annotations = property(lambda self: self._get_annotations(), - lambda self,annot: self._set_annotations(annot)) + lambda self, annot: self._set_annotations(annot)) channel_info = property(lambda self: self._get_channel_info(), - lambda self,chan_info: self._set_channel_info(chan_info)) + lambda self, chan_info: self._set_channel_info(chan_info)) channels = property(lambda self: self._get_channel_info(), - lambda self,chan_info: self._set_channel_info(chan_info)) + lambda self, chan_info: self._set_channel_info(chan_info)) data = property(lambda self: self.get_all_data()) diff --git a/ptsa/data/bvwrapper.py b/ptsa/data/bvwrapper.py index 6ce0049..bfa25a7 100644 --- a/ptsa/data/bvwrapper.py +++ b/ptsa/data/bvwrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,18 +8,20 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # local imports -from basewrapper import BaseWrapper +from .basewrapper import BaseWrapper # global imports import numpy as np import os.path -from ConfigParser import SafeConfigParser +from configparser import SafeConfigParser import io + class BVWrapper(BaseWrapper): """ Interface to data stored in a BrainVision Data Format. """ + def __init__(self, filepath): """ Initialize the interface to the data. @@ -35,17 +37,17 @@ def __init__(self, filepath): self.filepath = filepath self.filedir = os.path.split(filepath)[0] else: - raise IOError(str(filepath)+'\n does not exist!\n'+ + raise IOError(str(filepath) + '\n does not exist!\n' + 'Valid path to data file is needed!') # read in the info about the data from the header cp = SafeConfigParser() - lines = open(filepath,'r').readlines() + lines = open(filepath, 'r').readlines() # must remove the first lines b/c they are not standard INI format # also remove everything after [Comment] b/c it doesn't parse either ind = None - for i,line in enumerate(lines): + for i, line in enumerate(lines): if line.strip() == '[Comment]': ind = i break @@ -64,15 +66,18 @@ def __init__(self, filepath): hdr_string = ''.join(lines[1:ind]) # now read it in - cp.readfp(io.BytesIO(hdr_string)) + cp.readfp(io.StringIO(hdr_string)) # extract the info we need - self._binaryformat = cp.get('Binary Infos','binaryformat') - self._nchannels = int(cp.get('Common Infos','numberofchannels')) - self._data_orient = cp.get('Common Infos','dataorientation') - self._data_file = os.path.join(self.filedir,cp.get('Common Infos','datafile')) - self._samplerate = float(10e5)/int(cp.get('Common Infos','samplinginterval')) - self._markerfile = os.path.join(self.filedir,cp.get('Common Infos','markerfile')) + self._binaryformat = cp.get('Binary Infos', 'binaryformat') + self._nchannels = int(cp.get('Common Infos', 'numberofchannels')) + self._data_orient = cp.get('Common Infos', 'dataorientation') + self._data_file = os.path.join( + self.filedir, cp.get('Common Infos', 'datafile')) + self._samplerate = float( + 10e5) / int(cp.get('Common Infos', 'samplinginterval')) + self._markerfile = os.path.join( + self.filedir, cp.get('Common Infos', 'markerfile')) # read in scale factors for each channel (and other info) numbers = [] @@ -81,28 +86,28 @@ def __init__(self, filepath): units = [] #self._channel_scale = np.ones(self._nchannels) for i in range(self._nchannels): - info = cp.get('Channel Infos','Ch%d'%(i+1)).split(',') + info = cp.get('Channel Infos', 'Ch%d' % (i + 1)).split(',') #self._channel_scale[i] = float(info[2]) - numbers.append(i+1) + numbers.append(i + 1) names.append(info[0]) scales.append(float(info[2])) - units.append(unicode(info[3],'utf-8')) + units.append(str(info[3])) # try and get the impedances - impedances = np.ones(len(names))*-1 - for i,line in enumerate(lines[ind:]): + impedances = np.ones(len(names)) * -1 + for i, line in enumerate(lines[ind:]): if 'Impedance' in line: # found impedances, try and read them skipped = 0 - for l,li in enumerate(lines[ind+i+1:]): + for l, li in enumerate(lines[ind + i + 1:]): info = li.strip().split(' ') cname = info[1][:-1] if cname in names: impedances[names.index(cname)] = int(info[2]) break - self._channel_info = np.rec.fromarrays([numbers,names,scales, - units,impedances], + self._channel_info = np.rec.fromarrays([numbers, names, scales, + units, impedances], names='number,name,scale,unit,impedance') - + # process the binary format if self._binaryformat == 'INT_16': self._samplesize = 2 @@ -111,13 +116,13 @@ def __init__(self, filepath): self._samplesize = 4 self._dtype = np.dtype(np.float32) else: - raise ValueError('Unknown binary format: %s\n' % self._binaryformat) + raise ValueError('Unknown binary format: %s\n' % + self._binaryformat) # open the file to figure out the nsamples - mm = np.memmap(self._data_file,dtype=self._dtype, + mm = np.memmap(self._data_file, dtype=self._dtype, mode='r') - self._nsamples = mm.shape[0]/self._nchannels - + self._nsamples = mm.shape[0] / self._nchannels def _get_nchannels(self): return self._nchannels @@ -134,8 +139,8 @@ def _get_samplerate(self, channel=None): def _get_annotations(self): # read in from annotations file (must strip off first lines) cp = SafeConfigParser() - lines = open(self._markerfile,'r').readlines() - cp.readfp(io.BytesIO(''.join(lines[2:]))) + lines = open(self._markerfile, 'r').readlines() + cp.readfp(io.StringIO(''.join(lines[2:]))) # get the marker info markers = cp.items('Marker Infos') @@ -153,48 +158,45 @@ def _get_annotations(self): info = markers[i][1].split(',') annots.append(info[1]) # convert onset to seconds (subtracting 1 for actual offset) - onsets[i] = (long(info[2]))/self._samplerate + onsets[i] = (int(info[2])) / self._samplerate # save duration (for now, keep as string like in EDF) durations.append(info[3]) - if sub_one == False and info[0] == 'New Segment' and long(info[2])==1: + if sub_one == False and info[0] == 'New Segment' and int(info[2]) == 1: # we need to sub_one sub_one = True if sub_one: - onsets -= long(1)/self._samplerate - + onsets -= int(1) / self._samplerate + # convert to rec array - annotations = np.rec.fromarrays([onsets,durations,annots], + annotations = np.rec.fromarrays([onsets, durations, annots], names='onsets,durations,annotations') # sort by index and return return annotations[np.argsort(index)] - - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): - """ + + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): + """ """ # allocate for data - eventdata = np.empty((len(channels),len(event_offsets),dur_samp), - dtype=np.float64)*np.nan + eventdata = np.empty((len(channels), len(event_offsets), int(dur_samp)), + dtype=np.float64) * np.nan # Memmap to the file - mm = np.memmap(self._data_file,dtype=self._dtype, - mode='r',shape=(self._nsamples,self._nchannels)) - - # loop over events - for e,ev_offset in enumerate(event_offsets): + mm = np.memmap(self._data_file, dtype=self._dtype, + mode='r', shape=(int(self._nsamples), self._nchannels)) + + # loop over events + for e, ev_offset in enumerate(event_offsets): # set the range - ssamp = offset_samp+ev_offset + ssamp = offset_samp + ev_offset if (ssamp + dur_samp - 1) > self._nsamples: - raise IOError('Event with offset '+str(ev_offset)+ + raise IOError('Event with offset ' + str(ev_offset) + ' is outside the bounds of the data.') # only pick the channels of interest and scale - dat = np.multiply(mm[ssamp:ssamp+dur_samp,channels], + dat = np.multiply(mm[ssamp:int(ssamp + dur_samp), channels], self._channel_info['scale'][channels]) - #self._channel_scale[channels]) - eventdata[:,e,:] = dat.T - - return eventdata - + eventdata[:, e, :] = dat.T + return eventdata diff --git a/ptsa/data/datawrapper.py b/ptsa/data/datawrapper.py index 01b3698..70e0fee 100644 --- a/ptsa/data/datawrapper.py +++ b/ptsa/data/datawrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,17 +7,19 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + class DataWrapper(object): """ - Base class to provide interface to timeseries data. + Base class to provide interface to timeseries data. """ - def _load_data(self,channel,eventOffsets,dur_samp,offset_samp): + + def _load_data(self, channel, eventOffsets, dur_samp, offset_samp): raise NotImplementedError - - def get_event_data(self,channels,eventOffsets, - dur,offset,buf, + + def get_event_data(self, channels, eventOffsets, + dur, offset, buf, resampledRate=None, - filtFreq=None,filtType='stop',filtOrder=4, + filtFreq=None, filtType='stop', filtOrder=4, keepBuffer=False): """ Return an dictionary containing data for the specified channel @@ -37,39 +39,42 @@ def get_event_data(self,channels,eventOffsets, filtOrder: Order of the filter. keepBuffer: Whether to keep the buffer when returning the data. """ - + # set event durations from rate # get the samplesize in ms - samplesize = 1./self.samplerate + samplesize = 1. / self.samplerate # get the number of buffer samples - buf_samp = int(np.ceil(buf/samplesize)) + buf_samp = int(np.ceil(buf / samplesize)) # calculate the offset samples that contains the desired offsetMS - offset_samp = int(np.ceil((np.abs(offset)-samplesize*.5)/samplesize)*np.sign(offset)) + offset_samp = int( + np.ceil((np.abs(offset) - samplesize * .5) / samplesize) * np.sign(offset)) # finally get the duration necessary to cover the desired span - dur_samp = int(np.ceil((dur+offset - samplesize*.5)/samplesize)) - offset_samp + 1 - + dur_samp = int( + np.ceil((dur + offset - samplesize * .5) / samplesize)) - offset_samp + 1 + # add in the buffer - dur_samp += 2*buf_samp + dur_samp += 2 * buf_samp offset_samp -= buf_samp # load the timeseries (this must be implemented by subclasses) - eventdata = self._load_data(channel,eventOffsets,dur_samp,offset_samp) + eventdata = self._load_data( + channel, eventOffsets, dur_samp, offset_samp) # calc the time range - sampStart = offset_samp*samplesize - sampEnd = sampStart + (dur_samp-1)*samplesize - timeRange = np.linspace(sampStart,sampEnd,dur_samp) + sampStart = offset_samp * samplesize + sampEnd = sampStart + (dur_samp - 1) * samplesize + timeRange = np.linspace(sampStart, sampEnd, dur_samp) - # make it a timeseries + # make it a timeseries # if isinstance(eventInfo,TsEvents): # dims = [Dim('event', eventInfo.data, 'event'), # Dim('time',timeRange)] # else: # dims = [Dim('eventOffsets', eventOffsets, 'samples'), - # Dim('time',timeRange)] - dims = [Dim(eventOffsets,'eventOffsets'), - Dim(timeRange,'time')] + # Dim('time',timeRange)] + dims = [Dim(eventOffsets, 'eventOffsets'), + Dim(timeRange, 'time')] eventdata = TimeSeries(np.asarray(eventdata), dims, tdim='time', @@ -77,21 +82,21 @@ def get_event_data(self,channels,eventOffsets, return eventdata + # filter if desired + if not(filtFreq is None): + # filter that data + eventdata = eventdata.filter( + filtFreq, filtType=filtType, order=filtOrder) - # filter if desired - if not(filtFreq is None): - # filter that data - eventdata = eventdata.filter(filtFreq,filtType=filtType,order=filtOrder) - - # resample if desired - if not(resampledRate is None) and \ - not(resampledRate == eventdata.samplerate): - # resample the data + # resample if desired + if not(resampledRate is None) and \ + not(resampledRate == eventdata.samplerate): + # resample the data eventdata = eventdata.resampled(resampledRate) # remove the buffer and set the time range - if buf > 0 and not(keepBuffer): - # remove the buffer + if buf > 0 and not(keepBuffer): + # remove the buffer eventdata = eventdata.removeBuf(buf) # return the timeseries diff --git a/ptsa/data/edf/__init__.py b/ptsa/data/edf/__init__.py index e7dfee5..dfbc9e5 100644 --- a/ptsa/data/edf/__init__.py +++ b/ptsa/data/edf/__init__.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,4 +9,3 @@ from edf import read_samples, read_number_of_samples from edf import read_samplerate, read_annotations, read_number_of_signals - diff --git a/ptsa/data/edf/setup.py b/ptsa/data/edf/setup.py index 1e39da3..178b147 100644 --- a/ptsa/data/edf/setup.py +++ b/ptsa/data/edf/setup.py @@ -4,10 +4,10 @@ sourcefiles = ["edf.pyx", "edfwrap.c", "edflib.c"] setup( - cmdclass = {'build_ext': build_ext}, - ext_modules = [Extension("edf", - sources = sourcefiles, - define_macros = [('_LARGEFILE64_SOURCE', None), - ('_LARGEFILE_SOURCE', None)]), - ] + cmdclass={'build_ext': build_ext}, + ext_modules=[Extension("edf", + sources=sourcefiles, + define_macros=[('_LARGEFILE64_SOURCE', None), + ('_LARGEFILE_SOURCE', None)]), + ] ) diff --git a/ptsa/data/edfwrapper.py b/ptsa/data/edfwrapper.py index 2e3ade0..c9a9197 100644 --- a/ptsa/data/edfwrapper.py +++ b/ptsa/data/edfwrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,19 +8,21 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # local imports -from basewrapper import BaseWrapper -from edf import read_samples, read_number_of_samples -from edf import read_samplerate, read_annotations -from edf import read_number_of_signals +from .basewrapper import BaseWrapper +from .edf import read_samples, read_number_of_samples +from .edf import read_samplerate, read_annotations +from .edf import read_number_of_signals # global imports import numpy as np import os.path + class EdfWrapper(BaseWrapper): """ Interface to data stored in a EDF file and related formats (such as BDF). """ + def __init__(self, filepath): """ Initialize the interface to the data. @@ -35,20 +37,20 @@ def __init__(self, filepath): if os.path.exists(filepath): self.filepath = filepath else: - raise IOError(str(filepath)+'\n does not exist!'+ + raise IOError(str(filepath) + '\n does not exist!' + 'Valid path to data file is needed!') self._nchannels = read_number_of_signals(self.filepath) numbers = [] - names = [] + names = [] for i in range(self._nchannels): - numbers.append(i+1) + numbers.append(i + 1) # CTW: Here we should use the actual channel labels as # name -- requires additional cython code to interface # with the EDF library: - names.append(str(i+1)) + names.append(str(i + 1)) self._channel_info = np.rec.fromarrays( - [numbers, names], names='number,name') + [numbers, names], names='number,name') def _get_nchannels(self): return self._nchannels @@ -71,14 +73,14 @@ def _get_samplerate(self, channel=None): def _get_annotations(self): return read_annotations(self.filepath) - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): """ """ # allocate for data - eventdata = np.empty((len(channels),len(event_offsets),dur_samp), - dtype=np.float64)*np.nan + eventdata = np.empty((len(channels), len(event_offsets), dur_samp), + dtype=np.float64) * np.nan - # loop over events + # loop over events # PBS: eventually move this to the cython file for c, channel in enumerate(channels): for e, ev_offset in enumerate(event_offsets): @@ -92,10 +94,8 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # check the ranges if len(dat) < dur_samp: - raise IOError('Event with offset '+str(ev_offset)+ + raise IOError('Event with offset ' + str(ev_offset) + ' is outside the bounds of the data.') eventdata[c, e, :] = dat - return eventdata - - + return eventdata diff --git a/ptsa/data/events.py b/ptsa/data/events.py index 2a125d7..f444543 100644 --- a/ptsa/data/events.py +++ b/ptsa/data/events.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,10 +9,11 @@ # global imports import numpy as np -from timeseries import TimeSeries,Dim +from .timeseries import TimeSeries, Dim #import pdb + class Events(np.recarray): """ A recarray with the events to be analyzed. Includes convenience @@ -23,14 +24,14 @@ class Events(np.recarray): # def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None, # formats=None, names=None, titles=None, # byteorder=None, aligned=False): - + # def __new__(*args,**kwargs): # return np.recarray.__new__(*args,**kwargs) def __new__(subtype, data): return data.view(subtype) - - def remove_fields(self,*fields_to_remove): + + def remove_fields(self, *fields_to_remove): """ Return a new instance of the recarray with specified fields removed. @@ -50,7 +51,7 @@ def remove_fields(self,*fields_to_remove): # loop over fields, keeping if not matching fieldName for field in self.dtype.names: # don't add the field if in fields_to_remove list - #if sum(map(lambda x: x==field,fields_to_remove)) == 0: + # if sum(map(lambda x: x==field,fields_to_remove)) == 0: if not field in fields_to_remove: # append the data arrays.append(self[field]) @@ -59,9 +60,9 @@ def remove_fields(self,*fields_to_remove): # return the new Events if len(arrays) == 0: arrays.append([]) - return np.rec.fromarrays(arrays,names=','.join(names)).view(self.__class__) - - def add_fields(self,**fields): + return np.rec.fromarrays(arrays, names=','.join(names)).view(self.__class__) + + def add_fields(self, **fields): """ Add fields from the keyword args provided and return a new instance. @@ -76,51 +77,51 @@ def add_fields(self,**fields): Returns ------- New Events instance with the specified new fields. - + Examples -------- events.add_fields(name1=array1, name2=dtype('i4')) - + """ # list of current dtypes to which new dtypes will be added: # new_dtype = [(name,self[name].dtype) for name in self.dtype.names] - + # sequence of arrays and names from starting recarray #arrays = map(lambda x: self[x], self.dtype.names) arrays = [self[x] for x in self.dtype.names] names = ','.join(self.dtype.names) - + # loop over the kwargs of field - for name,data in fields.iteritems(): + for name, data in fields.items(): # see if already there, error if so - if self.dtype.fields.has_key(name): + if name in self.dtype.fields: # already exists - raise ValueError('Field "'+name+'" already exists.') - + raise ValueError('Field "' + name + '" already exists.') + # append the array and name - if(isinstance(data,np.dtype)| - isinstance(data,type)|isinstance(data,str)): + if(isinstance(data, np.dtype) | + isinstance(data, type) | isinstance(data, str)): # add empty array the length of the data - arrays.append(np.empty(len(self),data)) + arrays.append(np.empty(len(self), data)) else: # add the data as an array arrays.append(data) # add the name - if len(names)>0: + if len(names) > 0: # append , - names = names+',' + names = names + ',' # append the name - names = names+name + names = names + name # return the new Events - return np.rec.fromarrays(arrays,names=names).view(self.__class__) + return np.rec.fromarrays(arrays, names=names).view(self.__class__) - def get_data(self,channels,start_time,end_time,buffer_time=0.0, + def get_data(self, channels, start_time, end_time, buffer_time=0.0, resampled_rate=None, - filt_freq=None,filt_type='stop',filt_order=4, - keep_buffer=False,esrc='esrc',eoffset='eoffset', - loop_axis=None,num_mp_procs=0, + filt_freq=None, filt_type='stop', filt_order=4, + keep_buffer=False, esrc='esrc', eoffset='eoffset', + loop_axis=None, num_mp_procs=0, eoffset_in_time=True): """ Return the requested range of data for each event by using the @@ -155,25 +156,25 @@ def get_data(self,channels,start_time,end_time,buffer_time=0.0, eoffset: {string},optional Name for the field containing the offset (in seconds) for the event within the specified source. - eoffset_in_time: {boolean},optional + eoffset_in_time: {boolean},optional If True, the unit of the event offsets is taken to be time (unit of the data), otherwise samples. - + Returns ------- A TimeSeries instance with dimensions (channels,events,time). """ - + # check for necessary fields if not (esrc in self.dtype.names and eoffset in self.dtype.names): - raise ValueError(esrc+' and '+eoffset+' must be valid fieldnames '+ + raise ValueError(esrc + ' and ' + eoffset + ' must be valid fieldnames ' + 'specifying source and offset for the data.') - - # get ready to load dat - eventdata = [] + + # get ready to load dat + eventdata = [] events = [] - + # speed up by getting unique event sources first usources = np.unique(self[esrc]) @@ -181,16 +182,16 @@ def get_data(self,channels,start_time,end_time,buffer_time=0.0, eventdata = None for src in usources: # get the eventOffsets from that source - ind = np.atleast_1d(self[esrc]==src) - + ind = np.atleast_1d(self[esrc] == src) + if len(ind) == 1: - event_offsets=self[eoffset] + event_offsets = self[eoffset] events.append(self) else: event_offsets = self[ind][eoffset] events.append(self[ind]) - #print "Loading %d events from %s" % (ind.sum(),src) + # print "Loading %d events from %s" % (ind.sum(),src) # get the timeseries for those events newdat = src.get_event_data(channels, event_offsets, @@ -209,8 +210,8 @@ def get_data(self,channels,start_time,end_time,buffer_time=0.0, if eventdata is None: eventdata = newdat else: - eventdata = eventdata.extend(newdat,axis=1) - + eventdata = eventdata.extend(newdat, axis=1) + # concatenate (must eventually check that dims match) tdim = eventdata['time'] cdim = eventdata['channels'] @@ -218,9 +219,6 @@ def get_data(self,channels,start_time,end_time,buffer_time=0.0, events = np.concatenate(events).view(self.__class__) eventdata = TimeSeries(eventdata, 'time', srate, - dims=[cdim,Dim(events,'events'),tdim]) - - return eventdata - - + dims=[cdim, Dim(events, 'events'), tdim]) + return eventdata diff --git a/ptsa/data/events_old.py b/ptsa/data/events_old.py index 4323769..1c7531b 100644 --- a/ptsa/data/events_old.py +++ b/ptsa/data/events_old.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,11 +11,12 @@ import numpy as np from dimarray import Dim -from timeseries import TimeSeries -from basewrapper import BaseWrapper +from .timeseries import TimeSeries +from .basewrapper import BaseWrapper #import pdb + class Events(np.recarray): _required_fields = None @@ -24,31 +25,31 @@ class Events(np.recarray): # def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None, # formats=None, names=None, titles=None, # byteorder=None, aligned=False): - def __new__(*args,**kwargs): - self=np.recarray.__new__(*args,**kwargs) + def __new__(*args, **kwargs): + self = np.recarray.__new__(*args, **kwargs) if not(self._required_fields is None): for req_field in self._required_fields: if not(req_field in self.dtype.names): raise ValueError( - req_field+' is a required field!') + req_field + ' is a required field!') if not( - self[req_field].dtype==self._required_fields[req_field]): + self[req_field].dtype == self._required_fields[req_field]): raise ValueError( - req_field+' needs to be '+ - str(self._required_fields[req_field])+ - '. Provided dtype was '+str(self[req_field].dtype)) + req_field + ' needs to be ' + + str(self._required_fields[req_field]) + + '. Provided dtype was ' + str(self[req_field].dtype)) return self - - def remove_fields(self,*fields_to_remove): + + def remove_fields(self, *fields_to_remove): """ Return a new instance of the array with specified fields removed. """ if not(self._required_fields is None): for req_field in self._required_fields: - if sum(map(lambda x: x==req_field,fields_to_remove)) > 0: - raise ValueError(req_field+' is a required field!') + if sum([x == req_field for x in fields_to_remove]) > 0: + raise ValueError(req_field + ' is a required field!') # sequence of arrays and names arrays = [] names = '' @@ -56,83 +57,82 @@ def remove_fields(self,*fields_to_remove): # loop over fields, keeping if not matching fieldName for field in self.dtype.names: # don't add the field if in fields_to_remove list - if sum(map(lambda x: x==field,fields_to_remove)) == 0: + if sum([x == field for x in fields_to_remove]) == 0: # append the data arrays.append(self[field]) - if len(names)>0: + if len(names) > 0: # append , - names = names+',' + names = names + ',' # append the name - names = names+field + names = names + field # return the new Events - return np.rec.fromarrays(arrays,names=names).view(self.__class__) + return np.rec.fromarrays(arrays, names=names).view(self.__class__) - def add_fields(self,**fields): + def add_fields(self, **fields): """ Add fields from the keyword args provided and return a new instance. To add an empty field, pass a dtype as the array. addFields(name1=array1, name2=dtype('i4')) - + """ # list of current dtypes to which new dtypes will be added: # new_dtype = [(name,self[name].dtype) for name in self.dtype.names] - + # sequence of arrays and names from starting recarray - arrays = map(lambda x: self[x], self.dtype.names) + arrays = [self[x] for x in self.dtype.names] names = ','.join(self.dtype.names) - + # loop over the kwargs of field - for name,data in fields.iteritems(): + for name, data in fields.items(): # see if already there, error if so - if self.dtype.fields.has_key(name): + if name in self.dtype.fields: # already exists - raise ValueError('Field "'+name+'" already exists.') - + raise ValueError('Field "' + name + '" already exists.') + # append the array and name - if(isinstance(data,np.dtype)| - isinstance(data,type)|isinstance(data,str)): + if(isinstance(data, np.dtype) | + isinstance(data, type) | isinstance(data, str)): # add empty array the length of the data - arrays.append(np.empty(len(self),data)) + arrays.append(np.empty(len(self), data)) else: # add the data as an array arrays.append(data) # add the name - if len(names)>0: + if len(names) > 0: # append , - names = names+',' + names = names + ',' # append the name - names = names+name + names = names + name # return the new Events - return np.rec.fromarrays(arrays,names=names).view(self.__class__) + return np.rec.fromarrays(arrays, names=names).view(self.__class__) def __getitem__(self, index): # try block to ensure the _skip_dim_check flag gets reset # in the following finally block - try: + try: # skip the dim check b/c we're going to fiddle with them self._skip_field_check = True - ret = np.recarray.__getitem__(self,index) + ret = np.recarray.__getitem__(self, index) finally: # reset the _skip_dim_check flag: self._skip_field_check = False return ret - - def __getslice__(self,i,j): - try: + + def __getslice__(self, i, j): + try: # skip the field check self._skip_field_check = True - ret = np.recarray.__getslice__(self,i,j) + ret = np.recarray.__getslice__(self, i, j) finally: # reset the _skip_field_check flag: self._skip_field_check = False return ret - - def __array_finalize__(self,obj): + def __array_finalize__(self, obj): # print self.shape,obj.shape # # print self._skip_field_check,obj._skip_field_check self._skip_field_check = False @@ -140,20 +140,22 @@ def __array_finalize__(self,obj): # check fields: # PBS: Also don't check if obj is None (implies it's unpickling) # We must find out if there are other instances of it being None - if (isinstance(obj,Events) and obj._skip_field_check) or \ - obj is None: return + if (isinstance(obj, Events) and obj._skip_field_check) or \ + obj is None: + return # ensure that the fields are valid: if not(self._required_fields is None): for req_field in self._required_fields: if not(req_field in obj.dtype.names): raise ValueError( - req_field+' is a required field!') + req_field + ' is a required field!') if not( - obj[req_field].dtype==self._required_fields[req_field]): + obj[req_field].dtype == self._required_fields[req_field]): raise ValueError( - req_field+' needs to be '+ - str(self._required_fields[req_field])+ - '. Provided dtype was '+str(obj[req_field].dtype)) + req_field + ' needs to be ' + + str(self._required_fields[req_field]) + + '. Provided dtype was ' + str(obj[req_field].dtype)) + class TsEvents(Events): """ @@ -162,12 +164,12 @@ class TsEvents(Events): so that the class can know how to retrieve data for each event. """ - _required_fields = {'tssrc':BaseWrapper,'tsoffset':int} - + _required_fields = {'tssrc': BaseWrapper, 'tsoffset': int} + # def get_data(self,channel,dur,offset,buf,resampled_rate=None, # filt_freq=None,filt_type='stop', # filt_order=4,keep_buffer=False): - def get_data(self,*args,**kwargs): + def get_data(self, *args, **kwargs): """ Return the requested range of data for each event by using the proper data retrieval mechanism for each event. @@ -176,9 +178,9 @@ def get_data(self,*args,**kwargs): (events,time) for the data and also some information about the data returned. """ - # get ready to load dat - eventdata = None - + # get ready to load dat + eventdata = None + # events = self.data # speed up by getting unique event sources first @@ -187,15 +189,15 @@ def get_data(self,*args,**kwargs): # loop over unique sources for src in usources: # get the eventOffsets from that source - ind = np.atleast_1d(self['tssrc']==src) - + ind = np.atleast_1d(self['tssrc'] == src) + if len(ind) == 1: - src_events=self + src_events = self else: src_events = self[ind] - #print "Loading %d events from %s" % (ind.sum(),src) - # get the timeseries for those events + # print "Loading %d events from %s" % (ind.sum(),src) + # get the timeseries for those events # newdat = src.get_event_data(channel, # src_events, # dur, @@ -206,7 +208,7 @@ def get_data(self,*args,**kwargs): # filt_type, # filt_order, # keep_buffer) - newdat = src.get_event_data(*args,**kwargs) + newdat = src.get_event_data(*args, **kwargs) # see if concatenate if eventdata is None: @@ -214,13 +216,11 @@ def get_data(self,*args,**kwargs): eventdata = newdat else: # append it to the existing - np.concatenate(eventdata,newdat,eventdata.tdim) + np.concatenate(eventdata, newdat, eventdata.tdim) if eventdata is None: dims = [Dim(np.array(None), 'event'), Dim(np.array(None), 'time')] eventdata = TimeSeries(np.atleast_2d(np.array(None)), - samplerate=None,tdim='time',dims=dims) + samplerate=None, tdim='time', dims=dims) return eventdata - - diff --git a/ptsa/data/events_old2.py b/ptsa/data/events_old2.py index 9d6e8c7..5b911ff 100644 --- a/ptsa/data/events_old2.py +++ b/ptsa/data/events_old2.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -10,11 +10,12 @@ # global imports import numpy as np -from timeseries import TimeSeries,Dim -from basewrapper import BaseWrapper +from .timeseries import TimeSeries, Dim +from .basewrapper import BaseWrapper #import pdb + class Events(np.recarray): """ Docs here. @@ -26,23 +27,23 @@ class Events(np.recarray): # def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None, # formats=None, names=None, titles=None, # byteorder=None, aligned=False): - def __new__(*args,**kwargs): - self=np.recarray.__new__(*args,**kwargs) + def __new__(*args, **kwargs): + self = np.recarray.__new__(*args, **kwargs) if not(self._required_fields is None): for req_field in self._required_fields: if not(req_field in self.dtype.names): raise ValueError( - req_field+' is a required field!') + req_field + ' is a required field!') if not( - self[req_field].dtype==self._required_fields[req_field]): + self[req_field].dtype == self._required_fields[req_field]): raise ValueError( - req_field+' needs to be '+ - str(self._required_fields[req_field])+ - '. Provided dtype was '+str(self[req_field].dtype)) + req_field + ' needs to be ' + + str(self._required_fields[req_field]) + + '. Provided dtype was ' + str(self[req_field].dtype)) return self - - def remove_fields(self,*fields_to_remove): + + def remove_fields(self, *fields_to_remove): """ Return a new instance of the array with specified fields removed. @@ -54,7 +55,7 @@ def remove_fields(self,*fields_to_remove): # loop over fields, keeping if not matching fieldName for field in self.dtype.names: # don't add the field if in fields_to_remove list - #if sum(map(lambda x: x==field,fields_to_remove)) == 0: + # if sum(map(lambda x: x==field,fields_to_remove)) == 0: if not field in fields_to_remove: # append the data arrays.append(self[field]) @@ -63,73 +64,73 @@ def remove_fields(self,*fields_to_remove): # return the new Events if len(arrays) == 0: arrays.append([]) - return np.rec.fromarrays(arrays,names=','.join(names)).view(self.__class__) - - def add_fields(self,**fields): + return np.rec.fromarrays(arrays, names=','.join(names)).view(self.__class__) + + def add_fields(self, **fields): """ Add fields from the keyword args provided and return a new instance. To add an empty field, pass a dtype as the array. addFields(name1=array1, name2=dtype('i4')) - + """ # list of current dtypes to which new dtypes will be added: # new_dtype = [(name,self[name].dtype) for name in self.dtype.names] - + # sequence of arrays and names from starting recarray #arrays = map(lambda x: self[x], self.dtype.names) arrays = [self[x] for x in self.dtype.names] names = ','.join(self.dtype.names) - + # loop over the kwargs of field - for name,data in fields.iteritems(): + for name, data in fields.items(): # see if already there, error if so - if self.dtype.fields.has_key(name): + if name in self.dtype.fields: # already exists - raise ValueError('Field "'+name+'" already exists.') - + raise ValueError('Field "' + name + '" already exists.') + # append the array and name - if(isinstance(data,np.dtype)| - isinstance(data,type)|isinstance(data,str)): + if(isinstance(data, np.dtype) | + isinstance(data, type) | isinstance(data, str)): # add empty array the length of the data - arrays.append(np.empty(len(self),data)) + arrays.append(np.empty(len(self), data)) else: # add the data as an array arrays.append(data) # add the name - if len(names)>0: + if len(names) > 0: # append , - names = names+',' + names = names + ',' # append the name - names = names+name + names = names + name # return the new Events - return np.rec.fromarrays(arrays,names=names).view(self.__class__) + return np.rec.fromarrays(arrays, names=names).view(self.__class__) def __getitem__(self, index): # try block to ensure the _skip_dim_check flag gets reset # in the following finally block - try: + try: # skip the dim check b/c we're going to fiddle with them self._skip_field_check = True - ret = np.recarray.__getitem__(self,index) + ret = np.recarray.__getitem__(self, index) finally: # reset the _skip_dim_check flag: self._skip_field_check = False return ret - - def __getslice__(self,i,j): - try: + + def __getslice__(self, i, j): + try: # skip the field check self._skip_field_check = True - ret = np.recarray.__getslice__(self,i,j) + ret = np.recarray.__getslice__(self, i, j) finally: # reset the _skip_field_check flag: self._skip_field_check = False return ret - def __array_finalize__(self,obj): + def __array_finalize__(self, obj): # print self.shape,obj.shape # # print self._skip_field_check,obj._skip_field_check self._skip_field_check = False @@ -137,20 +138,21 @@ def __array_finalize__(self,obj): # check fields: # PBS: Also don't check if obj is None (implies it's unpickling) # We must find out if there are other instances of it being None - if (isinstance(obj,Events) and obj._skip_field_check) or \ - obj is None: return + if (isinstance(obj, Events) and obj._skip_field_check) or \ + obj is None: + return # ensure that the fields are valid: if not(self._required_fields is None): for req_field in self._required_fields: if not(req_field in obj.dtype.names): raise ValueError( - req_field+' is a required field!') + req_field + ' is a required field!') if not( - obj[req_field].dtype==self._required_fields[req_field]): + obj[req_field].dtype == self._required_fields[req_field]): raise ValueError( - req_field+' needs to be '+ - str(self._required_fields[req_field])+ - '. Provided dtype was '+str(obj[req_field].dtype)) + req_field + ' needs to be ' + + str(self._required_fields[req_field]) + + '. Provided dtype was ' + str(obj[req_field].dtype)) class TsEvents(Events): @@ -159,39 +161,39 @@ class TsEvents(Events): both tssrc (time series source) and tsoffset (time series offset) so that the class can know how to retrieve data for each event. """ - _required_fields = {'esrc':BaseWrapper,'eoffset':int} + _required_fields = {'esrc': BaseWrapper, 'eoffset': int} - def get_data(self,channel,dur,offset,buf,resampled_rate=None, - filt_freq=None,filt_type='stop', - filt_order=4,keep_buffer=False): + def get_data(self, channel, dur, offset, buf, resampled_rate=None, + filt_freq=None, filt_type='stop', + filt_order=4, keep_buffer=False): """ Return the requested range of data for each event by using the proper data retrieval mechanism for each event. The result will be an TimeSeries instance with dimensions (events,time). - """ - # get ready to load dat - eventdata = [] + """ + # get ready to load dat + eventdata = [] events = [] - + # speed up by getting unique event sources first usources = np.unique1d(self['esrc']) # loop over unique sources for src in usources: # get the eventOffsets from that source - ind = np.atleast_1d(self['esrc']==src) - + ind = np.atleast_1d(self['esrc'] == src) + if len(ind) == 1: - event_offsets=self['eoffset'] + event_offsets = self['eoffset'] events.append(self) else: event_offsets = self[ind]['eoffset'] events.append(self[ind]) - #print "Loading %d events from %s" % (ind.sum(),src) - # get the timeseries for those events + # print "Loading %d events from %s" % (ind.sum(),src) + # get the timeseries for those events eventdata.append(src.get_event_data(channel, event_offsets, dur, @@ -202,15 +204,13 @@ def get_data(self,channel,dur,offset,buf,resampled_rate=None, filt_type, filt_order, keep_buffer)) - + # concatenate (must eventually check that dims match) tdim = eventdata[0]['time'] srate = eventdata[0].samplerate events = np.concatenate(events).view(self.__class__) eventdata = TimeSeries(np.concatenate(eventdata), 'time', srate, - dims=[Dim(events,'events'),tdim]) - - return eventdata - + dims=[Dim(events, 'events'), tdim]) + return eventdata diff --git a/ptsa/data/hdf5wrapper.py b/ptsa/data/hdf5wrapper.py index 11d3121..6359d22 100644 --- a/ptsa/data/hdf5wrapper.py +++ b/ptsa/data/hdf5wrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -12,13 +12,15 @@ import h5py # local imports -from basewrapper import BaseWrapper -from timeseries import TimeSeries +from .basewrapper import BaseWrapper +from .timeseries import TimeSeries + class HDF5Wrapper(BaseWrapper): """ Interface to data stored in an HDF5 file. """ + def __init__(self, filepath, dataset_name='data', annotations_name='annotations', channel_info_name='channel_info', @@ -32,7 +34,7 @@ def __init__(self, filepath, dataset_name='data', For example, here is one way to create an HDF5 dataset from a TimeSeries instance: - + HDF5Wrapper('data.hdf5', data=data, compression='gzip') Now let's say the TimeSeries is float64, but you want to save @@ -42,7 +44,7 @@ def __init__(self, filepath, dataset_name='data', save the data in int16: HDF5Wrapper('data.hdf5', data=data, file_dtype=np.int16, compression='gzip') - + """ # set up the basic params of the data self.filepath = filepath @@ -53,15 +55,15 @@ def __init__(self, filepath, dataset_name='data', self.gain_buffer = gain_buffer self.gain = None self.hdf5opts = hdf5opts - + self.file_dtype = file_dtype self.data_dtype = None - + # see if create dataset if not data is None: # must provide samplerate and data # connect to the file and get the dataset - f = h5py.File(self.filepath,'a') + f = h5py.File(self.filepath, 'a') # use the data to create a dataset self.data_dtype = data.dtype @@ -80,7 +82,7 @@ def __init__(self, filepath, dataset_name='data', raise ValueError("You must specify a samplerate " + "if the dataset does not already exist.") # set the samplerate - d.attrs['samplerate'] = samplerate + d.attrs['samplerate'] = samplerate # create annotations if necessary if not annotations is None: @@ -104,12 +106,12 @@ def __init__(self, filepath, dataset_name='data', f.close() else: # connect to the file and get info - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') d = f[self.dataset_name] self.data_dtype = np.dtype(d.attrs['data_dtype']) self.file_dtype = d.dtype self.gain = d.attrs['gain'] - + def _data_to_file(self, data): # process the datatypes if self.file_dtype is None: @@ -129,35 +131,35 @@ def _data_to_file(self, data): self.gain = 1.0 # calc it if we are going from float to int if (self.file_dtype.kind == 'i') and (self.data_dtype.kind == 'f'): - fr = np.iinfo(self.file_dtype).max*2 - dr = np.abs(data).max()*2 * (1.+self.gain_buffer) - self.gain = dr/fr - + fr = np.iinfo(self.file_dtype).max * 2 + dr = np.abs(data).max() * 2 * (1. + self.gain_buffer) + self.gain = dr / fr + # calc and apply gain if necessary if self.apply_gain and self.gain != 1.0: - return np.asarray(data/self.gain,dtype=self.file_dtype) + return np.asarray(data / self.gain, dtype=self.file_dtype) else: - return np.asarray(data,dtype=self.file_dtype) + return np.asarray(data, dtype=self.file_dtype) def _data_from_file(self, data): # see if apply gain we've already calculated if self.apply_gain and self.gain != 1.0: - return np.asarray(data*self.gain, dtype=self.data_dtype) + return np.asarray(data * self.gain, dtype=self.data_dtype) else: return np.asarray(data, dtype=self.data_dtype) def _get_samplerate(self, channel=None): # Same samplerate for all channels. # get the samplerate property of the dataset - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') data = f[self.dataset_name] samplerate = data.attrs['samplerate'] f.close() return samplerate - def _get_nsamples(self,channel=None): + def _get_nsamples(self, channel=None): # get the dimensions of the data - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') data = f[self.dataset_name] nsamples = data.shape[1] f.close() @@ -165,7 +167,7 @@ def _get_nsamples(self,channel=None): def _get_nchannels(self): # get the dimensions of the data - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') data = f[self.dataset_name] nchannels = data.shape[0] f.close() @@ -173,7 +175,7 @@ def _get_nchannels(self): def _get_annotations(self): # get the dimensions of the data - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') if self.annotations_name in f: annot = f[self.annotations_name][:] else: @@ -183,7 +185,7 @@ def _get_annotations(self): def _set_annotations(self, annotations): # get the dimensions of the data - f = h5py.File(self.filepath,'a') + f = h5py.File(self.filepath, 'a') if self.annotations_name in f: del f[self.annotations_name] @@ -193,7 +195,7 @@ def _set_annotations(self, annotations): def _get_channel_info(self): # get the dimensions of the data - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') if self.channel_info_name in f: chan_info = f[self.channel_info_name][:] else: @@ -203,7 +205,7 @@ def _get_channel_info(self): def _set_channel_info(self, channel_info): # get the dimensions of the data - f = h5py.File(self.filepath,'a') + f = h5py.File(self.filepath, 'a') if self.channel_info_name in f: del f[self.channel_info_name] @@ -211,32 +213,33 @@ def _set_channel_info(self, channel_info): data=channel_info, **self.hdf5opts) f.close() - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): - """ + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): + """ """ # connect to the file and get the dataset - f = h5py.File(self.filepath,'r') + f = h5py.File(self.filepath, 'r') data = f[self.dataset_name] - + # allocate for data - eventdata = np.empty((len(channels),len(event_offsets),dur_samp), - dtype=self.data_dtype)*np.nan + eventdata = np.empty((len(channels), len(event_offsets), dur_samp), + dtype=self.data_dtype) * np.nan - # loop over events - for e,evOffset in enumerate(event_offsets): - # set the range - ssamp = offset_samp+evOffset + # loop over events + for e, evOffset in enumerate(event_offsets): + # set the range + ssamp = offset_samp + evOffset esamp = ssamp + dur_samp - + # check the ranges if ssamp < 0 or esamp > data.shape[1]: - raise IOError('Event with offset '+str(evOffset)+ + raise IOError('Event with offset ' + str(evOffset) + ' is outside the bounds of the data.') - eventdata[:,e,:] = self._data_from_file(data[channels,ssamp:esamp]) + eventdata[:, e, :] = self._data_from_file( + data[channels, ssamp:esamp]) # close the file f.close() - + return eventdata def append_data(self, data): @@ -244,7 +247,7 @@ def append_data(self, data): Must be all channels. """ # connect to the file and get the dataset - f = h5py.File(self.filepath,'a') + f = h5py.File(self.filepath, 'a') # get the dataset (must already exist) d = f[self.dataset_name] @@ -257,10 +260,10 @@ def append_data(self, data): # reshape to hold new data cursamp = d.shape[1] newsamp = data.shape[1] - d.shape = (d.shape[0], cursamp+newsamp) + d.shape = (d.shape[0], cursamp + newsamp) # append the data - d[:,cursamp:cursamp+newsamp] = self._data_to_file(data) + d[:, cursamp:cursamp + newsamp] = self._data_to_file(data) # close the file f.close() @@ -271,11 +274,11 @@ def set_channel_data(self, channel, data): of the entire dataset to match, throwing out data if smaller. """ # connect to the file and get the dataset - f = h5py.File(self.filepath,'a') + f = h5py.File(self.filepath, 'a') # get the dataset (must already exist) d = f[self.dataset_name] - + # reshape if necessary cursamp = d.shape[1] newsamp = len(data) @@ -283,7 +286,7 @@ def set_channel_data(self, channel, data): d.shape = (d.shape[0], newsamp) # set the data - d[channel,:] = self._data_to_file(data) + d[channel, :] = self._data_to_file(data) # close the file f.close() diff --git a/ptsa/data/process_log.py b/ptsa/data/process_log.py new file mode 100644 index 0000000..2b1da02 --- /dev/null +++ b/ptsa/data/process_log.py @@ -0,0 +1,118 @@ + +import numpy as np +import numpy.core.numerictypes as nt +_typestr = nt._typestr + +import yaml + +defaults = {str: '', + np.string_: '', + np.float64: np.float64(0), + np.int64: np.int64(0), + int: int(0), + int: int(0), + float: float(0.0), + bool: False, + type(None): None} + + +def _addrow(i, d, cols, tofill, to_ignore=[], prefix=''): + for k in d: + # add prefix + key = prefix + k + if key in to_ignore: + # skip it + continue + + # see if dict + if isinstance(d[k], dict): + _addrow(i, d[k], cols, tofill, + to_ignore=to_ignore, prefix=key + '_') + continue + + # see if tuple + if isinstance(d[k], tuple): + # turn into indexed dict + tdict = {} + for j in range(len(d[k])): + tdict[str(j)] = d[k][j] + _addrow(i, tdict, cols, tofill, + to_ignore=to_ignore, prefix=key + '_') + continue + + # see if already there + if not key in cols: + # add and fill it with default vals + cols[key] = [defaults[type(d[k])]] * i + + # append it + cols[key].append(d[k]) + + # remove from list to fill + if key in tofill: + del tofill[key] + + +def load_yaml(yaml_file, **append_cols): + # load the dictlist + dictlist = yaml.load(open(yaml_file, 'r')) + for i in range(len(dictlist)): + dictlist[i].update(append_cols) + return dictlist + + +def log2rec(dictlist, to_ignore=[], force_format=None, **append_cols): + """ + Convert a list of dicts (possibly in a yaml file) to a recarray. + + You can use the append_cols kwargs to add fields with the value + replicated across all rows. This is a good way to add subject ids + and personality variables. + """ + + # see if dictlist is a yaml file + if isinstance(dictlist, str): + # assume it's a file and read it in + dictlist = yaml.load(open(dictlist, 'r')) + + # make a dict of columns + cols = {} + + for i, d in enumerate(dictlist): + # get list to fill + tofill = dict([(k, defaults[type(cols[k][-1])]) + for k in list(cols.keys())]) + + # loop over keys and fill cols + _addrow(i, d, cols, tofill, to_ignore=to_ignore) + + # anything not filled add the default val + for k in tofill: + cols[k].append(tofill[k]) + + # add in the additional columns + for k in append_cols: + cols[k] = [append_cols[k]] * len(cols[list(cols.keys())[0]]) + + # get the formats + formats = {} + for k in list(cols.keys()): + obj = np.asarray(cols[k]) + if not isinstance(obj, np.ndarray): + raise ValueError("item in the array list must be an ndarray.") + oformat = _typestr[obj.dtype.type] + if issubclass(obj.dtype.type, nt.flexible): + oformat += repr(obj.itemsize) + formats[k] = oformat + + # enforce formats + if isinstance(force_format, dict): + for k in force_format: + if k in formats: + formats[k] = force_format[k] + + # make the rec from arrays + rec = np.rec.fromarrays([cols[k] for k in list(cols.keys())], + names=','.join(list(cols.keys())), + formats=','.join([formats[k] for k in list(cols.keys())])) + return rec diff --git a/ptsa/data/rawbinarydata.py b/ptsa/data/rawbinarydata.py index 1fda8a8..08fa2dc 100644 --- a/ptsa/data/rawbinarydata.py +++ b/ptsa/data/rawbinarydata.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,9 +8,9 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # local imports -from datawrapper import DataWrapper -from events import Events,TsEvents -from timeseries import TimeSeries,Dim +from .datawrapper import DataWrapper +from .events import Events, TsEvents +from .timeseries import TimeSeries, Dim # global imports import numpy as np @@ -19,12 +19,14 @@ import os from scipy.io import loadmat + class RawBinaryEEG(DataWrapper): """ Interface to data stored in binary format with a separate file for each channel. """ - def __init__(self,dataroot,samplerate=None,format='int16',gain=1): + + def __init__(self, dataroot, samplerate=None, format='int16', gain=1): """Initialize the interface to the data. You must specify the dataroot, which is a string that contains the path to and root, up to the channel numbers, where the data are stored.""" @@ -37,14 +39,14 @@ def __init__(self,dataroot,samplerate=None,format='int16',gain=1): # see if can find them from a params file in dataroot self.params = self._getParams(dataroot) - # set what we can from the params - if self.params.has_key('samplerate'): + # set what we can from the params + if 'samplerate' in self.params: self.samplerate = self.params['samplerate'] - if self.params.has_key('format'): + if 'format' in self.params: self.format = self.params['format'] - if self.params.has_key('dataformat'): + if 'dataformat' in self.params: self.format = self.params['dataformat'] - if self.params.has_key('gain'): + if 'gain' in self.params: self.gain = self.params['gain'] # set the nBytes and format str @@ -58,51 +60,50 @@ def __init__(self,dataroot,samplerate=None,format='int16',gain=1): self.nBytes = 8 self.fmtStr = 'd' - def _getParams(self,dataroot): + def _getParams(self, dataroot): """Get parameters of the data from the dataroot.""" # set default params - params = {'samplerate':256.03,'gain':1.} + params = {'samplerate': 256.03, 'gain': 1.} # first look for dataroot.params file paramFile = dataroot + '.params' if not os.path.isfile(paramFile): # see if it's params.txt - paramFile = os.path.join(os.path.dirname(dataroot),'params.txt') + paramFile = os.path.join(os.path.dirname(dataroot), 'params.txt') if not os.path.isfile(paramFile): - #raise "file not found" # fix this + # raise "file not found" # fix this return params - + # we have a file, so open and process it - for line in open(paramFile,'r').readlines(): + for line in open(paramFile, 'r').readlines(): # get the columns by splitting cols = line.strip().split() # set the params params[cols[0]] = eval(string.join(cols[1:])) - + # return the params dict return params - - def _load_timeseries(self,channel,eventOffsets,dur_samp,offset_samp): + def _load_timeseries(self, channel, eventOffsets, dur_samp, offset_samp): """ - + """ # determine the file - eegfname = '%s.%03i' % (self.dataroot,channel) - if os.path.isfile(eegfname): - efile = open(eegfname,'rb') - else: - # try unpadded lead - eegfname = '%s.%i' % (self.dataroot,channel) - if os.path.isfile(eegfname): - efile = open(eegfname,'rb') - else: - raise IOError('EEG file not found for channel %i and file root %s\n' - % (channel,self.dataroot)) - - # loop over events - eventdata = [] + eegfname = '%s.%03i' % (self.dataroot, channel) + if os.path.isfile(eegfname): + efile = open(eegfname, 'rb') + else: + # try unpadded lead + eegfname = '%s.%i' % (self.dataroot, channel) + if os.path.isfile(eegfname): + efile = open(eegfname, 'rb') + else: + raise IOError('EEG file not found for channel %i and file root %s\n' + % (channel, self.dataroot)) + + # loop over events + eventdata = [] # # get the eventOffsets # if isinstance(eventInfo,TsEvents): @@ -110,49 +111,49 @@ def _load_timeseries(self,channel,eventOffsets,dur_samp,offset_samp): # else: # eventOffsets = eventInfo # eventOffsets = np.asarray(eventOffsets) - # if len(eventOffsets.shape)==0: - # eventOffsets = [eventOffsets] - for evOffset in eventOffsets: - # seek to the position in the file - thetime = offset_samp+evOffset - efile.seek(self.nBytes*thetime,0) + # if len(eventOffsets.shape)==0: + # eventOffsets = [eventOffsets] + for evOffset in eventOffsets: + # seek to the position in the file + thetime = offset_samp + evOffset + efile.seek(self.nBytes * thetime, 0) - # read the data - data = efile.read(int(self.nBytes*dur_samp)) + # read the data + data = efile.read(int(self.nBytes * dur_samp)) - # convert from string to array based on the format - # hard codes little endian - data = np.array(struct.unpack('<'+str(len(data)/self.nBytes)+self.fmtStr,data)) + # convert from string to array based on the format + # hard codes little endian + data = np.array(struct.unpack( + '<' + str(len(data) / self.nBytes) + self.fmtStr, data)) - # make sure we got some data - if len(data) < dur_samp: - raise IOError('Event with offset %d is outside the bounds of file %s.\n' - % (evOffset,eegfname)) + # make sure we got some data + if len(data) < dur_samp: + raise IOError('Event with offset %d is outside the bounds of file %s.\n' + % (evOffset, eegfname)) - # append it to the events - eventdata.append(data) + # append it to the events + eventdata.append(data) # calc the time range - sampStart = offset_samp*samplesize - sampEnd = sampStart + (dur_samp-1)*samplesize - timeRange = np.linspace(sampStart,sampEnd,dur_samp) + sampStart = offset_samp * samplesize + sampEnd = sampStart + (dur_samp - 1) * samplesize + timeRange = np.linspace(sampStart, sampEnd, dur_samp) - # make it a timeseries - if isinstance(eventInfo,TsEvents): + # make it a timeseries + if isinstance(eventInfo, TsEvents): dims = [Dim('event', eventInfo.data, 'event'), - Dim('time',timeRange)] + Dim('time', timeRange)] else: dims = [Dim('eventOffsets', eventOffsets, 'samples'), - Dim('time',timeRange)] + Dim('time', timeRange)] eventdata = TimeSeries(np.array(eventdata), dims, tdim='time', self.samplerate) # multiply by the gain - eventdata *= self.gain + eventdata *= self.gain - return eventdata @@ -162,60 +163,59 @@ def createEventsFromMatFile(matfile): # load the mat file mat = loadmat(matfile) - if 'events' not in mat.keys(): + if 'events' not in list(mat.keys()): raise "\nError processing the Matlab file: %s\n" + \ "This file must contain an events structure" + \ "with the name \"events\" (case sensitive)!\n" +\ - "(All other content of the file is ignored.)" % matfile - + "(All other content of the file is ignored.)" % matfile + # get num events numEvents = len(mat['events']) # determine the fieldnames and formats fields = mat['events'][0]._fieldnames - + # create list with array for each field data = [] hasEEGInfo = False - for f,field in enumerate(fields): - # handle special cases - if field == 'eegfile': - # we have eeg info - hasEEGInfo = True - - # get unique files - eegfiles = np.unique(map(lambda x: str(x.eegfile),mat['events'])) - - # make dictionary of data wrapers for the eeg files - efile_dict = {} - for eegfile in eegfiles: - efile_dict[eegfile] = RawBinaryEEG(eegfile) - - # Handle when the eegfile field is blank - efile_dict[''] = None - - # set the eegfile to the correct data wrapper - newdat = np.array(map(lambda x: efile_dict[str(x.__getattribute__(field))], - mat['events'])) - - # change field name to eegsrc - fields[f] = 'eegsrc' - else: - # get the data in normal fashion - newdat = np.array(map(lambda x: x.__getattribute__(field),mat['events'])) - - # append the data - data.append(newdat) + for f, field in enumerate(fields): + # handle special cases + if field == 'eegfile': + # we have eeg info + hasEEGInfo = True + + # get unique files + eegfiles = np.unique([str(x.eegfile) for x in mat['events']]) + + # make dictionary of data wrapers for the eeg files + efile_dict = {} + for eegfile in eegfiles: + efile_dict[eegfile] = RawBinaryEEG(eegfile) + + # Handle when the eegfile field is blank + efile_dict[''] = None + + # set the eegfile to the correct data wrapper + newdat = np.array( + [efile_dict[str(x.__getattribute__(field))] for x in mat['events']]) + + # change field name to eegsrc + fields[f] = 'eegsrc' + else: + # get the data in normal fashion + newdat = np.array([x.__getattribute__(field) + for x in mat['events']]) + + # append the data + data.append(newdat) # allocate for new array - newrec = np.rec.fromarrays(data,names=fields) + newrec = np.rec.fromarrays(data, names=fields) # see if process into DataArray or Events if hasEEGInfo: - newrec = TsEvents(newrec) + newrec = TsEvents(newrec) else: - newrec = Events(newrec) + newrec = Events(newrec) return newrec - - diff --git a/ptsa/data/rawbinwrapper.py b/ptsa/data/rawbinwrapper.py index 4f4ec3e..42bca27 100644 --- a/ptsa/data/rawbinwrapper.py +++ b/ptsa/data/rawbinwrapper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,7 +8,7 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # local imports -from basewrapper import BaseWrapper +from .basewrapper import BaseWrapper # global imports import numpy as np @@ -17,12 +17,14 @@ import os from glob import glob + class RawBinWrapper(BaseWrapper): """ Interface to data stored in binary format with a separate file for each channel. """ - def __init__(self,dataroot,samplerate=None,format='int16',gain=1): + + def __init__(self, dataroot, samplerate=None, format='int16', gain=1): """Initialize the interface to the data. You must specify the dataroot, which is a string that contains the path to and root, up to the channel numbers, where the data are stored.""" @@ -35,14 +37,14 @@ def __init__(self,dataroot,samplerate=None,format='int16',gain=1): # see if can find them from a params file in dataroot self._params = self._get_params(dataroot) - # set what we can get from the params - if self._params.has_key('samplerate'): + # set what we can get from the params + if 'samplerate' in self._params: self._samplerate = self._params['samplerate'] - if self._params.has_key('format'): + if 'format' in self._params: self._format = self._params['format'] - if self._params.has_key('dataformat'): + if 'dataformat' in self._params: self._format = self._params['dataformat'] - if self._params.has_key('gain'): + if 'gain' in self._params: self._gain = self._params['gain'] # set the nBytes and format str @@ -56,7 +58,7 @@ def __init__(self,dataroot,samplerate=None,format='int16',gain=1): self._nbytes = 8 self._fmt_str = 'd' - self._chanfiles = glob(self._dataroot+'.*[0-9]') + self._chanfiles = glob(self._dataroot + '.*[0-9]') # sorting because the order of the output from glob is # arbitrary (not strictly necessary, but nice to have # consistency): @@ -72,20 +74,20 @@ def __init__(self,dataroot,samplerate=None,format='int16',gain=1): names.append(self._chanfiles[i].split('.')[-1]) self._channel_info = np.rec.fromarrays( [numbers, names], names='number,name') - + def _get_dataroot(self, channel=None): # Same dataroot for all channels: - return self._dataroot + return self._dataroot def _get_samplerate(self, channel=None): # Same samplerate for all channels: return self._samplerate - def _get_nsamples(self,channel=None): + def _get_nsamples(self, channel=None): # get the dimensions of the data # must open a valid channel and seek to the end if channel is not None: - raise NotImplementedError('Channel cannot be specified!') + raise NotImplementedError('Channel cannot be specified!') if self._nsamples is None: chanfile = open(self._chanfiles[0], 'rb') chanfile.seek(0, 2) @@ -93,7 +95,7 @@ def _get_nsamples(self,channel=None): raise ValueError( 'File length does not correspond to data format!') else: - self._nsamples = chanfile.tell()/self._nbytes + self._nsamples = chanfile.tell() / self._nbytes return self._nsamples def _get_nchannels(self): @@ -103,13 +105,13 @@ def _get_nchannels(self): def _get_channel_info(self): return self._channel_info - + def _get_annotations(self): # no annotations for raw data annot = None return annot - def _get_params(self,dataroot): + def _get_params(self, dataroot): """Get parameters of the data from the dataroot.""" params = {} @@ -120,50 +122,50 @@ def _get_params(self,dataroot): param_file = os.path.join(os.path.dirname(dataroot), 'params.txt') if not os.path.isfile(param_file): raise IOError( - 'No params file found in '+str(dataroot)+ + 'No params file found in ' + str(dataroot) + '. Params files must be in the same directory ' + 'as the EEG data and must be named \".params\" ' + - 'or \"params.txt\".') + 'or \"params.txt\".') # we have a file, so open and process it - for line in open(param_file,'r').readlines(): + for line in open(param_file, 'r').readlines(): # get the columns by splitting cols = line.strip().split() # set the params params[cols[0]] = eval(string.join(cols[1:])) - if (not params.has_key('samplerate')) or (not params.has_key('gain')): + if ('samplerate' not in params) or ('gain' not in params): raise ValueError( 'Params file must contain samplerate and gain!\n' + - 'The following fields were supplied:\n' + str(params.keys())) + 'The following fields were supplied:\n' + str(list(params.keys()))) # return the params dict return params - - def _load_data(self,channels,event_offsets,dur_samp,offset_samp): + def _load_data(self, channels, event_offsets, dur_samp, offset_samp): """ """ # allocate for data - eventdata = np.empty((len(channels),len(event_offsets),dur_samp), - dtype=np.float)*np.nan + eventdata = np.empty((len(channels), len(event_offsets), dur_samp), + dtype=np.float) * np.nan # loop over channels for c, channel in enumerate(channels): # determine the file - eegfname = self._dataroot+'.'+self._channel_info['name'][channel] + eegfname = self._dataroot + '.' + \ + self._channel_info['name'][channel] # eegfname = '{}.{:0>3}'.format(self._dataroot,channel) if os.path.isfile(eegfname): - efile = open(eegfname,'rb') + efile = open(eegfname, 'rb') else: raise IOError( - 'EEG file not found: '+eegfname) - # 'EEG file not found for channel {:0>3} '.format(channel) + - # 'and file root {}\n'.format(self._dataroot)) + 'EEG file not found: ' + eegfname) + # 'EEG file not found for channel {:0>3} '.format(channel) + + # 'and file root {}\n'.format(self._dataroot)) # loop over events for e, ev_offset in enumerate(event_offsets): # seek to the position in the file thetime = offset_samp + ev_offset - efile.seek(self._nbytes * thetime,0) + efile.seek(self._nbytes * thetime, 0) # read the data data = efile.read(int(self._nbytes * dur_samp)) @@ -184,8 +186,8 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): eventdata[c, e, :] = data # multiply by the gain - eventdata *= self._gain - + eventdata *= self._gain + return eventdata dataroot = property(lambda self: self._get_dataroot()) @@ -201,11 +203,11 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # raise "\nError processing the Matlab file: %s\n" + \ # "This file must contain an events structure" + \ # "with the name \"events\" (case sensitive)!\n" +\ -# "(All other content of the file is ignored.)" % matfile +# "(All other content of the file is ignored.)" % matfile # # get the events # events = mat['events'][0] - + # # get num events # numEvents = len(events) @@ -221,7 +223,7 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # else: # data.append(dtype(dat[0])) # return data - + # # create list with array for each field # data = [] # for f,field in enumerate(fields): @@ -232,7 +234,7 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # #eegfiles = np.unique([str(x.eegfile[0]) for x in events]) # eegfiles = np.unique(loadfield(events,field)) # eegfiles = eegfiles[eegfiles!=None] - + # # make dictionary of data wrapers for the unique eeg files # efile_dict = {} # for eegfile in eegfiles: @@ -240,15 +242,15 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # # Handle when the eegfile field is blank # efile_dict[''] = None - + # # set the eegfile to the correct data wrapper - + # # newdat = np.array( # # map(lambda x: efile_dict[str(x.__getattribute__(field))], # # events)) # newdat = np.array([efile_dict[str(x.__getattribute__(field)[0])] # for x in events]) - + # # change field name to esrc # fields[f] = 'esrc' # elif field == 'eegoffset': @@ -270,6 +272,3 @@ def _load_data(self,channels,event_offsets,dur_samp,offset_samp): # newrec = np.rec.fromarrays(data,names=fields).view(Events) # return newrec - - - diff --git a/ptsa/data/tests/test_events.py b/ptsa/data/tests/test_events.py index 5e51334..140dad5 100644 --- a/ptsa/data/tests/test_events.py +++ b/ptsa/data/tests/test_events.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -10,13 +10,12 @@ import numpy as np from numpy.testing import TestCase, assert_array_equal,\ - assert_array_almost_equal + assert_array_almost_equal -from ptsa.data import ArrayWrapper,BaseWrapper +from ptsa.data import ArrayWrapper, BaseWrapper from ptsa.data.events import Events - class Setup(): def __init__(self): self.test1xyz = np.array([(1.0, 2, 'bla1'), (3.0, 4, 'bla2')], @@ -83,7 +82,7 @@ def __init__(self): (dw, 6, 11., 4)]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('x', float),('y',int)]) + ('x', float), ('y', int)]) self.test2soxyz = np.array([[(dw, 2, 42., 1, 'z1'), (dw, 4, 33., 2, 'z2')], @@ -91,14 +90,14 @@ def __init__(self): (dw, 6, 11., 4, 'z4')]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('x', float),('y',int),('z','|S2')]) + ('x', float), ('y', int), ('z', '|S2')]) self.test2soy = np.array([[(dw, 2, 1), - (dw, 4, 2)], - [(dw, 5, 3), - (dw, 6, 4)]], + (dw, 4, 2)], + [(dw, 5, 3), + (dw, 6, 4)]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('y',int)]) + ('y', int)]) self.test2soz = np.array([[(dw, 2, 'z1'), (dw, 4, 'z2')], @@ -106,7 +105,7 @@ def __init__(self): (dw, 6, 'z4')]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('z','|S2')]) + ('z', '|S2')]) self.test2soyz = np.array([[(dw, 2, 1, 'z1'), (dw, 4, 2, 'z2')], @@ -114,7 +113,7 @@ def __init__(self): (dw, 6, 4, 'z4')]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('y', int),('z', '|S2')]) + ('y', int), ('z', '|S2')]) self.test2soxz = np.array([[(dw, 2, 42., 'z1'), (dw, 4, 33., 'z2')], @@ -122,14 +121,14 @@ def __init__(self): (dw, 6, 11., 'z4')]], dtype=[('esrc', BaseWrapper), ('eoffset', int), - ('x', float),('z', '|S2')]) + ('x', float), ('z', '|S2')]) class test_Events(TestCase): def setUp(self): - self.dat = np.random.rand(10,1000) - self.aw = ArrayWrapper(self.dat,200) - self.eoffsets = [80,140,270] + self.dat = np.random.rand(10, 1000) + self.aw = ArrayWrapper(self.dat, 200) + self.eoffsets = [80, 140, 270] def test_new(self): tst = Setup() @@ -158,241 +157,240 @@ def test_remove_fields(self): tst = Setup() test_a = tst.test1xyz.view(Events).remove_fields('z') test_b = tst.test1xy.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields('y') test_b = tst.test1xz.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields('x') test_b = tst.test1yz.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'y').remove_fields('z') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('y','z') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('y', 'z') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('z','y') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('z', 'y') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'z').remove_fields('y') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'y').remove_fields('x') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('y','x') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('y', 'x') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('x','y') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('x', 'y') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'x').remove_fields('y') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'x').remove_fields('z') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('x','z') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('x', 'z') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test1xyz.view(Events).remove_fields('z','x') + assert_array_equal(test_a, test_b) + test_a = tst.test1xyz.view(Events).remove_fields('z', 'x') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events).remove_fields( 'z').remove_fields('x') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xy.view(Events).remove_fields('y') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xy.view(Events).remove_fields('x') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xz.view(Events).remove_fields('z') test_b = tst.test1x.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xz.view(Events).remove_fields('x') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1yz.view(Events).remove_fields('z') test_b = tst.test1y.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1yz.view(Events).remove_fields('y') test_b = tst.test1z.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields('z') test_b = tst.test2soxy.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields('y') test_b = tst.test2soxz.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields('x') test_b = tst.test2soyz.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('y','z') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('y', 'z') test_b = tst.test2sox.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('z','y') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('z', 'y') test_b = tst.test2sox.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'z').remove_fields('y') test_b = tst.test2sox.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'y').remove_fields('z') test_b = tst.test2sox.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('x','y','z') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('x', 'y', 'z') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('x','z','y') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('x', 'z', 'y') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('y','x','z') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('y', 'x', 'z') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('y','z','x') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('y', 'z', 'x') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('z','y','x') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('z', 'y', 'x') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) - test_a = tst.test2soxyz.view(Events).remove_fields('z','x','y') + assert_array_equal(test_a, test_b) + test_a = tst.test2soxyz.view(Events).remove_fields('z', 'x', 'y') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'x').remove_fields('y').remove_fields('z') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'x').remove_fields('z').remove_fields('y') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'y').remove_fields('x').remove_fields('z') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'y').remove_fields('z').remove_fields('x') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'z').remove_fields('x').remove_fields('y') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events).remove_fields( 'z').remove_fields('y').remove_fields('x') test_b = tst.test2so.view(Events) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) def test_add_fields(self): tst = Setup() - x = np.array([1.0,3.0]) - y = np.array([2,4]) - z = np.array(['bla1','bla2']) + x = np.array([1.0, 3.0]) + y = np.array([2, 4]) + z = np.array(['bla1', 'bla2']) test_a = tst.test1xyz.view(Events) - self.assertRaises(ValueError,tst.test1xyz.view(Events).add_fields,x=x) - self.assertRaises(ValueError,tst.test1xyz.view(Events).add_fields,y=int) + self.assertRaises( + ValueError, tst.test1xyz.view(Events).add_fields, x=x) + self.assertRaises(ValueError, tst.test1xyz.view( + Events).add_fields, y=int) test_b = tst.test1xy.view(Events).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = test_a.add_fields(a=int) test_b = test_b.add_fields(a=int) - self.assertTrue(test_a.shape==test_b.shape) + self.assertTrue(test_a.shape == test_b.shape) assert_array_equal(np.sort(test_a.dtype.names), np.sort(test_b.dtype.names)) - for f,v in test_a.dtype.fields.iteritems(): - if f!='a': - assert_array_equal(test_a[f],test_b[f]) - self.assertTrue(test_a.dtype[f]==test_b.dtype[f]) + for f, v in test_a.dtype.fields.items(): + if f != 'a': + assert_array_equal(test_a[f], test_b[f]) + self.assertTrue(test_a.dtype[f] == test_b.dtype[f]) test_a = tst.test1xyz.view(Events) test_b = tst.test1x.view(Events).add_fields(y=y).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xyz.view(Events) - test_b = tst.test1x.view(Events).add_fields(y=y,z=z) + test_b = tst.test1x.view(Events).add_fields(y=y, z=z) assert_array_equal(np.sort(test_a.dtype.names), np.sort(test_b.dtype.names)) - for f,v in test_a.dtype.fields.iteritems(): - assert_array_equal(test_a[f],test_b[f]) - self.assertTrue(test_a.dtype[f]==test_b.dtype[f]) + for f, v in test_a.dtype.fields.items(): + assert_array_equal(test_a[f], test_b[f]) + self.assertTrue(test_a.dtype[f] == test_b.dtype[f]) for field in test_a.dtype.names: - assert_array_equal(test_a[field],test_b[field]) + assert_array_equal(test_a[field], test_b[field]) test_a = tst.test1xy.view(Events) test_b = tst.test1x.view(Events).add_fields(y=y) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1xz.view(Events) test_b = tst.test1x.view(Events).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test1yz.view(Events) test_b = tst.test1y.view(Events).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) - x = np.array([[42.,33.],[22.,11.]]) - y = np.array([[1,2],[3,4]]) - z = np.array([['z1','z2'],['z3','z4']]) + x = np.array([[42., 33.], [22., 11.]]) + y = np.array([[1, 2], [3, 4]]) + z = np.array([['z1', 'z2'], ['z3', 'z4']]) test_a = tst.test2soxyz.view(Events) test_b = tst.test2soxy.view(Events).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events) - test_b = tst.test2sox.view(Events).add_fields(y=y,z=z) + test_b = tst.test2sox.view(Events).add_fields(y=y, z=z) assert_array_equal(np.sort(test_a.dtype.names), np.sort(test_b.dtype.names)) - for f,v in test_a.dtype.fields.iteritems(): - assert_array_equal(test_a[f],test_b[f]) - self.assertTrue(test_a.dtype[f]==test_b.dtype[f]) + for f, v in test_a.dtype.fields.items(): + assert_array_equal(test_a[f], test_b[f]) + self.assertTrue(test_a.dtype[f] == test_b.dtype[f]) for field in test_a.dtype.names: - assert_array_equal(test_a[field],test_b[field]) + assert_array_equal(test_a[field], test_b[field]) test_a = tst.test2soxyz.view(Events) test_b = tst.test2sox.view(Events).add_fields(y=y).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events) test_b = tst.test2so.view(Events).add_fields(x=x).add_fields( y=y).add_fields(z=z) - assert_array_equal(test_a,test_b) + assert_array_equal(test_a, test_b) test_a = tst.test2soxyz.view(Events) - test_b = tst.test2so.view(Events).add_fields(x=x,y=y,z=z) + test_b = tst.test2so.view(Events).add_fields(x=x, y=y, z=z) assert_array_equal(np.sort(test_a.dtype.names), np.sort(test_b.dtype.names)) - for f,v in test_a.dtype.fields.iteritems(): - assert_array_equal(test_a[f],test_b[f]) - self.assertTrue(test_a.dtype[f]==test_b.dtype[f]) + for f, v in test_a.dtype.fields.items(): + assert_array_equal(test_a[f], test_b[f]) + self.assertTrue(test_a.dtype[f] == test_b.dtype[f]) - def test_get_data(self): # get data directly from the wrapper - ed = self.aw.get_event_data(3,self.eoffsets,.5,-.1,.25) + ed = self.aw.get_event_data(3, self.eoffsets, .5, -.1, .25) # create same array by hand: - ed2 = np.array([self.dat[3,60:161],self.dat[3,120:221], - self.dat[3,250:351]]) - assert_array_equal(ed,ed2) + ed2 = np.array([self.dat[3, 60:161], self.dat[3, 120:221], + self.dat[3, 250:351]]) + assert_array_equal(ed, ed2) # get data from a events # make some events - events = np.rec.fromarrays(([self.aw]*len(self.eoffsets),self.eoffsets), + events = np.rec.fromarrays(([self.aw] * len(self.eoffsets), self.eoffsets), names='esrc,eoffset').view(Events) - ed3 = events.get_data(3,.5,-.1,.25) + ed3 = events.get_data(3, .5, -.1, .25) - assert_array_almost_equal(ed[:],ed3[:],decimal=6) + assert_array_almost_equal(ed[:], ed3[:], decimal=6) # get data directly from the wrapper - ed = self.aw.get_event_data(3,self.eoffsets,.5,.1,.25) + ed = self.aw.get_event_data(3, self.eoffsets, .5, .1, .25) # create same array by hand: - ed2 = np.array([self.dat[3,100:201],self.dat[3,160:261], - self.dat[3,290:391]]) - assert_array_equal(ed,ed2) + ed2 = np.array([self.dat[3, 100:201], self.dat[3, 160:261], + self.dat[3, 290:391]]) + assert_array_equal(ed, ed2) # get data from a events # make some events - events = np.rec.fromarrays(([self.aw]*len(self.eoffsets),self.eoffsets), + events = np.rec.fromarrays(([self.aw] * len(self.eoffsets), self.eoffsets), names='esrc,eoffset').view(Events) - ed3 = events.get_data(3,.5,.1,.25) - - assert_array_almost_equal(ed[:],ed3[:],decimal=6) - + ed3 = events.get_data(3, .5, .1, .25) + assert_array_almost_equal(ed[:], ed3[:], decimal=6) diff --git a/ptsa/data/tests/test_timeseries.py b/ptsa/data/tests/test_timeseries.py index a3ee404..21aadee 100644 --- a/ptsa/data/tests/test_timeseries.py +++ b/ptsa/data/tests/test_timeseries.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,7 +11,7 @@ import re from numpy.testing import TestCase -from ptsa.data import Dim,DimArray,TimeSeries +from ptsa.data import Dim, DimArray, TimeSeries from ptsa import filt # from numpy.testing import NumpyTest, TestCase @@ -26,7 +26,7 @@ # def test_bar(self): pass # #print "testing bar" - + # if __name__ == '__main__': # NumpyTest.main() @@ -38,44 +38,46 @@ class TestData: def __init__(self): # create 10 Hz sine waves at 200 and 50 Hz 4000ms long numSecs = 4. - numPoints = int(numSecs*200.) + numPoints = int(numSecs * 200.) Hz = 10 - d200_10 = np.sin(np.arange(numPoints,dtype=np.float)*2*np.pi* - Hz*numSecs/numPoints) + d200_10 = np.sin(np.arange(numPoints, dtype=np.float) * 2 * np.pi * + Hz * numSecs / numPoints) Hz = 5 - d200_5 = np.sin(np.arange(numPoints,dtype=np.float)*2*np.pi* - Hz*numSecs/numPoints) - self.dat200 = np.array([d200_10,d200_5]) + d200_5 = np.sin(np.arange(numPoints, dtype=np.float) * 2 * np.pi * + Hz * numSecs / numPoints) + self.dat200 = np.array([d200_10, d200_5]) # calc the time range offset = -200 duration = numPoints - samplesize = 1./200. - sampStart = offset*samplesize - sampEnd = sampStart + (duration-1)*samplesize - timeRange = np.linspace(sampStart,sampEnd,duration) - self.dims200 = [Dim(np.arange(self.dat200.shape[0]),'channel'), - Dim(timeRange,'time',unit='ms')] - + samplesize = 1. / 200. + sampStart = offset * samplesize + sampEnd = sampStart + (duration - 1) * samplesize + timeRange = np.linspace(sampStart, sampEnd, duration) + self.dims200 = [Dim(np.arange(self.dat200.shape[0]), 'channel'), + Dim(timeRange, 'time', unit='ms')] + numSecs = 4. - numPoints = int(numSecs*50.) + numPoints = int(numSecs * 50.) Hz = 10 - d50_10 = np.sin(np.arange(numPoints,dtype=np.float)*2*np.pi* - Hz*numSecs/numPoints) + d50_10 = np.sin(np.arange(numPoints, dtype=np.float) * 2 * np.pi * + Hz * numSecs / numPoints) Hz = 5 - d50_5 = np.sin(np.arange(numPoints,dtype=np.float)*2*np.pi* - Hz*numSecs/numPoints) - self.dat50 = np.array([d50_10,d50_5]) + d50_5 = np.sin(np.arange(numPoints, dtype=np.float) * 2 * np.pi * + Hz * numSecs / numPoints) + self.dat50 = np.array([d50_10, d50_5]) # calc the time range in MS offset = -50 duration = numPoints - samplesize = 1000./50. - sampStart = offset*samplesize - sampEnd = sampStart + (duration-1)*samplesize - timeRange = np.linspace(sampStart,sampEnd,duration) - self.dims50 = [Dim(np.arange(self.dat50.shape[0]),'channel'), - Dim(timeRange,'time',unit='ms')] + samplesize = 1000. / 50. + sampStart = offset * samplesize + sampEnd = sampStart + (duration - 1) * samplesize + timeRange = np.linspace(sampStart, sampEnd, duration) + self.dims50 = [Dim(np.arange(self.dat50.shape[0]), 'channel'), + Dim(timeRange, 'time', unit='ms')] # test TimeSeries + + class test_TimeSeries(TestCase): def setUp(self): td = TestData() @@ -83,65 +85,63 @@ def setUp(self): self.dims200 = td.dims200 self.dat50 = td.dat50 self.dims50 = td.dims50 + def test_init(self): # init a TimeSeries with all combos of options and verify that # the attributes are correct # fewest params - ts = TimeSeries(self.dat200,'time',200,dims = self.dims200) + ts = TimeSeries(self.dat200, 'time', 200, dims=self.dims200) np.testing.assert_equal(ts[:], self.dat200[:]) - self.assertEquals(ts.shape, self.dat200.shape) - self.assertEquals(ts.taxis, len(self.dat200.shape)-1) - self.assertEquals(ts.samplerate,200) - self.assertRaises(ValueError,TimeSeries,self.dat200, - 'bla',200,dims=self.dims200) - self.assertRaises(ValueError,TimeSeries,self.dat200, - 'time',-200,dims=self.dims200) - - + self.assertEqual(ts.shape, self.dat200.shape) + self.assertEqual(ts.taxis, len(self.dat200.shape) - 1) + self.assertEqual(ts.samplerate, 200) + self.assertRaises(ValueError, TimeSeries, self.dat200, + 'bla', 200, dims=self.dims200) + self.assertRaises(ValueError, TimeSeries, self.dat200, + 'time', -200, dims=self.dims200) + def test_remove_buffer(self): buf = 200 - numsamp = 4*200 - ts = TimeSeries(self.dat200,'time',200, dims=self.dims200) + numsamp = 4 * 200 + ts = TimeSeries(self.dat200, 'time', 200, dims=self.dims200) ts_nobuff = ts.remove_buffer(1) - self.assertEquals(ts_nobuff.shape[ts_nobuff.taxis],numsamp-2*buf) - self.assertEquals(len(ts_nobuff['time']),numsamp-2*buf) - ts_nobuff = ts.remove_buffer((1,1)) - self.assertEquals(ts_nobuff.shape[ts_nobuff.taxis],numsamp-2*buf) - self.assertEquals(len(ts_nobuff['time']),numsamp-2*buf) + self.assertEqual(ts_nobuff.shape[ts_nobuff.taxis], numsamp - 2 * buf) + self.assertEqual(len(ts_nobuff['time']), numsamp - 2 * buf) + ts_nobuff = ts.remove_buffer((1, 1)) + self.assertEqual(ts_nobuff.shape[ts_nobuff.taxis], numsamp - 2 * buf) + self.assertEqual(len(ts_nobuff['time']), numsamp - 2 * buf) # make sure that negative durations throw exception - self.assertRaises(ValueError,ts.remove_buffer,-1) + self.assertRaises(ValueError, ts.remove_buffer, -1) def tst_setattr(self): - ts = TimeSeries(self.dat200,'time',200,dims=self.dims200) - self.assertRaises(ValueError,ts.__setattr__,'tdim','bla') - self.assertRaises(ValueError,ts.__setattr__,'samplerate',-1) + ts = TimeSeries(self.dat200, 'time', 200, dims=self.dims200) + self.assertRaises(ValueError, ts.__setattr__, 'tdim', 'bla') + self.assertRaises(ValueError, ts.__setattr__, 'samplerate', -1) def test_filter(self): samplerate = 200 - filtType='stop' - freqRange = [10,20] + filtType = 'stop' + freqRange = [10, 20] order = 4 - ts = TimeSeries(self.dat200,'time',samplerate,dims=self.dims200) + ts = TimeSeries(self.dat200, 'time', samplerate, dims=self.dims200) ts_filt = ts.filtered(freqRange, filtType, order) - test = filt.buttfilt(self.dat200,freqRange,samplerate,filtType, - order,axis=ts.taxis) - np.testing.assert_array_almost_equal(ts_filt[:],test[:],decimal=6) + test = filt.buttfilt(self.dat200, freqRange, samplerate, filtType, + order, axis=ts.taxis) + np.testing.assert_array_almost_equal(ts_filt[:], test[:], decimal=6) def test_resample(self): - ts200 = TimeSeries(self.dat200,'time',200,dims=self.dims200) + ts200 = TimeSeries(self.dat200, 'time', 200, dims=self.dims200) ts50 = TimeSeries( - self.dat50,'time',50,dims=self.dims50).remove_buffer(1.0) + self.dat50, 'time', 50, dims=self.dims50).remove_buffer(1.0) ts50_200 = ts200.resampled(50).remove_buffer(1.0) - np.testing.assert_equal(ts50_200.shape[:],ts50.shape[:]) - #print type(ts200['time']) - #print type(ts50['time']) + np.testing.assert_equal(ts50_200.shape[:], ts50.shape[:]) + # print type(ts200['time']) + # print type(ts50['time']) np.testing.assert_array_almost_equal( - ts50_200['time']*1000,ts50['time'],decimal=6) - np.testing.assert_array_almost_equal(ts50_200[:],ts50[:],decimal=6) + ts50_200['time'] * 1000, ts50['time'], decimal=6) + np.testing.assert_array_almost_equal(ts50_200[:], ts50[:], decimal=6) def test_remove_tdim(self): - ts200 = TimeSeries(self.dat200,'time',200,dims=self.dims200) - self.assertTrue(isinstance(ts200.mean('time'),DimArray)) - - + ts200 = TimeSeries(self.dat200, 'time', 200, dims=self.dims200) + self.assertTrue(isinstance(ts200.mean('time'), DimArray)) diff --git a/ptsa/data/tests/testdata.py b/ptsa/data/tests/testdata.py index ee937a6..17c6b4e 100644 --- a/ptsa/data/tests/testdata.py +++ b/ptsa/data/tests/testdata.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -13,40 +13,44 @@ import numpy as N + class TestData(): def __init__(self): # create 10 Hz sine waves at 200 and 50 Hz 4000ms long numSecs = 4. - numPoints = int(numSecs*200.) + numPoints = int(numSecs * 200.) Hz = 10 - d200_10 = N.sin(N.arange(numPoints,dtype=N.float)*2*N.pi*Hz*numSecs/numPoints) + d200_10 = N.sin(N.arange(numPoints, dtype=N.float) * + 2 * N.pi * Hz * numSecs / numPoints) Hz = 5 - d200_5 = N.sin(N.arange(numPoints,dtype=N.float)*2*N.pi*Hz*numSecs/numPoints) - self.dat200 = N.array([d200_10,d200_5]) + d200_5 = N.sin(N.arange(numPoints, dtype=N.float) * + 2 * N.pi * Hz * numSecs / numPoints) + self.dat200 = N.array([d200_10, d200_5]) # calc the time range in MS offset = -200 duration = numPoints - samplesize = 1000./200. - sampStart = offset*samplesize - sampEnd = sampStart + (duration-1)*samplesize - timeRange = N.linspace(sampStart,sampEnd,duration) - self.dims200 = [Dim('channel',N.arange(self.dat200.shape[0])), - Dim('time',timeRange,'ms')] - + samplesize = 1000. / 200. + sampStart = offset * samplesize + sampEnd = sampStart + (duration - 1) * samplesize + timeRange = N.linspace(sampStart, sampEnd, duration) + self.dims200 = [Dim('channel', N.arange(self.dat200.shape[0])), + Dim('time', timeRange, 'ms')] + numSecs = 4. - numPoints = int(numSecs*50.) + numPoints = int(numSecs * 50.) Hz = 10 - d50_10 = N.sin(N.arange(numPoints,dtype=N.float)*2*N.pi*Hz*numSecs/numPoints) + d50_10 = N.sin(N.arange(numPoints, dtype=N.float) * + 2 * N.pi * Hz * numSecs / numPoints) Hz = 5 - d50_5 = N.sin(N.arange(numPoints,dtype=N.float)*2*N.pi*Hz*numSecs/numPoints) - self.dat50 = N.array([d50_10,d50_5]) + d50_5 = N.sin(N.arange(numPoints, dtype=N.float) * + 2 * N.pi * Hz * numSecs / numPoints) + self.dat50 = N.array([d50_10, d50_5]) # calc the time range in MS offset = -50 duration = numPoints - samplesize = 1000./50. - sampStart = offset*samplesize - sampEnd = sampStart + (duration-1)*samplesize - timeRange = N.linspace(sampStart,sampEnd,duration) - self.dims50 = [Dim('channel',N.arange(self.dat50.shape[0])), - Dim('time',timeRange,'ms')] - + samplesize = 1000. / 50. + sampStart = offset * samplesize + sampEnd = sampStart + (duration - 1) * samplesize + timeRange = N.linspace(sampStart, sampEnd, duration) + self.dims50 = [Dim('channel', N.arange(self.dat50.shape[0])), + Dim('time', timeRange, 'ms')] diff --git a/ptsa/data/timeseries.py b/ptsa/data/timeseries.py index 795c0b5..08bd641 100644 --- a/ptsa/data/timeseries.py +++ b/ptsa/data/timeseries.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,7 +7,7 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -from dimarray import Dim,DimArray,AttrArray +from dimarray import Dim, DimArray, AttrArray from ptsa import filt from ptsa.helper import next_pow2, pad_to_next_pow2 @@ -32,7 +32,7 @@ class TimeSeries(DimArray): time dimension and associated sample rate). It also provides methods for manipulating the time dimension, such as resampling and filtering the data. - + Parameters ---------- data : {array_like} @@ -76,9 +76,9 @@ class TimeSeries(DimArray): 'time' """ - _required_attrs = {'dims':np.ndarray, - 'tdim':str, - 'samplerate':float} + _required_attrs = {'dims': np.ndarray, + 'tdim': str, + 'samplerate': float} taxis = property(lambda self: self.get_axis(self.tdim), doc="Numeric time axis (read only).") @@ -90,8 +90,8 @@ def __new__(cls, data, tdim, samplerate, *args, **kwargs): # ensure that tdim is a valid dimension name: if not(tdim in ts.dim_names): raise ValueError( - 'Provided time dimension name (tdim) is invalid!\n'+ - 'Provided value: '+ str(tdim)+'\nAvailable dimensions: '+ + 'Provided time dimension name (tdim) is invalid!\n' + + 'Provided value: ' + str(tdim) + '\nAvailable dimensions: ' + str(ts.dim_names)) ts.tdim = tdim # ensure that sample rate is a float: @@ -99,29 +99,29 @@ def __new__(cls, data, tdim, samplerate, *args, **kwargs): # ensure that sample rate is postive: if samplerate <= 0: raise ValueError( - 'Samplerate must be positive! Provided value: '+ + 'Samplerate must be positive! Provided value: ' + str(samplerate)) ts.samplerate = samplerate - + # convert to TimeSeries and return: return ts.view(cls) - + def __setattr__(self, name, value): # ensure that tdim is a valid dimension name: if name == 'tdim': if not(value in self.dim_names): raise ValueError( - 'Provided time dimension name (tdim) is invalid!\n'+ - 'Provided value: '+ str(value)+'\nAvailable dimensions: '+ + 'Provided time dimension name (tdim) is invalid!\n' + + 'Provided value: ' + str(value) + '\nAvailable dimensions: ' + str(self.dim_names)) # ensure that sample rate is a postive float: elif name == 'samplerate': value = float(value) if value <= 0: raise ValueError( - 'Samplerate must be positive! Provided value: '+ + 'Samplerate must be positive! Provided value: ' + str(value)) - DimArray.__setattr__(self,name,value) + DimArray.__setattr__(self, name, value) def _ret_func(self, ret, axis): """ @@ -137,53 +137,51 @@ def _ret_func(self, ret, axis): if self.get_axis(axis) == self.taxis: return_as_dimarray = True # pop the dim - ret.dims = ret.dims[np.arange(len(ret.dims))!=axis] + ret.dims = ret.dims[np.arange(len(ret.dims)) != axis] if return_as_dimarray: # The function removed the time dimension, so we return a # DimArray instead of a TimeSeries return ret.view(DimArray) else: return ret.view(self.__class__) - - + def remove_buffer(self, duration): - """ - Remove the desired buffer duration (in seconds) and reset the - time range. - - Parameter - --------- - duration : {int,float,({int,float},{int,float})} - The duration to be removed. The units depend on the samplerate: - E.g., if samplerate is specified in Hz (i.e., samples per second), - the duration needs to be specified in seconds and if samplerate is - specified in kHz (i.e., samples per millisecond), the duration needs - to be specified in milliseconds. - A single number causes the specified duration to be removed from the - beginning and end. A 2-tuple can be passed in to specify different - durations to be removed from the beginning and the end respectively. - - Returns - ------- - ts : {TimeSeries} - A TimeSeries instance with the requested durations removed from the - beginning and/or end. """ - # see if we need to remove anything + Remove the desired buffer duration (in seconds) and reset the + time range. + + Parameter + --------- + duration : {int,float,({int,float},{int,float})} + The duration to be removed. The units depend on the samplerate: + E.g., if samplerate is specified in Hz (i.e., samples per second), + the duration needs to be specified in seconds and if samplerate is + specified in kHz (i.e., samples per millisecond), the duration needs + to be specified in milliseconds. + A single number causes the specified duration to be removed from the + beginning and end. A 2-tuple can be passed in to specify different + durations to be removed from the beginning and the end respectively. + + Returns + ------- + ts : {TimeSeries} + A TimeSeries instance with the requested durations removed from the + beginning and/or end. + """ + # see if we need to remove anything duration = np.atleast_1d(duration) if len(duration) != 2: duration = duration.repeat(2) num_samp = np.round(self.samplerate * duration) # ensure that the number of samples are >= 0: - if np.any(num_samp<0): - raise ValueError('Duration must not be negative!'+ - 'Provided values: '+str(duration)) + if np.any(num_samp < 0): + raise ValueError('Duration must not be negative!' + + 'Provided values: ' + str(duration)) # remove the buffer from the data - return self.take(range(int(num_samp[0]), - self.shape[self.taxis]-int(num_samp[1])), - self.taxis) + return (self.take(list(range(int(num_samp[0]), + self.shape[self.taxis] - int(num_samp[1]))), self.taxis)) - def filtered(self,freq_range,filt_type='stop',order=4): + def filtered(self, freq_range, filt_type='stop', order=4): """ Filter the data using a Butterworth filter and return a new TimeSeries instance. @@ -204,12 +202,12 @@ def filtered(self,freq_range,filt_type='stop',order=4): """ filtered_array = filt.buttfilt(np.asarray(self), - freq_range,self.samplerate,filt_type, - order,axis=self.taxis) + freq_range, self.samplerate, filt_type, + order, axis=self.taxis) attrs = self._attrs.copy() - for k in self._required_attrs.keys(): - attrs.pop(k,None) - return TimeSeries(filtered_array,self.tdim, self.samplerate, + for k in list(self._required_attrs.keys()): + attrs.pop(k, None) + return TimeSeries(filtered_array, self.tdim, self.samplerate, dims=self.dims.copy(), **attrs) def resampled(self, resampled_rate, window=None, @@ -248,24 +246,26 @@ def resampled(self, resampled_rate, window=None, """ # resample the data, getting new time range time_range = self[self.tdim] - new_length = int(np.round(len(time_range)*resampled_rate/self.samplerate)) + new_length = int(np.round(len(time_range) * + resampled_rate / self.samplerate)) if pad_to_pow2: padded_length = 2**next_pow2(len(time_range)) - padded_new_length = int(np.round(padded_length*resampled_rate/self.samplerate)) - time_range = np.hstack([time_range, - (np.arange(1,padded_length-len(time_range)+1)*np.diff(time_range[-2:]))+time_range[-1]]) + padded_new_length = int( + np.round(padded_length * resampled_rate / self.samplerate)) + time_range = np.hstack([time_range, + (np.arange(1, padded_length - len(time_range) + 1) * np.diff(time_range[-2:])) + time_range[-1]]) if loop_axis is None: # just do standard method on all data at once if pad_to_pow2: - newdat,new_time_range = resample(pad_to_next_pow2(np.asarray(self),axis=self.taxis), - padded_new_length, t=time_range, - axis=self.taxis, window=window) + newdat, new_time_range = resample(pad_to_next_pow2(np.asarray(self), axis=self.taxis), + padded_new_length, t=time_range, + axis=self.taxis, window=window) else: - newdat,new_time_range = resample(np.asarray(self), - new_length, t=time_range, - axis=self.taxis, window=window) + newdat, new_time_range = resample(np.asarray(self), + new_length, t=time_range, + axis=self.taxis, window=window) else: # loop over specified axis @@ -273,14 +273,14 @@ def resampled(self, resampled_rate, window=None, loop_dim = self.get_dim_name(loop_axis) loop_dim_len = len(self[loop_dim]) # specify empty boolean index - ind = np.zeros(loop_dim_len,dtype=np.bool) + ind = np.zeros(loop_dim_len, dtype=np.bool) newdat = [] if has_mp and num_mp_procs != 0: po = mp.Pool(num_mp_procs) for i in range(loop_dim_len): ind[i] = True - dat = self.select(**{loop_dim:ind}) + dat = self.select(**{loop_dim: ind}) taxis = dat.taxis if has_mp and num_mp_procs != 0: # start async proc @@ -295,24 +295,24 @@ def resampled(self, resampled_rate, window=None, taxis, window))) else: # just call on that dataset - sys.stdout.write('%d '%i) + sys.stdout.write('%d ' % i) sys.stdout.flush() if pad_to_pow2: dat = pad_to_next_pow2(np.asarray(dat), axis=dat.taxis) - ndat,new_time_range = resample(np.asarray(dat), padded_new_length, t=time_range, - axis=taxis, window=window) + ndat, new_time_range = resample(np.asarray(dat), padded_new_length, t=time_range, + axis=taxis, window=window) else: - ndat,new_time_range = resample(np.asarray(dat), new_length, t=time_range, - axis=taxis, window=window) + ndat, new_time_range = resample(np.asarray(dat), new_length, t=time_range, + axis=taxis, window=window) newdat.append(ndat) ind[i] = False if has_mp and num_mp_procs != 0: # aggregate mp results po.close() - #po.join() + # po.join() out = [] for i in range(len(newdat)): - sys.stdout.write('%d '%i) + sys.stdout.write('%d ' % i) sys.stdout.flush() out.append(newdat[i].get()) #out = [newdat[i].get() for i in range(len(newdat))] @@ -320,28 +320,28 @@ def resampled(self, resampled_rate, window=None, new_time_range = out[i][1] # concatenate the new data - newdat = np.concatenate(newdat,axis=self.get_axis(loop_axis)) + newdat = np.concatenate(newdat, axis=self.get_axis(loop_axis)) sys.stdout.write('\n') sys.stdout.flush() # remove pad if we padded it if pad_to_pow2: - newdat = newdat.take(range(new_length),axis=self.taxis) + newdat = newdat.take(list(range(new_length)), axis=self.taxis) new_time_range = new_time_range[:new_length] # set the time dimension newdims = self.dims.copy() attrs = self.dims[self.taxis]._attrs.copy() - for k in self.dims[self.taxis]._required_attrs.keys(): - attrs.pop(k,None) + for k in list(self.dims[self.taxis]._required_attrs.keys()): + attrs.pop(k, None) newdims[self.taxis] = Dim(new_time_range, self.dims[self.taxis].name, **attrs) attrs = self._attrs.copy() - for k in self._required_attrs.keys(): - attrs.pop(k,None) + for k in list(self._required_attrs.keys()): + attrs.pop(k, None) return TimeSeries(newdat, self.tdim, resampled_rate, dims=newdims, **attrs) @@ -355,7 +355,7 @@ def baseline_corrected(self, base_range): Parameters ---------- base_range: {tuple} - Tuple specifying the start and end time range (inclusive) + Tuple specifying the start and end time range (inclusive) for the baseline. Returns @@ -365,7 +365,8 @@ def baseline_corrected(self, base_range): """ # get the average of baseline range - baseline = self['time >= %f'%base_range[0],'time <= %f'%base_range[1]].mean('time') + baseline = self['time >= %f' % base_range[0], + 'time <= %f' % base_range[1]].mean('time') # replicate over the time dimension baseline = baseline.add_dim(self['time']).transpose(self.dim_names) @@ -375,8 +376,7 @@ def baseline_corrected(self, base_range): # return a new timeseries attrs = self._attrs.copy() - for k in self._required_attrs.keys(): - attrs.pop(k,None) - return TimeSeries(new_dat,self.tdim, self.samplerate, + for k in list(self._required_attrs.keys()): + attrs.pop(k, None) + return TimeSeries(new_dat, self.tdim, self.samplerate, dims=self.dims.copy(), **attrs) - diff --git a/ptsa/emd.py b/ptsa/emd.py index 05d8757..85be3e6 100644 --- a/ptsa/emd.py +++ b/ptsa/emd.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -15,17 +15,18 @@ import scipy.interpolate import scipy.signal -def emd(data,max_modes=10): + +def emd(data, max_modes=10): """Calculate the Emprical Mode Decomposition of a signal.""" # initialize modes - modes=[] - + modes = [] + # perform sifts until we have all modes - residue=data + residue = data while not _done_sifting(residue): # perform a sift - imf,residue = _do_sift(residue) - + imf, residue = _do_sift(residue) + # append the imf modes.append(imf) @@ -33,13 +34,14 @@ def emd(data,max_modes=10): if len(modes) == max_modes: # we have all we wanted break - + # append the residue modes.append(residue) # return an array of modes return np.asarray(modes) + def eemd(data, noise_std=0.2, num_ensembles=100, num_sifts=10): """ Ensemble Empirical Mode Decomposition (EEMD) @@ -48,23 +50,23 @@ def eemd(data, noise_std=0.2, num_ensembles=100, num_sifts=10): """ # get modes to generate num_samples = len(data) - num_modes = int(np.fix(np.log2(num_samples)))-1 + num_modes = int(np.fix(np.log2(num_samples))) - 1 # normalize incomming data dstd = data.std() - y = data/dstd - + y = data / dstd + # allocate for starting value - all_modes = np.zeros((num_modes+2,num_samples)) - + all_modes = np.zeros((num_modes + 2, num_samples)) + # loop over num_ensembles for e in range(num_ensembles): # perturb starting data - x0 = y + np.random.randn(num_samples)*noise_std + x0 = y + np.random.randn(num_samples) * noise_std # save the starting value all_modes[0] += x0 - + # loop over modes for m in range(num_modes): # do the sifts @@ -73,20 +75,22 @@ def eemd(data, noise_std=0.2, num_ensembles=100, num_sifts=10): imf = _do_one_sift(imf) # save the imf - all_modes[m+1] += imf - + all_modes[m + 1] += imf + # set the residual x0 = x0 - imf # save the final residual all_modes[-1] += x0 - + # average everything out and renormalize - return all_modes*dstd/np.float64(num_ensembles) - + return all_modes * dstd / np.float64(num_ensembles) + + def _done_sifting(d): """We are done sifting is there a monotonic function.""" - return np.sum(_localmax(d))+np.sum(_localmax(-d))<=2 + return np.sum(_localmax(d)) + np.sum(_localmax(-d)) <= 2 + def _do_sift(data): """ @@ -98,14 +102,14 @@ def _do_sift(data): least five sifts.""" # save the data (may have to copy) - imf=data + imf = data # sift until num extrema and ZC differ by at most 1 while True: - imf=_do_one_sift(imf) - numExtrema,numZC = _analyze_imf(imf) - #print 'numextrema=%d, numZC=%d' % (numExtrema, numZC) - if abs(numExtrema-numZC)<=1: + imf = _do_one_sift(imf) + numExtrema, numZC = _analyze_imf(imf) + # print 'numextrema=%d, numZC=%d' % (numExtrema, numZC) + if abs(numExtrema - numZC) <= 1: break # then continue until numExtrema and ZCs are constant for at least @@ -115,19 +119,19 @@ def _do_sift(data): lastNumExtrema = numExtrema lastNumZC = numZC while numConstant < desiredNumConstant: - imf=_do_one_sift(imf) - numExtrema,numZC = _analyze_imf(imf) + imf = _do_one_sift(imf) + numExtrema, numZC = _analyze_imf(imf) if numExtrema == lastNumExtrema and \ numZC == lastNumZC: # is the same so increment - numConstant+=1 + numConstant += 1 else: # different, so reset numConstant = 0 # save the last extrema and ZC lastNumExtrema = numExtrema lastNumZC = numZC - + # FIX THIS # while True: # imf = _do_one_sift(imf) @@ -139,27 +143,27 @@ def _do_sift(data): # break # calc the residue - residue=data-imf + residue = data - imf # return the imf and residue - return imf,residue + return imf, residue def _do_one_sift(data): - upper=_get_upper_spline(data) - lower=-_get_upper_spline(-data) - #upper=jinterp(find(maxes),data(maxes),xs); - #lower=jinterp(find(mins),data(mins),xs); + upper = _get_upper_spline(data) + lower = -_get_upper_spline(-data) + # upper=jinterp(find(maxes),data(maxes),xs); + # lower=jinterp(find(mins),data(mins),xs); - #imf=mean([upper;lower],1) - imf = (upper+lower)*.5 + # imf=mean([upper;lower],1) + imf = (upper + lower) * .5 - detail=data-imf + detail = data - imf # plot(xs,data,'b-',xs,upper,'r--',xs,lower,'r--',xs,imf,'k-') - return detail # imf + return detail # imf def _get_upper_spline(data): @@ -171,64 +175,63 @@ def _get_upper_spline(data): if len(maxInds) == 1: # Special case: if there is just one max, then entire spline # is that number - #s=repmat(data(maxInds),size(data)); - s = np.ones(len(data))*data[maxInds] + # s=repmat(data(maxInds),size(data)); + s = np.ones(len(data)) * data[maxInds] return s # Start points - if maxInds[0]==0: + if maxInds[0] == 0: # first point is a local max - preTimes=1-maxInds[1] - preData=data[maxInds[1]] + preTimes = 1 - maxInds[1] + preData = data[maxInds[1]] else: # first point is NOT local max - preTimes=1-maxInds[[1,0]] - preData=data[maxInds[[1,0]]] + preTimes = 1 - maxInds[[1, 0]] + preData = data[maxInds[[1, 0]]] # end points - if maxInds[-1]==len(data)-1: + if maxInds[-1] == len(data) - 1: # last point is a local max - postTimes=2*len(data)-maxInds[-2]-1; - postData=data[maxInds[-2]]; + postTimes = 2 * len(data) - maxInds[-2] - 1 + postData = data[maxInds[-2]] else: # last point is NOT a local max - postTimes=2*len(data)-maxInds[[-1,-2]]; - postData=data[maxInds[[-1,-2]]] + postTimes = 2 * len(data) - maxInds[[-1, -2]] + postData = data[maxInds[[-1, -2]]] # perform the spline fit - t=np.r_[preTimes,maxInds,postTimes]; - d2=np.r_[preData, data[maxInds], postData]; - #s=interp1(t,d2,1:length(data),'spline'); + t = np.r_[preTimes, maxInds, postTimes] + d2 = np.r_[preData, data[maxInds], postData] + # s=interp1(t,d2,1:length(data),'spline'); # XXX verify the 's' argument # needed to change so that fMRI dat would work - rep = scipy.interpolate.splrep(t,d2,s=.0) - s = scipy.interpolate.splev(range(len(data)),rep) - # plot(1:length(data),data,'b-',1:length(data),s,'k-',t,d2,'r--'); + rep = scipy.interpolate.splrep(t, d2, s=.0) + s = scipy.interpolate.splev(list(range(len(data))), rep) + # plot(1:length(data),data,'b-',1:length(data),s,'k-',t,d2,'r--'); return s def _analyze_imf(d): - numExtrema = np.sum(_localmax(d))+np.sum(_localmax(-d)) - numZC = np.sum(np.diff(np.sign(d))!=0) - return numExtrema,numZC + numExtrema = np.sum(_localmax(d)) + np.sum(_localmax(-d)) + numZC = np.sum(np.diff(np.sign(d)) != 0) + return numExtrema, numZC # % if debug # % clf # % a1=subplot(2,1,1); # % plot(xs,d,'b-',xs,upper,'k-',xs,lower,'k-'); # % axis tight; - + # % a2=subplot(2,1,2); # % plot(xs,stopScore,'b-',[0 length(d)],[thresh1 thresh1],'k--',[0 length(d)],[thresh2 ... # % thresh2],'r--'); # % axis tight; -# % xlabel(sprintf('score = %.3g',s)); +# % xlabel(sprintf('score = %.3g',s)); # % linkaxes([a1 a2],'x') # % keyboard - -# % end +# % end # function yi=jinterp(x,y,xi); @@ -238,39 +241,40 @@ def _analyze_imf(d): # yi=interp1(x,y,xi,'spline'); # end - def _localmax(d): """Calculate the local maxima of a vector.""" # this gets a value of -2 if it is an unambiguous local max # value -1 denotes that the run its a part of may contain a local max - diffvec = np.r_[-np.inf,d,-np.inf] - diffScore=np.diff(np.sign(np.diff(diffvec))) - + diffvec = np.r_[-np.inf, d, -np.inf] + diffScore = np.diff(np.sign(np.diff(diffvec))) + # Run length code with help from: # http://home.online.no/~pjacklam/matlab/doc/mtt/index.html # (this is all painfully complicated, but I did it in order to avoid loops...) # here calculate the position and length of each run - runEndingPositions=np.r_[np.nonzero(d[0:-1]!=d[1:])[0],len(d)-1] + runEndingPositions = np.r_[np.nonzero(d[0:-1] != d[1:])[0], len(d) - 1] runLengths = np.diff(np.r_[-1, runEndingPositions]) - runStarts=runEndingPositions-runLengths + 1 + runStarts = runEndingPositions - runLengths + 1 # Now concentrate on only the runs with length>1 - realRunStarts = runStarts[runLengths>1] - realRunStops = runEndingPositions[runLengths>1] - realRunLengths = runLengths[runLengths>1] + realRunStarts = runStarts[runLengths > 1] + realRunStops = runEndingPositions[runLengths > 1] + realRunLengths = runLengths[runLengths > 1] # save only the runs that are local maxima - maxRuns=(diffScore[realRunStarts]==-1) & (diffScore[realRunStops]==-1) + maxRuns = (diffScore[realRunStarts] == - + 1) & (diffScore[realRunStops] == -1) # If a run is a local max, then count the middle position (rounded) as the 'max' # CHECK THIS - maxRunMiddles=np.round(realRunStarts[maxRuns]+realRunLengths[maxRuns]/2.)-1 + maxRunMiddles = np.round( + realRunStarts[maxRuns] + realRunLengths[maxRuns] / 2.) - 1 # get all the maxima - maxima=(diffScore==-2) + maxima = (diffScore == -2) maxima[maxRunMiddles.astype(np.int32)] = True return maxima @@ -279,33 +283,31 @@ def _localmax(d): #%maxima([1 end])=false; -def calc_inst_info(modes,samplerate): +def calc_inst_info(modes, samplerate): """ Calculate the instantaneous frequency, amplitude, and phase of each mode. """ - amp=np.zeros(modes.shape,np.float32); - phase=np.zeros(modes.shape,np.float32); - f=np.zeros(modes.shape,np.float32); + amp = np.zeros(modes.shape, np.float32) + phase = np.zeros(modes.shape, np.float32) + f = np.zeros(modes.shape, np.float32) for m in range(len(modes)): - h=scipy.signal.hilbert(modes[m]); - amp[m,:]=np.abs(h); - phase[m,:]=np.angle(h); - f[m,:] = np.r_[np.nan, - 0.5*(np.angle(-h[2:]*np.conj(h[0:-2]))+np.pi)/(2*np.pi) * samplerate, - np.nan] - - #f(m,:) = [nan 0.5*(angle(-h(t+1).*conj(h(t-1)))+pi)/(2*pi) * sr nan]; - + h = scipy.signal.hilbert(modes[m]) + amp[m, :] = np.abs(h); + phase[m, :] = np.angle(h); + f[m, :] = np.r_[np.nan, + 0.5 * (np.angle(-h[2:] * np.conj(h[0:-2]) + ) + np.pi) / (2 * np.pi) * samplerate, + np.nan] + + # f(m,:) = [nan 0.5*(angle(-h(t+1).*conj(h(t-1)))+pi)/(2*pi) * sr nan]; + # calc the freqs (old way) - #f=np.diff(np.unwrap(phase[:,np.r_[0,0:len(modes[0])]]))/(2*np.pi)*samplerate + # f=np.diff(np.unwrap(phase[:,np.r_[0,0:len(modes[0])]]))/(2*np.pi)*samplerate # clip the freqs so they don't go below zero #f = f.clip(0,f.max()) - return f,amp,phase - - - + return f, amp, phase diff --git a/ptsa/filt.py b/ptsa/filt.py index b172371..c3f46d7 100644 --- a/ptsa/filt.py +++ b/ptsa/filt.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -8,22 +8,23 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from scipy.signal import butter, cheby1, firwin, lfilter -from numpy import asarray, vstack, hstack, eye, ones, zeros, linalg, newaxis, r_, flipud, convolve, matrix, array,concatenate +from numpy import asarray, vstack, hstack, eye, ones, zeros, linalg, newaxis, r_, flipud, convolve, matrix, array, concatenate import numpy as np from scipy.special import sinc -from helper import reshape_to_2d, reshape_from_2d, repeat_to_match_dims +from .helper import reshape_to_2d, reshape_from_2d, repeat_to_match_dims -from filtfilt import filtfilt as filtfilt_future +from .filtfilt import filtfilt as filtfilt_future import pdb -def buttfilt(dat,freq_range,sample_rate,filt_type,order,axis=-1): + +def buttfilt(dat, freq_range, sample_rate, filt_type, order, axis=-1): """Wrapper for a Butterworth filter. """ - + # make sure dat is an array dat = asarray(dat) @@ -35,16 +36,16 @@ def buttfilt(dat,freq_range,sample_rate,filt_type,order,axis=-1): freq_range = asarray(freq_range) # Nyquist frequency - nyq=sample_rate/2.; + nyq = sample_rate / 2. # generate the butterworth filter coefficients - [b,a]=butter(order,freq_range/nyq,filt_type) + [b, a] = butter(order, freq_range / nyq, filt_type) # loop over final dimension - #for i in xrange(dat.shape[0]): + # for i in xrange(dat.shape[0]): # dat[i] = filtfilt(b,a,dat[i]) #dat = filtfilt2(b,a,dat,axis=axis) - dat = filtfilt_future(b,a,dat,axis=axis) + dat = filtfilt_future(b, a, dat, axis=axis) # reshape the data back #dat = reshape_from_2d(dat,axis,origshape) @@ -55,11 +56,12 @@ def buttfilt(dat,freq_range,sample_rate,filt_type,order,axis=-1): ###### # from scipy.signal import cheby1, firwin, lfilter -# this import is now at the top of the file +# this import is now at the top of the file + def decimate(x, q, n=None, ftype='iir', axis=-1): """Downsample the signal x by an integer factor q, using an order n filter - + By default, an order 8 Chebyshev type I filter is used or a 30 point FIR filter with hamming window if ftype is 'fir'. @@ -72,14 +74,14 @@ def decimate(x, q, n=None, ftype='iir', axis=-1): 'fir' filter) ftype -- type of the filter; can be 'iir' or 'fir' axis -- the axis along which the filter should be applied - + Outputs: y -- the downsampled signal """ if type(q) != type(1): - raise TypeError, "q should be an integer" + raise TypeError("q should be an integer") if n is None: if ftype == 'fir': @@ -88,27 +90,27 @@ def decimate(x, q, n=None, ftype='iir', axis=-1): n = 8 if ftype == 'fir': # PBS - This method must be verified - b = firwin(n+1, 1./q, window='hamming') + b = firwin(n + 1, 1. / q, window='hamming') y = lfilter(b, 1., x, axis=axis) else: - (b, a) = cheby1(n, 0.05, 0.8/q) + (b, a) = cheby1(n, 0.05, 0.8 / q) # reshape the data to 2D with time on the 2nd dimension origshape = x.shape - y = reshape_to_2d(x,axis) + y = reshape_to_2d(x, axis) # loop over final dimension - for i in xrange(y.shape[0]): - y[i] = filtfilt(b,a,y[i]) + for i in range(y.shape[0]): + y[i] = filtfilt(b, a, y[i]) #y = filtfilt2(b,a,y) # reshape the data back - y = reshape_from_2d(y,axis,origshape) + y = reshape_from_2d(y, axis, origshape) # This needs to be filtfilt eventually #y = lfilter(b, a, x, axis=axis) - return y.swapaxes(0,axis)[::q].swapaxes(0,axis) + return y.swapaxes(0, axis)[::q].swapaxes(0, axis) ############ @@ -120,109 +122,112 @@ def decimate(x, q, n=None, ftype='iir', axis=-1): # from scipy.signal import lfilter # imports now at top of file -def lfilter_zi(b,a): - #compute the zi state from the filter parameters. see [Gust96]. +def lfilter_zi(b, a): + # compute the zi state from the filter parameters. see [Gust96]. - #Based on: - # [Gust96] Fredrik Gustafsson, Determining the initial states in forward-backward - # filtering, IEEE Transactions on Signal Processing, pp. 988--992, April 1996, + # Based on: + # [Gust96] Fredrik Gustafsson, Determining the initial states in forward-backward + # filtering, IEEE Transactions on Signal Processing, pp. 988--992, April 1996, # Volume 44, Issue 4 - n=max(len(a),len(b)) + n = max(len(a), len(b)) - zin = ( eye(n-1) - hstack( (-a[1:n,newaxis], - vstack((eye(n-2), zeros(n-2)))))) + zin = (eye(n - 1) - hstack((-a[1:n, newaxis], + vstack((eye(n - 2), zeros(n - 2)))))) - zid= b[1:n] - a[1:n]*b[0] + zid = b[1:n] - a[1:n] * b[0] - zi_matrix=linalg.inv(zin)*(matrix(zid).transpose()) - zi_return=[] + zi_matrix = linalg.inv(zin) * (matrix(zid).transpose()) + zi_return = [] - #convert the result into a regular array (not a matrix) + # convert the result into a regular array (not a matrix) for i in range(len(zi_matrix)): zi_return.append(float(zi_matrix[i][0])) return array(zi_return) -def filtfilt(b,a,x): - #For now only accepting 1d arrays - ntaps=max(len(a),len(b)) - edge=ntaps*3 + +def filtfilt(b, a, x): + # For now only accepting 1d arrays + ntaps = max(len(a), len(b)) + edge = ntaps * 3 if x.ndim != 1: - raise ValueError, "Filtflit is only accepting 1 dimension arrays." + raise ValueError("Filtflit is only accepting 1 dimension arrays.") - #x must be bigger than edge + # x must be bigger than edge if x.size < edge: - raise ValueError, "Input vector needs to be bigger than 3 * max(len(a),len(b)." + raise ValueError( + "Input vector needs to be bigger than 3 * max(len(a),len(b).") + if len(a) != len(b): + b = r_[b, zeros(len(a) - len(b))] - if len(a)!=len(b): - b=r_[b,zeros(len(a)-len(b))] + zi = lfilter_zi(b, a) - - zi=lfilter_zi(b,a) - - #Grow the signal to have edges for stabilizing - #the filter with inverted replicas of the signal - s=r_[2*x[0]-x[edge:1:-1],x,2*x[-1]-x[-1:-edge:-1]] - #in the case of one go we only need one of the extrems + # Grow the signal to have edges for stabilizing + # the filter with inverted replicas of the signal + s = r_[2 * x[0] - x[edge:1:-1], x, 2 * x[-1] - x[-1:-edge:-1]] + # in the case of one go we only need one of the extrems # both are needed for filtfilt - (y,zf)=lfilter(b,a,s,-1,zi*s[0]) - - (y,zf)=lfilter(b,a,flipud(y),-1,zi*y[-1]) + (y, zf) = lfilter(b, a, s, -1, zi * s[0]) - return flipud(y[edge-1:-edge+1]) + (y, zf) = lfilter(b, a, flipud(y), -1, zi * y[-1]) + return flipud(y[edge - 1:-edge + 1]) -def filtfilt2(b,a,x,axis=-1): +def filtfilt2(b, a, x, axis=-1): # trying to accept N-dimensional arrays # calculate the edge needed - ntaps=max(len(a),len(b)) - edge=ntaps*3 + ntaps = max(len(a), len(b)) + edge = ntaps * 3 - #x must be bigger than edge + # x must be bigger than edge if x.shape[axis] < edge: - raise ValueError, "Input vector needs to be bigger than 3 * max(len(a),len(b)." + raise ValueError( + "Input vector needs to be bigger than 3 * max(len(a),len(b).") # fill out b if necessary - if len(a)!=len(b): - b=r_[b,zeros(len(a)-len(b))] + if len(a) != len(b): + b = r_[b, zeros(len(a) - len(b))] # calculate the initial conditions scaling factor - zi=lfilter_zi(b,a) + zi = lfilter_zi(b, a) + + # Grow the signal to have edges for stabilizing + # the filter with inverted replicas of the signal + # s=r_[2*x[0]-x[edge:1:-1],x,2*x[-1]-x[-1:-edge:-1]] - #Grow the signal to have edges for stabilizing - #the filter with inverted replicas of the signal - #s=r_[2*x[0]-x[edge:1:-1],x,2*x[-1]-x[-1:-edge:-1]] - - bRange = range(edge,1,-1) - sBeg = 2*x.take([0],axis).repeat(len(bRange),axis) - x.take(bRange,axis) - eRange = range(-1,-edge,-1) - sEnd = 2*x.take([-1],axis).repeat(len(eRange),axis) - x.take(eRange,axis) + bRange = list(range(edge, 1, -1)) + sBeg = 2 * x.take([0], axis).repeat(len(bRange), + axis) - x.take(bRange, axis) + eRange = list(range(-1, -edge, -1)) + sEnd = 2 * x.take([-1], axis).repeat(len(eRange), + axis) - x.take(eRange, axis) - s = concatenate((sBeg,x,sEnd),axis) + s = concatenate((sBeg, x, sEnd), axis) - #in the case of one go we only need one of the extremes + # in the case of one go we only need one of the extremes # both are needed for filtfilt # peform filter in forward direction - sBeg = s.take([0],axis).repeat(len(zi),axis) - ziBeg = repeat_to_match_dims(zi,sBeg,axis) * sBeg - (y,zf)=lfilter(b,a,s,axis,ziBeg) + sBeg = s.take([0], axis).repeat(len(zi), axis) + ziBeg = repeat_to_match_dims(zi, sBeg, axis) * sBeg + (y, zf) = lfilter(b, a, s, axis, ziBeg) # perform filter in reverse direction - sEnd = y.take([-1],axis).repeat(len(zi),axis) - ziEnd = repeat_to_match_dims(zi,sEnd,axis) * sEnd - (y,zf)=lfilter(b,a,y.take(range(y.shape[axis]-1,-1,-1),axis),axis,ziEnd) + sEnd = y.take([-1], axis).repeat(len(zi), axis) + ziEnd = repeat_to_match_dims(zi, sEnd, axis) * sEnd + (y, zf) = lfilter(b, a, y.take( + list(range(y.shape[axis] - 1, -1, -1)), axis), axis, ziEnd) # flip it back - y = y.take(range(y.shape[axis]-1,-1,-1),axis) - return y.take(range(edge-1,y.shape[axis]-edge+1),axis) - + y = y.take(list(range(y.shape[axis] - 1, -1, -1)), axis) + return y.take(list(range(edge - 1, y.shape[axis] - edge + 1)), axis) + # if __name__=='__main__': @@ -242,7 +247,6 @@ def filtfilt2(b,a,x,axis=-1): # y=filtfilt(b,a,xn) - # plot(x,'c') # hold(True) # plot(xn,'k') @@ -266,26 +270,27 @@ def firls(N, f, D=None): if D is None: D = [1, 0] assert len(D) == len(f), "must have one desired response per band" - assert N%2 == 1, 'filter length must be odd' - L = (N-1)//2 + assert N % 2 == 1, 'filter length must be odd' + L = (N - 1) // 2 - k = np.arange(L+1) - k.shape = (1, L+1) + k = np.arange(L + 1) + k.shape = (1, L + 1) j = k.T R = 0 r = 0 for i, (f0, f1) in enumerate(f): - R += np.pi*f1*sinc(2*(j-k)*f1) - np.pi*f0*sinc(2*(j-k)*f0) + \ - np.pi*f1*sinc(2*(j+k)*f1) - np.pi*f0*sinc(2*(j+k)*f0) + R += np.pi * f1 * sinc(2 * (j - k) * f1) - np.pi * f0 * sinc(2 * (j - k) * f0) + \ + np.pi * f1 * sinc(2 * (j + k) * f1) - np.pi * \ + f0 * sinc(2 * (j + k) * f0) - r += D[i]*(2*np.pi*f1*sinc(2*j*f1) - 2*np.pi*f0*sinc(2*j*f0)) + r += D[i] * (2 * np.pi * f1 * sinc(2 * j * f1) - + 2 * np.pi * f0 * sinc(2 * j * f0)) a = np.dot(np.linalg.inv(R), r) a.shape = (-1,) h = np.zeros(N) - h[:L] = a[:0:-1]/2. + h[:L] = a[:0:-1] / 2. h[L] = a[0] - h[L+1:] = a[1:]/2. + h[L + 1:] = a[1:] / 2. return h - diff --git a/ptsa/filtfilt.py b/ptsa/filtfilt.py index 46ef172..3470a28 100644 --- a/ptsa/filtfilt.py +++ b/ptsa/filtfilt.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -12,7 +12,7 @@ from scipy import linalg from scipy.signal import lfilter import numpy as np -from _arraytools import axis_slice, axis_reverse, odd_ext, even_ext, const_ext +from ._arraytools import axis_slice, axis_reverse, odd_ext, even_ext, const_ext def lfilter_zi(b, a): @@ -222,8 +222,8 @@ def filtfilt(b, a, x, axis=-1, padtype='odd', padlen=None): if padtype not in ['even', 'odd', 'constant', None]: raise ValueError(("Unknown value '%s' given to padtype. padtype must " - "be 'even', 'odd', 'constant', or None.") % - padtype) + "be 'even', 'odd', 'constant', or None.") % + padtype) b = np.asarray(b) a = np.asarray(a) diff --git a/ptsa/fixed_scipy.py b/ptsa/fixed_scipy.py index 88c733d..a287575 100644 --- a/ptsa/fixed_scipy.py +++ b/ptsa/fixed_scipy.py @@ -7,13 +7,14 @@ ################################################################################ ################################################################################ ### -### scipy.signal.wavelets.morlet +# scipy.signal.wavelets.morlet ### ################################################################################ ################################################################################ from scipy import linspace, pi, exp, zeros + def morlet(M, w=5.0, s=1.0, complete=True): """Complex Morlet wavelet. @@ -51,18 +52,18 @@ def morlet(M, w=5.0, s=1.0, complete=True): by f = 2*s*w*r / M where r is the sampling rate. """ - x = linspace(-s*2*pi,s*2*pi,M) - output = exp(1j*w*x) - + x = linspace(-s * 2 * pi, s * 2 * pi, M) + output = exp(1j * w * x) + if complete: - output -= exp(-0.5*(w**2)) - - output *= exp(-0.5*(x**2)) * pi**(-0.25) - + output -= exp(-0.5 * (w**2)) + + output *= exp(-0.5 * (x**2)) * pi**(-0.25) + return output ### -### scipy.signal.wavelets.morlet() +# scipy.signal.wavelets.morlet() ### ################################################################################ ################################################################################ diff --git a/ptsa/helper.py b/ptsa/helper.py index 0e03f38..1f532d8 100644 --- a/ptsa/helper.py +++ b/ptsa/helper.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,120 +11,127 @@ import numpy as np import os.path -def reshape_to_2d(data,axis): + +def reshape_to_2d(data, axis): """Reshape data to 2D with specified axis as the 2nd dimension.""" # get the shape, rank, and the length of the chosen axis dshape = data.shape rnk = len(dshape) n = dshape[axis] # convert negative axis to positive axis - if axis < 0: + if axis < 0: axis = axis + rnk # determine the new order of the axes - newdims = np.r_[0:axis,axis+1:rnk,axis] + newdims = np.r_[0:axis, axis + 1:rnk, axis] # reshape and transpose the data - newdata = np.reshape(np.transpose(data,tuple(newdims)), - (np.prod(dshape,axis=0)/n,n)) - + newdata = np.reshape(np.transpose(data, tuple(newdims)), + (np.prod(dshape, axis=0) / n, n)) + # make sure we have a copy #newdata = newdata.copy() return newdata -def reshape_from_2d(data,axis,dshape): + +def reshape_from_2d(data, axis, dshape): """Reshape data from 2D back to specified dshape.""" # set the rank of the array rnk = len(dshape) # fix negative axis to be positive - if axis < 0: + if axis < 0: axis = axis + rnk # determine the dims from reshape_to_2d call - newdims = np.r_[0:axis,axis+1:rnk,axis] + newdims = np.r_[0:axis, axis + 1:rnk, axis] # determine the transposed shape and reshape it back - tdshape = np.take(dshape,newdims,0) - ret = np.reshape(data,tuple(tdshape)) + tdshape = np.take(dshape, newdims, 0) + ret = np.reshape(data, tuple(tdshape)) # figure out how to retranspose the matrix - vals = range(rnk) - olddims = vals[:axis] + [rnk-1] +vals[axis:rnk-1] - ret = np.transpose(ret,tuple(olddims)) - + vals = list(range(rnk)) + olddims = vals[:axis] + [rnk - 1] + vals[axis:rnk - 1] + ret = np.transpose(ret, tuple(olddims)) + # make sure we have a copy #ret = ret.copy() return ret -def repeat_to_match_dims(x,y,axis=-1): - + +def repeat_to_match_dims(x, y, axis=-1): + rnk = len(y.shape) - + # convert negative axis to positive axis - if axis < 0: + if axis < 0: axis = axis + rnk - for d in range(axis)+range(axis+1,rnk): + for d in list(range(axis)) + list(range(axis + 1, rnk)): # add the dimension - x = np.expand_dims(x,d) + x = np.expand_dims(x, d) # repeat to fill that dim - x = x.repeat(y.shape[d],d) + x = x.repeat(y.shape[d], d) return x def deg2rad(degrees): """Convert degrees to radians.""" - return degrees/180.*np.math.pi + return degrees / 180. * np.math.pi + def rad2deg(radians): """Convert radians to degrees.""" - return radians/np.math.pi*180. + return radians / np.math.pi * 180. -def pol2cart(theta,radius,z=None,radians=True): + +def pol2cart(theta, radius, z=None, radians=True): """Converts corresponding angles (theta), radii, and (optional) height (z) from polar (or, when height is given, cylindrical) coordinates to Cartesian coordinates x, y, and z. Theta is assumed to be in radians, but will be converted from degrees if radians==False.""" if radians: - x = radius*np.cos(theta) - y = radius*np.sin(theta) + x = radius * np.cos(theta) + y = radius * np.sin(theta) else: - x = radius*np.cos(deg2rad(theta)) - y = radius*np.sin(deg2rad(theta)) + x = radius * np.cos(deg2rad(theta)) + y = radius * np.sin(deg2rad(theta)) if z is not None: # make sure we have a copy - z=z.copy() - return x,y,z + z = z.copy() + return x, y, z else: - return x,y + return x, y + -def cart2pol(x,y,z=None,radians=True): +def cart2pol(x, y, z=None, radians=True): """Converts corresponding Cartesian coordinates x, y, and (optional) z to polar (or, when z is given, cylindrical) coordinates angle (theta), radius, and z. By default theta is returned in radians, but will be converted - to degrees if radians==False.""" + to degrees if radians==False.""" if radians: - theta = np.arctan2(y,x) + theta = np.arctan2(y, x) else: - theta = rad2deg(np.arctan2(y,x)) - radius = np.hypot(x,y) + theta = rad2deg(np.arctan2(y, x)) + radius = np.hypot(x, y) if z is not None: # make sure we have a copy - z=z.copy() - return theta,radius,z + z = z.copy() + return theta, radius, z else: - return theta,radius + return theta, radius -def lock_file(filename,lockdirpath=None,lockdirname=None): + +def lock_file(filename, lockdirpath=None, lockdirname=None): if lockdirname is None: - lockdirname=filename+'.lock' + lockdirname = filename + '.lock' if not(lockdirpath is None): - lockdirname = lockdirpath+lockdirname + lockdirname = lockdirpath + lockdirname if os.path.exists(lockdirname): return False else: @@ -134,27 +141,30 @@ def lock_file(filename,lockdirpath=None,lockdirname=None): return False return True -def release_file(filename,lockdirpath=None,lockdirname=None): + +def release_file(filename, lockdirpath=None, lockdirname=None): if lockdirname is None: - lockdirname=filename+'.lock' + lockdirname = filename + '.lock' if not(lockdirpath is None): - lockdirname = lockdirpath+lockdirname + lockdirname = lockdirpath + lockdirname try: os.rmdir(lockdirname) except: return False return True - + + def next_pow2(n): """ Returns p such that 2 ** p >= n """ p = int(np.floor(np.log2(n))) - if 2 ** p == n: + if 2 ** p == n: return p else: return p + 1 + def pad_to_next_pow2(x, axis=0): """ Pad an array with zeros to the next power of two along the @@ -170,7 +180,7 @@ def pad_to_next_pow2(x, axis=0): shape = list(x.shape) shape[axis] = to_pad padding = np.zeros(shape, dtype=x.dtype) - return np.concatenate([x,padding], axis=axis) + return np.concatenate([x, padding], axis=axis) else: # nothing needs to be done return x @@ -190,11 +200,11 @@ def centered(arr, newsize): Returns ------- A center slice into the input array - + Note ---- Adapted from scipy.signal.signaltools._centered - + """ # Don't make a copy of newsize when creating array: newsize = np.asarray(newsize) @@ -208,6 +218,8 @@ def centered(arr, newsize): import inspect + + def getargspec(obj): """Get the names and default values of a callable's arguments @@ -231,7 +243,7 @@ def getargspec(obj): See http://kbyanc.blogspot.com/2007/07/python-more-generic-getargspec.html """ if not callable(obj): - raise TypeError, "%s is not callable" % type(obj) + raise TypeError("%s is not callable" % type(obj)) try: if inspect.isfunction(obj): return inspect.getargspec(obj) @@ -243,13 +255,13 @@ def getargspec(obj): # inspect.getargspec() returns for methods. # NB: We use im_func so we work with # instancemethod objects also. - spec = list(inspect.getargspec(obj.im_func)) + spec = list(inspect.getargspec(obj.__func__)) spec[0] = spec[0][1:] return spec elif inspect.isclass(obj): return getargspec(obj.__init__) elif isinstance(obj, object) and \ - not isinstance(obj, type(arglist.__get__)): + not isinstance(obj, type(arglist.__get__)): # We already know the instance is callable, # so it must have a __call__ method defined. # Return the arguments it expects. @@ -263,6 +275,5 @@ def getargspec(obj): # care what aspect(s) of that object we actually # examined). pass - raise NotImplementedError, \ - "do not know how to get argument list for %s" % \ - type(obj) + raise NotImplementedError("do not know how to get argument list for %s" % + type(obj)) diff --git a/ptsa/hilbert.py b/ptsa/hilbert.py index 4057fa7..245827f 100644 --- a/ptsa/hilbert.py +++ b/ptsa/hilbert.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,15 +11,17 @@ import numpy as np from scipy.signal import hilbert -from ptsa.data.timeseries import TimeSeries,Dim +from ptsa.data.timeseries import TimeSeries, Dim from ptsa.helper import next_pow2 -freq_bands = [('delta', [2.0,4.0]), - ('theta', [4.0,8.0]), - ('alpha', [9.0,14.0]), - ('beta', [16.0,26.0]), - ('gamma_1', [28.0,42.0]), - ('gamma_2', [44.0,100.0])] +freq_bands = [('delta', [2.0, 4.0]), + ('theta', [4.0, 8.0]), + ('alpha', [9.0, 14.0]), + ('beta', [16.0, 26.0]), + ('gamma_1', [28.0, 42.0]), + ('gamma_2', [44.0, 100.0])] + + def hilbert_pow(dat_ts, bands=None, pad_to_pow2=False, verbose=True): """ """ @@ -42,14 +44,14 @@ def hilbert_pow(dat_ts, bands=None, pad_to_pow2=False, verbose=True): pow = None for band in bands: if verbose: - sys.stdout.write('%s '%band[0]) + sys.stdout.write('%s ' % band[0]) sys.stdout.flush() - p = TimeSeries(np.abs(hilbert(dat_ts.filtered(band[1], + p = TimeSeries(np.abs(hilbert(dat_ts.filtered(band[1], filt_type='pass'), N=npts, axis=taxis).take(np.arange(npts_orig), - axis=taxis)), - tdim=dat_ts.tdim, samplerate=dat_ts.samplerate, - dims=dat_ts.dims.copy()).add_dim(Dim([band[0]],'freqs')) + axis=taxis)), + tdim=dat_ts.tdim, samplerate=dat_ts.samplerate, + dims=dat_ts.dims.copy()).add_dim(Dim([band[0]], 'freqs')) if pow is None: pow = p else: diff --git a/ptsa/iwasobi.py b/ptsa/iwasobi.py index d19ed13..fb06664 100644 --- a/ptsa/iwasobi.py +++ b/ptsa/iwasobi.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,11 +9,12 @@ # global imports import numpy as np -from scipy.linalg import toeplitz,hankel +from scipy.linalg import toeplitz, hankel #import pdb #from IPython.Debugger import Tracer; debug_here = Tracer() + class IWASOBI(): """ Implements algorithm WASOBI for blind source separation of @@ -22,68 +23,68 @@ class IWASOBI(): Ported from MATLAB code by Jakub Petkov / Petr Tichavsky """ + def __init__(self, ar_max=10, rmax=0.99, eps0=5.0e-7): """ """ self.ar_max = ar_max self.rmax = rmax self.eps0 = eps0 - + def __call__(self, data): x = data num_iterations = 3 # get the shape - d,N = x.shape + d, N = x.shape # get mean and remove it - Xmean=x.astype(np.float64).mean(1) + Xmean = x.astype(np.float64).mean(1) # x=x-Xmean*ones(1,N); %%%%%%%%% removing the sample mean - x = np.subtract(x.astype(np.float64).T,Xmean).T - + x = np.subtract(x.astype(np.float64).T, Xmean).T + # T=length(x(1,:))-AR_order; - T = N-self.ar_max - + T = N - self.ar_max + # C0=corr_est(x,T,AR_order); - C0 = self.corr_est(x.astype(np.float64),T,self.ar_max) - + C0 = self.corr_est(x.astype(np.float64), T, self.ar_max) + # for k=2:AR_order+1 # ik=d*(k-1); # C0(:,ik+1:ik+d)=0.5*(C0(:,ik+1:ik+d)+C0(:,ik+1:ik+d)'); # end %%%%%%%%% symmetrization - for k in xrange(1,self.ar_max+1): - ik = d*(k) - C0[:,ik:ik+d] = 0.5*(C0[:,ik:ik+d]+C0[:,ik:ik+d].T) + for k in range(1, self.ar_max + 1): + ik = d * (k) + C0[:, ik:ik + d] = 0.5 * (C0[:, ik:ik + d] + C0[:, ik:ik + d].T) # [Winit Ms] = uwajd(C0,20); %%% compute initial separation # %%% using uniform weights - Winit,Ms = self.uwajd(C0,20) - + Winit, Ms = self.uwajd(C0, 20) + # %conver # %t1 = cputime-time_start; # W=Winit; W = Winit.copy() - + # for in = 1:num_iterations # [H ARC]=weights(Ms,rmax,eps0); # [W Ms]=wajd(C0,H,W,5); # end - for i in xrange(num_iterations): - H,ARC = self.weights(Ms,self.rmax,self.eps0) - W,Ms = self.wajd(C0,H,W,5) + for i in range(num_iterations): + H, ARC = self.weights(Ms, self.rmax, self.eps0) + W, Ms = self.wajd(C0, H, W, 5) - # ISR=CRLB4(ARC)/N; - ISR = self.CRLB4(ARC)/np.float(N) - + ISR = self.CRLB4(ARC) / np.float(N) + # %t1 = [t1 cputime-time_start]; # signals=W*x+(W*Xmean)*ones(1,N); - signals = np.add(np.dot(W,x).T,np.dot(W,Xmean)).T + signals = np.add(np.dot(W, x).T, np.dot(W, Xmean)).T - return (W,Winit,ISR,signals) + return (W, Winit, ISR, signals) - def THinv5(self,phi,K,M,eps): + def THinv5(self, phi, K, M, eps): """ function G=THinv5(phi,K,M,eps) % @@ -103,18 +104,18 @@ def THinv5(self,phi,K,M,eps): # %C=[]; C = [] - # %for im=1:M + # %for im=1:M # % A=toeplitz(phi(1:K,im),phi(1:K,im)')+hankel(phi(1:K,im),phi(K:2*K-1,im)')+eps(im)*eye(K); # % C=[C inv(A)]; # %end - for im in xrange(M): - A = (toeplitz(phi[:K,im],phi[:K,im].T) + - hankel(phi[:K,im],phi[K-1:2*K,im].T) + - eps[im]*np.eye(K)) + for im in range(M): + A = (toeplitz(phi[:K, im], phi[:K, im].T) + + hankel(phi[:K, im], phi[K - 1:2 * K, im].T) + + eps[im] * np.eye(K)) C.append(np.linalg.inv(A)) - return np.concatenate(C,axis=1) - + return np.concatenate(C, axis=1) + # # phi(2*K,1:M)=0; # phi[2*K-1,:M] = 0 # # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -143,10 +144,10 @@ def THinv5(self,phi,K,M,eps): # # f2o=phi(k+1:-1:2,:)+phi(k+1:2*k,:); # f2o = phi[k+1:0:-1,:] + phi[k+1:2*k+1,:] # # alm=sum(f2o.*x4(1:k,:),1)+phi(1,:)+eps+phi(2*k+1,:); - # # a0=zeros(1,M); + # # a0=zeros(1,M); # # if k rmax: - v = v*rmax/vmax + v = v * rmax / vmax # AR(:,id)=real(poly(v)'); %%% reconstructs back the covariance function - AR[:,id] = np.real(np.poly(v).T) - # end + AR[:, id] = np.real(np.poly(v).T) + # end # Rs=ar2r(AR); Rs = self.ar2r(AR) # sigmy=R(1,:)./Rs(1,:); - sigmy = R[0,:]/Rs[0,:] + sigmy = R[0, :] / Rs[0, :] # % [v1; v2] # end %%%%%%%%%%%%%%%%%%%%%%% of armodel - return AR,sigmy + return AR, sigmy def ar2r(self, a): """ @@ -245,13 +245,13 @@ def ar2r(self, a): # end if a.shape[0] == 1: a = a.T - + # [p m] = size(a); % pocet vektoru koef.AR modelu - p,m = a.shape + p, m = a.shape # alfa = a; alfa = a.copy() # K=zeros(p,m); - K = np.zeros((p,m)) + K = np.zeros((p, m)) # p = p-1; p -= 1 # for n=p:-1:1 @@ -262,21 +262,22 @@ def ar2r(self, a): # a=alfa; # end # XXX Check here if broken - for n in range(p)[::-1]: #range(p-1,-1,-1): - K[n,:] = -a[n+1,:] - for k in xrange(n): - alfa[k+1,:] = (a[k+1,:]+K[n,:]*a[n-k,:])/(1-K[n,:]**2) + for n in range(p)[::-1]: # range(p-1,-1,-1): + K[n, :] = -a[n + 1, :] + for k in range(n): + alfa[k + 1, :] = (a[k + 1, :] + K[n, :] * + a[n - k, :]) / (1 - K[n, :]**2) a = alfa.copy() - # % + # % # r = zeros(p+1,m); - r = np.zeros((p+1,m)) + r = np.zeros((p + 1, m)) # r(1,:) = 1./prod(1-K.^2); - r[0,:] = 1/np.prod(1-K**2,0) + r[0, :] = 1 / np.prod(1 - K**2, 0) # f = r; f = r.copy() # b=f; b = f.copy() - # for k=1:p + # for k=1:p # for n=k:-1:1 # K_n = K(n,:); # f(n,:)=f(n+1,:)+K_n.*b(k-n+1,:); @@ -286,18 +287,18 @@ def ar2r(self, a): # r(k+1,:) = f(1,:); # end # XXX Check here if broken - for k in xrange(p): - for n in range(k+1)[::-1]: #range(k-1:-1,-1): - K_n = K[n,:] - f[n,:] = f[n+1,:] + K_n*b[k-n,:] - b[k-n,:] = -K_n*f[n+1,:]+(1-K_n**2)*b[k-n,:] - b[k+1,:] = f[0,:] - r[k+1,:] = f[0,:] + for k in range(p): + for n in range(k + 1)[::-1]: # range(k-1:-1,-1): + K_n = K[n, :] + f[n, :] = f[n + 1, :] + K_n * b[k - n, :] + b[k - n, :] = -K_n * f[n + 1, :] + (1 - K_n**2) * b[k - n, :] + b[k + 1, :] = f[0, :] + r[k + 1, :] = f[0, :] # end %%%%%%%%%%%%%%%%%%%%%%%%%%% of ar2r # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% return r - def corr_est(self,x,T,q): + def corr_est(self, x, T, q): """ # function R_est=corr_est(x,T,q) # % @@ -305,63 +306,64 @@ def corr_est(self,x,T,q): # NumOfSources = size(x,1); NumOfSources = x.shape[0] # R_est = zeros(NumOfSources,(q+1)*NumOfSources); - R_est = np.zeros((NumOfSources,(q+1)*NumOfSources)) + R_est = np.zeros((NumOfSources, (q + 1) * NumOfSources)) # for index=1:q+1 # R_est(:,NumOfSources*(index-1) + (1:NumOfSources)) = 1/T*(x(:,1:T)*x(:,index:T+index-1)'); # end - for index in xrange(q+1): + for index in range(q + 1): #irange = NumOfSources*(index) + np.arange(NumOfSources) - i = NumOfSources*(index) - R_est[:,i:i+NumOfSources] = (1/np.float(T))*(np.dot(x[:,:T],x[:,index:T+index].T)) + i = NumOfSources * (index) + R_est[:, i:i + NumOfSources] = (1 / np.float(T)) * \ + (np.dot(x[:, :T], x[:, index:T + index].T)) return R_est - def weights(self,Ms,rmax,eps0): + def weights(self, Ms, rmax, eps0): """ function [H ARC]=weights(Ms,rmax,eps0) % """ # [d,Ld]=size(Ms); - d,Ld = Ms.shape + d, Ld = Ms.shape # L=floor(Ld/d); - L = np.int32(np.floor(Ld/np.float(d))) + L = np.int32(np.floor(Ld / np.float(d))) # d2=d*(d-1)/2; - d2 = np.int32(d*(d-1)/2.) + d2 = np.int32(d * (d - 1) / 2.) # R=zeros(L,d); - R = np.zeros((L,d)) + R = np.zeros((L, d)) # for index=1:L # id=(index-1)*d; - # R(index,:)=diag(Ms(:,id+1:id+d)).'; %%% columns of R will contain + # R(index,:)=diag(Ms(:,id+1:id+d)).'; %%% columns of R will contain # %%% covariance function of the separated components # end - for index in xrange(L): - id = index*d - R[index,:] = np.diag(Ms[:,id:id+d]) + for index in range(L): + id = index * d + R[index, :] = np.diag(Ms[:, id:id + d]) # % # [ARC,sigmy]=armodel(R,rmax); %%% compute AR models of estimated components - ARC,sigmy = self.armodel(R,rmax) + ARC, sigmy = self.armodel(R, rmax) # % # AR3=zeros(2*L-1,d2); - AR3 = np.zeros((2*L-1,d2)) + AR3 = np.zeros((2 * L - 1, d2)) # ll = 1; # for i=2:d # for k=1:i-1 # AR3(:,ll) = conv(ARC(:,i),ARC(:,k)); # ll = ll+1; # % AR3=[AR3 conv(AR(:,i),AR(:,k))]; - # end + # end # end ll = 0 - for i in xrange(1,d): - for k in xrange(i): - AR3[:,ll] = np.convolve(ARC[:,i],ARC[:,k]) + for i in range(1, d): + for k in range(i): + AR3[:, ll] = np.convolve(ARC[:, i], ARC[:, k]) ll += 1 # phi=ar2r(AR3); %%%%%%%%%% functions phi to evaluate CVinv phi = self.ar2r(AR3) - # H=THinv5(phi,L,d2,eps0*phi(1,:)); %%%% to compute inversions of CV + # H=THinv5(phi,L,d2,eps0*phi(1,:)); %%%% to compute inversions of CV # %%%% It has dimension zeros(M,M*d2). - H = self.THinv5(phi,L,d2,eps0*phi[0,:]) - # im=1; + H = self.THinv5(phi, L, d2, eps0 * phi[0, :]) + # im=1; # for i=2:d # for k=1:i-1 # fact=1/(sigmy(1,i)*sigmy(1,k)); @@ -371,17 +373,17 @@ def weights(self,Ms,rmax,eps0): # end # end im = 0 - for i in xrange(1,d): - for k in xrange(i): - fact = 1/(sigmy[i]*sigmy[k]) - imm = im*L - H[:,imm:imm+L] = H[:,imm:imm+L]*fact - im+=1 + for i in range(1, d): + for k in range(i): + fact = 1 / (sigmy[i] * sigmy[k]) + imm = im * L + H[:, imm:imm + L] = H[:, imm:imm + L] * fact + im += 1 # end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% of weights - return (H,ARC) + return (H, ARC) - def CRLB4(self,ARC): + def CRLB4(self, ARC): """ function ISR = CRLB4(ARC) % @@ -391,34 +393,36 @@ def CRLB4(self,ARC): % are stored as columns in matrix ARC. """ # [M K]=size(ARC); - M,K = ARC.shape - + M, K = ARC.shape + # Rs=ar2r(ARC); Rs = self.ar2r(ARC) - + # sum_Rs_s=zeros(K,K); - sum_Rs_s = np.zeros((K,K)) - + sum_Rs_s = np.zeros((K, K)) + # for s=0:M-1 # for t=0:M-1 # sum_Rs_s=sum_Rs_s+(ARC(s+1,:).*ARC(t+1,:))'*Rs(abs(s-t)+1,:); # end # end - for s in xrange(M): - for t in xrange(M): - sum_Rs_s += np.dot((ARC[s,:]*ARC[t,:])[np.newaxis,:].T, - Rs[np.abs(s-t),:][np.newaxis,:]) + for s in range(M): + for t in range(M): + sum_Rs_s += np.dot((ARC[s, :] * ARC[t, :])[np.newaxis, :].T, + Rs[np.abs(s - t), :][np.newaxis, :]) # denom=sum_Rs_s'.*sum_Rs_s+eye(K)-1; - denom = sum_Rs_s.T*sum_Rs_s+np.eye(K)-1 + denom = sum_Rs_s.T * sum_Rs_s + np.eye(K) - 1 # ISR=sum_Rs_s'./denom.*(ones(K,1)*Rs(1,:))./(Rs(1,:)'*ones(1,K)); - ISR = sum_Rs_s.T/denom*np.outer(np.ones((K,1)),Rs[0,:])/np.outer(Rs[0,:].T,np.ones((1,K))) + ISR = sum_Rs_s.T / denom * \ + np.outer(np.ones((K, 1)), Rs[0, :]) / \ + np.outer(Rs[0, :].T, np.ones((1, K))) # ISR(eye(K)==1)=0; - ISR[np.eye(K)==1] = 0 + ISR[np.eye(K) == 1] = 0 # end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% of CRLB4 return ISR - def uwajd(self,M,maxnumiter=20,W_est0=None): + def uwajd(self, M, maxnumiter=20, W_est0=None): """ function [W_est Ms]=uwajd(M,maxnumiter,W_est0) % @@ -435,11 +439,11 @@ def uwajd(self,M,maxnumiter=20,W_est0=None): % """ # [d Md]=size(M); - d,Md = M.shape + d, Md = M.shape # L=floor(Md/d); - L = np.int32(np.floor(Md/np.float(d))) + L = np.int32(np.floor(Md / np.float(d))) # Md=L*d; - Md = L*d + Md = L * d # iter=0; iter = 0 # eps=1e-7; @@ -453,39 +457,44 @@ def uwajd(self,M,maxnumiter=20,W_est0=None): # W_est=W_est0; # end if W_est0 is None: - E,H = np.linalg.eig(M[:,:d]) + E, H = np.linalg.eig(M[:, :d]) H = np.real_if_close(H, tol=100) - W_est = np.dot(np.diag(1./np.sqrt(E)),H.T) + W_est = np.dot(np.diag(1. / np.sqrt(E)), H.T) else: W_est = W_est0 # if nargin<2 # maxnumiter=20; - # end + # end # Ms=M; Ms = M.copy() # Rs=zeros(d,L); - Rs = np.zeros((d,L)) + Rs = np.zeros((d, L)) # for k=1:L # ini=(k-1)*d; # M(:,ini+1:ini+d)=0.5*(M(:,ini+1:ini+d)+M(:,ini+1:ini+d)'); # Ms(:,ini+1:ini+d)=W_est*M(:,ini+1:ini+d)*W_est'; # Rs(:,k)=diag(Ms(:,ini+1:ini+d)); # end - for k in xrange(L): - ini = k*d - M[:,ini:ini+d] = 0.5*(M[:,ini:ini+d]+M[:,ini:ini+d].T) - Ms[:,ini:ini+d] = np.dot(np.dot(W_est,M[:,ini:ini+d]),W_est.T) - Rs[:,k] = np.diag(Ms[:,ini:ini+d]) + for k in range(L): + ini = k * d + M[:, ini:ini + d] = 0.5 * (M[:, ini:ini + d] + M[:, ini:ini + d].T) + Ms[:, ini:ini + + d] = np.dot(np.dot(W_est, M[:, ini:ini + d]), W_est.T) + Rs[:, k] = np.diag(Ms[:, ini:ini + d]) # crit=sum(Ms(:).^2)-sum(Rs(:).^2); crit = (Ms**2).sum() - (Rs**2).sum() # while improve>eps && iter eps and iter eps and iter < maxnumiter: # b11=[]; b12=[]; b22=[]; c1=[]; c2=[]; - b11=[]; b12=[]; b22=[]; c1=[]; c2=[]; - # for id=2:d + b11 = [] + b12 = [] + b22 = [] + c1 = [] + c2 = [] + # for id=2:d # Yim=Ms(1:id-1,id:d:Md); # b22=[b22; sum(Rs(id,:).^2)*ones(id-1,1)]; # b12=[b12; (Rs(id,:)*Rs(1:id-1,:)')']; @@ -493,27 +502,27 @@ def uwajd(self,M,maxnumiter=20,W_est0=None): # c2=[c2; (Rs(id,:)*Yim')']; # c1=[c1; sum(Rs(1:id-1,:).*Yim,2)]; # end - for id in xrange(1,d): - Yim = Ms[0:id,id:Md:d] - b22.append(np.dot((Rs[id,:]**2).sum(0),np.ones((id,1)))) - b12.append(np.dot(Rs[id,:],Rs[:id,:].T).T) - b11.append((Rs[:id,:]**2).sum(1)) - c2.append(np.dot(Rs[id,:],Yim.T).T) - c1.append((Rs[:id,:]*Yim).sum(1)) + for id in range(1, d): + Yim = Ms[0:id, id:Md:d] + b22.append(np.dot((Rs[id, :]**2).sum(0), np.ones((id, 1)))) + b12.append(np.dot(Rs[id, :], Rs[:id, :].T).T) + b11.append((Rs[:id, :]**2).sum(1)) + c2.append(np.dot(Rs[id, :], Yim.T).T) + c1.append((Rs[:id, :] * Yim).sum(1)) b22 = np.squeeze(np.vstack(b22)) b12 = np.hstack(b12) b11 = np.hstack(b11) c2 = np.hstack(c2) c1 = np.hstack(c1) # det0=b11.*b22-b12.^2; - det0 = b11*b22-b12**2 + det0 = b11 * b22 - b12**2 # d1=(c1.*b22-b12.*c2)./det0; - d1 = (c1*b22-b12*c2)/det0 + d1 = (c1 * b22 - b12 * c2) / det0 # d2=(b11.*c2-b12.*c1)./det0; - d2 = (b11*c2-b12*c1)/det0 + d2 = (b11 * c2 - b12 * c1) / det0 # % value=norm([d1; d2]) # m=0; - m=0 + m = 0 # A0=eye(d); A0 = np.eye(d) # for id=2:d @@ -521,30 +530,31 @@ def uwajd(self,M,maxnumiter=20,W_est0=None): # A0(1:id-1,id)=d2(m+1:m+id-1,1); # m=m+id-1; # end - for id in xrange(1,d): - A0[id,0:id] = d1[m:m+id] - A0[0:id,id] = d2[m:m+id] + for id in range(1, d): + A0[id, 0:id] = d1[m:m + id] + A0[0:id, id] = d2[m:m + id] m += id # Ainv=inv(A0); Ainv = np.linalg.inv(A0) # W_est=Ainv*W_est; - W_est = np.dot(Ainv,W_est) + W_est = np.dot(Ainv, W_est) # Raux=W_est*M(:,1:d)*W_est'; - Raux = np.dot(np.dot(W_est,M[:,:d]),W_est.T) + Raux = np.dot(np.dot(W_est, M[:, :d]), W_est.T) # aux=1./sqrt(diag(Raux)); - aux = 1/np.sqrt(np.diag(Raux)) + aux = 1 / np.sqrt(np.diag(Raux)) # W_est=diag(aux)*W_est; % normalize the result - W_est = np.dot(np.diag(aux),W_est) + W_est = np.dot(np.diag(aux), W_est) # for k=1:L # ini=(k-1)*d; # Ms(:,ini+1:ini+d) = W_est*M(:,ini+1:ini+d)*W_est'; # Rs(:,k)=diag(Ms(:,ini+1:ini+d)); # end - for k in xrange(L): - ini = k*d - Ms[:,ini:ini+d] = np.dot(np.dot(W_est,M[:,ini:ini+d]),W_est.T) - Rs[:,k] = np.diag(Ms[:,ini:ini+d]) + for k in range(L): + ini = k * d + Ms[:, ini:ini + + d] = np.dot(np.dot(W_est, M[:, ini:ini + d]), W_est.T) + Rs[:, k] = np.diag(Ms[:, ini:ini + d]) # critic=sum(Ms(:).^2)-sum(Rs(:).^2); critic = (Ms**2).sum() - (Rs**2).sum() # % improve=abs(critic-crit(end)); @@ -558,9 +568,9 @@ def uwajd(self,M,maxnumiter=20,W_est0=None): # end %%%%%% of while # end %%%%%%%%%%%%%%%%%%% of uwajd - return W_est,Ms + return W_est, Ms - def wajd(self,M,H,W_est0=None,maxnumit=100): + def wajd(self, M, H, W_est0=None, maxnumit=100): """ function [W_est Ms]=wajd(M,H,W_est0,maxnumit) % @@ -581,16 +591,16 @@ def wajd(self,M,H,W_est0=None,maxnumit=100): % """ # [d Md]=size(M); - d,Md = M.shape + d, Md = M.shape # L=floor(Md/d); - L = np.int32(np.floor(Md/np.float(d))) + L = np.int32(np.floor(Md / np.float(d))) # dd2=d*(d-1)/2; - dd2 = np.int32(d*(d-1)/2.); + dd2 = np.int32(d * (d - 1) / 2.) # Md=L*d; - Md = L*d + Md = L * d # if nargin<4 # maxnumit=100; - # end + # end # if nargin<3 # [H E]=eig(M(:,1:d)); # W_est=diag(1./sqrt(diag(E)))*H'; @@ -598,39 +608,40 @@ def wajd(self,M,H,W_est0=None,maxnumit=100): # W_est=W_est0; # end if W_est0 is None: - E,H = np.linalg.eig(M[:,:d]) + E, H = np.linalg.eig(M[:, :d]) H = np.real_if_close(H, tol=100) - W_est = np.dot(np.diag(1/np.sqrt(E)),H.T) + W_est = np.dot(np.diag(1 / np.sqrt(E)), H.T) else: W_est = W_est0 # Ms=M; Ms = M.copy() # Rs=zeros(d,L); - Rs = np.zeros((d,L)) + Rs = np.zeros((d, L)) # for k=1:L # ini=(k-1)*d; # M(:,ini+1:ini+d)=0.5*(M(:,ini+1:ini+d)+M(:,ini+1:ini+d)'); # Ms(:,ini+1:ini+d)=W_est*M(:,ini+1:ini+d)*W_est'; # Rs(:,k)=diag(Ms(:,ini+1:ini+d)); - # end - for k in xrange(L): - ini = k*d - M[:,ini:ini+d] = 0.5*(M[:,ini:ini+d]+M[:,ini:ini+d].T) - Ms[:,ini:ini+d] = np.dot(np.dot(W_est,M[:,ini:ini+d]),W_est.T) - Rs[:,k] = np.diag(Ms[:,ini:ini+d]) + # end + for k in range(L): + ini = k * d + M[:, ini:ini + d] = 0.5 * (M[:, ini:ini + d] + M[:, ini:ini + d].T) + Ms[:, ini:ini + + d] = np.dot(np.dot(W_est, M[:, ini:ini + d]), W_est.T) + Rs[:, k] = np.diag(Ms[:, ini:ini + d]) # for iter=1:maxnumit - for iter in xrange(maxnumit): + for iter in range(maxnumit): # b11=zeros(dd2,1); b12=b11; b22=b11; c1=b11; c2=c1; - b11 = np.zeros((dd2,1)) - b12 = np.zeros((dd2,1)) - b22 = np.zeros((dd2,1)) - c1 = np.zeros((dd2,1)) - c2 = np.zeros((dd2,1)) + b11 = np.zeros((dd2, 1)) + b12 = np.zeros((dd2, 1)) + b22 = np.zeros((dd2, 1)) + c1 = np.zeros((dd2, 1)) + c2 = np.zeros((dd2, 1)) # m=0; - m=0 - # for id=2:d + m = 0 + # for id=2:d # for id2=1:id-1 # m=m+1; im=(m-1)*L; # Wm=H(:,im+1:im+L); @@ -646,29 +657,29 @@ def wajd(self,M,H,W_est0=None,maxnumit=100): # c2(m)=Wlam1'*Yim'; # end # end - for id in xrange(1,d): - for id2 in xrange(id): - im = m*L - Wm = H[:,im:im+L] - Yim = Ms[id,id2:Md:d] - Rs_id = Rs[id,:] - Rs_id2 = Rs[id2,:] - Wlam1 = np.dot(Wm,Rs_id.T) - Wlam2 = np.dot(Wm,Rs_id2.T) - b11[m] = np.dot(Rs_id2,Wlam2) - b12[m] = np.dot(Rs_id,Wlam2) - b22[m] = np.dot(Rs_id,Wlam1) - c1[m] = np.dot(Wlam2.T,Yim.T) - c2[m] = np.dot(Wlam1.T,Yim.T) - m+=1 + for id in range(1, d): + for id2 in range(id): + im = m * L + Wm = H[:, im:im + L] + Yim = Ms[id, id2:Md:d] + Rs_id = Rs[id, :] + Rs_id2 = Rs[id2, :] + Wlam1 = np.dot(Wm, Rs_id.T) + Wlam2 = np.dot(Wm, Rs_id2.T) + b11[m] = np.dot(Rs_id2, Wlam2) + b12[m] = np.dot(Rs_id, Wlam2) + b22[m] = np.dot(Rs_id, Wlam1) + c1[m] = np.dot(Wlam2.T, Yim.T) + c2[m] = np.dot(Wlam1.T, Yim.T) + m += 1 # det0=b11.*b22-b12.^2; - det0 = b11*b22-b12**2 + det0 = b11 * b22 - b12**2 # d1=(c1.*b22-b12.*c2)./det0; - d1 = (c1*b22-b12*c2)/det0 + d1 = (c1 * b22 - b12 * c2) / det0 # d2=(b11.*c2-b12.*c1)./det0; - d2 = (b11*c2-b12*c1)/det0 + d2 = (b11 * c2 - b12 * c1) / det0 # m=0; - m=0 + m = 0 # A0=eye(d); A0 = np.eye(d) # for id=2:d @@ -676,38 +687,36 @@ def wajd(self,M,H,W_est0=None,maxnumit=100): # A0(1:id-1,id)=d2(m+1:m+id-1,1); # m=m+id-1; # end - for id in xrange(1,d): - A0[id,0:id] = d1[m:m+id,0] - A0[0:id,id] = d2[m:m+id,0] + for id in range(1, d): + A0[id, 0:id] = d1[m:m + id, 0] + A0[0:id, id] = d2[m:m + id, 0] m += id # Ainv=inv(A0); Ainv = np.linalg.inv(A0) # W_est=Ainv*W_est; - W_est = np.dot(Ainv,W_est) + W_est = np.dot(Ainv, W_est) # Raux=W_est*M(:,1:d)*W_est'; - Raux = np.dot(np.dot(W_est,M[:,:d]),W_est.T) + Raux = np.dot(np.dot(W_est, M[:, :d]), W_est.T) # aux=1./sqrt(diag(Raux)); - aux = 1/np.sqrt(np.diag(Raux)) + aux = 1 / np.sqrt(np.diag(Raux)) # W_est=diag(aux)*W_est; % normalize the result - W_est = np.dot(np.diag(aux),W_est) + W_est = np.dot(np.diag(aux), W_est) # for k=1:L # ini=(k-1)*d; # Ms(:,ini+1:ini+d) = W_est*M(:,ini+1:ini+d)*W_est'; # Rs(:,k)=diag(Ms(:,ini+1:ini+d)); # end - for k in xrange(L): - ini = k*d - Ms[:,ini:ini+d] = np.dot(np.dot(W_est,M[:,ini:ini+d]),W_est.T) - Rs[:,k] = np.diag(Ms[:,ini:ini+d]) + for k in range(L): + ini = k * d + Ms[:, ini:ini + + d] = np.dot(np.dot(W_est, M[:, ini:ini + d]), W_est.T) + Rs[:, k] = np.diag(Ms[:, ini:ini + d]) # end %%%%%%%%%%% of for # end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% of wajd - return W_est,Ms - + return W_est, Ms + def iwasobi(data, ar_max=10, rmax=0.99, eps0=5.0e-7): """ """ return IWASOBI(ar_max=ar_max, rmax=rmax, eps0=eps0)(data) - - - diff --git a/ptsa/pca.py b/ptsa/pca.py index a1d310c..8fe53ea 100644 --- a/ptsa/pca.py +++ b/ptsa/pca.py @@ -1,45 +1,44 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -# -# See the COPYING file distributed along with the PTSA package for the -# copyright and license terms. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## - -# global imports -import numpy as np - -def pca(X, ncomps=None, eigratio=1e6): - """ - Principal components analysis - -% [W,Y] = pca(X,NBC,EIGRATIO) returns the PCA matrix W and the principal -% components Y corresponding to the data matrix X (realizations -% columnwise). The number of components is NBC components unless the -% ratio between the maximum and minimum covariance eigenvalue is below -% EIGRATIO. In such a case, the function will return as few components as -% are necessary to guarantee that such ratio is greater than EIGRATIO. - - """ - - if ncomps is None: - ncomps = X.shape[0] - - C = np.cov(X) - D,V = np.linalg.eigh(C) - val = np.abs(D) - I = np.argsort(val)[::-1] - val = val[I] - - while (val[0]/val[ncomps-1])>eigratio: - ncomps -= 1 - - V = V[:,I[:ncomps]] - D = np.diag(D[I[:ncomps]]**(-.5)) - W = np.dot(D,V.T) - Y = np.dot(W,X) - - return W,Y - - +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See the COPYING file distributed along with the PTSA package for the +# copyright and license terms. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + +# global imports +import numpy as np + + +def pca(X, ncomps=None, eigratio=1e6): + """ + Principal components analysis + +% [W,Y] = pca(X,NBC,EIGRATIO) returns the PCA matrix W and the principal +% components Y corresponding to the data matrix X (realizations +% columnwise). The number of components is NBC components unless the +% ratio between the maximum and minimum covariance eigenvalue is below +% EIGRATIO. In such a case, the function will return as few components as +% are necessary to guarantee that such ratio is greater than EIGRATIO. + + """ + + if ncomps is None: + ncomps = X.shape[0] + + C = np.cov(X) + D, V = np.linalg.eigh(C) + val = np.abs(D) + I = np.argsort(val)[::-1] + val = val[I] + + while (val[0] / val[ncomps - 1]) > eigratio: + ncomps -= 1 + + V = V[:, I[:ncomps]] + D = np.diag(D[I[:ncomps]]**(-.5)) + W = np.dot(D, V.T) + Y = np.dot(W, X) + + return W, Y diff --git a/ptsa/plotting/__init__.py b/ptsa/plotting/__init__.py index f1518ec..5f08baa 100644 --- a/ptsa/plotting/__init__.py +++ b/ptsa/plotting/__init__.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,6 +7,5 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -from topo import topoplot -from misc import errorfill - +from .topo import topoplot +from .misc import errorfill diff --git a/ptsa/plotting/logo.py b/ptsa/plotting/logo.py index 4506f89..8e1c1e8 100644 --- a/ptsa/plotting/logo.py +++ b/ptsa/plotting/logo.py @@ -13,33 +13,35 @@ mpl.rcParams['axes.edgecolor'] = 'gray' axalpha = 0.05 -#figcolor = '#EFEFEF' +# figcolor = '#EFEFEF' figcolor = 'white' dpi = 80 -fig = plt.figure(figsize=(4, 1.1),dpi=dpi) +fig = plt.figure(figsize=(4, 1.1), dpi=dpi) fig.figurePatch.set_edgecolor(figcolor) fig.figurePatch.set_facecolor(figcolor) def add_timeseries(): ax = fig.add_axes([0., 0., 1., 1.]) - x = np.linspace(0,1,1000) - freqs = [8,16,32,64] + x = np.linspace(0, 1, 1000) + freqs = [8, 16, 32, 64] # y = np.zeros(1000) # for f in freqs: # y = y + np.sin(x*np.pi*f*4 + f/60.)*(10.0/(f)) # y = y+.5 - y = np.sin(x*np.pi*32)*.45 + .5 - lines = plt.plot(x,y, + y = np.sin(x * np.pi * 32) * .45 + .5 + lines = plt.plot(x, y, transform=ax.transAxes, color="#11557c", alpha=0.25,) ax.set_axis_off() return ax + def add_ptsa_text(ax): ax.text(0.95, 0.5, 'PTSA', color='#11557c', fontsize=65, - ha='right', va='center', alpha=1.0, transform=ax.transAxes) + ha='right', va='center', alpha=1.0, transform=ax.transAxes) + def add_pizza(): ax = fig.add_axes([0.025, 0.075, 0.3, 0.85], polar=True, resolution=50) @@ -48,15 +50,15 @@ def add_pizza(): ax.set_axisbelow(True) N = 8 arc = 2. * np.pi - theta = np.arange(0.0, arc, arc/N) + theta = np.arange(0.0, arc, arc / N) radii = 10 * np.array([0.79, 0.81, 0.78, 0.77, 0.79, 0.78, 0.83, 0.78]) - width = np.pi / 4 * np.array([1.0]*N) + width = np.pi / 4 * np.array([1.0] * N) theta = theta[1:] radii = radii[1:] width = width[1:] bars = ax.bar(theta, radii, width=width, bottom=0.0) for r, bar in zip(radii, bars): - bar.set_facecolor(cm.hot(r/10.)) + bar.set_facecolor(cm.hot(r / 10.)) bar.set_edgecolor('r') bar.set_alpha(0.6) @@ -72,19 +74,18 @@ def add_pizza(): # add some veggie peperoni #theta = np.array([.08,.18,.32,.46,.54,.68,.77,.85,.96]) * np.pi * 2.0 #radii = 10*np.array([.6,.38,.58,.5,.62,.42,.58,.67,.45]) - theta = np.array([.18,.32,.46,.54,.68,.77,.85,.96]) * np.pi * 2.0 - radii = 10*np.array([.38,.58,.5,.62,.42,.58,.67,.45]) - c = plt.scatter(theta,radii,c='r',s=7**2) + theta = np.array([.18, .32, .46, .54, .68, .77, .85, .96]) * np.pi * 2.0 + radii = 10 * np.array([.38, .58, .5, .62, .42, .58, .67, .45]) + c = plt.scatter(theta, radii, c='r', s=7**2) c.set_alpha(0.75) ax.set_yticks(np.arange(1, 9, 2)) ax.set_rmax(9) - + if __name__ == '__main__': main_axes = add_timeseries() add_pizza() - #add_ptsa_text(main_axes) - #plt.show() + # add_ptsa_text(main_axes) + # plt.show() plt.savefig('logo.png') - diff --git a/ptsa/plotting/misc.py b/ptsa/plotting/misc.py index 6fd9769..1043545 100644 --- a/ptsa/plotting/misc.py +++ b/ptsa/plotting/misc.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,15 +11,16 @@ import numpy as np import pylab as pl -def errorfill(xvals,yvals,errvals,**kwargs): + +def errorfill(xvals, yvals, errvals, **kwargs): """ Plot an errorbar as a filled polygon that can be transparent. See the pylab.fill method for kwargs. """ # set the xrange - x_range = np.concatenate((xvals,np.flipud(xvals))) - y_range = np.concatenate((yvals+errvals,np.flipud(yvals-errvals))) + x_range = np.concatenate((xvals, np.flipud(xvals))) + y_range = np.concatenate((yvals + errvals, np.flipud(yvals - errvals))) # do the fill - return pl.fill(x_range,y_range,**kwargs) + return pl.fill(x_range, y_range, **kwargs) diff --git a/ptsa/plotting/topo.py b/ptsa/plotting/topo.py index 42694ea..4d51d40 100644 --- a/ptsa/plotting/topo.py +++ b/ptsa/plotting/topo.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -18,22 +18,23 @@ 'head_linecolor': 'black', 'nose_linewidth': 2, 'ear_linewidth': 2, - } + } default_label_props = {'ha': 'center', 'va': 'center'} default_sensor_props = {'marker': 'o', - 'c': 'k', + 'c': 'k', 's': 8} default_contour_props = {'linewidths': 0, 'linestyle': '-', - 'colors': 'black',} + 'colors': 'black', } -def topoplot(values=None, labels=None, sensors=None, axes=None, - center=(0,0), nose_dir=0., radius=0.5, + +def topoplot(values=None, labels=None, sensors=None, axes=None, + center=(0, 0), nose_dir=0., radius=0.5, head_props=None, sensor_props=None, - label_props=None, + label_props=None, contours=15, contour_props=None, - resolution=400, axis_props='off', + resolution=400, axis_props='off', plot_mask='circular', plot_radius_buffer=.2, **kwargs): """ @@ -90,35 +91,35 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, Optional keyword arguments to be passed on to contourf. """ - if axes is not None: # axes are given - a=axes - else: # a new subplot is created - a=plt.subplot(1, 1, 1, aspect='equal') + if axes is not None: # axes are given + a = axes + else: # a new subplot is created + a = plt.subplot(1, 1, 1, aspect='equal') a.axis(axis_props) - - if True: # head should be plotted + + if True: # head should be plotted # deal with the head props hprops = default_head_props.copy() if not head_props is None: hprops.update(head_props) # Set up head - head = plt.Circle(center, radius, fill=False, + head = plt.Circle(center, radius, fill=False, linewidth=hprops['head_linewidth'], edgecolor=hprops['head_linecolor'], axes=a) # Nose: - nose_width = 0.18*radius + nose_width = 0.18 * radius # Distance from the center of the head to the point where the # nose touches the outline of the head: - nose_dist = np.cos(np.arcsin((nose_width/2.)/radius))*radius + nose_dist = np.cos(np.arcsin((nose_width / 2.) / radius)) * radius # Distance from the center of the head to the tip of the nose: - nose_tip_dist = 1.15*radius + nose_tip_dist = 1.15 * radius # Convert to polar coordinates for rotating: nose_polar_angle, nose_polar_radius = cart2pol( - np.array([-nose_width/2, 0, nose_width/2]), + np.array([-nose_width / 2, 0, nose_width / 2]), np.array([nose_dist, nose_tip_dist, nose_dist])) nose_polar_angle = nose_polar_angle + deg2rad(nose_dir) # And back to cartesian coordinates for plotting: @@ -133,30 +134,30 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, axes=a) # Ears: - q = .04 # ear lengthening + q = .04 # ear lengthening ear_x = np.array( - [.497-.005, .510,.518, .5299, .5419, .54, .547, - .532, .510, .489-.005])*(radius/0.5) + [.497 - .005, .510, .518, .5299, .5419, .54, .547, + .532, .510, .489 - .005]) * (radius / 0.5) ear_y = np.array( - [q+.0555, q+.0775, q+.0783, q+.0746, q+.0555, - -.0055, -.0932, -.1313, -.1384, -.1199])*(radius/0.5) + [q + .0555, q + .0775, q + .0783, q + .0746, q + .0555, + -.0055, -.0932, -.1313, -.1384, -.1199]) * (radius / 0.5) # Convert to polar coordinates for rotating: rightear_polar_angle, rightear_polar_radius = cart2pol(ear_x, ear_y) leftear_polar_angle, leftear_polar_radius = cart2pol(-ear_x, ear_y) - rightear_polar_angle = rightear_polar_angle+deg2rad(nose_dir) - leftear_polar_angle = leftear_polar_angle+deg2rad(nose_dir) + rightear_polar_angle = rightear_polar_angle + deg2rad(nose_dir) + leftear_polar_angle = leftear_polar_angle + deg2rad(nose_dir) # And back to cartesian coordinates for plotting: rightear_x, rightear_y = pol2cart(rightear_polar_angle, - rightear_polar_radius) + rightear_polar_radius) leftear_x, leftear_y = pol2cart(leftear_polar_angle, leftear_polar_radius) - + # Move ears with head: rightear_x = rightear_x + center[0] rightear_y = rightear_y + center[1] leftear_x = leftear_x + center[0] leftear_y = leftear_y + center[1] - + ear_right = plt.Line2D(rightear_x, rightear_y, color=hprops['head_linecolor'], linewidth=hprops['ear_linewidth'], @@ -168,7 +169,7 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, linewidth=hprops['ear_linewidth'], solid_joinstyle='round', solid_capstyle='round', - axes=a) + axes=a) a.add_artist(head) a.add_artist(nose) a.add_artist(ear_right) @@ -176,31 +177,31 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, if sensors is None: if axes is None: - a.set_xlim(-radius*1.2+center[0], radius*1.2+center[0]) - a.set_ylim(-radius*1.2+center[1], radius*1.2+center[1]) + a.set_xlim(-radius * 1.2 + center[0], radius * 1.2 + center[0]) + a.set_ylim(-radius * 1.2 + center[1], radius * 1.2 + center[1]) return("No sensor locations specified!") - + # Convert & rotate sensor locations: - angles = -sensors[0]+90 - angles = angles+nose_dir + angles = -sensors[0] + 90 + angles = angles + nose_dir angles = deg2rad(angles) radii = sensors[1] # expand or shrink electrode locations with radius of head: - radii = radii*(radius/0.5) + radii = radii * (radius / 0.5) # plotting radius is determined by largest sensor radius: - plot_radius = max(radii)*(1.0+plot_radius_buffer) - + plot_radius = max(radii) * (1.0 + plot_radius_buffer) + # convert electrode locations to cartesian coordinates for plotting: x, y = pol2cart(angles, radii) x = x + center[0] y = y + center[1] - if True: # plot electrodes + if True: # plot electrodes sprops = default_sensor_props.copy() if not sensor_props is None: sprops.update(sensor_props) - #a.plot(x,y,markerfacecolor=colors[1],marker='o',linestyle='') + # a.plot(x,y,markerfacecolor=colors[1],marker='o',linestyle='') a.scatter(x, y, zorder=10, **sprops) if not labels is None: @@ -209,30 +210,29 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, lprops.update(label_props) for i in range(len(labels)): - a.text(x[i],y[i],labels[i],**lprops) + a.text(x[i], y[i], labels[i], **lprops) - if values is None: - return #('No values to plot specified!') - if np.size(values) != np.size(sensors,1): - return('Numer of values to plot is different from number of sensors!'+ + return # ('No values to plot specified!') + if np.size(values) != np.size(sensors, 1): + return('Numer of values to plot is different from number of sensors!' + '\nNo values have been plotted!') # set the values z = values # resolution determines the number of interpolated points per unit - nx = round(resolution*plot_radius) - ny = round(resolution*plot_radius) + nx = round(resolution * plot_radius) + ny = round(resolution * plot_radius) # now set up the grid: - xi, yi = np.meshgrid(np.linspace(-plot_radius, plot_radius,nx), - np.linspace(-plot_radius, plot_radius,ny)) + xi, yi = np.meshgrid(np.linspace(-plot_radius, plot_radius, nx), + np.linspace(-plot_radius, plot_radius, ny)) # and move the center to coincide with the center of the head: xi = xi + center[0] yi = yi + center[1] # interploate points: - if plot_mask=='linear': + if plot_mask == 'linear': # masked = True means that no extrapolation outside the # electrode boundaries is made this effectively creates a mask # with a linear boundary (connecting the outer electrode @@ -240,11 +240,11 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, #zi = griddata(x,y,z,xi,yi,masked=True) #zi = griddata(x,y,z,xi,yi) pass - elif plot_mask=='circular': - npts = np.mean((nx,ny))*2 - t = np.linspace(0, 2*np.pi,npts)[:-1] - x = np.r_[x, np.cos(t)*plot_radius] - y = np.r_[y, np.sin(t)*plot_radius] + elif plot_mask == 'circular': + npts = np.mean((nx, ny)) * 2 + t = np.linspace(0, 2 * np.pi, npts)[:-1] + x = np.r_[x, np.cos(t) * plot_radius] + y = np.r_[y, np.sin(t) * plot_radius] z = np.r_[z, np.zeros(len(t))] else: # we need a custom mask: @@ -272,16 +272,15 @@ def topoplot(values=None, labels=None, sensors=None, axes=None, # # If no colormap is specified, use default colormap: # if cmap is None: # cmap = plt.get_cmap() - + # make contours cprops = default_contour_props.copy() if not contour_props is None: cprops.update(contour_props) - + if np.any(cprops['linewidths'] > 0): a.contour(xi, yi, zi, contours, **cprops) # make countour color patches: # a.contourf(xi, yi, zi, contours, cmap=cmap, extend='both') a.contourf(xi, yi, zi, contours, extend='both', **kwargs) - diff --git a/ptsa/sandbox.py b/ptsa/sandbox.py index b077604..6f92c4d 100644 --- a/ptsa/sandbox.py +++ b/ptsa/sandbox.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -7,54 +7,62 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -# BaseDict code from: +# BaseDict code from: # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473790 # import cPickle # import moved to top of file + class BaseDict(dict): ''' A dict allows inputting data as adict.xxx as well as adict['xxx'] In python obj: - + obj.var=x ---> 'var' be in obj.__dict__ (this is python default) In python dict: - + dict['var']=x ---> 'var' be in dict.items(this is dict behavior) In BaseDict: let bd=BaseDict() - + Both bd.var and bd['var'] will save x to bd.items and bd.setDict('var', x) will save x to bd.__dict__ This allows an easier access of the variables. - + ''' + def __init__(self, data=None): - if data: dict.__init__(self, data) - else: dict.__init__(self) + if data: + dict.__init__(self, data) + else: + dict.__init__(self) dic = self.__dict__ - dic['__ver__'] ='20041208_1' - dic['__author__']='Runsun Pan' - + dic['__ver__'] = '20041208_1' + dic['__author__'] = 'Runsun Pan' + def __setattr__(self, name, val): - if name in self.__dict__: self.__dict__[name]= val - else: self[name] = val - + if name in self.__dict__: + self.__dict__[name] = val + else: + self[name] = val + def __getattr__(self, name): - if name in self.__dict__: return self.__dict__[name] - else: return self[name] - - def setDict(self, name, val): + if name in self.__dict__: + return self.__dict__[name] + else: + return self[name] + + def setDict(self, name, val): ''' setDict(name, val): Assign *val* to the key *name* of __dict__. - + :Usage: - + >>> bd = BaseDict() >>> bd.getDict()['height'] Traceback (most recent call last): @@ -67,14 +75,14 @@ def setDict(self, name, val): ''' self.__dict__[name] = val - return self + return self - def getDict(self): + def getDict(self): ''' Return the internal __dict__. - + :Usage: - + >>> bd = BaseDict() >>> bd.getDict()['height'] Traceback (most recent call last): @@ -86,14 +94,14 @@ def getDict(self): 160 ''' return self.__dict__ - - def setItem(self, name, val): + + def setItem(self, name, val): ''' Set the value of dict key *name* to *val*. Note this dict is not the __dict__. :Usage: - + >>> bd = BaseDict() >>> bd {} @@ -105,21 +113,21 @@ def setItem(self, name, val): ''' self[name] = val return self - - def __getstate__(self): + + def __getstate__(self): ''' Needed for cPickle in .copy() ''' - return self.__dict__.copy() + return self.__dict__.copy() - def __setstate__(self,dict): + def __setstate__(self, dict): ''' Needed for cPickle in .copy() ''' - self.__dict__.update(dict) + self.__dict__.update(dict) - def copy(self): + def copy(self): ''' Return a copy. - + :Usage: - + >>> bd = BaseDict() >>> bd['name']=[1,2,3] >>> bd @@ -141,44 +149,43 @@ def copy(self): {'name': [1, 2, 3]} >>> bd2 {'name': ['aa', 2, 3], 'height': 60} - + ''' return cPickle.loads(cPickle.dumps(self)) - class DataDict(BaseDict): """ Dictionary where you can access the values as attributes, but with added features for manipulating the data inside. """ - def removeBuffer(self,fields,axis=-1): - """Use the information contained in the data dictionary to remove the - buffer from the specified fields and reset the time range. If - bufLen is 0, no action is performed.""" - # see if remove the anything - if self.bufLen>0: - # make sure it's a list - fields = N.asarray(fields) - if len(fields.shape)==0: - fields = [fields] - for field in fields: - # remove the buffer - self[field] = self[field].take(range(self.bufLen, - self[field].shape[axis]-self.bufLen), - axis) - # set the time range with no buffer - self.time = N.linspace(self.OffsetMS, - self.OffsetMS+self.DurationMS, - self[fields[0]].shape[axis]) - # reset buffer to indicate it was removed - self.bufLen = 0 + def removeBuffer(self, fields, axis=-1): + """Use the information contained in the data dictionary to remove the + buffer from the specified fields and reset the time range. If + bufLen is 0, no action is performed.""" + # see if remove the anything + if self.bufLen > 0: + # make sure it's a list + fields = N.asarray(fields) + if len(fields.shape) == 0: + fields = [fields] + for field in fields: + # remove the buffer + self[field] = self[field].take(list(range(self.bufLen, + self[field].shape[axis] - self.bufLen)), + axis) + # set the time range with no buffer + self.time = N.linspace(self.OffsetMS, + self.OffsetMS + self.DurationMS, + self[fields[0]].shape[axis]) + # reset buffer to indicate it was removed + self.bufLen = 0 class InfoArray(N.ndarray): def __new__(subtype, data, info=None, dtype=None, copy=True): # When data is an InfoArray if isinstance(data, InfoArray): - if not copy and dtype==data.dtype: + if not copy and dtype == data.dtype: return data.view(subtype) else: return data.astype(dtype).view(subtype) @@ -186,7 +193,7 @@ def __new__(subtype, data, info=None, dtype=None, copy=True): subtype.info = subtype._info return N.array(data).view(subtype) - def __array_finalize__(self,obj): + def __array_finalize__(self, obj): if hasattr(obj, "info"): # The object already has an info tag: just use it self.info = obj.info @@ -195,11 +202,12 @@ def __array_finalize__(self,obj): self.info = self._info def __repr__(self): - desc="""\ + desc = """\ array(data= %(data)s, tag=%(tag)s)""" - return desc % {'data': str(self), 'tag':self.info } + return desc % {'data': str(self), 'tag': self.info} + class EegTimeSeries_old(object): """ @@ -207,7 +215,8 @@ class EegTimeSeries_old(object): Keeps track of time dimension, samplerate, buffer, offset, duration, time. """ - def __init__(self,data,samplerate,tdim=-1,offsetMS=None,offset=None,bufferMS=None,buffer=None): + + def __init__(self, data, samplerate, tdim=-1, offsetMS=None, offset=None, bufferMS=None, buffer=None): """ """ # set the initial values @@ -216,7 +225,7 @@ def __init__(self,data,samplerate,tdim=-1,offsetMS=None,offset=None,bufferMS=Non self.dtype = data.dtype self.shape = data.shape self.ndim = len(data.shape) - + # get the time dimension if tdim >= 0: # use it @@ -230,49 +239,51 @@ def __init__(self,data,samplerate,tdim=-1,offsetMS=None,offset=None,bufferMS=Non if not offset is None: # set offsetMS from offset self.offset = offset - self.offsetMS = float(offset)*1000./self.samplerate + self.offsetMS = float(offset) * 1000. / self.samplerate elif not offsetMS is None: # set the offset from the MS self.offsetMS = offsetMS - self.offset = int(N.round(float(offsetMS)*self.samplerate/1000.)) + self.offset = int( + N.round(float(offsetMS) * self.samplerate / 1000.)) else: # default to no offset self.offset = 0 self.offsetMS = 0 - + # set the buffer if not buffer is None: # set bufferMS from buffer self.buffer = buffer - self.bufferMS = float(buffer)*1000./self.samplerate + self.bufferMS = float(buffer) * 1000. / self.samplerate elif not bufferMS is None: # set the buffer from the MS self.bufferMS = bufferMS - self.buffer = int(N.round(float(bufferMS)*self.samplerate/1000.)) + self.buffer = int( + N.round(float(bufferMS) * self.samplerate / 1000.)) else: # default to no buffer self.buffer = 0 self.bufferMS = 0 # set the duration (does not include the buffer) - self.durationMS = (self.shape[self.tdim]-2*self.buffer)*1000./self.samplerate + self.durationMS = (self.shape[self.tdim] - + 2 * self.buffer) * 1000. / self.samplerate # set the time range - self.trangeMS = N.linspace(self.offsetMS-self.bufferMS, - self.offsetMS+self.durationMS+self.bufferMS, + self.trangeMS = N.linspace(self.offsetMS - self.bufferMS, + self.offsetMS + self.durationMS + self.bufferMS, self.shape[self.tdim]) - def __getitem__(self, item): """ :Parameters: item : ``slice`` The slice of the data to take. - + :Returns: ``numpy.ndarray`` """ return self.data[item] - + def __setitem__(self, item, value): """ :Parameters: @@ -280,55 +291,57 @@ def __setitem__(self, item, value): The slice of the data to write to value : A single value or array of type ``self.dtype`` The value to be set. - + :Returns: ``None`` """ self.data[item] = value def removeBuffer(self): - """Use the information contained in the time series to remove the - buffer reset the time range. If buffer is 0, no action is - performed.""" - # see if remove the anything - if self.buffer>0: + """Use the information contained in the time series to remove the + buffer reset the time range. If buffer is 0, no action is + performed.""" + # see if remove the anything + if self.buffer > 0: # remove the buffer - self.data = self.data.take(range(self.buffer, - self.shape[self.tdim]-self.buffer),self.tdim) + self.data = self.data.take(list(range(self.buffer, + self.shape[self.tdim] - self.buffer)), self.tdim) # reset buffer to indicate it was removed - self.buffer = 0 + self.buffer = 0 self.bufferMS = 0 # reset the shape self.shape = self.data.shape - # set the time range with no buffer - self.trangeMS = N.linspace(self.offsetMS-self.bufferMS, - self.offsetMS+self.durationMS+self.bufferMS, + # set the time range with no buffer + self.trangeMS = N.linspace(self.offsetMS - self.bufferMS, + self.offsetMS + self.durationMS + self.bufferMS, self.shape[self.tdim]) - - - def filter(self,freqRange,filtType='stop',order=4): + def filter(self, freqRange, filtType='stop', order=4): """ Filter the data using a Butterworth filter. """ - self.data = filter.buttfilt(self.data,freqRange,self.samplerate,filtType, - order,axis=self.tdim) + self.data = filter.buttfilt(self.data, freqRange, self.samplerate, filtType, + order, axis=self.tdim) - def resample(self,resampledRate,window=None): + def resample(self, resampledRate, window=None): """ Resample the data and reset all the time ranges. Uses the resample function from scipy. This method seems to be more accurate than the decimate method. """ # resample the data - newLength = N.fix(self.data.shape[self.tdim]*resampledRate/float(self.samplerate)) - self.data = resample(self.data,newLength,axis=self.tdim,window=window) + newLength = N.fix( + self.data.shape[self.tdim] * resampledRate / float(self.samplerate)) + self.data = resample(self.data, newLength, + axis=self.tdim, window=window) # set the new offset and buffer lengths - self.buffer = int(N.round(float(self.buffer)*resampledRate/float(self.samplerate))) - self.offset = int(N.round(float(self.offset)*resampledRate/float(self.samplerate))) + self.buffer = int(N.round(float(self.buffer) * + resampledRate / float(self.samplerate))) + self.offset = int(N.round(float(self.offset) * + resampledRate / float(self.samplerate))) # set the new samplerate self.samplerate = resampledRate @@ -337,25 +350,27 @@ def resample(self,resampledRate,window=None): self.shape = self.data.shape # set the time range with no buffer - self.trangeMS = N.linspace(self.offsetMS-self.bufferMS, - self.offsetMS+self.durationMS+self.bufferMS, + self.trangeMS = N.linspace(self.offsetMS - self.bufferMS, + self.offsetMS + self.durationMS + self.bufferMS, self.shape[self.tdim]) - - def decimate(self,resampledRate, order=None, ftype='iir'): + def decimate(self, resampledRate, order=None, ftype='iir'): """ Decimate the data and reset the time ranges. """ # set the downfact - downfact = int(N.round(float(self.samplerate)/resampledRate)) + downfact = int(N.round(float(self.samplerate) / resampledRate)) # do the decimation - self.data = filter.decimate(self.data,downfact, n=order, ftype=ftype, axis=self.tdim) + self.data = filter.decimate( + self.data, downfact, n=order, ftype=ftype, axis=self.tdim) # set the new offset and buffer lengths - self.buffer = int(N.round(float(self.buffer)*resampledRate/float(self.samplerate))) - self.offset = int(N.round(float(self.offset)*resampledRate/float(self.samplerate))) + self.buffer = int(N.round(float(self.buffer) * + resampledRate / float(self.samplerate))) + self.offset = int(N.round(float(self.offset) * + resampledRate / float(self.samplerate))) # set the new samplerate self.samplerate = resampledRate @@ -364,10 +379,9 @@ def decimate(self,resampledRate, order=None, ftype='iir'): self.shape = self.data.shape # set the time range with no buffer - self.trangeMS = N.linspace(self.offsetMS-self.bufferMS, - self.offsetMS+self.durationMS+self.bufferMS, + self.trangeMS = N.linspace(self.offsetMS - self.bufferMS, + self.offsetMS + self.durationMS + self.bufferMS, self.shape[self.tdim]) - # class EventRecord(N.record): diff --git a/ptsa/stats/__init__.py b/ptsa/stats/__init__.py index 34f151c..d7dccdb 100644 --- a/ptsa/stats/__init__.py +++ b/ptsa/stats/__init__.py @@ -1,10 +1,8 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## - - diff --git a/ptsa/stats/cluster.py b/ptsa/stats/cluster.py index b4b2b93..3d58a0e 100644 --- a/ptsa/stats/cluster.py +++ b/ptsa/stats/cluster.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -15,6 +15,8 @@ from ptsa.helper import pol2cart # some functions from MNE + + def _get_components(x_in, connectivity): """get connected components from a mask and a connectivity matrix""" cs_graph_components = sparse.cs_graph_components @@ -78,7 +80,7 @@ def find_clusters(x, threshold, tail=0, connectivity=None): if x.ndim == 1: clusters = ndimage.find_objects(labels, n_labels) sums = ndimage.measurements.sum(x, labels, - index=range(1, n_labels + 1)) + index=list(range(1, n_labels + 1))) else: clusters = list() sums = np.empty(n_labels) @@ -167,13 +169,14 @@ def sparse_dim_connectivity(dim_con): return cmat + def simple_neighbors_1d(n): """ Return connectivity for simple 1D neighbors. """ - c = np.zeros((n,n)) - c[np.triu_indices(n,1)] = 1 - c[np.triu_indices(n,2)] = 0 + c = np.zeros((n, n)) + c[np.triu_indices(n, 1)] = 1 + c[np.triu_indices(n, 2)] = 0 return c @@ -186,28 +189,28 @@ def sensor_neighbors(sensor_locs): sensor locs. """ # see if loading from file - if isinstance(sensor_locs,str): + if isinstance(sensor_locs, str): # load from file locs = np.loadtxt(sensor_locs) theta = -locs[0] + 90 radius = locs[1] - x,y = pol2cart(theta,radius,radians=False) - sensor_locs = np.vstack((x,y)).T + x, y = pol2cart(theta, radius, radians=False) + sensor_locs = np.vstack((x, y)).T # get info about the sensors nsens = len(sensor_locs) - + # do the triangulation d = spatial.Delaunay(sensor_locs) # determine the neighbors - n = [np.unique(d.vertices[np.nonzero(d.vertices==i)[0]]) + n = [np.unique(d.vertices[np.nonzero(d.vertices == i)[0]]) for i in range(nsens)] # make the symmetric connectivity matrix - cn = np.zeros((nsens,nsens)) + cn = np.zeros((nsens, nsens)) for r in range(nsens): - cn[r,n[r]] = 1 + cn[r, n[r]] = 1 # only keep the upper cn[np.tril_indices(nsens)] = 0 @@ -216,7 +219,7 @@ def sensor_neighbors(sensor_locs): return cn -def tfce(x, dt=.1, E=2/3., H=2.0, tail=0, connectivity=None): +def tfce(x, dt=.1, E=2 / 3., H=2.0, tail=0, connectivity=None): """ Threshold-Free Cluster Enhancement. """ @@ -232,26 +235,26 @@ def tfce(x, dt=.1, E=2/3., H=2.0, tail=0, connectivity=None): if tail == -1: sign = -1.0 if (x < 0).sum() > 0: - trange = np.arange(x[x < 0].max(), x.min()-dt, -dt) + trange = np.arange(x[x < 0].max(), x.min() - dt, -dt) elif tail == 1: sign = 1.0 - if (x > 0).sum()>0: - trange = np.arange(x[x > 0].min(), x.max()+dt, dt) + if (x > 0).sum() > 0: + trange = np.arange(x[x > 0].min(), x.max() + dt, dt) else: sign = 1.0 - trange = np.arange(np.abs(x).min(), np.abs(x).max()+dt, dt) + trange = np.arange(np.abs(x).min(), np.abs(x).max() + dt, dt) # make own connectivity if not provided so that we have consistent # return values if connectivity is None: xr = x - #connectivity = sparse_dim_connectivity([simple_neighbors_1d(n) + # connectivity = sparse_dim_connectivity([simple_neighbors_1d(n) # for n in x.shape]) else: # integrate in steps of dt over the threshold # do reshaping once xr = x.reshape(np.prod(x.shape)) - + # get starting values for data (reshaped if needed) xt = np.zeros_like(xr) for thresh in trange: @@ -263,7 +266,8 @@ def tfce(x, dt=.1, E=2/3., H=2.0, tail=0, connectivity=None): # add to values in clusters for c in clusts: # take into account direction of test - xt[c] += sign * np.power(c.sum(),E) * np.power(sign*thresh,H) * dt + xt[c] += sign * np.power(c.sum(), E) * \ + np.power(sign * thresh, H) * dt # return the enhanced data, reshaped back if connectivity is None: diff --git a/ptsa/stats/lmer.py b/ptsa/stats/lmer.py index 8525f27..0183689 100644 --- a/ptsa/stats/lmer.py +++ b/ptsa/stats/lmer.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -17,7 +17,7 @@ from scipy.linalg import diagsvd from scipy.stats import rankdata -from joblib import Parallel,delayed +from joblib import Parallel, delayed # Connect to an R session import rpy2.robjects @@ -28,24 +28,24 @@ from rpy2.robjects import Formula, FactorVector from rpy2.robjects.environments import Environment from rpy2.robjects.vectors import DataFrame, Vector, FloatVector -from rpy2.rinterface import MissingArg,SexpVector +from rpy2.rinterface import MissingArg, SexpVector # Make it so we can send numpy arrays to R import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() # load some required packages -# PBS: Eventually we should try/except these to get people +# PBS: Eventually we should try/except these to get people # to install missing packages lme4 = importr('lme4') rstats = importr('stats') fdrtool = importr('fdrtool') ssvd = importr('ssvd') -if hasattr(lme4,'coef'): - r_coef = lme4.coef +if hasattr(lme4, 'coef'): + r_coef = lme4.coef else: r_coef = rstats.coef -if hasattr(lme4,'model_matrix'): +if hasattr(lme4, 'model_matrix'): r_model_matrix = lme4.model_matrix else: r_model_matrix = rstats.model_matrix @@ -54,15 +54,15 @@ #import pandas.rpy.common as com # load ptsa clustering -import cluster +from . import cluster -def lmer_feature(formula_str, dat, perms=None, +def lmer_feature(formula_str, dat, perms=None, val=None, factors=None, **kwargs): """ Run LMER on a number of permutations of the predicted data. - + """ # get the perm_var perm_var = formula_str.split('~')[0].strip() @@ -76,13 +76,12 @@ def lmer_feature(formula_str, dat, perms=None, factors = [] # convert the recarray to a DataFrame - rdf = DataFrame({k:(FactorVector(dat[k]) - if (k in factors) or isinstance(dat[k][0],str) - else dat[k]) + rdf = DataFrame({k: (FactorVector(dat[k]) + if (k in factors) or isinstance(dat[k][0], str) + else dat[k]) for k in dat.dtype.names}) - + #rdf = com.convert_to_r_dataframe(pd.DataFrame(dat),strings_as_factors=True) - # get the column index col_ind = list(rdf.colnames).index(perm_var) @@ -97,17 +96,17 @@ def lmer_feature(formula_str, dat, perms=None, # run on each permutation tvals = None - for i,perm in enumerate(perms): + for i, perm in enumerate(perms): if not perm is None: # set the perm - rdf[col_ind] = rdf[col_ind].rx(perm+1) + rdf[col_ind] = rdf[col_ind].rx(perm + 1) # inside try block to catch convergence errors try: ms = lme4.lmer(rformula, data=rdf, **kwargs) except: continue - #tvals.append(np.array([np.nan])) + # tvals.append(np.array([np.nan])) # extract the result df = r['data.frame'](r_coef(r['summary'](ms))) @@ -115,13 +114,14 @@ def lmer_feature(formula_str, dat, perms=None, # init the data # get the row names rows = list(r['row.names'](df)) - tvals = np.rec.fromarrays([np.ones(len(perms))*np.nan + tvals = np.rec.fromarrays([np.ones(len(perms)) * np.nan for ro in range(len(rows))], names=','.join(rows)) tvals[i] = tuple(df.rx2('t.value')) return tvals + class LMER(): """ Wrapper for the lmer method provided by lme4 in R. @@ -130,7 +130,8 @@ class LMER(): and extracting the associated t-stat. """ - def __init__(self, formula_str, df, factors=None, + + def __init__(self, formula_str, df, factors=None, resid_formula_str=None, **lmer_opts): """ """ @@ -140,32 +141,31 @@ def __init__(self, formula_str, df, factors=None, # add column if necessary if not pred_var in df.dtype.names: # must add it - df = append_fields(df, pred_var, [0.0]*len(df), usemask=False) + df = append_fields(df, pred_var, [0.0] * len(df), usemask=False) # make factor list if necessary if factors is None: factors = {} # add in missingarg for any potential factor not provided for k in df.dtype.names: - if isinstance(df[k][0],str) and not factors.has_key(k): + if isinstance(df[k][0], str) and k not in factors: factors[k] = MissingArg - + for f in factors: if factors[f] is None: factors[f] = MissingArg # checking for both types of R Vectors for rpy2 variations - elif (not isinstance(factors[f],Vector) and + elif (not isinstance(factors[f], Vector) and not factors[f] == MissingArg): factors[f] = Vector(factors[f]) # convert the recarray to a DataFrame (releveling if desired) - self._rdf = DataFrame({k:(FactorVector(df[k], levels=factors[k]) - if (k in factors) or isinstance(df[k][0],str) - else df[k]) + self._rdf = DataFrame({k: (FactorVector(df[k], levels=factors[k]) + if (k in factors) or isinstance(df[k][0], str) + else df[k]) for k in df.dtype.names}) #self._rdf = com.convert_to_r_dataframe(pd.DataFrame(df),strings_as_factors=True) - # get the column index self._col_ind = list(self._rdf.colnames).index(pred_var) @@ -198,24 +198,26 @@ def run(self, vals=None, perms=None): # run on each permutation tvals = None log_likes = None - for i,perm in enumerate(perms): + for i, perm in enumerate(perms): if not perm is None: # set the perm - self._rdf[self._col_ind] = self._rdf[self._col_ind].rx(perm+1) + self._rdf[self._col_ind] = self._rdf[self._col_ind].rx( + perm + 1) # inside try block to catch convergence errors try: if self._rformula_resid: # get resid first - msr = lme4.lmer(self._rformula_resid, data=self._rdf, + msr = lme4.lmer(self._rformula_resid, data=self._rdf, **self._lmer_opts) self._rdf[self._col_ind] = lme4.resid(msr) # run the model (possibly on the residuals from above) - ms = lme4.lmer(self._rformula, data=self._rdf, **self._lmer_opts) + ms = lme4.lmer(self._rformula, data=self._rdf, + **self._lmer_opts) except: continue - #tvals.append(np.array([np.nan])) - + # tvals.append(np.array([np.nan])) + # save the model if self._ms is None: self._ms = ms @@ -228,7 +230,7 @@ def run(self, vals=None, perms=None): # init the data # get the row names rows = list(r['row.names'](df)) - tvals = np.rec.fromarrays([np.ones(len(perms))*np.nan + tvals = np.rec.fromarrays([np.ones(len(perms)) * np.nan for ro in range(len(rows))], names=','.join(rows)) log_likes = np.zeros(len(perms)) @@ -240,8 +242,6 @@ def run(self, vals=None, perms=None): return tvals, log_likes - - """LMER-PLS/MELD Notes We can optionally have this run on rank data, too, so that it's @@ -271,78 +271,84 @@ def run(self, vals=None, perms=None): """ + def procrustes(orig_lv, boot_lv): # define coordinate space between orignal and bootstrap LVs - temp=np.dot(orig_lv,boot_lv.T) + temp = np.dot(orig_lv, boot_lv.T) # orthogonalze space - U,s,V = np.linalg.svd(temp) + U, s, V = np.linalg.svd(temp) # determine procrustean transform - #rotatemat=U*V' - return np.dot(U,V) + # rotatemat=U*V' + return np.dot(U, V) + def pick_stable_features(R, shape, nboot=500, connectivity=None): # generate the boots - boots = [np.random.random_integers(0,len(R)-1,len(R)) - for i in xrange(nboot)] + boots = [np.random.random_integers(0, len(R) - 1, len(R)) + for i in range(nboot)] # run tfce on each subj and cond if True: - Rt = np.zeros([R.shape[0],R.shape[1]]+list(shape)) + Rt = np.zeros([R.shape[0], R.shape[1]] + list(shape)) for i in range(Rt.shape[0]): for j in range(Rt.shape[1]): - Rt[i,j] = cluster.tfce(np.arctanh(R[i,j]).reshape(*shape), - dt=.05,tail=1,connectivity=connectivity, - E=1.0,H=2.0) - Rt[i,j] += cluster.tfce(np.arctanh(R[i,j]).reshape(*shape), - dt=.05,tail=-1,connectivity=connectivity, - E=1.0,H=2.0) + Rt[i, j] = cluster.tfce(np.arctanh(R[i, j]).reshape(*shape), + dt=.05, tail=1, connectivity=connectivity, + E=1.0, H=2.0) + Rt[i, j] += cluster.tfce(np.arctanh(R[i, j]).reshape(*shape), + dt=.05, tail=-1, connectivity=connectivity, + E=1.0, H=2.0) else: # convert to Z Rt = np.arctanh(R) # calc bootstrap ratio Rtb = np.array([Rt[boots[b]].mean(0) for b in range(len(boots))]) - Rtbr = Rt.mean(0)/Rtb.std(0) + Rtbr = Rt.mean(0) / Rtb.std(0) return Rtbr + def sparse_stable_svd(R, nboot=50): # generate the boots - boots = [np.random.random_integers(0,len(R)-1,len(R)) - for i in xrange(nboot)] + boots = [np.random.random_integers(0, len(R) - 1, len(R)) + for i in range(nboot)] # calc the original SVD U, s, Vh = np.linalg.svd(np.concatenate(R), full_matrices=False) - + # do the boots rVs = [] for i in range(len(boots)): - Ub, sb, Vhb = np.linalg.svd(np.concatenate(R[boots[i]]), full_matrices=False) + Ub, sb, Vhb = np.linalg.svd( + np.concatenate(R[boots[i]]), full_matrices=False) + + rmat = procrustes(U, Ub) - rmat = procrustes(U,Ub) + rVs.append(np.dot(rmat, np.dot(diagsvd(sb, len(sb), len(sb)), Vhb))) - rVs.append(np.dot(rmat,np.dot(diagsvd(sb,len(sb),len(sb)),Vhb))) - # get the bootstrap ratios rVs = np.array(rVs) - Vs = np.dot(diagsvd(s,len(s),len(s)),Vh) - boot_ratio = Vs/rVs.std(0) - + Vs = np.dot(diagsvd(s, len(s), len(s)), Vh) + boot_ratio = Vs / rVs.std(0) + # pass the boot ratios through fdrtool to pick stable features - fachist = np.histogram(boot_ratio.flatten(),bins=500) - peak = fachist[1][fachist[0]==np.max(fachist[0])][0] - results = fdrtool.fdrtool(FloatVector(boot_ratio.flatten()-peak), statistic='normal', + fachist = np.histogram(boot_ratio.flatten(), bins=500) + peak = fachist[1][fachist[0] == np.max(fachist[0])][0] + results = fdrtool.fdrtool(FloatVector(boot_ratio.flatten() - peak), statistic='normal', plot=False, verbose=False) qv = np.array(results.rx('qval')).reshape(boot_ratio.shape) #qv = None # apply the thresh - return U,s,Vh,qv,boot_ratio + return U, s, Vh, qv, boot_ratio + # global container so that we can use joblib with a smaller memory # footprint (i.e., we don't need to duplicate everything) _global_meld = {} + def _eval_model(model_id, perm=None, boot=None): # arg check if not perm is None and not boot is None: @@ -357,14 +363,14 @@ def _eval_model(model_id, perm=None, boot=None): R = [] # set the boot shuffle - if boot is None: + if boot is None: ind_b = np.arange(len(mm._groups)) else: ind_b = boot # loop over group vars ind = {} - for i,k in enumerate(mm._groups[ind_b]): + for i, k in enumerate(mm._groups[ind_b]): # grab the A and M A = mm._A[k] M = mm._M[k] @@ -373,14 +379,14 @@ def _eval_model(model_id, perm=None, boot=None): if perm is None: ind[k] = np.arange(len(A)) else: - ind[k] = perm[k] + ind[k] = perm[k] if perm is None and not _R is None: # reuse R we already calculated R.append(mm._R[ind_b[i]]) else: # calc the correlation - R.append(np.inner(A.T,M[ind[k]].T)) + R.append(np.inner(A.T, M[ind[k]].T)) # save R before concatenating if not a permutation if perm is None and boot is None and _R is None: @@ -401,14 +407,14 @@ def _eval_model(model_id, perm=None, boot=None): R[feat_mask] = 0.0 # pick only stable features - Rtbr = pick_stable_features(R_nocat, mm._feat_shape, connectivity = None) + Rtbr = pick_stable_features(R_nocat, mm._feat_shape, connectivity=None) # conjunction across conditions #stable_ind = ((np.abs(Rtbr)>2.575).sum(0)>0).flatten() - stable_ind = np.abs(Rtbr)>2.575 - stable_ind = stable_ind.reshape((stable_ind.shape[0],-1)) + stable_ind = np.abs(Rtbr) > 2.575 + stable_ind = stable_ind.reshape((stable_ind.shape[0], -1)) # zero out non-stable - R_nocat[:,~stable_ind] = 0.0 + R_nocat[:, ~stable_ind] = 0.0 R = np.concatenate([R_nocat[i] for i in range(len(R_nocat))]) # perform SVD @@ -421,13 +427,13 @@ def _eval_model(model_id, perm=None, boot=None): U, s, Vh, qv, boot_ratio = sparse_stable_svd(R_nocat) ssV_full = np.sqrt((Vh**2).sum(1)) #Vh[np.abs(boot_ratio)<2.95] = 0.0 - qvt_br = qv[np.abs(boot_ratio) > + qvt_br = qv[np.abs(boot_ratio) > np.percentile(np.abs(boot_ratio).flat, 99.5)].max() - qvt = max(qv.min()+.01, .5) - qvt = min(qvt,qvt_br) - Vh[qv>qvt] = 0.0 + qvt = max(qv.min() + .01, .5) + qvt = min(qvt, qvt_br) + Vh[qv > qvt] = 0.0 ssV_sparse = np.sqrt((Vh**2).sum(1)) - s[s>0] *= ssV_sparse[s>0]/ssV_full[s>0] + s[s > 0] *= ssV_sparse[s > 0] / ssV_full[s > 0] #s[np.abs(Vh).sum(1)==0.0] = 0.0 else: U, s, Vh = np.linalg.svd(R, full_matrices=False) @@ -448,11 +454,11 @@ def _eval_model(model_id, perm=None, boot=None): O = [mm._O[i].copy() for i in ind_b] if not boot is None: # replace the group - for i,k in enumerate(mm._groups): + for i, k in enumerate(mm._groups): O[i][mm._re_group] = k lmer = LMER(mm._formula_str, np.concatenate(O), - factors=mm._factors, + factors=mm._factors, resid_formula_str=mm._resid_formula_str, **mm._lmer_opts) mer = None else: @@ -462,13 +468,13 @@ def _eval_model(model_id, perm=None, boot=None): run_null = True if mm._mer_null is None: lmer_null = LMER(mm._null_formula_str, np.concatenate(O), - factors=mm._factors, + factors=mm._factors, resid_formula_str=mm._resid_formula_str, **mm._lmer_opts) mer_null is None else: run_null = False mer_null = mm._mer_null - + # loop over LVs performing LMER #Dw = [] res = [] @@ -477,12 +483,12 @@ def _eval_model(model_id, perm=None, boot=None): ll_full_boot = [] for i in range(len(Vh)): if ss[i] <= 0.0: - #print 'skipped ',str(i) + # print 'skipped ',str(i) continue # flatten then weigh features via dot product - Dw = np.concatenate([np.dot(mm._D[k][ind[k]],Vh[i]) - for g,k in enumerate(mm._groups[ind_b])]) + Dw = np.concatenate([np.dot(mm._D[k][ind[k]], Vh[i]) + for g, k in enumerate(mm._groups[ind_b])]) # run the main model if mer is None: @@ -495,7 +501,7 @@ def _eval_model(model_id, perm=None, boot=None): new_tvals = np.rec.fromarrays([[tv] for tv in tuple(df.rx2('t.value'))], names=','.join(rows)) new_ll = float(r['logLik'](mer)[0]) - res.append((new_tvals,np.array([new_ll]))) + res.append((new_tvals, np.array([new_ll]))) # see if run null model if perm is None and boot is None and mm._null_formula_str: @@ -504,15 +510,17 @@ def _eval_model(model_id, perm=None, boot=None): m_full = lmer._ms ll_null = [] ll_full = [] - for bi in xrange(mm._num_null_boot): + for bi in range(mm._num_null_boot): # sim new data #Dw_sim = np.array(r['simulate'](m_null))[0] Dw_sim = r['simulate'](m_null) # run full and null models, saving the loglike - #ll_null.append(lmer_null.run(vals=Dw_sim)[1]) - #ll_full.append(lmer.run(vals=Dw_sim)[1]) - ll_null.append(float(r['logLik'](r['refit'](m_null,Dw_sim))[0])) - ll_full.append(float(r['logLik'](r['refit'](m_full,Dw_sim))[0])) + # ll_null.append(lmer_null.run(vals=Dw_sim)[1]) + # ll_full.append(lmer.run(vals=Dw_sim)[1]) + ll_null.append( + float(r['logLik'](r['refit'](m_null, Dw_sim))[0])) + ll_full.append( + float(r['logLik'](r['refit'](m_full, Dw_sim))[0])) # append to full list ll_null_boot.append(ll_null) ll_full_boot.append(ll_full) @@ -523,49 +531,52 @@ def _eval_model(model_id, perm=None, boot=None): O = [mm._O[i].copy() for i in ind_b] if not boot is None: # replace the group - for i,k in enumerate(mm._groups): + for i, k in enumerate(mm._groups): O[i][mm._re_group] = k lmer = LMER(mm._formula_str, np.concatenate(O), - factors=mm._factors, + factors=mm._factors, resid_formula_str=mm._resid_formula_str, **mm._lmer_opts) Dw = np.random.randn(len(np.concatenate(O))) - temp_t,temp_ll = lmer.run(vals=Dw) + temp_t, temp_ll = lmer.run(vals=Dw) - for n in temp_t.dtype.names: temp_t[n] = 0.0 + for n in temp_t.dtype.names: + temp_t[n] = 0.0 temp_ll[0] = 0.0 - res.append((temp_t,temp_ll)) + res.append((temp_t, temp_ll)) - tvals,log_likes = zip(*res) + tvals, log_likes = list(zip(*res)) tvals = np.concatenate(tvals) log_likes = np.concatenate(log_likes) if perm is None and boot is None and mm._null_formula_str: - tvals_null,log_likes_null = zip(*res_null) + tvals_null, log_likes_null = list(zip(*res_null)) log_likes_null = np.concatenate(log_likes_null) - print "Like Ratio: ", 2*(log_likes.sum() - log_likes_null.sum()) - print "Like Ratio Scaled: ", 2*(np.dot(log_likes,ss[ss>0.0]/_ss) - - np.dot(log_likes_null,ss[ss>0.0]/_ss)) - print "Like Ratio Comb: ", np.dot(2*(log_likes-log_likes_null),ss[ss>0.0]/_ss) + print("Like Ratio: ", 2 * (log_likes.sum() - log_likes_null.sum())) + print("Like Ratio Scaled: ", 2 * (np.dot(log_likes, ss[ss > 0.0] / _ss) - + np.dot(log_likes_null, ss[ss > 0.0] / _ss))) + print("Like Ratio Comb: ", np.dot( + 2 * (log_likes - log_likes_null), ss[ss > 0.0] / _ss)) ll_null_boot = np.vstack(ll_null_boot) ll_full_boot = np.vstack(ll_full_boot) - lr = 2*(log_likes-log_likes_null) - lr_boot = 2*(ll_full_boot-ll_null_boot) - print "Like Boot: ", np.dot(lr_boot, ss[ss>0.0]/_ss) - sscale = ss/ss.sum() - mg = np.dot([((lr_boot[:,ii]-lr[ii])>0).sum() for ii in range(20)],sscale) - print "prop: ", mg/float(mm._num_null_boot) - 1/0 + lr = 2 * (log_likes - log_likes_null) + lr_boot = 2 * (ll_full_boot - ll_null_boot) + print("Like Boot: ", np.dot(lr_boot, ss[ss > 0.0] / _ss)) + sscale = ss / ss.sum() + mg = np.dot([((lr_boot[:, ii] - lr[ii]) > 0).sum() + for ii in range(20)], sscale) + print("prop: ", mg / float(mm._num_null_boot)) + 1 / 0 # # get the term names (skip the intercept) # names = res.dtype.names[1:] # # calc the SSU for each component # nterms = len(names) - # #ssU = {n:np.array([s[c]*np.sqrt((U[c][i::nterms]**2).sum()) + # #ssU = {n:np.array([s[c]*np.sqrt((U[c][i::nterms]**2).sum()) # # for c in range(len(s))]) # # for i,n in enumerate(names)} # ssU = {} @@ -579,9 +590,9 @@ def _eval_model(model_id, perm=None, boot=None): # recombine and scale the tvals across components if boot is None: - ts = np.rec.fromarrays([np.dot(tvals[k],ss[ss>0.0]/_ss) #/(ss>0.).sum() + ts = np.rec.fromarrays([np.dot(tvals[k], ss[ss > 0.0] / _ss) # /(ss>0.).sum() for k in tvals.dtype.names], - names= ','.join(tvals.dtype.names)) + names=','.join(tvals.dtype.names)) # tvals = np.rec.fromarrays([np.dot(tvals[k],ssU[k][ss>0.0]) #/(ss>0.).sum() # for k in names], # names= ','.join(names)) @@ -591,8 +602,8 @@ def _eval_model(model_id, perm=None, boot=None): tfs = [] for k in tvals.dtype.names: tfs.append(np.dot(tvals[k], - np.dot(diagsvd(ss[ss>0], len(ss[ss>0]),len(ss[ss>0])), - Vh[ss>0,...]))) #/(ss>0).sum()) + np.dot(diagsvd(ss[ss > 0], len(ss[ss > 0]), len(ss[ss > 0])), + Vh[ss > 0, ...]))) # /(ss>0).sum()) tfs = np.rec.fromarrays(tfs, names=','.join(tvals.dtype.names)) # for k in names: # tfs.append(np.dot(res[k], @@ -600,11 +611,10 @@ def _eval_model(model_id, perm=None, boot=None): # Vh[ss>0,...]))) #/(ss>0).sum()) # tfs = np.rec.fromarrays(tfs, names=','.join(names)) - # decide what to return if perm is None and boot is None: # return tvals, tfs, and R for actual non-permuted data - out = (ts,tfs,_R,feat_mask,_ss,mer,mer_null) + out = (ts, tfs, _R, feat_mask, _ss, mer, mer_null) elif perm is None: # return the boot features out = tfs @@ -614,6 +624,7 @@ def _eval_model(model_id, perm=None, boot=None): return out + def lmer_feat(mer, Dw): mer = r['refit'](mer, FloatVector(Dw)) df = r['data.frame'](r_coef(r['summary'](mer))) @@ -623,13 +634,12 @@ def lmer_feat(mer, Dw): return new_tvals - def _memmap_array(x, memmap_dir=None): if memmap_dir is None: memmap_dir = tempfile.gettempdir() - filename = os.path.join(memmap_dir,str(id(x))+'.npy') - np.save(filename,x) - return np.load(filename,'r') + filename = os.path.join(memmap_dir, str(id(x)) + '.npy') + np.save(filename, x) + return np.load(filename, 'r') class MELD(object): @@ -645,20 +655,21 @@ class MELD(object): """ + def __init__(self, fe_formula, re_formula, - re_group, dep_data, ind_data, + re_group, dep_data, ind_data, factors=None, row_mask=None, use_ranks=False, use_norm=True, memmap=False, memmap_dir=None, resid_formula=None, null_formula=None, num_null_boot=0, svd_terms=None, use_ssvd=False, - #nperms=500, nboot=100, + # nperms=500, nboot=100, n_jobs=1, verbose=10, lmer_opts=None): """ """ - if verbose>0: + if verbose > 0: sys.stdout.write('Initializing...') sys.stdout.flush() start_time = time.time() @@ -708,43 +719,43 @@ def __init__(self, fe_formula, re_formula, self._re_group = re_group if isinstance(ind_data, dict): # groups are the keys - self._groups = np.array(ind_data.keys()) + self._groups = np.array(list(ind_data.keys())) else: # groups need to be extracted from the recarray self._groups = np.unique(ind_data[re_group]) for g in self._groups: # get that subj inds - if isinstance(ind_data,dict): + if isinstance(ind_data, dict): # the index is just the group into that dict ind_ind = g else: # select the rows based on the group - ind_ind = ind_data[re_group]==g + ind_ind = ind_data[re_group] == g # process the row mask if row_mask is None: # no mask, so all good - row_ind = np.ones(len(ind_data[ind_ind]),dtype=np.bool) + row_ind = np.ones(len(ind_data[ind_ind]), dtype=np.bool) elif isinstance(row_mask, dict): # pull the row_mask from the dict row_ind = row_mask[g] else: # index into it with ind_ind row_ind = row_mask[ind_ind] - + # extract that group's A,M,O # first save the observations (rows of A) self._O[g] = ind_data[ind_ind][row_ind] if use_ranks: # loop over non-factors and rank them for n in self._O[g].dtype.names: - if (n in factors) or isinstance(self._O[g][n][0],str): + if (n in factors) or isinstance(self._O[g][n][0], str): continue self._O[g][n] = rankdata(self._O[g][n]) O.append(self._O[g]) # eventually allow for dict of data files for dep_data - if isinstance(dep_data,dict): + if isinstance(dep_data, dict): # the index is just the group into that dict dep_ind = g else: @@ -758,28 +769,29 @@ def __init__(self, fe_formula, re_formula, # Save D index into data self._D[g] = dep_data[dep_ind][row_ind] # reshape it - self._D[g] = self._D[g].reshape((self._D[g].shape[0],-1)) + self._D[g] = self._D[g].reshape((self._D[g].shape[0], -1)) if use_ranks: - if verbose>0: - sys.stdout.write('Ranking %s...'%(str(g))) + if verbose > 0: + sys.stdout.write('Ranking %s...' % (str(g))) sys.stdout.flush() - for i in xrange(self._D[g].shape[1]): - self._D[g][:,i] = rankdata(self._D[g][:,i]) + for i in range(self._D[g].shape[1]): + self._D[g][:, i] = rankdata(self._D[g][:, i]) # reshape M, so we don't have to do it repeatedly - self._M[g] = self._D[g].copy() #dep_data[ind].reshape((dep_data[ind].shape[0],-1)) - + # dep_data[ind].reshape((dep_data[ind].shape[0],-1)) + self._M[g] = self._D[g].copy() + # normalize M if use_norm: self._M[g] -= self._M[g].mean(0) self._M[g] /= np.sqrt((self._M[g]**2).sum(0)) # determine A from the model.matrix - rdf = DataFrame({k:(FactorVector(self._O[g][k]) - if k in factors else self._O[g][k]) + rdf = DataFrame({k: (FactorVector(self._O[g][k]) + if k in factors else self._O[g][k]) for k in self._O[g].dtype.names}) - + # model spec as data frame ms = r['data.frame'](r_model_matrix(Formula(fe_formula), data=rdf)) @@ -788,16 +800,16 @@ def __init__(self, fe_formula, re_formula, self._svd_terms = [c for c in cols if not 'Intercept' in c] else: self._svd_terms = svd_terms - self._A[g] = np.concatenate([np.array(ms.rx(c)) + self._A[g] = np.concatenate([np.array(ms.rx(c)) for c in self._svd_terms]).T - #for c in cols if not 'Intercept' in c]).T + # for c in cols if not 'Intercept' in c]).T if use_ranks: - for i in xrange(self._A[g].shape[1]): - self._A[g][:,i] = rankdata(self._A[g][:,i]) + for i in range(self._A[g].shape[1]): + self._A[g][:, i] = rankdata(self._A[g][:, i]) # normalize A - if True: #use_norm: + if True: # use_norm: self._A[g] -= self._A[g].mean(0) self._A[g] /= np.sqrt((self._A[g]**2).sum(0)) @@ -823,8 +835,8 @@ def __init__(self, fe_formula, re_formula, self._tp = [] self._tb = [] - if verbose>0: - sys.stdout.write('Done (%.2g sec)\n'%(time.time()-start_time)) + if verbose > 0: + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.write('Processing actual data...') sys.stdout.flush() start_time = time.time() @@ -837,7 +849,8 @@ def __init__(self, fe_formula, re_formula, self._ss = None self._mer = None self._mer_null = None - tp,tb,R,feat_mask,ss,mer,mer_null = _eval_model(id(self),None, None) + tp, tb, R, feat_mask, ss, mer, mer_null = _eval_model( + id(self), None, None) self._R = R self._tp.append(tp) self._tb.append(tb) @@ -846,8 +859,8 @@ def __init__(self, fe_formula, re_formula, self._mer = mer self._mer_null = mer_null - if verbose>0: - sys.stdout.write('Done (%.2g sec)\n'%(time.time()-start_time)) + if verbose > 0: + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.flush() def __del__(self): @@ -856,16 +869,16 @@ def __del__(self): # clean up memmapping files if self._memmap: - for g in self._M.keys(): + for g in list(self._M.keys()): try: - filename = self._M[g].filename + filename = self._M[g].filename del self._M[g] os.remove(filename) except OSError: pass - for g in self._D.keys(): + for g in list(self._D.keys()): try: - filename = self._D[g].filename + filename = self._D[g].filename del self._D[g] os.remove(filename) except OSError: @@ -873,11 +886,9 @@ def __del__(self): # clean self out of global model list global _global_meld - if _global_meld and _global_meld.has_key(my_id): + if _global_meld and my_id in _global_meld: del _global_meld[my_id] - - def run_perms(self, perms, n_jobs=None, verbose=None): """Run the specified permutations. @@ -890,13 +901,13 @@ def run_perms(self, perms, n_jobs=None, verbose=None): if verbose is None: verbose = self._verbose - if not isinstance(perms,list): + if not isinstance(perms, list): # perms is nperms nperms = perms # gen the perms ahead of time perms = [] - for p in xrange(nperms): + for p in range(nperms): ind = {} for k in self._groups: # gen a perm for that subj @@ -907,8 +918,8 @@ def run_perms(self, perms, n_jobs=None, verbose=None): # calc nperms nperms = len(perms) - if verbose>0: - sys.stdout.write('Running %d permutations...\n'%nperms) + if verbose > 0: + sys.stdout.write('Running %d permutations...\n' % nperms) sys.stdout.flush() start_time = time.time() @@ -922,16 +933,15 @@ def run_perms(self, perms, n_jobs=None, verbose=None): # sys.stdout.flush() # res.append(self._eval_model(perm=perm, boot=None, # n_jobs=n_jobs, verbose=0)) - res = Parallel(n_jobs=n_jobs, - verbose=verbose)(delayed(_eval_model)(id(self),perm,None) + res = Parallel(n_jobs=n_jobs, + verbose=verbose)(delayed(_eval_model)(id(self), perm, None) for perm in perms) self._tp.extend(res) - if verbose>0: - sys.stdout.write('Done (%.2g sec)\n'%(time.time()-start_time)) + if verbose > 0: + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.flush() - def run_boots(self, boots, n_jobs=None, verbose=None): """Run the specified bootstraps. @@ -944,7 +954,7 @@ def run_boots(self, boots, n_jobs=None, verbose=None): if verbose is None: verbose = self._verbose - if isinstance(boots,list): + if isinstance(boots, list): # get the nboots nboots = len(boots) else: @@ -952,11 +962,11 @@ def run_boots(self, boots, n_jobs=None, verbose=None): nboots = boots # calculate the boots with replacement - boots = [np.random.random_integers(0,len(self._R)-1,len(self._R)) - for i in xrange(nboots)] + boots = [np.random.random_integers(0, len(self._R) - 1, len(self._R)) + for i in range(nboots)] - if verbose>0: - sys.stdout.write('Running %d bootstraps...\n'%nboots) + if verbose > 0: + sys.stdout.write('Running %d bootstraps...\n' % nboots) sys.stdout.flush() start_time = time.time() @@ -971,16 +981,15 @@ def run_boots(self, boots, n_jobs=None, verbose=None): # sys.stdout.flush() # res.append(self._eval_model(perm=None, boot=boot, # n_jobs=n_jobs, verbose=0)) - res = Parallel(n_jobs=n_jobs, - verbose=verbose)(delayed(_eval_model)(id(self),None,boot) - for boot in boots) + res = Parallel(n_jobs=n_jobs, + verbose=verbose)(delayed(_eval_model)(id(self), None, boot) + for boot in boots) self._tb.extend(res) - if verbose>0: - sys.stdout.write('Done (%.2g sec)\n'%(time.time()-start_time)) + if verbose > 0: + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.flush() - @property def terms(self): return self._tp[0].dtype.names @@ -1003,13 +1012,13 @@ def pvals_uncorrected(self): """ # fancy way to stack recarrays tvals = self._tp[0].__array_wrap__(np.hstack(self._tp)) - #allt = np.abs(np.vstack([tvals[n] + # allt = np.abs(np.vstack([tvals[n] # for n in tvals.dtype.names # if n != '(Intercept)'])) - #pvals = {n:np.mean(allt.flatten()>=np.abs(tvals[n][0])) + # pvals = {n:np.mean(allt.flatten()>=np.abs(tvals[n][0])) names = [n for n in tvals.dtype.names if n != '(Intercept)'] - pvals = [np.mean(np.abs(tvals[n])>=np.abs(tvals[n][0])) + pvals = [np.mean(np.abs(tvals[n]) >= np.abs(tvals[n][0])) for n in names] pvals = np.rec.fromarrays(pvals, names=','.join(names)) return pvals @@ -1021,18 +1030,18 @@ def pvals(self): pvals = self.pvals_uncorrected names = pvals.dtype.names # scale the pvals - ind = np.argsort([1-pvals[n] for n in names]).argsort()+1 - for i,n in enumerate(names): - pvals[n] = (pvals[n]*ind[i]).clip(0,1) + ind = np.argsort([1 - pvals[n] for n in names]).argsort() + 1 + for i, n in enumerate(names): + pvals[n] = (pvals[n] * ind[i]).clip(0, 1) # ensure monotonicity # reorder ind to go from smallest to largest p ind = (-ind).argsort() - for i in range(1,len(ind)): - if pvals[names[ind[i]]] < pvals[names[ind[i-1]]]: - pvals[names[ind[i]]] = pvals[names[ind[i-1]]] - + for i in range(1, len(ind)): + if pvals[names[ind[i]]] < pvals[names[ind[i - 1]]]: + pvals[names[ind[i]]] = pvals[names[ind[i - 1]]] + return pvals - + @property def boot_ratio(self): """Return the bootstrap ratio for each feature. @@ -1050,9 +1059,9 @@ def boot_ratio(self): brs = [] for n in names: # reshape back to number of bootstraps - tf = tfs[n].reshape((len(self._tb),-1)) + tf = tfs[n].reshape((len(self._tb), -1)) # calculate the bootstrap ratio and shape to feature space - brs.append((tf[0]/tf.std(0)).reshape(self._feat_shape)) + brs.append((tf[0] / tf.std(0)).reshape(self._feat_shape)) brs = np.rec.fromarrays(brs, names=','.join(names)) @@ -1077,12 +1086,12 @@ def fdr_boot(self): br = FloatVector(br[good_ind]) # calc the fdr - results = fdrtool.fdrtool(br, statistic='normal', + results = fdrtool.fdrtool(br, statistic='normal', plot=False, verbose=False) # append the qvals qv[good_ind] = np.array(results.rx('qval')) - + qvals.append(qv.reshape(self._feat_shape)) # convert to recarray @@ -1100,35 +1109,35 @@ def fdr_boot(self): # generate some fake data nobs = 100 nsubj = 10 - nfeat = (50,100) + nfeat = (50, 100) nperms = 50 nboots = 50 use_ranks = False - s = np.concatenate([np.array(['subj%02d'%i]*nobs) for i in range(nsubj)]) - beh = np.concatenate([np.array([1]*(nobs/2) + [0]*(nobs/2)) + s = np.concatenate([np.array(['subj%02d' % i] * nobs) + for i in range(nsubj)]) + beh = np.concatenate([np.array([1] * (nobs / 2) + [0] * (nobs / 2)) for i in range(nsubj)]) # observations (data frame) ind_data = np.rec.fromarrays((np.zeros(len(s)), beh, - np.random.rand(len(s)),s), + np.random.rand(len(s)), s), names='val,beh,beh2,subj') # data with observations in first dimension and features on the remaining - dep_data = np.random.rand(len(s),*nfeat) - print 'Data shape:',dep_data.shape + dep_data = np.random.rand(len(s), *nfeat) + print('Data shape:', dep_data.shape) # now with signal # add in some signal dep_data_s = dep_data.copy() - for i in range(0,20,2): + for i in range(0, 20, 2): for j in range(2): - dep_data_s[:,4,i+j] += (ind_data['beh'] * (i+1)/200.) + dep_data_s[:, 4, i + j] += (ind_data['beh'] * (i + 1) / 200.) # smooth the data import scipy.ndimage - dep_data = scipy.ndimage.gaussian_filter(dep_data, [0,1,1]) - dep_data_s = scipy.ndimage.gaussian_filter(dep_data_s, [0,1,1]) - + dep_data = scipy.ndimage.gaussian_filter(dep_data, [0, 1, 1]) + dep_data_s = scipy.ndimage.gaussian_filter(dep_data_s, [0, 1, 1]) # # run without signal # # set up the lmer_pht @@ -1142,12 +1151,11 @@ def fdr_boot(self): # run with signal # set up the lmer_pht me_s = MELD('val ~ beh+beh2', '(1|subj)', 'subj', - dep_data_s, ind_data, factors = {'subj':None}, + dep_data_s, ind_data, factors={'subj': None}, use_ranks=use_ranks, use_ssvd=False, n_jobs=n_jobs) me_s.run_perms(nperms) - #me.run_boots(nboots) - + # me.run_boots(nboots) # # explore the results # print @@ -1161,15 +1169,13 @@ def fdr_boot(self): # #print "BR num sig:",[(n,(me.fdr_boot[n]<=.05).sum()) # # for n in brs.dtype.names] - print - print "Now with signal!" - print "----------------" - print "Terms:",me_s.terms - print "t-vals:",me_s.t_terms - print "term p-vals:",me_s.pvals + print() + print("Now with signal!") + print("----------------") + print("Terms:", me_s.terms) + print("t-vals:", me_s.t_terms) + print("term p-vals:", me_s.pvals) #brs_s = me_s.boot_ratio - #print "Bootstrap ratio shape:",brs_s.shape - #print "BR num sig:",[(n,(me_s.fdr_boot[n]<=.05).sum()) + # print "Bootstrap ratio shape:",brs_s.shape + # print "BR num sig:",[(n,(me_s.fdr_boot[n]<=.05).sum()) # for n in brs_s.dtype.names] - - diff --git a/ptsa/stats/meld.py b/ptsa/stats/meld.py index d8cedde..84346ba 100644 --- a/ptsa/stats/meld.py +++ b/ptsa/stats/meld.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -18,7 +18,7 @@ from scipy.stats import rankdata import scipy.stats.distributions as dists -from joblib import Parallel,delayed +from joblib import Parallel, delayed # Connect to an R session import rpy2.robjects @@ -29,7 +29,7 @@ from rpy2.robjects import Formula, FactorVector from rpy2.robjects.environments import Environment from rpy2.robjects.vectors import DataFrame, Vector, FloatVector -from rpy2.rinterface import MissingArg,SexpVector +from rpy2.rinterface import MissingArg, SexpVector # Make it so we can send numpy arrays to R import rpy2.robjects.numpy2ri @@ -37,35 +37,37 @@ #import rpy2.robjects as ro #import rpy2.robjects.numpy2ri as numpy2ri #ro.conversion.py2ri = numpy2ri -#numpy2ri.activate() +# numpy2ri.activate() # load some required packages -# PBS: Eventually we should try/except these to get people +# PBS: Eventually we should try/except these to get people # to install missing packages lme4 = importr('lme4') rstats = importr('stats') -if hasattr(lme4,'coef'): - r_coef = lme4.coef +if hasattr(lme4, 'coef'): + r_coef = lme4.coef else: r_coef = rstats.coef -if hasattr(lme4,'model_matrix'): +if hasattr(lme4, 'model_matrix'): r_model_matrix = lme4.model_matrix else: r_model_matrix = rstats.model_matrix # load ptsa clustering -import cluster -from stat_helper import fdr_correction +from . import cluster +from .stat_helper import fdr_correction # deal with warnings for bootstrap import warnings + class InstabilityWarning(UserWarning): """Issued when results may be unstable.""" pass + # On import, make sure that InstabilityWarnings are not filtered out. -warnings.simplefilter('always',InstabilityWarning) +warnings.simplefilter('always', InstabilityWarning) class LMER(): @@ -76,7 +78,8 @@ class LMER(): and extracting the associated t-stat. """ - def __init__(self, formula_str, df, factors=None, + + def __init__(self, formula_str, df, factors=None, resid_formula_str=None, **lmer_opts): """ """ @@ -86,28 +89,28 @@ def __init__(self, formula_str, df, factors=None, # add column if necessary if not pred_var in df.dtype.names: # must add it - df = append_fields(df, pred_var, [0.0]*len(df), usemask=False) + df = append_fields(df, pred_var, [0.0] * len(df), usemask=False) # make factor list if necessary if factors is None: factors = {} # add in missingarg for any potential factor not provided for k in df.dtype.names: - if isinstance(df[k][0],str) and not factors.has_key(k): + if isinstance(df[k][0], str) and k not in factors: factors[k] = MissingArg - + for f in factors: if factors[f] is None: factors[f] = MissingArg # checking for both types of R Vectors for rpy2 variations - elif (not isinstance(factors[f],Vector) and + elif (not isinstance(factors[f], Vector) and not factors[f] == MissingArg): factors[f] = Vector(factors[f]) # convert the recarray to a DataFrame (releveling if desired) - self._rdf = DataFrame({k:(FactorVector(df[k], levels=factors[k]) - if (k in factors) or isinstance(df[k][0],str) - else df[k]) + self._rdf = DataFrame({k: (FactorVector(df[k], levels=factors[k]) + if (k in factors) or isinstance(df[k][0], str) + else df[k]) for k in df.dtype.names}) # get the column index @@ -142,10 +145,11 @@ def run(self, vals=None, perms=None): # run on each permutation tvals = None log_likes = None - for i,perm in enumerate(perms): + for i, perm in enumerate(perms): if not perm is None: # set the perm - self._rdf[self._col_ind] = self._rdf[self._col_ind].rx(perm+1) + self._rdf[self._col_ind] = self._rdf[self._col_ind].rx( + perm + 1) # inside try block to catch convergence errors try: @@ -159,8 +163,8 @@ def run(self, vals=None, perms=None): **self._lmer_opts) except: continue - #tvals.append(np.array([np.nan])) - + # tvals.append(np.array([np.nan])) + # save the model if self._ms is None: self._ms = ms @@ -173,7 +177,7 @@ def run(self, vals=None, perms=None): # init the data # get the row names rows = list(r['row.names'](df)) - tvals = np.rec.fromarrays([np.ones(len(perms))*np.nan + tvals = np.rec.fromarrays([np.ones(len(perms)) * np.nan for ro in range(len(rows))], names=','.join(rows)) log_likes = np.zeros(len(perms)) @@ -211,7 +215,7 @@ def std(self): def R_to_tfce(R, connectivity=None, shape=None, - dt=.01, E=2/3., H=2.0): + dt=.01, E=2 / 3., H=2.0): """Apply TFCE to the R values.""" # allocate for tfce Rt = np.zeros_like(R) @@ -240,8 +244,8 @@ def pick_stable_features(Z, nboot=500): """Use a bootstrap to pick stable features. """ # generate the boots - boots = [np.random.random_integers(0, len(Z)-1, len(Z)) - for i in xrange(nboot)] + boots = [np.random.random_integers(0, len(Z) - 1, len(Z)) + for i in range(nboot)] # calc bootstrap ratio # calc the bootstrap std in efficient way @@ -251,13 +255,13 @@ def pick_stable_features(Z, nboot=500): ov = OnlineVariance(ddof=0) for b in range(len(boots)): ov.include(Z[boots[b]].mean(0)) - Zbr = Z.mean(0)/ov.std + Zbr = Z.mean(0) / ov.std # ignore any nans Zbr[np.isnan(Zbr)] = 0. # bootstrap ratios are supposedly t-distributed, so test sig - Zbr = dists.t(len(Z)-1).cdf(-1*np.abs(Zbr))*2. + Zbr = dists.t(len(Z) - 1).cdf(-1 * np.abs(Zbr)) * 2. Zbr[Zbr > 1] = 1 return Zbr @@ -286,7 +290,7 @@ def blockwise_dot(A, B, max_elements=int(2**26), out=None): if len(A.shape) == 1: A = A[np.newaxis, :] if len(B.shape) == 1: - B = B[:,np.newaxis] + B = B[:, np.newaxis] m, n = A.shape n1, o = B.shape @@ -336,7 +340,7 @@ def _eval_model(model_id, perm=None): # loop over group vars ind = {} - for i,k in enumerate(mm._groups[ind_b]): + for i, k in enumerate(mm._groups[ind_b]): # grab the A and M A = mm._A[k] M = mm._M[k] @@ -352,7 +356,7 @@ def _eval_model(model_id, perm=None): R.append(mm._R[ind_b[i]]) else: # calc the correlation - #R.append(np.dot(A.T,M[ind[k]].copy())) + # R.append(np.dot(A.T,M[ind[k]].copy())) #R.append(blockwise_dot(M[:][ind[k]].T, A).T) R.append(blockwise_dot(M[ind[k]].T, A).T) @@ -438,10 +442,10 @@ def _eval_model(model_id, perm=None): continue # flatten then weigh features via dot product - Dw = np.concatenate([#np.dot(mm._D[k][ind[k]].copy(),Vh[i]) - #blockwise_dot(mm._D[k][:][ind[k]], Vh[i]) - blockwise_dot(mm._D[k][ind[k]], Vh[i]) - for g, k in enumerate(mm._groups[ind_b])]) + Dw = np.concatenate([ # np.dot(mm._D[k][ind[k]].copy(),Vh[i]) + #blockwise_dot(mm._D[k][:][ind[k]], Vh[i]) + blockwise_dot(mm._D[k][ind[k]], Vh[i]) + for g, k in enumerate(mm._groups[ind_b])]) # run the main model if mer is None: @@ -486,7 +490,7 @@ def _eval_model(model_id, perm=None): # print "perm fail" # pull out data from all the components - tvals, log_likes = zip(*res) + tvals, log_likes = list(zip(*res)) tvals = np.concatenate(tvals) log_likes = np.concatenate(log_likes) @@ -506,9 +510,9 @@ def _eval_model(model_id, perm=None): # Vh[ss > 0, ...]))) # /(ss>0).sum()) tfs.append(blockwise_dot(tvals[k], blockwise_dot(diagsvd(ss[ss > 0], - len(ss[ss > 0]), - len(ss[ss > 0])), - Vh[ss > 0, ...]))) + len(ss[ss > 0]), + len(ss[ss > 0])), + Vh[ss > 0, ...]))) tfs = np.rec.fromarrays(tfs, names=','.join(tvals.dtype.names)) # decide what to return @@ -556,6 +560,7 @@ class MELD(object): """ + def __init__(self, fe_formula, re_formula, re_group, dep_data, ind_data, factors=None, row_mask=None, @@ -566,7 +571,7 @@ def __init__(self, fe_formula, re_formula, svd_terms=None, feat_thresh=0.05, feat_nboot=1000, do_tfce=False, connectivity=None, shape=None, - dt=.01, E=2/3., H=2.0, + dt=.01, E=2 / 3., H=2.0, n_jobs=1, verbose=10, lmer_opts=None): """ @@ -578,7 +583,7 @@ def __init__(self, fe_formula, re_formula, with a grouping variable. """ - if verbose>0: + if verbose > 0: sys.stdout.write('Initializing...') sys.stdout.flush() start_time = time.time() @@ -629,7 +634,7 @@ def __init__(self, fe_formula, re_formula, self._re_group = re_group if isinstance(ind_data, dict): # groups are the keys - self._groups = np.array(ind_data.keys()) + self._groups = np.array(list(ind_data.keys())) else: # groups need to be extracted from the recarray self._groups = np.unique(ind_data[re_group]) @@ -699,7 +704,7 @@ def __init__(self, fe_formula, re_formula, sys.stdout.write('Ranking %s...' % (str(g))) sys.stdout.flush() - for i in xrange(self._D[g].shape[1]): + for i in range(self._D[g].shape[1]): # rank it self._D[g][:, i] = rankdata(self._D[g][:, i]) @@ -738,7 +743,7 @@ def __init__(self, fe_formula, re_formula, for c in self._svd_terms]).T if use_ranks: - for i in xrange(self._A[g].shape[1]): + for i in range(self._A[g].shape[1]): # rank it self._A[g][:, i] = rankdata(self._A[g][:, i]) @@ -767,7 +772,8 @@ def __init__(self, fe_formula, re_formula, # mask the connectivity if self._do_tfce and (len(self._dep_mask.flatten()) > self._dep_mask.sum()): - self._connectivity = self._connectivity.tolil()[self._dep_mask.flatten()][:,self._dep_mask.flatten()].tocoo() + self._connectivity = self._connectivity.tolil( + )[self._dep_mask.flatten()][:, self._dep_mask.flatten()].tocoo() # prepare for the perms and boots and jackknife self._perms = [] @@ -777,7 +783,7 @@ def __init__(self, fe_formula, re_formula, self._pfmask = [] if verbose > 0: - sys.stdout.write('Done (%.2g sec)\n' % (time.time()-start_time)) + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.write('Processing actual data...') sys.stdout.flush() start_time = time.time() @@ -800,7 +806,7 @@ def __init__(self, fe_formula, re_formula, self._mer = mer if verbose > 0: - sys.stdout.write('Done (%.2g sec)\n' % (time.time()-start_time)) + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.flush() def __del__(self): @@ -809,14 +815,14 @@ def __del__(self): # clean up memmapping files if self._memmap: - for g in self._M.keys(): + for g in list(self._M.keys()): try: filename = self._M[g].filename del self._M[g] os.remove(filename) except OSError: pass - for g in self._D.keys(): + for g in list(self._D.keys()): try: filename = self._D[g].filename del self._D[g] @@ -847,7 +853,7 @@ def run_perms(self, perms, n_jobs=None, verbose=None): # gen the perms ahead of time perms = [] - for p in xrange(nperms): + for p in range(nperms): ind = {} for k in self._groups: # gen a perm for that subj @@ -871,13 +877,13 @@ def run_perms(self, perms, n_jobs=None, verbose=None): # backend='threading', verbose=verbose)(delayed(_eval_model)(id(self), perm) for perm in perms) - tp, tfs, feat_mask = zip(*res) + tp, tfs, feat_mask = list(zip(*res)) self._tp.extend(tp) self._tb.extend(tfs) self._pfmask.extend(feat_mask) if verbose > 0: - sys.stdout.write('Done (%.2g sec)\n' % (time.time()-start_time)) + sys.stdout.write('Done (%.2g sec)\n' % (time.time() - start_time)) sys.stdout.flush() @property @@ -910,7 +916,7 @@ def p_features(self): def get_p_features(self, names=None, conj=None): tpf = self._tb[0].__array_wrap__(np.hstack(self._tb)) pfmasks = np.array(self._pfmask).transpose((1, 0, 2)) - nperms = np.float(len(self._perms)+1) + nperms = np.float(len(self._perms) + 1) pfs = [] if names is None: @@ -934,7 +940,7 @@ def get_p_features(self, names=None, conj=None): # then divide by number of perms # got this from: # http://stackoverflow.com/questions/18875970/comparing-two-numpy-arrays-of-different-length - pf = ((nperms-np.searchsorted(nullTdist, tf.flatten(), 'left')) / + pf = ((nperms - np.searchsorted(nullTdist, tf.flatten(), 'left')) / nperms).reshape((int(nperms), -1)) pfs.append(pf) @@ -956,7 +962,7 @@ def get_p_features(self, names=None, conj=None): # get pvalues for each feature for each term pfts = np.searchsorted(nullPdist, pfs[:, 0, :].flatten(), - 'right').reshape(len(pfs), -1)/nperms + 'right').reshape(len(pfs), -1) / nperms pfeats = [] for n in range(len(names)): pfeat = np.ones(self._feat_shape) @@ -969,7 +975,7 @@ def get_p_features(self, names=None, conj=None): if __name__ == '__main__': - np.random.RandomState(seed = 42) + np.random.RandomState(seed=42) # test some MELD n_jobs = -1 @@ -986,7 +992,7 @@ def get_p_features(self, names=None, conj=None): s = np.concatenate([np.array(['subj%02d' % i] * nobs) for i in range(nsubj)]) - beh = np.concatenate([np.array([1] * (nobs/2) + [0]*(nobs / 2)) + beh = np.concatenate([np.array([1] * (nobs / 2) + [0] * (nobs / 2)) for i in range(nsubj)]) # observations (data frame) ind_data = np.rec.fromarrays((np.zeros(len(s)), @@ -996,7 +1002,7 @@ def get_p_features(self, names=None, conj=None): # data with observations in first dimension and features on the remaining dep_data = np.random.randn(len(s), *nfeat) - print 'Data shape:', dep_data.shape + print('Data shape:', dep_data.shape) # make a mask dep_mask = np.ones(nfeat, dtype=np.bool) @@ -1008,10 +1014,10 @@ def get_p_features(self, names=None, conj=None): dep_data_s = dep_data.copy() for i in range(0, 20, 2): for j in range(2): - dep_data_s[:, 4, i+j] += (ind_data['beh'] * (i+1)/50.) - dep_data_s[:, 5, i+j] += (ind_data['beh'] * (i+1)/50.) - dep_data_s[:, 5, i+j] += (ind_data['beh2'] * (i+1)/50.) - dep_data_s[:, 6, i+j] += (ind_data['beh2'] * (i+1)/50.) + dep_data_s[:, 4, i + j] += (ind_data['beh'] * (i + 1) / 50.) + dep_data_s[:, 5, i + j] += (ind_data['beh'] * (i + 1) / 50.) + dep_data_s[:, 5, i + j] += (ind_data['beh2'] * (i + 1) / 50.) + dep_data_s[:, 6, i + j] += (ind_data['beh2'] * (i + 1) / 50.) # smooth the data if smoothed: @@ -1019,8 +1025,8 @@ def get_p_features(self, names=None, conj=None): dep_data = scipy.ndimage.gaussian_filter(dep_data, [0, 1, 1]) dep_data_s = scipy.ndimage.gaussian_filter(dep_data_s, [0, 1, 1]) - print "Starting MELD test" - print "beh has signal, beh2 does not" + print("Starting MELD test") + print("beh has signal, beh2 does not") me_s = MELD('val ~ beh+beh2', '(1|subj)', 'subj', dep_data_s, ind_data, factors={'subj': None}, use_ranks=use_ranks, @@ -1028,15 +1034,15 @@ def get_p_features(self, names=None, conj=None): feat_nboot=1000, feat_thresh=0.05, do_tfce=True, connectivity=None, shape=None, - dt=.01, E=2/3., H=2.0, + dt=.01, E=2 / 3., H=2.0, n_jobs=n_jobs, verbose=verbose, memmap=memmap, # lmer_opts={'control':lme4.lmerControl(optimizer="nloptwrap", # #optimizer="Nelder_Mead", # optCtrl=r['list'](maxfun=100000)) # } - ) + ) me_s.run_perms(nperms) pfts = me_s.p_features - print "Number of signifcant features:", [(n, (pfts[n] <= .05).sum()) - for n in pfts.dtype.names] + print("Number of signifcant features:", [(n, (pfts[n] <= .05).sum()) + for n in pfts.dtype.names]) diff --git a/ptsa/stats/nonparam.py b/ptsa/stats/nonparam.py index eac9d14..1a62ff9 100644 --- a/ptsa/stats/nonparam.py +++ b/ptsa/stats/nonparam.py @@ -1,119 +1,121 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -# -# See the COPYING file distributed along with the PTSA package for the -# copyright and license terms. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## - -# global imports -import numpy as np -from scipy.stats import ttest_ind, ttest_1samp, norm -import sys - -def gen_perms(dat, group_var, nperms): - """ - Generate permutations within a group variable, but across conditions. - - There is no need to sort your data as this method will shuffle the - indices properly. - - """ - # grab the unique groups - ugrp = np.unique(dat[group_var]) - - # save indices for each unique group - grpind = {u:np.nonzero(dat[group_var]==u)[0] for u in ugrp} - - # set the base permutation indices for each unique group - p_ind = {u:np.arange(len(grpind[u])) for u in ugrp} - - # start with actual data - perms = [np.arange(len(dat))] - - # loop and shuffle for each perm - for p in xrange(nperms): - # set the starting indices - ind = np.arange(len(dat)) - - # loop over each group - for u in ugrp: - # permute the indices for that group - perm = np.random.permutation(p_ind[u]) - - # insert the permuted group indices into the base index - np.put(ind,grpind[u],grpind[u][perm]) - - # append the shuffled perm to the list of permutations - perms.append(ind) - - # turn the final perms into an array - perms = np.array(perms) - return perms - - - -def ttest_ind_z_one_sided(X,Y): - # do the test - t,p = ttest_ind(X,Y) - - # convert the pvals to one-sided tests based on the t - p = (p/2.)+np.finfo(p.dtype).eps - p[t>0] = 1-p[t>0] - - # convert the p to a z - z = norm.ppf(p) - - return z - - -def permutation_test(X, Y=None, parametric=True, iterations=1000): - """ - Perform a permutation test on paired or non-paired data. - - Observations must be on the first axis. - """ - # see if paired or not and concat data - if Y is None: - paired = True - data = X - nX = len(X) - else: - paired = False - data = np.r_[X,Y] - nX = len(X) - nY = len(Y) - - # currently no non-parametric - if not parametric: - raise NotImplementedError("Currently only parametric stats are supported.") - - # perform stats - z_boot = [] - if paired: - # paired stat - raise NotImplementedError("Currently only non-paired stats are supported.") - # first on actual data - #t,p = ttest_1samp(data) - else: - # non-paired - # first on actual data - z = ttest_ind_z_one_sided(data[:nX],data[nX:]) - - # now on random shuffles - sys.stdout.write('%d: '%iterations) - for i in xrange(iterations): - # shuffle it - sys.stdout.write('%d '%i) - sys.stdout.flush() - np.random.shuffle(data) - z_boot.append(ttest_ind_z_one_sided(data[:nX],data[nX:])) - sys.stdout.write('\n') - sys.stdout.flush() - - # convert z_boot to array - z_boot = np.asarray(z_boot) - - # return those z values - return z, z_boot +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See the COPYING file distributed along with the PTSA package for the +# copyright and license terms. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + +# global imports +import numpy as np +from scipy.stats import ttest_ind, ttest_1samp, norm +import sys + + +def gen_perms(dat, group_var, nperms): + """ + Generate permutations within a group variable, but across conditions. + + There is no need to sort your data as this method will shuffle the + indices properly. + + """ + # grab the unique groups + ugrp = np.unique(dat[group_var]) + + # save indices for each unique group + grpind = {u: np.nonzero(dat[group_var] == u)[0] for u in ugrp} + + # set the base permutation indices for each unique group + p_ind = {u: np.arange(len(grpind[u])) for u in ugrp} + + # start with actual data + perms = [np.arange(len(dat))] + + # loop and shuffle for each perm + for p in range(nperms): + # set the starting indices + ind = np.arange(len(dat)) + + # loop over each group + for u in ugrp: + # permute the indices for that group + perm = np.random.permutation(p_ind[u]) + + # insert the permuted group indices into the base index + np.put(ind, grpind[u], grpind[u][perm]) + + # append the shuffled perm to the list of permutations + perms.append(ind) + + # turn the final perms into an array + perms = np.array(perms) + return perms + + +def ttest_ind_z_one_sided(X, Y): + # do the test + t, p = ttest_ind(X, Y) + + # convert the pvals to one-sided tests based on the t + p = (p / 2.) + np.finfo(p.dtype).eps + p[t > 0] = 1 - p[t > 0] + + # convert the p to a z + z = norm.ppf(p) + + return z + + +def permutation_test(X, Y=None, parametric=True, iterations=1000): + """ + Perform a permutation test on paired or non-paired data. + + Observations must be on the first axis. + """ + # see if paired or not and concat data + if Y is None: + paired = True + data = X + nX = len(X) + else: + paired = False + data = np.r_[X, Y] + nX = len(X) + nY = len(Y) + + # currently no non-parametric + if not parametric: + raise NotImplementedError( + "Currently only parametric stats are supported.") + + # perform stats + z_boot = [] + if paired: + # paired stat + raise NotImplementedError( + "Currently only non-paired stats are supported.") + # first on actual data + #t,p = ttest_1samp(data) + else: + # non-paired + # first on actual data + z = ttest_ind_z_one_sided(data[:nX], data[nX:]) + + # now on random shuffles + sys.stdout.write('%d: ' % iterations) + for i in range(iterations): + # shuffle it + sys.stdout.write('%d ' % i) + sys.stdout.flush() + np.random.shuffle(data) + z_boot.append(ttest_ind_z_one_sided(data[:nX], data[nX:])) + sys.stdout.write('\n') + sys.stdout.flush() + + # convert z_boot to array + z_boot = np.asarray(z_boot) + + # return those z values + return z, z_boot diff --git a/ptsa/stats/stat_helper.py b/ptsa/stats/stat_helper.py index 0419b9e..4f7f565 100644 --- a/ptsa/stats/stat_helper.py +++ b/ptsa/stats/stat_helper.py @@ -1,5 +1,6 @@ import numpy as np + def _ecdf(x): '''no frills empirical cdf used in fdrcorrection ''' @@ -7,7 +8,6 @@ def _ecdf(x): return np.arange(1, nobs + 1) / float(nobs) - def fdr_correction(pvals, alpha=0.05, method='indep'): """P-value correction with False Discovery Rate (FDR) @@ -69,4 +69,3 @@ def fdr_correction(pvals, alpha=0.05, method='indep'): pvals_corrected = pvals_corrected[sortrevind].reshape(shape_init) reject = reject[sortrevind].reshape(shape_init) return reject, pvals_corrected - diff --git a/ptsa/tests/test_fixed_scipy.py b/ptsa/tests/test_fixed_scipy.py index 74d6031..52f4a5f 100644 --- a/ptsa/tests/test_fixed_scipy.py +++ b/ptsa/tests/test_fixed_scipy.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -16,55 +16,55 @@ class TestFixed_Scipy(TestCase): def test_morlet(self): - x = wavelets.morlet(50,4.1,complete=True) - y = wavelets.morlet(50,4.1,complete=False) + x = wavelets.morlet(50, 4.1, complete=True) + y = wavelets.morlet(50, 4.1, complete=False) # Test if complete and incomplete wavelet have same lengths: - assert_equal(len(x),len(y)) + assert_equal(len(x), len(y)) # Test if complete wavelet is less than incomplete wavelet: - assert_array_less(x,y) - - x = wavelets.morlet(10,50,complete=False) - y = wavelets.morlet(10,50,complete=True) + assert_array_less(x, y) + + x = wavelets.morlet(10, 50, complete=False) + y = wavelets.morlet(10, 50, complete=True) # For large widths complete and incomplete wavelets should be # identical within numerical precision: - assert_equal(x,y) + assert_equal(x, y) # miscellaneous tests: - x = N.array([1.73752399e-09 +9.84327394e-25j, - 6.49471756e-01 +0.00000000e+00j, - 1.73752399e-09 -9.84327394e-25j]) - y = wavelets.morlet(3,w=2,complete=True) - assert_array_almost_equal(x,y) - - x = N.array([2.00947715e-09 +9.84327394e-25j, - 7.51125544e-01 +0.00000000e+00j, - 2.00947715e-09 -9.84327394e-25j]) - y = wavelets.morlet(3,w=2,complete=False) - assert_array_almost_equal(x,y,decimal=2) - - x = wavelets.morlet(10000,s=4,complete=True) - y = wavelets.morlet(20000,s=8,complete=True)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = N.array([1.73752399e-09 + 9.84327394e-25j, + 6.49471756e-01 + 0.00000000e+00j, + 1.73752399e-09 - 9.84327394e-25j]) + y = wavelets.morlet(3, w=2, complete=True) + assert_array_almost_equal(x, y) + + x = N.array([2.00947715e-09 + 9.84327394e-25j, + 7.51125544e-01 + 0.00000000e+00j, + 2.00947715e-09 - 9.84327394e-25j]) + y = wavelets.morlet(3, w=2, complete=False) + assert_array_almost_equal(x, y, decimal=2) + + x = wavelets.morlet(10000, s=4, complete=True) + y = wavelets.morlet(20000, s=8, complete=True)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) - x = wavelets.morlet(10000,s=4,complete=False) - assert_array_almost_equal(y,x,decimal=2) - y = wavelets.morlet(20000,s=8,complete=False)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = wavelets.morlet(10000, s=4, complete=False) + assert_array_almost_equal(y, x, decimal=2) + y = wavelets.morlet(20000, s=8, complete=False)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) - x = wavelets.morlet(10000,w=3,s=5,complete=True) - y = wavelets.morlet(20000,w=3,s=10,complete=True)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = wavelets.morlet(10000, w=3, s=5, complete=True) + y = wavelets.morlet(20000, w=3, s=10, complete=True)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) - x = wavelets.morlet(10000,w=3,s=5,complete=False) - assert_array_almost_equal(y,x,decimal=2) - y = wavelets.morlet(20000,w=3,s=10,complete=False)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = wavelets.morlet(10000, w=3, s=5, complete=False) + assert_array_almost_equal(y, x, decimal=2) + y = wavelets.morlet(20000, w=3, s=10, complete=False)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) - x = wavelets.morlet(10000,w=7,s=10,complete=True) - y = wavelets.morlet(20000,w=7,s=20,complete=True)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = wavelets.morlet(10000, w=7, s=10, complete=True) + y = wavelets.morlet(20000, w=7, s=20, complete=True)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) - x = wavelets.morlet(10000,w=7,s=10,complete=False) - assert_array_almost_equal(x,y,decimal=2) - y = wavelets.morlet(20000,w=7,s=20,complete=False)[5000:15000] - assert_array_almost_equal(x,y,decimal=2) + x = wavelets.morlet(10000, w=7, s=10, complete=False) + assert_array_almost_equal(x, y, decimal=2) + y = wavelets.morlet(20000, w=7, s=20, complete=False)[5000:15000] + assert_array_almost_equal(x, y, decimal=2) diff --git a/ptsa/tests/test_wavelet.py b/ptsa/tests/test_wavelet.py index e77b1e5..93c37d7 100644 --- a/ptsa/tests/test_wavelet.py +++ b/ptsa/tests/test_wavelet.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -9,367 +9,369 @@ import numpy as np import re -from numpy.testing import * #NumpyTest, NumpyTestCase +from numpy.testing import * # NumpyTest, NumpyTestCase from ptsa.wavelet import * -from ptsa.data import TimeSeries,Dim - +from ptsa.data import TimeSeries, Dim class test_morlet_multi(TestCase): def test_morlet_multi(self): # make sure errors are raised when not called with enough or # the right kind of arguments: - self.assertRaises(TypeError,morlet_multi) - self.assertRaises(ValueError,morlet_multi,[],[],200) - self.assertRaises(ValueError,morlet_multi,[1],[],200) - self.assertRaises(ValueError,morlet_multi,[],[1],200) - self.assertRaises(ValueError,morlet_multi,[1],[1],[]) - self.assertRaises(ValueError,morlet_multi,[1,2],[1,2,3],200) - self.assertRaises(ValueError,morlet_multi,[1,2,3],[1,2],200) - self.assertRaises(ValueError,morlet_multi,[1],[1,2],200) - self.assertRaises(ValueError,morlet_multi,[1],[1],200,[1,2]) - self.assertRaises(ValueError,morlet_multi,[1,2,3],[1],200,[1,2]) - - x = morlet_multi(2,4,200,complete=True) - y = morlet_multi(2,4,200,complete=False) + self.assertRaises(TypeError, morlet_multi) + self.assertRaises(ValueError, morlet_multi, [], [], 200) + self.assertRaises(ValueError, morlet_multi, [1], [], 200) + self.assertRaises(ValueError, morlet_multi, [], [1], 200) + self.assertRaises(ValueError, morlet_multi, [1], [1], []) + self.assertRaises(ValueError, morlet_multi, [1, 2], [1, 2, 3], 200) + self.assertRaises(ValueError, morlet_multi, [1, 2, 3], [1, 2], 200) + self.assertRaises(ValueError, morlet_multi, [1], [1, 2], 200) + self.assertRaises(ValueError, morlet_multi, [1], [1], 200, [1, 2]) + self.assertRaises(ValueError, morlet_multi, [ + 1, 2, 3], [1], 200, [1, 2]) + + x = morlet_multi(2, 4, 200, complete=True) + y = morlet_multi(2, 4, 200, complete=False) # Make sure we got one wavelet in each case: - self.assertEqual(len(x),1) - self.assertEqual(len(y),1) + self.assertEqual(len(x), 1) + self.assertEqual(len(y), 1) # Test if complete and incomplete wavelet have same lengths: - self.assertEqual(len(x[0]),len(y[0])) + self.assertEqual(len(x[0]), len(y[0])) # Test if complete wavelet is less than incomplete wavelet: - assert_array_less(x[0],y[0]) + assert_array_less(x[0], y[0]) - x = morlet_multi([2,2,2],5,100) + x = morlet_multi([2, 2, 2], 5, 100) # len(freqs) wavelets are expected: - self.assertEqual(len(x),3) + self.assertEqual(len(x), 3) # when the same frequencies and widths are given, the same # wavelets should result: - assert_array_equal(x[0],x[1]) - assert_array_equal(x[0],x[2]) + assert_array_equal(x[0], x[1]) + assert_array_equal(x[0], x[2]) - - x = morlet_multi([2,4,6],5,100) + x = morlet_multi([2, 4, 6], 5, 100) # len(freqs) wavelets are expected: - self.assertEqual(len(x),3) + self.assertEqual(len(x), 3) # length of wavelet should be longer for lower frequencies # (all else being equal) - self.assertTrue(len(x[0])>len(x[1]) and len(x[1])>len(x[2])) + self.assertTrue(len(x[0]) > len(x[1]) and len(x[1]) > len(x[2])) - x = morlet_multi([2,2,2],[3,6,9],100) + x = morlet_multi([2, 2, 2], [3, 6, 9], 100) # len(freqs) wavelets are expected: - self.assertEqual(len(x),3) + self.assertEqual(len(x), 3) # length of wavelet should be longer for greater widths # (all else being equal) - self.assertTrue(len(x[0])= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - x_ts = phase_pow_multi(1,dat_ts) + + x_ts = phase_pow_multi(1, dat_ts) # ensure correct output shape: - self.assertEqual(np.shape(x_ts),(2,1,2,1000)) + self.assertEqual(np.shape(x_ts), (2, 1, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],x_ts[0][0][1]) - assert_array_equal(x_ts[1][0][0],x_ts[1][0][1]) + assert_array_equal(x_ts[0][0][0], x_ts[0][0][1]) + assert_array_equal(x_ts[1][0][0], x_ts[1][0][1]) # ensure valid output values: phaseTest = np.abs(x_ts[0]) <= np.pi powerTest = x_ts[1] >= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - y = phase_pow_multi([1],dat,100,to_return='phase') + + y = phase_pow_multi([1], dat, 100, to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y),(1,2,1000)) + self.assertEqual(np.shape(y), (1, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[0][0][0],y[0][1]) + assert_array_equal(x[0][0][0], y[0][1]) # ensure valid output values: phaseTest = np.abs(y[0]) <= np.pi self.assertTrue(phaseTest.all()) - y_ts = phase_pow_multi([1],dat_ts,to_return='phase') + y_ts = phase_pow_multi([1], dat_ts, to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y_ts),(1,2,1000)) + self.assertEqual(np.shape(y_ts), (1, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],y_ts[0][1]) + assert_array_equal(x_ts[0][0][0], y_ts[0][1]) # ensure valid output values: phaseTest = np.abs(y_ts[0]) <= np.pi self.assertTrue(phaseTest.all()) - z = phase_pow_multi(1,dat,[100],to_return='power') + z = phase_pow_multi(1, dat, [100], to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z),(1,2,1000)) + self.assertEqual(np.shape(z), (1, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[1][0][0],z[0][1]) + assert_array_equal(x[1][0][0], z[0][1]) # ensure valid output values: powerTest = z >= 0 self.assertTrue(powerTest.all()) - z_ts = phase_pow_multi(1,dat_ts,to_return='power') + z_ts = phase_pow_multi(1, dat_ts, to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z_ts),(1,2,1000)) + self.assertEqual(np.shape(z_ts), (1, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[1][0][0],z_ts[0][1]) + assert_array_equal(x_ts[1][0][0], z_ts[0][1]) # ensure valid output values: powerTest = z_ts >= 0 self.assertTrue(powerTest.all()) - x = phase_pow_multi([1,2,3],dat,100,widths=6) + x = phase_pow_multi([1, 2, 3], dat, 100, widths=6) # ensure correct output shape: - self.assertEqual(np.shape(x),(2,3,2,1000)) + self.assertEqual(np.shape(x), (2, 3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[0][0][0],x[0][0][1]) - assert_array_equal(x[1][0][0],x[1][0][1]) - assert_array_equal(x[0][1][0],x[0][1][1]) - assert_array_equal(x[1][1][0],x[1][1][1]) - assert_array_equal(x[0][2][0],x[0][2][1]) - assert_array_equal(x[1][2][0],x[1][2][1]) + assert_array_equal(x[0][0][0], x[0][0][1]) + assert_array_equal(x[1][0][0], x[1][0][1]) + assert_array_equal(x[0][1][0], x[0][1][1]) + assert_array_equal(x[1][1][0], x[1][1][1]) + assert_array_equal(x[0][2][0], x[0][2][1]) + assert_array_equal(x[1][2][0], x[1][2][1]) # ensure valid output values: phaseTest = np.abs(x[0]) <= np.pi powerTest = x[1] >= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - x_ts = phase_pow_multi([1,2,3],dat_ts,widths=6) + + x_ts = phase_pow_multi([1, 2, 3], dat_ts, widths=6) # ensure correct output shape: - self.assertEqual(np.shape(x_ts),(2,3,2,1000)) + self.assertEqual(np.shape(x_ts), (2, 3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],x_ts[0][0][1]) - assert_array_equal(x_ts[1][0][0],x_ts[1][0][1]) - assert_array_equal(x_ts[0][1][0],x_ts[0][1][1]) - assert_array_equal(x_ts[1][1][0],x_ts[1][1][1]) - assert_array_equal(x_ts[0][2][0],x_ts[0][2][1]) - assert_array_equal(x_ts[1][2][0],x_ts[1][2][1]) + assert_array_equal(x_ts[0][0][0], x_ts[0][0][1]) + assert_array_equal(x_ts[1][0][0], x_ts[1][0][1]) + assert_array_equal(x_ts[0][1][0], x_ts[0][1][1]) + assert_array_equal(x_ts[1][1][0], x_ts[1][1][1]) + assert_array_equal(x_ts[0][2][0], x_ts[0][2][1]) + assert_array_equal(x_ts[1][2][0], x_ts[1][2][1]) # ensure valid output values: phaseTest = np.abs(x_ts[0]) <= np.pi powerTest = x_ts[1] >= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - y = phase_pow_multi([1,2,3],dat,[100],widths=6,to_return='phase') + + y = phase_pow_multi([1, 2, 3], dat, [100], widths=6, to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y),(3,2,1000)) + self.assertEqual(np.shape(y), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[0][0][0],y[0][1]) - assert_array_equal(x[0][1][0],y[1][1]) - assert_array_equal(x[0][2][0],y[2][1]) + assert_array_equal(x[0][0][0], y[0][1]) + assert_array_equal(x[0][1][0], y[1][1]) + assert_array_equal(x[0][2][0], y[2][1]) # ensure valid output values: phaseTest = np.abs(y) <= np.pi self.assertTrue(phaseTest.all()) - y_ts = phase_pow_multi([1,2,3],dat,[100],widths=6,to_return='phase') + y_ts = phase_pow_multi([1, 2, 3], dat, [100], + widths=6, to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y_ts),(3,2,1000)) + self.assertEqual(np.shape(y_ts), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],y_ts[0][1]) - assert_array_equal(x_ts[0][1][0],y_ts[1][1]) - assert_array_equal(x_ts[0][2][0],y_ts[2][1]) + assert_array_equal(x_ts[0][0][0], y_ts[0][1]) + assert_array_equal(x_ts[0][1][0], y_ts[1][1]) + assert_array_equal(x_ts[0][2][0], y_ts[2][1]) # ensure valid output values: phaseTest = np.abs(y_ts) <= np.pi self.assertTrue(phaseTest.all()) - z = phase_pow_multi([1,2,3],dat,100,widths=[6],to_return='power') + z = phase_pow_multi([1, 2, 3], dat, 100, widths=[6], to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z),(3,2,1000)) + self.assertEqual(np.shape(z), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[1][0][0],z[0][1]) - assert_array_equal(x[1][1][0],z[1][1]) - assert_array_equal(x[1][2][0],z[2][1]) + assert_array_equal(x[1][0][0], z[0][1]) + assert_array_equal(x[1][1][0], z[1][1]) + assert_array_equal(x[1][2][0], z[2][1]) # ensure valid output values: powerTest = z >= 0 self.assertTrue(powerTest.all()) - z_ts = phase_pow_multi([1,2,3],dat,100,widths=[6],to_return='power') + z_ts = phase_pow_multi([1, 2, 3], dat, 100, widths=[ + 6], to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z_ts),(3,2,1000)) + self.assertEqual(np.shape(z_ts), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[1][0][0],z_ts[0][1]) - assert_array_equal(x_ts[1][1][0],z_ts[1][1]) - assert_array_equal(x_ts[1][2][0],z_ts[2][1]) + assert_array_equal(x_ts[1][0][0], z_ts[0][1]) + assert_array_equal(x_ts[1][1][0], z_ts[1][1]) + assert_array_equal(x_ts[1][2][0], z_ts[2][1]) # ensure valid output values: powerTest = z_ts >= 0 self.assertTrue(powerTest.all()) - x = phase_pow_multi([4,9,8],dat,[100,200,300],widths=[6,5,4]) + x = phase_pow_multi([4, 9, 8], dat, [100, 200, 300], widths=[6, 5, 4]) # ensure correct output shape: - self.assertEqual(np.shape(x),(2,3,2,1000)) + self.assertEqual(np.shape(x), (2, 3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[0][0][0],x[0][0][1]) - assert_array_equal(x[1][0][0],x[1][0][1]) - assert_array_equal(x[0][1][0],x[0][1][1]) - assert_array_equal(x[1][1][0],x[1][1][1]) - assert_array_equal(x[0][2][0],x[0][2][1]) - assert_array_equal(x[1][2][0],x[1][2][1]) + assert_array_equal(x[0][0][0], x[0][0][1]) + assert_array_equal(x[1][0][0], x[1][0][1]) + assert_array_equal(x[0][1][0], x[0][1][1]) + assert_array_equal(x[1][1][0], x[1][1][1]) + assert_array_equal(x[0][2][0], x[0][2][1]) + assert_array_equal(x[1][2][0], x[1][2][1]) # ensure valid output values: phaseTest = np.abs(x[0]) <= np.pi powerTest = x[1] >= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - x_ts = phase_pow_multi([4,9,8],dat,[100,200,300],widths=[6,5,4]) + + x_ts = phase_pow_multi( + [4, 9, 8], dat, [100, 200, 300], widths=[6, 5, 4]) # ensure correct output shape: - self.assertEqual(np.shape(x_ts),(2,3,2,1000)) + self.assertEqual(np.shape(x_ts), (2, 3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],x_ts[0][0][1]) - assert_array_equal(x_ts[1][0][0],x_ts[1][0][1]) - assert_array_equal(x_ts[0][1][0],x_ts[0][1][1]) - assert_array_equal(x_ts[1][1][0],x_ts[1][1][1]) - assert_array_equal(x_ts[0][2][0],x_ts[0][2][1]) - assert_array_equal(x_ts[1][2][0],x_ts[1][2][1]) + assert_array_equal(x_ts[0][0][0], x_ts[0][0][1]) + assert_array_equal(x_ts[1][0][0], x_ts[1][0][1]) + assert_array_equal(x_ts[0][1][0], x_ts[0][1][1]) + assert_array_equal(x_ts[1][1][0], x_ts[1][1][1]) + assert_array_equal(x_ts[0][2][0], x_ts[0][2][1]) + assert_array_equal(x_ts[1][2][0], x_ts[1][2][1]) # ensure valid output values: phaseTest = np.abs(x_ts[0]) <= np.pi powerTest = x_ts[1] >= 0 self.assertTrue(phaseTest.all()) self.assertTrue(powerTest.all()) - - y = phase_pow_multi([4,9,8],dat,[100,200,300], - widths=[6,5,4],to_return='phase') + + y = phase_pow_multi([4, 9, 8], dat, [100, 200, 300], + widths=[6, 5, 4], to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y),(3,2,1000)) + self.assertEqual(np.shape(y), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[0][0][0],y[0][1]) - assert_array_equal(x[0][1][0],y[1][1]) - assert_array_equal(x[0][2][0],y[2][1]) + assert_array_equal(x[0][0][0], y[0][1]) + assert_array_equal(x[0][1][0], y[1][1]) + assert_array_equal(x[0][2][0], y[2][1]) # ensure valid output values: phaseTest = np.abs(y) <= np.pi self.assertTrue(phaseTest.all()) - y_ts = phase_pow_multi([4,9,8],dat,[100,200,300], - widths=[6,5,4],to_return='phase') + y_ts = phase_pow_multi([4, 9, 8], dat, [100, 200, 300], + widths=[6, 5, 4], to_return='phase') # ensure correct output shape: - self.assertEqual(np.shape(y_ts),(3,2,1000)) + self.assertEqual(np.shape(y_ts), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[0][0][0],y_ts[0][1]) - assert_array_equal(x_ts[0][1][0],y_ts[1][1]) - assert_array_equal(x_ts[0][2][0],y_ts[2][1]) + assert_array_equal(x_ts[0][0][0], y_ts[0][1]) + assert_array_equal(x_ts[0][1][0], y_ts[1][1]) + assert_array_equal(x_ts[0][2][0], y_ts[2][1]) # ensure valid output values: phaseTest = np.abs(y_ts) <= np.pi self.assertTrue(phaseTest.all()) - z = phase_pow_multi([4,9,8],dat,[100,200,300], - widths=[6,5,4],to_return='power') + z = phase_pow_multi([4, 9, 8], dat, [100, 200, 300], + widths=[6, 5, 4], to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z),(3,2,1000)) + self.assertEqual(np.shape(z), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x[1][0][0],z[0][1]) - assert_array_equal(x[1][1][0],z[1][1]) - assert_array_equal(x[1][2][0],z[2][1]) + assert_array_equal(x[1][0][0], z[0][1]) + assert_array_equal(x[1][1][0], z[1][1]) + assert_array_equal(x[1][2][0], z[2][1]) # ensure valid output values: powerTest = z >= 0 self.assertTrue(powerTest.all()) - z_ts = phase_pow_multi([4,9,8],dat,[100,200,300], - widths=[6,5,4],to_return='power') + z_ts = phase_pow_multi([4, 9, 8], dat, [100, 200, 300], + widths=[6, 5, 4], to_return='power') # ensure correct output shape: - self.assertEqual(np.shape(z_ts),(3,2,1000)) + self.assertEqual(np.shape(z_ts), (3, 2, 1000)) # dat has two identical rows, ensure output has corresponding # identities: - assert_array_equal(x_ts[1][0][0],z_ts[0][1]) - assert_array_equal(x_ts[1][1][0],z_ts[1][1]) - assert_array_equal(x_ts[1][2][0],z_ts[2][1]) + assert_array_equal(x_ts[1][0][0], z_ts[0][1]) + assert_array_equal(x_ts[1][1][0], z_ts[1][1]) + assert_array_equal(x_ts[1][2][0], z_ts[2][1]) # ensure valid output values: powerTest = z_ts >= 0 self.assertTrue(powerTest.all()) diff --git a/ptsa/version.py b/ptsa/version.py index a14adf2..47f99df 100644 --- a/ptsa/version.py +++ b/ptsa/version.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -13,13 +13,14 @@ from distutils.version import StrictVersion -## !!!!! -import versionString +# !!!!! +from . import versionString #vstr = open('versionString.txt').readline().strip() -## !!!!! +# !!!!! ptsaVersion = StrictVersion(versionString.vstr) + def versionAtLeast(someString): """ Check that the current ptsa Version >= argument string's version. @@ -30,6 +31,7 @@ def versionAtLeast(someString): else: return False + def versionWithin(str1, str2): """ Check that the current ptsa version is in the version-range described diff --git a/ptsa/versionString.py b/ptsa/versionString.py index 82cbae2..76c0708 100644 --- a/ptsa/versionString.py +++ b/ptsa/versionString.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -12,4 +12,3 @@ # !!!!!!!!!!!! # Be sure to update the version before a release vstr = '0.0.1' - diff --git a/ptsa/wavelet.py b/ptsa/wavelet.py index 50592e8..646caaf 100644 --- a/ptsa/wavelet.py +++ b/ptsa/wavelet.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,12 +11,12 @@ import numpy as np # from scipy import unwrap # import scipy.stats as stats -from scipy.fftpack import fft,ifft +from scipy.fftpack import fft, ifft # import scipy.signal # import scipy.ndimage # from ptsa.filt import decimate -from ptsa.helper import reshape_to_2d,reshape_from_2d,centered,next_pow2 -from ptsa.data import TimeSeries,Dim +from ptsa.helper import reshape_to_2d, reshape_from_2d, centered, next_pow2 +from ptsa.data import TimeSeries, Dim from ptsa.fixed_scipy import morlet as morlet_wavelet import pywt @@ -35,8 +35,8 @@ def swt(data, wavelet, level=None): This version is 2 orders of magnitude faster than the one in pywt even though it uses pywt for all the calculations. - - Input parameters: + + Input parameters: data One-dimensional data to transform @@ -51,28 +51,28 @@ def swt(data, wavelet, level=None): num_levels = level idata = data.copy() res = [] - for j in range(1,num_levels+1): - step_size = int(math.pow(2, j-1)) + for j in range(1, num_levels + 1): + step_size = int(math.pow(2, j - 1)) last_index = step_size # allocate cA = np.empty_like(data) cD = np.empty_like(data) - for first in xrange(last_index): # 0 to last_index - 1 - # Getting the indices that we will transform + for first in range(last_index): # 0 to last_index - 1 + # Getting the indices that we will transform indices = np.arange(first, len(cD), step_size) # select the even indices - even_indices = indices[0::2] + even_indices = indices[0::2] # select the odd indices - odd_indices = indices[1::2] - + odd_indices = indices[1::2] + # get the even - (cA1,cD1) = pywt.dwt(idata[indices], wavelet, 'per') + (cA1, cD1) = pywt.dwt(idata[indices], wavelet, 'per') cA[even_indices] = cA1 cD[even_indices] = cD1 # then the odd - (cA1,cD1) = pywt.dwt(np.roll(idata[indices],-1), wavelet, 'per') + (cA1, cD1) = pywt.dwt(np.roll(idata[indices], -1), wavelet, 'per') cA[odd_indices] = cA1 cD[odd_indices] = cD1 @@ -80,7 +80,7 @@ def swt(data, wavelet, level=None): idata = cA # prepend the result - res.insert(0,(cA,cD)) + res.insert(0, (cA, cD)) return res @@ -88,11 +88,11 @@ def swt(data, wavelet, level=None): def iswt(coefficients, wavelet): """ Inverse Stationary Wavelet Transform - - Input parameters: + + Input parameters: coefficients - approx and detail coefficients, arranged in level value + approx and detail coefficients, arranged in level value exactly as output from swt: e.g. [(cA1, cD1), (cA2, cD2), ..., (cAn, cDn)] @@ -100,34 +100,36 @@ def iswt(coefficients, wavelet): Either the name of a wavelet or a Wavelet object """ - output = coefficients[0][0].copy() # Avoid modification of input data + output = coefficients[0][0].copy() # Avoid modification of input data - #num_levels, equivalent to the decomposition level, n + # num_levels, equivalent to the decomposition level, n num_levels = len(coefficients) - for j in range(num_levels,0,-1): - step_size = int(math.pow(2, j-1)) + for j in range(num_levels, 0, -1): + step_size = int(math.pow(2, j - 1)) last_index = step_size _, cD = coefficients[num_levels - j] - for first in xrange(last_index): # 0 to last_index - 1 + for first in range(last_index): # 0 to last_index - 1 - # Getting the indices that we will transform + # Getting the indices that we will transform indices = np.arange(first, len(cD), step_size) # select the even indices - even_indices = indices[0::2] + even_indices = indices[0::2] # select the odd indices - odd_indices = indices[1::2] + odd_indices = indices[1::2] # perform the inverse dwt on the selected indices, # making sure to use periodic boundary conditions - x1 = pywt.idwt(output[even_indices], cD[even_indices], wavelet, 'per') - x2 = pywt.idwt(output[odd_indices], cD[odd_indices], wavelet, 'per') + x1 = pywt.idwt(output[even_indices], + cD[even_indices], wavelet, 'per') + x2 = pywt.idwt(output[odd_indices], + cD[odd_indices], wavelet, 'per') # perform a circular shift right x2 = np.roll(x2, 1) # average and insert into the correct indices - output[indices] = (x1 + x2)/2. + output[indices] = (x1 + x2) / 2. return output @@ -136,12 +138,12 @@ def morlet_multi(freqs, widths, samplerates, sampling_windows=7, complete=True): """ Calculate Morlet wavelets with the total energy normalized to 1. - + Calls the scipy.signal.wavelet.morlet() function to generate Morlet wavelets with the specified frequencies, samplerates, and widths (in cycles); see the docstring for the scipy morlet function for details. These wavelets are normalized before they are returned. - + Parameters ---------- freqs : {float, array_like of floats} @@ -171,17 +173,17 @@ def morlet_multi(freqs, widths, samplerates, the complete version of a Morlet wavelet. Complete should be True, especially for low (<=5) values of width. See scipy.signal.wavelet.morlet() for details. - + Returns ------- A 2-D (frequencies * samples) array of Morlet wavelets. - + Notes ----- The in scipy versions <= 0.6.0, the scipy.signal.wavelet.morlet() code contains a bug. Until it is fixed in a stable release, this code calls a local fixed version of the scipy function. - + Examples -------- >>> wavelet = morlet_multi(10,5,200) @@ -203,61 +205,61 @@ def morlet_multi(freqs, widths, samplerates, # check input: if len(freqs) < 1: raise ValueError("At least one frequency must be specified!") - if len(widths) < 1 or len(freqs)%len(widths) != 0: - raise ValueError("Freqs and widths are not compatible: len(freqs) must "+ - "be evenly divisible by len(widths).\n"+ - "len(freqs) = "+str(len(freqs))+"\nlen(widths) = "+ + if len(widths) < 1 or len(freqs) % len(widths) != 0: + raise ValueError("Freqs and widths are not compatible: len(freqs) must " + + "be evenly divisible by len(widths).\n" + + "len(freqs) = " + str(len(freqs)) + "\nlen(widths) = " + str(len(widths))) - if len(samplerates) < 1 or len(freqs)%len(samplerates) != 0: - raise ValueError("Freqs and samplerates are not compatible:"+ - "len(freqs) must be evenly divisible by"+ - "len(samplerates).\nlen(freqs) = "+str(len(freqs))+ - "\nlen(samplerates) = "+str(len(samplerates))) - if len(sampling_windows) < 1 or len(freqs)%len(sampling_windows) != 0: - raise ValueError("Freqs and sampling_windows are not compatible:"+ - "len(freqs) must be evenly divisible by"+ - "len(sampling_windows).\nlen(freqs) = "+str(len(freqs))+ - "\nlen(sampling_windows) = "+str(len(sampling_windows))) - - + if len(samplerates) < 1 or len(freqs) % len(samplerates) != 0: + raise ValueError("Freqs and samplerates are not compatible:" + + "len(freqs) must be evenly divisible by" + + "len(samplerates).\nlen(freqs) = " + str(len(freqs)) + + "\nlen(samplerates) = " + str(len(samplerates))) + if len(sampling_windows) < 1 or len(freqs) % len(sampling_windows) != 0: + raise ValueError("Freqs and sampling_windows are not compatible:" + + "len(freqs) must be evenly divisible by" + + "len(sampling_windows).\nlen(freqs) = " + str(len(freqs)) + + "\nlen(sampling_windows) = " + str(len(sampling_windows))) + # make len(widths)==len(freqs): - widths = widths.repeat(len(freqs)/len(widths)) - + widths = widths.repeat(len(freqs) / len(widths)) + # make len(samplerates)==len(freqs): - samplerates = samplerates.repeat(len(freqs)/len(samplerates)) + samplerates = samplerates.repeat(len(freqs) / len(samplerates)) # make len(sampling_windows)==len(freqs): - sampling_windows = sampling_windows.repeat(len(freqs)/len(sampling_windows)) - + sampling_windows = sampling_windows.repeat( + len(freqs) / len(sampling_windows)) + # std. devs. in the time domain: - st = widths/(2*np.pi*freqs) - + st = widths / (2 * np.pi * freqs) + # determine number of samples needed: - samples = np.ceil(st*samplerates*sampling_windows) - + samples = np.ceil(st * samplerates * sampling_windows) + # each scale depends on frequency, samples, width, and samplerates: - scales = (freqs*samples)/(2.*widths*samplerates) - + scales = (freqs * samples) / (2. * widths * samplerates) + # generate list of unnormalized wavelets: - wavelets = [morlet_wavelet(samples[i],w=widths[i],s=scales[i], + wavelets = [morlet_wavelet(samples[i], w=widths[i], s=scales[i], complete=complete) - for i in xrange(len(scales))] - + for i in range(len(scales))] + # generate list of energies for the wavelets: - energies = [np.sqrt(np.sum(np.power(np.abs(wavelets[i]),2.))/samplerates[i]) - for i in xrange(len(scales))] - + energies = [np.sqrt(np.sum(np.power(np.abs(wavelets[i]), 2.)) / samplerates[i]) + for i in range(len(scales))] + # normalize the wavelets by dividing each one by its energy: - norm_wavelets = [wavelets[i]/energies[i] - for i in xrange(len(scales))] - + norm_wavelets = [wavelets[i] / energies[i] + for i in range(len(scales))] + return norm_wavelets -def convolve_wave(wav,eegdat): +def convolve_wave(wav, eegdat): wave_coef = [] for ev_dat in eegdat: - wave_coef.append(np.convolve(wav,ev_dat,'same')) + wave_coef.append(np.convolve(wav, ev_dat, 'same')) return wave_coef @@ -271,7 +273,7 @@ def fconv_multi(in1, in2, mode='full'): result. Therefore the output array has as many rows as the product of the number of rows in in1 and in2 (the number of colums depend on the mode). - + Parameters ---------- in1 : {array_like} @@ -283,55 +285,55 @@ def fconv_multi(in1, in2, mode='full'): mode : {'full','valid','same'},optional Specifies the size of the output. See the docstring for scipy.signal.convolve() for details. - + Returns ------- Array with in1.shape[0]*in2.shape[0] rows with the convolution of the 1-D signals in the rows of in1 and in2. - """ + """ # ensure proper number of dimensions in1 = np.atleast_2d(in1) in2 = np.atleast_2d(in2) # get the number of signals and samples in each input - num1,s1 = in1.shape - num2,s2 = in2.shape - + num1, s1 = in1.shape + num2, s2 = in2.shape + # see if we will be returning a complex result complex_result = (np.issubdtype(in1.dtype, np.complex) or np.issubdtype(in2.dtype, np.complex)) # determine the size based on the next power of 2 - actual_size = s1+s2-1 - size = np.power(2,next_pow2(actual_size)) + actual_size = s1 + s2 - 1 + size = np.power(2, next_pow2(actual_size)) # perform the fft of each row of in1 and in2: #in1_fft = np.empty((num1,size),dtype=np.complex128) - in1_fft = np.empty((num1,size),dtype=np.complex) - for i in xrange(num1): - in1_fft[i] = fft(in1[i],size) + in1_fft = np.empty((num1, size), dtype=np.complex) + for i in range(num1): + in1_fft[i] = fft(in1[i], size) #in2_fft = np.empty((num2,size),dtype=np.complex128) - in2_fft = np.empty((num2,size),dtype=np.complex) - for i in xrange(num2): - in2_fft[i] = fft(in2[i],size) - + in2_fft = np.empty((num2, size), dtype=np.complex) + for i in range(num2): + in2_fft[i] = fft(in2[i], size) + # duplicate the signals and multiply before taking the inverse - in1_fft = in1_fft.repeat(num2,axis=0) - in1_fft *= np.vstack([in2_fft]*num1) + in1_fft = in1_fft.repeat(num2, axis=0) + in1_fft *= np.vstack([in2_fft] * num1) ret = ifft(in1_fft) # ret = ifft(in1_fft.repeat(num2,axis=0) * \ # np.vstack([in2_fft]*num1)) - + # delete to save memory del in1_fft, in2_fft - + # strip of extra space if necessary - ret = ret[:,:actual_size] - + ret = ret[:, :actual_size] + # determine if complex, keeping only real if not if not complex_result: ret = ret.real - + # now only keep the requested portion if mode == "full": return ret @@ -340,10 +342,9 @@ def fconv_multi(in1, in2, mode='full'): osize = s1 else: osize = s2 - return centered(ret,(num1*num2,osize)) + return centered(ret, (num1 * num2, osize)) elif mode == "valid": - return centered(ret,(num1*num2,np.abs(s2-s1)+1)) - + return centered(ret, (num1 * num2, np.abs(s2 - s1) + 1)) def phase_pow_multi(freqs, dat, samplerates=None, widths=5, @@ -377,7 +378,7 @@ def phase_pow_multi(freqs, dat, samplerates=None, widths=5, The width(s) of the wavelets in cycles. See docstring of morlet_multi() for details. to_return : {'both','power','phase'}, optional - Specify whether to return power, phase, or both. + Specify whether to return power, phase, or both. time_axis : {int},optional Index of the time/samples dimension in dat. Must be specified if dat is not a TimeSeries instance. If dat is a TimeSeries @@ -394,7 +395,7 @@ def phase_pow_multi(freqs, dat, samplerates=None, widths=5, (only used if dat is a TimeSeries instance). **kwargs : {**kwargs},optional Additional key word arguments to be passed on to morlet_multi(). - + Returns ------- Array(s) of phase and/or power values as specified in to_return. The @@ -403,114 +404,112 @@ def phase_pow_multi(freqs, dat, samplerates=None, widths=5, dimension. """ - dat_is_ts = False # is dat a TimeSeries instance? - if isinstance(dat,TimeSeries): + dat_is_ts = False # is dat a TimeSeries instance? + if isinstance(dat, TimeSeries): samplerates = dat.samplerate time_axis = dat.get_axis(dat.tdim) dat_is_ts = True elif samplerates is None: - raise ValueError('Samplerate must be specified unless you provide a TimeSeries!') + raise ValueError( + 'Samplerate must be specified unless you provide a TimeSeries!') # convert the time_axis to positive index - if time_axis < 0: + if time_axis < 0: time_axis += len(dat.shape) - + # ensure proper dimensionality (needed for len call later): freqs = np.atleast_1d(freqs) - + # check input values: if to_return != 'both' and to_return != 'power' and to_return != 'phase': - raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to "+ - "specify whether power, phase, or both are to be "+ + raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to " + + "specify whether power, phase, or both are to be " + "returned. Invalid value: %s " % to_return) - if not np.issubdtype(conv_dtype,np.complex): - raise ValueError("conv_dtype must be a complex data type!\n"+ - "Invalid value: "+str(conv_dtype)) + if not np.issubdtype(conv_dtype, np.complex): + raise ValueError("conv_dtype must be a complex data type!\n" + + "Invalid value: " + str(conv_dtype)) # generate list of wavelets: - wavelets = morlet_multi(freqs,widths,samplerates,**kwargs) - + wavelets = morlet_multi(freqs, widths, samplerates, **kwargs) + # make sure we have at least as many data samples as wavelet samples - if (np.max([len(i) for i in wavelets]) > dat.shape[time_axis]): - raise ValueError("The number of data samples is insufficient compared "+ - "to the number of wavelet samples. Try increasing "+ - "data samples by using a (longer) buffer.\n data "+ - "samples: "+str(dat.shape[time_axis])+"\nmax wavelet "+ - "samples: "+str(np.max([len(i) for i in wavelets]))) - + if (np.max([len(i) for i in wavelets]) > dat.shape[time_axis]): + raise ValueError("The number of data samples is insufficient compared " + + "to the number of wavelet samples. Try increasing " + + "data samples by using a (longer) buffer.\n data " + + "samples: " + str(dat.shape[time_axis]) + "\nmax wavelet " + + "samples: " + str(np.max([len(i) for i in wavelets]))) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshape_to_2d(dat, time_axis) #.view(np.ndarray) + eegdat = reshape_to_2d(dat, time_axis) # .view(np.ndarray) # for efficiency pre-generate empty array for convolution: - wav_coef = np.empty((eegdat.shape[0]*len(freqs), - eegdat.shape[1]),dtype=conv_dtype) - + wav_coef = np.empty((eegdat.shape[0] * len(freqs), + eegdat.shape[1]), dtype=conv_dtype) + # populate this array with the convolutions: - i=0 + i = 0 step = len(eegdat) for wav in wavelets: - wc = fconv_multi(wav,eegdat,'same') - wav_coef[i:i+step] = wc - i+=step + wc = fconv_multi(wav, eegdat, 'same') + wav_coef[i:i + step] = wc + i += step # for ev_dat in eegdat: # wav_coef[i]=np.convolve(wav,ev_dat,'same') # #wav_coef[i]=scipy.signal.fftconvolve(ev_dat,wav,'same') # i+=1 - + # Determine shape for ouput arrays with added frequency dimension: newshape = list(origshape) # freqs must be first for reshape_from_2d to work - newshape.insert(0,len(freqs)) + newshape.insert(0, len(freqs)) newshape = tuple(newshape) # must add to the time axis, too time_axis += 1 if dat_is_ts: - freq_dim = Dim(freqs,freq_name) - dims_with_freq = np.empty(len(dat.dims)+1,dat.dims.dtype) + freq_dim = Dim(freqs, freq_name) + dims_with_freq = np.empty(len(dat.dims) + 1, dat.dims.dtype) dims_with_freq[0] = freq_dim dims_with_freq[1:] = dat.dims[:] - + if to_return == 'power' or to_return == 'both': # calculate power (wav_coef values are complex, so taking the # absolute value is necessary before taking the power): power = np.abs(wav_coef)**2 # reshape to new shape: - power = reshape_from_2d(power,time_axis,newshape) + power = reshape_from_2d(power, time_axis, newshape) if dat_is_ts: power = TimeSeries(power, tdim=dat.tdim, samplerate=dat.samplerate, dims=dims_with_freq) - - + if to_return == 'phase' or to_return == 'both': # normalize the phase estimates to length one taking care of # instances where they are zero: norm_factor = np.abs(wav_coef) ind = norm_factor == 0 norm_factor[ind] = 1. - wav_coef = wav_coef/norm_factor + wav_coef = wav_coef / norm_factor # wav_coef contains complex numbers, so we need to set these # to 0 when the absolute value 0. wav_coef[ind] = 0 # calculate phase: phase = np.angle(wav_coef) # reshape to new shape - phase = reshape_from_2d(phase,time_axis,newshape) + phase = reshape_from_2d(phase, time_axis, newshape) if dat_is_ts: phase = TimeSeries(phase, tdim=dat.tdim, samplerate=dat.samplerate, dims=dims_with_freq) - if to_return == 'power': return power elif to_return == 'phase': return phase elif to_return == 'both': - return phase,power - + return phase, power ################## @@ -556,7 +555,7 @@ def phase_pow_multi_old(freqs, dat, samplerates, widths=5, to_return='both', Higher complex dtypes produce higher float dtypes in the output. **kwargs : {**kwargs},optional Additional key word arguments to be passed on to morlet_multi(). - + Returns ------- Array(s) of phase and/or power values as specified in to_return. The @@ -566,184 +565,182 @@ def phase_pow_multi_old(freqs, dat, samplerates, widths=5, to_return='both', # ensure proper dimensionality (needed for len call later): freqs = np.atleast_1d(freqs) - + # check input values: if to_return != 'both' and to_return != 'power' and to_return != 'phase': - raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to "+ - "specify whether power, phase, or both are to be "+ + raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to " + + "specify whether power, phase, or both are to be " + "returned. Invalid value: %s " % to_return) - if not np.issubdtype(conv_dtype,np.complex): - raise ValueError("conv_dtype must be a complex data type!\n"+ - "Invalid value: "+str(conv_dtype)) + if not np.issubdtype(conv_dtype, np.complex): + raise ValueError("conv_dtype must be a complex data type!\n" + + "Invalid value: " + str(conv_dtype)) # generate list of wavelets: - wavelets = morlet_multi(freqs,widths,samplerates,**kwargs) - + wavelets = morlet_multi(freqs, widths, samplerates, **kwargs) + # make sure we have at least as many data samples as wavelet samples - if (np.max([len(i) for i in wavelets]) > dat.shape[time_axis]): - raise ValueError("The number of data samples is insufficient compared "+ - "to the number of wavelet samples. Try increasing "+ - "data samples by using a (longer) buffer.\n data "+ - "samples: "+str(dat.shape[time_axis])+"\nmax wavelet "+ - "samples: "+str(np.max([len(i) for i in wavelets]))) - + if (np.max([len(i) for i in wavelets]) > dat.shape[time_axis]): + raise ValueError("The number of data samples is insufficient compared " + + "to the number of wavelet samples. Try increasing " + + "data samples by using a (longer) buffer.\n data " + + "samples: " + str(dat.shape[time_axis]) + "\nmax wavelet " + + "samples: " + str(np.max([len(i) for i in wavelets]))) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshape_to_2d(dat,time_axis) + eegdat = reshape_to_2d(dat, time_axis) # for efficiency pre-generate empty array for convolution: - wavCoef = np.empty((eegdat.shape[time_axis-1]*len(freqs), - eegdat.shape[time_axis]),dtype=conv_dtype) - + wavCoef = np.empty((eegdat.shape[time_axis - 1] * len(freqs), + eegdat.shape[time_axis]), dtype=conv_dtype) + # populate this array with the convolutions: - i=0 + i = 0 for wav in wavelets: for evDat in dat: - wavCoef[i]=np.convolve(wav,evDat,'same') - i+=1 - + wavCoef[i] = np.convolve(wav, evDat, 'same') + i += 1 + # Determine shape for ouput arrays with added frequency dimension: newshape = list(origshape) # freqs must be first for reshapeFrom2D to work - newshape.insert(freq_axis,len(freqs)) + newshape.insert(freq_axis, len(freqs)) newshape = tuple(newshape) - + if to_return == 'power' or to_return == 'both': # calculate power: - power = np.power(np.abs(wavCoef),2) + power = np.power(np.abs(wavCoef), 2) # reshape to new shape: - power = reshape_from_2d(power,time_axis,newshape) - + power = reshape_from_2d(power, time_axis, newshape) + if to_return == 'phase' or to_return == 'both': # normalize the phase estimates to length one taking care of # instances where they are zero: norm_factor = np.abs(wavCoef) ind = norm_factor == 0 norm_factor[ind] = 1. - wavCoef = wavCoef/norm_factor + wavCoef = wavCoef / norm_factor wavCoef[ind] = 0 # calculate phase: phase = np.angle(wavCoef) # reshape to new shape - phase = reshape_from_2d(phase,time_axis,newshape) - + phase = reshape_from_2d(phase, time_axis, newshape) + if to_return == 'power': return power elif to_return == 'phase': return phase elif to_return == 'both': - return phase,power - - - - - + return phase, power -def morlet(freq,t,width): +def morlet(freq, t, width): """Generate a Morlet wavelet for specified frequncy for times t. The wavelet will be normalized so the total energy is 1. width defines the ``width'' of the wavelet in cycles. A value >= 5 is suggested. """ - sf = float(freq)/float(width) - st = 1./(2*np.pi*sf) - A = 1./np.sqrt(st*np.sqrt(np.pi)) - y = A*np.exp(-np.power(t,2)/(2*np.power(st,2)))*np.exp(2j*np.pi*freq*t) + sf = float(freq) / float(width) + st = 1. / (2 * np.pi * sf) + A = 1. / np.sqrt(st * np.sqrt(np.pi)) + y = A * np.exp(-np.power(t, 2) / (2 * np.power(st, 2))) * \ + np.exp(2j * np.pi * freq * t) return y -def phasePow1d(freq,dat,samplerate,width): +def phasePow1d(freq, dat, samplerate, width): """ Calculate phase and power for a single freq and 1d signal. """ # set the parameters for the wavelet - dt = 1./float(samplerate) - sf = float(freq)/float(width) - st = 1./(2*np.pi*sf) - + dt = 1. / float(samplerate) + sf = float(freq) / float(width) + st = 1. / (2 * np.pi * sf) + # get the morlet wavelet for the proper time range - t=np.arange(-3.5*st,3.5*st,dt) - m = morlet(freq,t,width) + t = np.arange(-3.5 * st, 3.5 * st, dt) + m = morlet(freq, t, width) # make sure we are not trying to get a too low a freq # for now it is up to them - #if len(t) > len(dat): - #raise + # if len(t) > len(dat): + # raise # convolve the wavelet and the signal - y = np.convolve(m,dat,'full') + y = np.convolve(m, dat, 'full') # cut off the extra - y = y[np.ceil(len(m)/2.)-1:len(y)-np.floor(len(m)/2.)]; + y = y[np.ceil(len(m) / 2.) - 1:len(y) - np.floor(len(m) / 2.)]; # get the power - power = np.power(np.abs(y),2) + power = np.power(np.abs(y), 2) # find where the power is zero - ind = power==0 - + ind = power == 0 + # normalize the phase estimates to length one y[ind] = 1. - y = y/np.abs(y) + y = y / np.abs(y) y[ind] = 0 - + # get the phase phase = np.angle(y) - return phase,power + return phase, power -def phasePow2d(freq,dat,samplerate,width): + +def phasePow2d(freq, dat, samplerate, width): """ Calculate phase and power for a single freq and 2d signal of shape (events,time). This will be slightly faster than phasePow1d for multiple events because it only calculates the Morlet wavelet once. """ # set the parameters for the wavelet - dt = 1./float(samplerate) - sf = float(freq)/float(width) - st = 1./(2*np.pi*sf) - + dt = 1. / float(samplerate) + sf = float(freq) / float(width) + st = 1. / (2 * np.pi * sf) + # get the morlet wavelet for the proper time range - t=np.arange(-3.5*st,3.5*st,dt) - m = morlet(freq,t,width) + t = np.arange(-3.5 * st, 3.5 * st, dt) + m = morlet(freq, t, width) # make sure is array dat = np.asarray(dat) # allocate for the necessary space - wCoef = np.empty(dat.shape,np.complex64) + wCoef = np.empty(dat.shape, np.complex64) #wCoef = np.empty(dat.shape,np.complex192) - for ev,evDat in enumerate(dat): - # convolve the wavelet and the signal - y = np.convolve(m,evDat,'full') + for ev, evDat in enumerate(dat): + # convolve the wavelet and the signal + y = np.convolve(m, evDat, 'full') - # cut off the extra - y = y[np.ceil(len(m)/2.)-1:len(y)-np.floor(len(m)/2.)]; + # cut off the extra + y = y[np.ceil(len(m) / 2.) - 1:len(y) - np.floor(len(m) / 2.)]; - # insert the data - wCoef[ev] = y + # insert the data + wCoef[ev] = y # get the power - power = np.power(np.abs(wCoef),2) + power = np.power(np.abs(wCoef), 2) # find where the power is zero - ind = power==0 - + ind = power == 0 + # normalize the phase estimates to length one wCoef[ind] = 1. - wCoef = wCoef/np.abs(wCoef) + wCoef = wCoef / np.abs(wCoef) wCoef[ind] = 0 - + # get the phase phase = np.angle(wCoef) - return phase,power + return phase, power -def tsPhasePow(freqs,tseries,width=5,resample=None,keepBuffer=False, - verbose=False,to_return='both',freqDimName='freq'): + +def tsPhasePow(freqs, tseries, width=5, resample=None, keepBuffer=False, + verbose=False, to_return='both', freqDimName='freq'): """ Calculate phase and/or power on an TimeSeries, returning new TimeSeries instances. @@ -752,75 +749,75 @@ def tsPhasePow(freqs,tseries,width=5,resample=None,keepBuffer=False, raise ValueError("to_return must be \'pow\', \'phase\', or \'both\' to\ specify whether power, phase, or both should be returned. Invalid\ value for to_return: %s " % to_return) - + # first get the phase and power as desired - res = calcPhasePow(freqs,tseries.data,tseries.samplerate,axis=tseries.tdim, - width=width,verbose=verbose,to_return=to_return) + res = calcPhasePow(freqs, tseries.data, tseries.samplerate, axis=tseries.tdim, + width=width, verbose=verbose, to_return=to_return) # handle the dims tsdims = tseries.dims.copy() # add in frequency dimension - freqDim = Dim(freqDimName,freqs,'Hz') - tsdims.insert(0,freqDim) - + freqDim = Dim(freqDimName, freqs, 'Hz') + tsdims.insert(0, freqDim) + # turn them into timeseries if to_return == 'pow' or to_return == 'both': # turn into a timeseries - powerAll = TimeSeries(res,tsdims, - tseries.samplerate,unit='XXX get pow unit', - tdim=-1,buf_samp=tseries.buf_samp) - powerAll.data[powerAll.data<=0] = np.finfo(powerAll.data.dtype).eps + powerAll = TimeSeries(res, tsdims, + tseries.samplerate, unit='XXX get pow unit', + tdim=-1, buf_samp=tseries.buf_samp) + powerAll.data[powerAll.data <= 0] = np.finfo(powerAll.data.dtype).eps # see if resample if resample: # must take log before the resample powerAll.data = np.log10(powerAll.data) powerAll.resample(resample) - powerAll.data = np.power(10,powerAll.data) + powerAll.data = np.power(10, powerAll.data) # see if remove buffer if not keepBuffer: powerAll.removeBuf() - + if to_return == 'phase' or to_return == 'both': # get the phase matrix - phaseAll = TimeSeries(res,tsdims, - tseries.samplerate,unit='radians', - tdim=-1,buf_samp=tseries.buf_samp) + phaseAll = TimeSeries(res, tsdims, + tseries.samplerate, unit='radians', + tdim=-1, buf_samp=tseries.buf_samp) if resample: # must unwrap before resampling phaseAll.data = np.unwrap(phaseAll.data) phaseAll.resample(resample) - phaseAll.data = np.mod(phaseAll.data+np.pi,2*np.pi)-np.pi; + phaseAll.data = np.mod(phaseAll.data + np.pi, 2 * np.pi) - np.pi # see if remove buffer if not keepBuffer: phaseAll.removeBuf() - + # see what to return if to_return == 'pow': return powerAll elif to_return == 'phase': return phaseAll elif to_return == 'both': - return phaseAll,powerAll - - + return phaseAll, powerAll + -def calcPhasePow(freqs,dat,samplerate,axis=-1,width=5,verbose=False,to_return='both'): +def calcPhasePow(freqs, dat, samplerate, axis=-1, width=5, verbose=False, to_return='both'): """Calculate phase and power over time with a Morlet wavelet. You can optionally pass in downsample, which is the samplerate to - decimate to following the power/phase calculation. + decimate to following the power/phase calculation. As always, it is best to pass in extra signal (a buffer) on either side of the signal of interest because power calculations and decimation have edge effects.""" if to_return != 'both' and to_return != 'pow' and to_return != 'phase': - raise ValueError("to_return must be \'pow\', \'phase\', or \'both\' to specify whether power, phase, or both are returned. Invalid value: %s " % to_return) - + raise ValueError( + "to_return must be \'pow\', \'phase\', or \'both\' to specify whether power, phase, or both are returned. Invalid value: %s " % to_return) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshape_to_2d(dat,axis) + eegdat = reshape_to_2d(dat, axis) # allocate phaseAll = [] @@ -828,46 +825,44 @@ def calcPhasePow(freqs,dat,samplerate,axis=-1,width=5,verbose=False,to_return='b # loop over freqs freqs = np.asarray(freqs) - if len(freqs.shape)==0: - freqs = np.array([freqs]) + if len(freqs.shape) == 0: + freqs = np.array([freqs]) if verbose: - sys.stdout.write('Calculating wavelet phase/power...\n') - sys.stdout.write('Freqs (%g to %g): ' % (np.min(freqs),np.max(freqs))) - for f,freq in enumerate(freqs): - if verbose: - sys.stdout.write('%g ' % (freq)) - sys.stdout.flush() - # get the phase and power for that freq - phase,power = phasePow2d(freq,eegdat,samplerate,width) - + sys.stdout.write('Calculating wavelet phase/power...\n') + sys.stdout.write('Freqs (%g to %g): ' % (np.min(freqs), np.max(freqs))) + for f, freq in enumerate(freqs): + if verbose: + sys.stdout.write('%g ' % (freq)) + sys.stdout.flush() + # get the phase and power for that freq + phase, power = phasePow2d(freq, eegdat, samplerate, width) + # reshape back do original data shape - if to_return == 'phase' or to_return == 'both': - phase = reshape_from_2d(phase,axis,origshape) - if to_return == 'pow' or to_return == 'both': - power = reshape_from_2d(power,axis,origshape) - - # see if allocate - if len(phaseAll) == 0 and len(powerAll) == 0: - if to_return == 'phase' or to_return == 'both': - phaseAll = np.empty(np.concatenate(([len(freqs)],phase.shape)), - dtype=phase.dtype) - if to_return == 'pow' or to_return == 'both': - powerAll = np.empty(np.concatenate(([len(freqs)],power.shape)), - dtype=power.dtype) - # insert into all - if to_return == 'phase' or to_return == 'both': - phaseAll[f] = phase - if to_return == 'pow' or to_return == 'both': - powerAll[f] = power + if to_return == 'phase' or to_return == 'both': + phase = reshape_from_2d(phase, axis, origshape) + if to_return == 'pow' or to_return == 'both': + power = reshape_from_2d(power, axis, origshape) + + # see if allocate + if len(phaseAll) == 0 and len(powerAll) == 0: + if to_return == 'phase' or to_return == 'both': + phaseAll = np.empty(np.concatenate(([len(freqs)], phase.shape)), + dtype=phase.dtype) + if to_return == 'pow' or to_return == 'both': + powerAll = np.empty(np.concatenate(([len(freqs)], power.shape)), + dtype=power.dtype) + # insert into all + if to_return == 'phase' or to_return == 'both': + phaseAll[f] = phase + if to_return == 'pow' or to_return == 'both': + powerAll[f] = power if verbose: - sys.stdout.write('\n') + sys.stdout.write('\n') if to_return == 'pow': return powerAll elif to_return == 'phase': return phaseAll elif to_return == 'both': - return phaseAll,powerAll - - + return phaseAll, powerAll diff --git a/ptsa/wavelet_obsolete.py b/ptsa/wavelet_obsolete.py index 7d35ecc..8125668 100644 --- a/ptsa/wavelet_obsolete.py +++ b/ptsa/wavelet_obsolete.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -11,23 +11,24 @@ import numpy as N from scipy import unwrap import scipy.stats as stats -from scipy.fftpack import fft,ifft +from scipy.fftpack import fft, ifft from ptsa.filt import decimate -from ptsa.helper import reshapeTo2D,reshapeFrom2D,nextPow2,centered -from ptsa.data import TimeSeries,Dim,Dims,DimData +from ptsa.helper import reshapeTo2D, reshapeFrom2D, nextPow2, centered +from ptsa.data import TimeSeries, Dim, Dims, DimData from ptsa.fixed_scipy import morlet as morlet_wavelet + def morlet_multi(freqs, widths, samplerate, sampling_window=7, complete=True): """ Calculate Morlet wavelets with the total energy normalized to 1. - + Calls the scipy.signal.wavelet.morlet() function to generate Morlet wavelets with the specified frequencies, samplerate, and widths (in cycles); see the docstring for the scipy morlet function for details. These wavelets are normalized before they are returned. - + Parameters ---------- freqs : {int, float, array_like of ints or floats} @@ -58,17 +59,17 @@ def morlet_multi(freqs, widths, samplerate, the complete version of a Morlet wavelet. Complete should be True, especially for low (<=5) values of width. See scipy.signal.wavelet.morlet() for details. - + Returns ------- A 2-D (frequencies * samples) array of Morlet wavelets. - + Notes ----- The in scipy versions <= 0.6.0, the scipy.signal.wavelet.morlet() code contains a bug. Until it is fixed in a stable release, this code calls a local fixed version of the scipy function. - + Examples -------- >>> wavelet = morlet_multi(10,5,200) @@ -86,47 +87,47 @@ def morlet_multi(freqs, widths, samplerate, widths = N.atleast_1d(widths) # make len(widths)==len(freqs): - widths = widths.repeat(len(freqs)/len(widths)) + widths = widths.repeat(len(freqs) / len(widths)) if len(widths) != len(freqs): - raise ValueError("Freqs and widths are not compatible: len(freqs) must "+ - "be evenly divisible by len(widths).\n"+ - "len(freqs) = "+str(len(freqs))+"\nlen(widths) = "+ - str(len(widths)/(len(freqs)/len(widths)))) - + raise ValueError("Freqs and widths are not compatible: len(freqs) must " + + "be evenly divisible by len(widths).\n" + + "len(freqs) = " + str(len(freqs)) + "\nlen(widths) = " + + str(len(widths) / (len(freqs) / len(widths)))) + # std. devs. in the time domain: - st = widths/(2*N.pi*freqs) - + st = widths / (2 * N.pi * freqs) + # determine number of samples needed based on wavelet with maximum # standard deviation in time domain - samples = N.ceil(N.max(st)*samplerate*sampling_window) - + samples = N.ceil(N.max(st) * samplerate * sampling_window) + # determine the scales of the wavelet (cf. # scipy.signal.wavelets.morlet docstring): - scales = (freqs*samples)/(2.*widths*samplerate) - + scales = (freqs * samples) / (2. * widths * samplerate) + #wavelets = N.empty((len(freqs),samples),dtype=N.complex128) - wavelets = N.empty((len(freqs),samples),dtype=N.complex) - for i in xrange(len(freqs)): - wavelets[i] = morlet_wavelet(samples,w=widths[i],s=scales[i], + wavelets = N.empty((len(freqs), samples), dtype=N.complex) + for i in range(len(freqs)): + wavelets[i] = morlet_wavelet(samples, w=widths[i], s=scales[i], complete=complete) - #wavelets = N.array([morlet_wavelet(samples,w=widths[i],s=scales[i], + # wavelets = N.array([morlet_wavelet(samples,w=widths[i],s=scales[i], # complete=complete) # for i in xrange(len(scales))]) - energy = N.sqrt(N.sum(N.power(N.abs(wavelets),2.),axis=1)/samplerate) - norm_factors = N.vstack([1./energy]*samples).T - return wavelets*norm_factors + energy = N.sqrt(N.sum(N.power(N.abs(wavelets), 2.), axis=1) / samplerate) + norm_factors = N.vstack([1. / energy] * samples).T + return wavelets * norm_factors -def morlet_multi2(freqs, widths, samplerate,fft_thresh=90, - sampling_window=7, complete=True): +def morlet_multi2(freqs, widths, samplerate, fft_thresh=90, + sampling_window=7, complete=True): """ Calculate Morlet wavelets with the total energy normalized to 1. - + Calls the scipy.signal.wavelet.morlet() function to generate Morlet wavelets with the specified frequencies, samplerate, and widths (in cycles); see the docstring for the scipy morlet function for details. These wavelets are normalized before they are returned. - + Parameters ---------- freqs : {int, float, array_like of ints or floats} @@ -159,17 +160,17 @@ def morlet_multi2(freqs, widths, samplerate,fft_thresh=90, the complete version of a Morlet wavelet. Complete should be True, especially for low (<=5) values of width. See scipy.signal.wavelet.morlet() for details. - + Returns ------- A 2-D (frequencies * samples) array of Morlet wavelets. - + Notes ----- The in scipy versions <= 0.6.0, the scipy.signal.wavelet.morlet() code contains a bug. Until it is fixed in a stable release, this code calls a local fixed version of the scipy function. - + Examples -------- >>> wavelet = morlet_multi(10,5,200) @@ -187,19 +188,19 @@ def morlet_multi2(freqs, widths, samplerate,fft_thresh=90, widths = N.atleast_1d(widths) # make len(widths)==len(freqs): - widths = widths.repeat(len(freqs)/len(widths)) + widths = widths.repeat(len(freqs) / len(widths)) if len(widths) != len(freqs): - raise ValueError("Freqs and widths are not compatible: len(freqs) must "+ - "be evenly divisible by len(widths).\n"+ - "len(freqs) = "+str(len(freqs))+"\nlen(widths) = "+ - str(len(widths)/(len(freqs)/len(widths)))) - + raise ValueError("Freqs and widths are not compatible: len(freqs) must " + + "be evenly divisible by len(widths).\n" + + "len(freqs) = " + str(len(freqs)) + "\nlen(widths) = " + + str(len(widths) / (len(freqs) / len(widths)))) + # std. devs. in the time domain: - st = widths/(2*N.pi*freqs) - + st = widths / (2 * N.pi * freqs) + # determine number of samples needed based on wavelet with maximum # standard deviation in time domain - samples = N.ceil(st*samplerate*sampling_window) + samples = N.ceil(st * samplerate * sampling_window) #samples = N.ceil(N.max(st)*samplerate*sampling_window) # get indices for wavelets that exceed the threshold for fft @@ -209,38 +210,38 @@ def morlet_multi2(freqs, widths, samplerate,fft_thresh=90, fft_samples = N.max(samples[fft_ind]) fft_freqs = freqs[fft_ind] fft_widths = widths[fft_ind] - + # determine the scales of the wavelet (cf. # scipy.signal.wavelets.morlet docstring): - fft_scales = (fft_freqs*fft_samples)/(2.*fft_widths*samplerate) - + fft_scales = (fft_freqs * fft_samples) / (2. * fft_widths * samplerate) + #fft_wavelets = N.empty((len(fft_freqs),fft_samples),dtype=N.complex128) - fft_wavelets = N.empty((len(fft_freqs),fft_samples),dtype=N.complex) - for i in xrange(len(fft_freqs)): - fft_wavelets[i] = morlet_wavelet(fft_samples,w=fft_widths[i], - s=fft_scales[i],complete=complete) - fft_energy = N.sqrt(N.sum(N.power(N.abs(fft_wavelets),2.), - axis=1)/samplerate) - fft_norm_factors = N.vstack([1./fft_energy]*fft_samples).T - fft_wavelets = fft_wavelets*fft_norm_factors + fft_wavelets = N.empty((len(fft_freqs), fft_samples), dtype=N.complex) + for i in range(len(fft_freqs)): + fft_wavelets[i] = morlet_wavelet(fft_samples, w=fft_widths[i], + s=fft_scales[i], complete=complete) + fft_energy = N.sqrt(N.sum(N.power(N.abs(fft_wavelets), 2.), + axis=1) / samplerate) + fft_norm_factors = N.vstack([1. / fft_energy] * fft_samples).T + fft_wavelets = fft_wavelets * fft_norm_factors else: fft_wavelets = N.array([[]]) - + reg_samples = samples[~fft_ind] reg_freqs = freqs[~fft_ind] reg_widths = widths[~fft_ind] - - reg_scales = (reg_freqs*reg_samples)/(2.*reg_widths*samplerate) - reg_wavelets = [morlet_wavelet(reg_samples[i],w=reg_widths[i], - s=reg_scales[i],complete=complete) - for i in xrange(len(reg_scales))] - reg_energy = [N.sqrt(N.sum(N.power(N.abs(reg_wavelets[i]),2.))/samplerate) - for i in xrange(len(reg_scales))] - reg_norm_wavelets = [reg_wavelets[i]/reg_energy[i] - for i in xrange(len(reg_scales))] + reg_scales = (reg_freqs * reg_samples) / (2. * reg_widths * samplerate) + + reg_wavelets = [morlet_wavelet(reg_samples[i], w=reg_widths[i], + s=reg_scales[i], complete=complete) + for i in range(len(reg_scales))] + reg_energy = [N.sqrt(N.sum(N.power(N.abs(reg_wavelets[i]), 2.)) / samplerate) + for i in range(len(reg_scales))] + reg_norm_wavelets = [reg_wavelets[i] / reg_energy[i] + for i in range(len(reg_scales))] - return (fft_wavelets,reg_norm_wavelets,fft_ind) + return (fft_wavelets, reg_norm_wavelets, fft_ind) def fconv_multi(in1, in2, mode='full'): @@ -253,7 +254,7 @@ def fconv_multi(in1, in2, mode='full'): result. Therefore the output array has as many rows as the product of the number of rows in in1 and in2 (the number of colums depend on the mode). - + Parameters ---------- in1 : {array_like} @@ -265,55 +266,55 @@ def fconv_multi(in1, in2, mode='full'): mode : {'full','valid','same'},optional Specifies the size of the output. See the docstring for scipy.signal.convolve() for details. - + Returns ------- Array with in1.shape[0]*in2.shape[0] rows with the convolution of the 1-D signals in the rows of in1 and in2. - """ + """ # ensure proper number of dimensions in1 = N.atleast_2d(in1) in2 = N.atleast_2d(in2) # get the number of signals and samples in each input - num1,s1 = in1.shape - num2,s2 = in2.shape - + num1, s1 = in1.shape + num2, s2 = in2.shape + # see if we will be returning a complex result complex_result = (N.issubdtype(in1.dtype, N.complex) or N.issubdtype(in2.dtype, N.complex)) # determine the size based on the next power of 2 - actual_size = s1+s2-1 - size = N.power(2,nextPow2(actual_size)) + actual_size = s1 + s2 - 1 + size = N.power(2, nextPow2(actual_size)) # perform the fft of each row of in1 and in2: #in1_fft = N.empty((num1,size),dtype=N.complex128) - in1_fft = N.empty((num1,size),dtype=N.complex) - for i in xrange(num1): - in1_fft[i] = fft(in1[i],size) + in1_fft = N.empty((num1, size), dtype=N.complex) + for i in range(num1): + in1_fft[i] = fft(in1[i], size) #in2_fft = N.empty((num2,size),dtype=N.complex128) - in2_fft = N.empty((num2,size),dtype=N.complex) - for i in xrange(num2): - in2_fft[i] = fft(in2[i],size) - + in2_fft = N.empty((num2, size), dtype=N.complex) + for i in range(num2): + in2_fft[i] = fft(in2[i], size) + # duplicate the signals and multiply before taking the inverse - in1_fft = in1_fft.repeat(num2,axis=0) - in1_fft *= N.vstack([in2_fft]*num1) + in1_fft = in1_fft.repeat(num2, axis=0) + in1_fft *= N.vstack([in2_fft] * num1) ret = ifft(in1_fft) # ret = ifft(in1_fft.repeat(num2,axis=0) * \ # N.vstack([in2_fft]*num1)) - + # delete to save memory del in1_fft, in2_fft - + # strip of extra space if necessary - ret = ret[:,:actual_size] - + ret = ret[:, :actual_size] + # determine if complex, keeping only real if not if not complex_result: ret = ret.real - + # now only keep the requested portion if mode == "full": return ret @@ -322,9 +323,9 @@ def fconv_multi(in1, in2, mode='full'): osize = s1 else: osize = s2 - return centered(ret,(num1*num2,osize)) + return centered(ret, (num1 * num2, osize)) elif mode == "valid": - return centered(ret,(num1*num2,N.abs(s2-s1)+1)) + return centered(ret, (num1 * num2, N.abs(s2 - s1) + 1)) def phase_pow_multi(freqs, dat, samplerate, widths=5, to_return='both', @@ -360,7 +361,7 @@ def phase_pow_multi(freqs, dat, samplerate, widths=5, to_return='both', Should be in {0, time_axis, time_axis+1,len(dat.shape)}. **kwargs : {**kwargs},optional Additional key word arguments to be passed on to morlet_multi(). - + Returns ------- Array(s) of phase and/or power values as specified in to_return. The @@ -368,40 +369,40 @@ def phase_pow_multi(freqs, dat, samplerate, widths=5, to_return='both', dimension is for the frequencies and is inserted at freq_axis. """ if to_return != 'both' and to_return != 'power' and to_return != 'phase': - raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to "+ - "specify whether power, phase, or both are to be "+ + raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to " + + "specify whether power, phase, or both are to be " + "returned. Invalid value: %s " % to_return) # generate array of wavelets: - wavelets = morlet_multi(freqs,widths,samplerate,**kwargs) + wavelets = morlet_multi(freqs, widths, samplerate, **kwargs) # make sure we have at least as many data samples as wavelet samples - if wavelets.shape[1]>dat.shape[time_axis]: - raise ValueError("The number of data samples is insufficient compared "+ - "to the number of wavelet samples. Try increasing "+ - "data samples by using a (longer) buffer.\n data "+ - "samples: "+str(dat.shape[time_axis])+"\nwavelet "+ - "samples: "+str(wavelets.shape[1])) - + if wavelets.shape[1] > dat.shape[time_axis]: + raise ValueError("The number of data samples is insufficient compared " + + "to the number of wavelet samples. Try increasing " + + "data samples by using a (longer) buffer.\n data " + + "samples: " + str(dat.shape[time_axis]) + "\nwavelet " + + "samples: " + str(wavelets.shape[1])) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshapeTo2D(dat,time_axis) + eegdat = reshapeTo2D(dat, time_axis) # calculate wavelet coefficients: - wavCoef = fconv_multi(wavelets,eegdat,mode='same') + wavCoef = fconv_multi(wavelets, eegdat, mode='same') # Determine shape for ouput arrays with added frequency dimension: newshape = list(origshape) # freqs must be first for reshapeFrom2D to work # XXX - newshape.insert(freq_axis,len(freqs)) + newshape.insert(freq_axis, len(freqs)) newshape = tuple(newshape) - + if to_return == 'power' or to_return == 'both': # calculate power: - power = N.power(N.abs(wavCoef),2) + power = N.power(N.abs(wavCoef), 2) # reshape to new shape: - power = reshapeFrom2D(power,time_axis,newshape) + power = reshapeFrom2D(power, time_axis, newshape) if to_return == 'phase' or to_return == 'both': # normalize the phase estimates to length one taking care of @@ -409,23 +410,23 @@ def phase_pow_multi(freqs, dat, samplerate, widths=5, to_return='both', norm_factor = N.abs(wavCoef) ind = norm_factor == 0 norm_factor[ind] = 1. - wavCoef = wavCoef/norm_factor + wavCoef = wavCoef / norm_factor wavCoef[ind] = 0 # calculate phase: phase = N.angle(wavCoef) # reshape to new shape - phase = reshapeFrom2D(phase,time_axis,newshape) + phase = reshapeFrom2D(phase, time_axis, newshape) if to_return == 'power': return power elif to_return == 'phase': return phase elif to_return == 'both': - return phase,power - + return phase, power + def phase_pow_multi2(freqs, dat, samplerate, widths=5, to_return='both', - time_axis=-1, freq_axis=0, **kwargs): + time_axis=-1, freq_axis=0, **kwargs): """ Calculate phase and power with wavelets across multiple events. @@ -457,7 +458,7 @@ def phase_pow_multi2(freqs, dat, samplerate, widths=5, to_return='both', Should be in {0, time_axis, time_axis+1,len(dat.shape)}. **kwargs : {**kwargs},optional Additional key word arguments to be passed on to morlet_multi(). - + Returns ------- Array(s) of phase and/or power values as specified in to_return. The @@ -465,61 +466,61 @@ def phase_pow_multi2(freqs, dat, samplerate, widths=5, to_return='both', dimension is for the frequencies and is inserted at freq_axis. """ if to_return != 'both' and to_return != 'power' and to_return != 'phase': - raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to "+ - "specify whether power, phase, or both are to be "+ + raise ValueError("to_return must be \'power\', \'phase\', or \'both\' to " + + "specify whether power, phase, or both are to be " + "returned. Invalid value: %s " % to_return) # generate array of wavelets: - fft_wavelets,reg_wavelets,fft_ind = morlet_multi2(freqs,widths,samplerate, - **kwargs) + fft_wavelets, reg_wavelets, fft_ind = morlet_multi2(freqs, widths, samplerate, + **kwargs) # make sure we have at least as many data samples as wavelet samples if ((fft_wavelets.shape[1] > dat.shape[time_axis]) or - ((len(reg_wavelets)>0) and - (N.max([len(i) for i in reg_wavelets]) > dat.shape[time_axis]))): - raise ValueError("The number of data samples is insufficient compared "+ - "to the number of wavelet samples. Try increasing "+ - "data samples by using a (longer) buffer.\n data "+ - "samples: "+str(dat.shape[time_axis])+"\nwavelet "+ - "samples: "+str(fft_wavelets.shape[1])) - + ((len(reg_wavelets) > 0) and + (N.max([len(i) for i in reg_wavelets]) > dat.shape[time_axis]))): + raise ValueError("The number of data samples is insufficient compared " + + "to the number of wavelet samples. Try increasing " + + "data samples by using a (longer) buffer.\n data " + + "samples: " + str(dat.shape[time_axis]) + "\nwavelet " + + "samples: " + str(fft_wavelets.shape[1])) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshapeTo2D(dat,time_axis) + eegdat = reshapeTo2D(dat, time_axis) # calculate wavelet coefficients: - #wavCoef = N.empty((eegdat.shape[time_axis-1]*len(freqs), + # wavCoef = N.empty((eegdat.shape[time_axis-1]*len(freqs), # eegdat.shape[time_axis]),dtype=N.complex128) - wavCoef = N.empty((eegdat.shape[time_axis-1]*len(freqs), - eegdat.shape[time_axis]),dtype=N.complex) + wavCoef = N.empty((eegdat.shape[time_axis - 1] * len(freqs), + eegdat.shape[time_axis]), dtype=N.complex) if fft_wavelets.shape[1] > 0: - fconv_ind = N.repeat(fft_ind,eegdat.shape[time_axis-1]) - wavCoef[fconv_ind] = fconv_multi(fft_wavelets,eegdat,mode='same') + fconv_ind = N.repeat(fft_ind, eegdat.shape[time_axis - 1]) + wavCoef[fconv_ind] = fconv_multi(fft_wavelets, eegdat, mode='same') - #reg_wavCoef = N.empty((eegdat.shape[time_axis-1]*N.sum(~fft_ind), + # reg_wavCoef = N.empty((eegdat.shape[time_axis-1]*N.sum(~fft_ind), # eegdat.shape[time_axis]),dtype=N.complex128) - reg_wavCoef = N.empty((eegdat.shape[time_axis-1]*N.sum(~fft_ind), - eegdat.shape[time_axis]),dtype=N.complex) - conv_ind = N.repeat(~fft_ind,eegdat.shape[time_axis-1]) - i=0 - for reg in xrange(len(reg_wavelets)): - for ev,evDat in enumerate(dat): - #print len(reg_wavelets), reg - reg_wavCoef[i] = N.convolve(reg_wavelets[reg],evDat,'same') + reg_wavCoef = N.empty((eegdat.shape[time_axis - 1] * N.sum(~fft_ind), + eegdat.shape[time_axis]), dtype=N.complex) + conv_ind = N.repeat(~fft_ind, eegdat.shape[time_axis - 1]) + i = 0 + for reg in range(len(reg_wavelets)): + for ev, evDat in enumerate(dat): + # print len(reg_wavelets), reg + reg_wavCoef[i] = N.convolve(reg_wavelets[reg], evDat, 'same') i += 1 wavCoef[conv_ind] = reg_wavCoef - + # Determine shape for ouput arrays with added frequency dimension: newshape = list(origshape) # freqs must be first for reshapeFrom2D to work # XXX - newshape.insert(freq_axis,len(freqs)) + newshape.insert(freq_axis, len(freqs)) newshape = tuple(newshape) - + if to_return == 'power' or to_return == 'both': # calculate power: - power = N.power(N.abs(wavCoef),2) + power = N.power(N.abs(wavCoef), 2) # reshape to new shape: - power = reshapeFrom2D(power,time_axis,newshape) + power = reshapeFrom2D(power, time_axis, newshape) if to_return == 'phase' or to_return == 'both': # normalize the phase estimates to length one taking care of @@ -527,129 +528,131 @@ def phase_pow_multi2(freqs, dat, samplerate, widths=5, to_return='both', norm_factor = N.abs(wavCoef) ind = norm_factor == 0 norm_factor[ind] = 1. - wavCoef = wavCoef/norm_factor + wavCoef = wavCoef / norm_factor wavCoef[ind] = 0 # calculate phase: phase = N.angle(wavCoef) # reshape to new shape - phase = reshapeFrom2D(phase,time_axis,newshape) + phase = reshapeFrom2D(phase, time_axis, newshape) if to_return == 'power': return power elif to_return == 'phase': return phase elif to_return == 'both': - return phase,power - + return phase, power ################## # Old wavelet code ################## -def morlet(freq,t,width): +def morlet(freq, t, width): """Generate a Morlet wavelet for specified frequncy for times t. The wavelet will be normalized so the total energy is 1. width defines the ``width'' of the wavelet in cycles. A value >= 5 is suggested. """ - sf = float(freq)/float(width) - st = 1./(2*N.pi*sf) - A = 1./N.sqrt(st*N.sqrt(N.pi)) - y = A*N.exp(-N.power(t,2)/(2*N.power(st,2)))*N.exp(2j*N.pi*freq*t) + sf = float(freq) / float(width) + st = 1. / (2 * N.pi * sf) + A = 1. / N.sqrt(st * N.sqrt(N.pi)) + y = A * N.exp(-N.power(t, 2) / (2 * N.power(st, 2))) * \ + N.exp(2j * N.pi * freq * t) return y -def phasePow1d(freq,dat,samplerate,width): +def phasePow1d(freq, dat, samplerate, width): """ Calculate phase and power for a single freq and 1d signal. """ # set the parameters for the wavelet - dt = 1./float(samplerate) - sf = float(freq)/float(width) - st = 1./(2*N.pi*sf) - + dt = 1. / float(samplerate) + sf = float(freq) / float(width) + st = 1. / (2 * N.pi * sf) + # get the morlet wavelet for the proper time range - t=N.arange(-3.5*st,3.5*st,dt) - m = morlet(freq,t,width) + t = N.arange(-3.5 * st, 3.5 * st, dt) + m = morlet(freq, t, width) # make sure we are not trying to get a too low a freq # for now it is up to them - #if len(t) > len(dat): - #raise + # if len(t) > len(dat): + # raise # convolve the wavelet and the signal - y = N.convolve(m,dat,'full') + y = N.convolve(m, dat, 'full') # cut off the extra - y = y[N.ceil(len(m)/2.)-1:len(y)-N.floor(len(m)/2.)]; + y = y[N.ceil(len(m) / 2.) - 1:len(y) - N.floor(len(m) / 2.)]; # get the power - power = N.power(N.abs(y),2) + power = N.power(N.abs(y), 2) # find where the power is zero - ind = power==0 - + ind = power == 0 + # normalize the phase estimates to length one y[ind] = 1. - y = y/N.abs(y) + y = y / N.abs(y) y[ind] = 0 - + # get the phase phase = N.angle(y) - return phase,power + return phase, power -def phasePow2d(freq,dat,samplerate,width): + +def phasePow2d(freq, dat, samplerate, width): """ Calculate phase and power for a single freq and 2d signal of shape (events,time). This will be slightly faster than phasePow1d for multiple events because it only calculates the Morlet wavelet once. """ # set the parameters for the wavelet - dt = 1./float(samplerate) - sf = float(freq)/float(width) - st = 1./(2*N.pi*sf) - + dt = 1. / float(samplerate) + sf = float(freq) / float(width) + st = 1. / (2 * N.pi * sf) + # get the morlet wavelet for the proper time range - t=N.arange(-3.5*st,3.5*st,dt) - m = morlet(freq,t,width) + t = N.arange(-3.5 * st, 3.5 * st, dt) + m = morlet(freq, t, width) # make sure is array dat = N.asarray(dat) # allocate for the necessary space #wCoef = N.empty(dat.shape,N.complex64) - wCoef = N.empty(dat.shape,N.complex128) + wCoef = N.empty(dat.shape, N.complex128) - for ev,evDat in enumerate(dat): - # convolve the wavelet and the signal - y = N.convolve(m,evDat,'full') + for ev, evDat in enumerate(dat): + # convolve the wavelet and the signal + y = N.convolve(m, evDat, 'full') - # cut off the extra - y = y[N.ceil(len(m)/2.)-1:len(y)-N.floor(len(m)/2.)]; + # cut off the extra + y = y[N.ceil(len(m) / 2.) - 1:len(y) - N.floor(len(m) / 2.)]; - # insert the data - wCoef[ev] = y + # insert the data + wCoef[ev] = y # get the power - power = N.power(N.abs(wCoef),2) + power = N.power(N.abs(wCoef), 2) # find where the power is zero - ind = power==0 - + ind = power == 0 + # normalize the phase estimates to length one wCoef[ind] = 1. - wCoef = wCoef/N.abs(wCoef) + wCoef = wCoef / N.abs(wCoef) wCoef[ind] = 0 - + # get the phase phase = N.angle(wCoef) - return phase,power + return phase, power -def tsPhasePow(freqs,tseries,width=5,resample=None,keepBuffer=False, - verbose=False,to_return='both',freqDimName='freq'): + +def tsPhasePow(freqs, tseries, width=5, resample=None, keepBuffer=False, + verbose=False, to_return='both', freqDimName='freq'): """ Calculate phase and/or power on an TimeSeries, returning new TimeSeries instances. @@ -658,60 +661,59 @@ def tsPhasePow(freqs,tseries,width=5,resample=None,keepBuffer=False, raise ValueError("to_return must be \'pow\', \'phase\', or \'both\' to\ specify whether power, phase, or both should be returned. Invalid\ value for to_return: %s " % to_return) - + # first get the phase and power as desired - res = calcPhasePow(freqs,tseries.data,tseries.samplerate,axis=tseries.tdim, - width=width,verbose=verbose,to_return=to_return) + res = calcPhasePow(freqs, tseries.data, tseries.samplerate, axis=tseries.tdim, + width=width, verbose=verbose, to_return=to_return) # handle the dims tsdims = tseries.dims.copy() # add in frequency dimension - freqDim = Dim(freqDimName,freqs,'Hz') - tsdims.insert(0,freqDim) - + freqDim = Dim(freqDimName, freqs, 'Hz') + tsdims.insert(0, freqDim) + # turn them into timeseries if to_return == 'pow' or to_return == 'both': # turn into a timeseries - powerAll = TimeSeries(res,tsdims, - tseries.samplerate,unit='XXX get pow unit', - tdim=-1,buf_samp=tseries.buf_samp) - powerAll.data[powerAll.data<=0] = N.finfo(powerAll.data.dtype).eps + powerAll = TimeSeries(res, tsdims, + tseries.samplerate, unit='XXX get pow unit', + tdim=-1, buf_samp=tseries.buf_samp) + powerAll.data[powerAll.data <= 0] = N.finfo(powerAll.data.dtype).eps # see if resample if resample: # must take log before the resample powerAll.data = N.log10(powerAll.data) powerAll.resample(resample) - powerAll.data = N.power(10,powerAll.data) + powerAll.data = N.power(10, powerAll.data) # see if remove buffer if not keepBuffer: powerAll.removeBuf() - + if to_return == 'phase' or to_return == 'both': # get the phase matrix - phaseAll = TimeSeries(res,tsdims, - tseries.samplerate,unit='radians', - tdim=-1,buf_samp=tseries.buf_samp) + phaseAll = TimeSeries(res, tsdims, + tseries.samplerate, unit='radians', + tdim=-1, buf_samp=tseries.buf_samp) if resample: # must unwrap before resampling phaseAll.data = N.unwrap(phaseAll.data) phaseAll.resample(resample) - phaseAll.data = N.mod(phaseAll.data+N.pi,2*N.pi)-N.pi; + phaseAll.data = N.mod(phaseAll.data + N.pi, 2 * N.pi) - N.pi # see if remove buffer if not keepBuffer: phaseAll.removeBuf() - + # see what to return if to_return == 'pow': return powerAll elif to_return == 'phase': return phaseAll elif to_return == 'both': - return phaseAll,powerAll - - + return phaseAll, powerAll -def calcPhasePow(freqs,dat,samplerate,axis=-1,width=5,verbose=False,to_return='both'): + +def calcPhasePow(freqs, dat, samplerate, axis=-1, width=5, verbose=False, to_return='both'): """Calculate phase and power over time with a Morlet wavelet. You can optionally pass in downsample, which is the samplerate to @@ -722,11 +724,12 @@ def calcPhasePow(freqs,dat,samplerate,axis=-1,width=5,verbose=False,to_return='b decimation have edge effects.""" if to_return != 'both' and to_return != 'pow' and to_return != 'phase': - raise ValueError("to_return must be \'pow\', \'phase\', or \'both\' to specify whether power, phase, or both are returned. Invalid value: %s " % to_return) - + raise ValueError( + "to_return must be \'pow\', \'phase\', or \'both\' to specify whether power, phase, or both are returned. Invalid value: %s " % to_return) + # reshape the data to 2D with time on the 2nd dimension origshape = dat.shape - eegdat = reshapeTo2D(dat,axis) + eegdat = reshapeTo2D(dat, axis) # allocate phaseAll = [] @@ -734,46 +737,44 @@ def calcPhasePow(freqs,dat,samplerate,axis=-1,width=5,verbose=False,to_return='b # loop over freqs freqs = N.asarray(freqs) - if len(freqs.shape)==0: - freqs = N.array([freqs]) + if len(freqs.shape) == 0: + freqs = N.array([freqs]) if verbose: - sys.stdout.write('Calculating wavelet phase/power...\n') - sys.stdout.write('Freqs (%g to %g): ' % (N.min(freqs),N.max(freqs))) - for f,freq in enumerate(freqs): - if verbose: - sys.stdout.write('%g ' % (freq)) - sys.stdout.flush() - # get the phase and power for that freq - phase,power = phasePow2d(freq,eegdat,samplerate,width) - + sys.stdout.write('Calculating wavelet phase/power...\n') + sys.stdout.write('Freqs (%g to %g): ' % (N.min(freqs), N.max(freqs))) + for f, freq in enumerate(freqs): + if verbose: + sys.stdout.write('%g ' % (freq)) + sys.stdout.flush() + # get the phase and power for that freq + phase, power = phasePow2d(freq, eegdat, samplerate, width) + # reshape back do original data shape - if to_return == 'phase' or to_return == 'both': - phase = reshapeFrom2D(phase,axis,origshape) - if to_return == 'pow' or to_return == 'both': - power = reshapeFrom2D(power,axis,origshape) - - # see if allocate - if len(phaseAll) == 0 and len(powerAll) == 0: - if to_return == 'phase' or to_return == 'both': - phaseAll = N.empty(N.concatenate(([len(freqs)],phase.shape)), - dtype=phase.dtype) - if to_return == 'pow' or to_return == 'both': - powerAll = N.empty(N.concatenate(([len(freqs)],power.shape)), - dtype=power.dtype) + if to_return == 'phase' or to_return == 'both': + phase = reshapeFrom2D(phase, axis, origshape) + if to_return == 'pow' or to_return == 'both': + power = reshapeFrom2D(power, axis, origshape) + + # see if allocate + if len(phaseAll) == 0 and len(powerAll) == 0: + if to_return == 'phase' or to_return == 'both': + phaseAll = N.empty(N.concatenate(([len(freqs)], phase.shape)), + dtype=phase.dtype) + if to_return == 'pow' or to_return == 'both': + powerAll = N.empty(N.concatenate(([len(freqs)], power.shape)), + dtype=power.dtype) # insert into all - if to_return == 'phase' or to_return == 'both': - phaseAll[f] = phase - if to_return == 'pow' or to_return == 'both': - powerAll[f] = power + if to_return == 'phase' or to_return == 'both': + phaseAll[f] = phase + if to_return == 'pow' or to_return == 'both': + powerAll[f] = power if verbose: - sys.stdout.write('\n') + sys.stdout.write('\n') if to_return == 'pow': return powerAll elif to_return == 'phase': return phaseAll elif to_return == 'both': - return phaseAll,powerAll - - + return phaseAll, powerAll diff --git a/ptsa/wica.py b/ptsa/wica.py index 76ef3cc..0e754c4 100644 --- a/ptsa/wica.py +++ b/ptsa/wica.py @@ -1,5 +1,5 @@ -#emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# ex: set sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See the COPYING file distributed along with the PTSA package for the @@ -13,7 +13,7 @@ from ptsa.pca import pca from ptsa.iwasobi import iwasobi -from ptsa.wavelet import iswt,swt +from ptsa.wavelet import iswt, swt try: import multiprocessing as mp @@ -28,34 +28,34 @@ def find_blinks(dat, L, fast_rate=.5, slow_rate=.975, thresh=None): """ # make the range to go around an eyeblink #L = np.int32(np.round(samplerate*0.1))*2 - + # process the data - #if thresh is None: + # if thresh is None: # zd = (dat-np.mean(dat))/np.std(dat) - #else: + # else: # zd = dat zdf = dat zdb = dat[::-1] - + # initialize the fast and slow running averages - fastf = np.zeros(len(dat)+1) - slowf = np.zeros(len(dat)+1) + fastf = np.zeros(len(dat) + 1) + slowf = np.zeros(len(dat) + 1) slowf[0] = np.mean(zdf[:10]) - fastb = np.zeros(len(dat)+1) - slowb = np.zeros(len(dat)+1) + fastb = np.zeros(len(dat) + 1) + slowb = np.zeros(len(dat) + 1) slowb[0] = np.mean(zdb[:10]) - + # params for running averages a = fast_rate - b = 1-a + b = 1 - a c = slow_rate - d = 1-c + d = 1 - c # first forward # calc running average - for i in xrange(len(zdf)): - fastf[i+1] = a*fastf[i] + b*(zdf[i]-slowf[i]) - slowf[i+1] = c*slowf[i] + d*(zdf[i]) + for i in range(len(zdf)): + fastf[i + 1] = a * fastf[i] + b * (zdf[i] - slowf[i]) + slowf[i + 1] = c * slowf[i] + d * (zdf[i]) # remove the first value fastf = fastf[1:] @@ -63,43 +63,45 @@ def find_blinks(dat, L, fast_rate=.5, slow_rate=.975, thresh=None): # then backward # calc running average - for i in xrange(len(zdb)): - fastb[i+1] = a*fastb[i] + b*(zdb[i]-slowb[i]) - slowb[i+1] = c*slowb[i] + d*(zdb[i]) + for i in range(len(zdb)): + fastb[i + 1] = a * fastb[i] + b * (zdb[i] - slowb[i]) + slowb[i + 1] = c * slowb[i] + d * (zdb[i]) # remove the first value fastb = fastb[1:] slowb = slowb[1:] # combine - fast = (fastf*fastb)/2. + fast = (fastf * fastb) / 2. # determine the thresh if thresh is None: thresh = np.std(np.abs(fast)) - + # determine the artifact indices - + # first apply a thresh - idx = np.nonzero(np.abs(fast)>thresh)[0] + idx = np.nonzero(np.abs(fast) > thresh)[0] inds = np.arange(len(dat), dtype=np.int32) # make sure to connect contiguous artifacts - idx_ext = np.zeros(len(idx)*(2*L+1), dtype=np.int32) - for k in xrange(len(idx)): - idx_ext[(2*L+1)*(k):(2*L+1)*(k+1)-1] = np.arange(idx[k]-L,idx[k]+L) + idx_ext = np.zeros(len(idx) * (2 * L + 1), dtype=np.int32) + for k in range(len(idx)): + idx_ext[(2 * L + 1) * (k):(2 * L + 1) * (k + 1) - + 1] = np.arange(idx[k] - L, idx[k] + L) id_noise = np.setdiff1d(inds, idx_ext) id_artef = np.setdiff1d(inds, id_noise) - - return id_artef,id_noise -def _clean_find_thresh(Y,Kthr,wavelet,L): + return id_artef, id_noise + + +def _clean_find_thresh(Y, Kthr, wavelet, L): # init xn = None thld = 0.0 N = len(Y) - + # find the outliers # need to replace this with blink-finding code @@ -107,27 +109,28 @@ def _clean_find_thresh(Y,Kthr,wavelet,L): # Sig = median(abs(Y)/0.6745); #Sig = np.median(np.abs(Y)/0.6745) #Sig = np.median(np.abs(icaEEG[Comp[c],pure_range[0]:pure_range[1]])/0.6745) - Sig = np.median(np.abs(Y)/0.6745) + Sig = np.median(np.abs(Y) / 0.6745) # Thr = 4*Sig; - Thr = 3*Sig + Thr = 3 * Sig # idx = find(abs(Y) > Thr); idx = np.nonzero(np.abs(Y) > Thr)[0] # idx_ext = zeros(1,length(idx)*(2*L+1)); - idx_ext = np.zeros(len(idx)*(2*L+1), dtype=np.int32) + idx_ext = np.zeros(len(idx) * (2 * L + 1), dtype=np.int32) # for k=1:length(idx), # idx_ext((2*L+1)*(k-1)+1:(2*L+1)*k) = [idx(k)-L:idx(k)+L]; # end - for k in xrange(len(idx)): - idx_ext[(2*L+1)*(k):(2*L+1)*(k+1)-1] = np.arange(idx[k]-L,idx[k]+L) + for k in range(len(idx)): + idx_ext[(2 * L + 1) * (k):(2 * L + 1) * (k + 1) - + 1] = np.arange(idx[k] - L, idx[k] + L) # id_noise=setdiff((1:N), idx_ext); - id_noise = np.setdiff1d(range(N), idx_ext) + id_noise = np.setdiff1d(list(range(N)), idx_ext) # id_artef=setdiff((1:N), id_noise); - id_artef = np.setdiff1d(range(N), id_noise) + id_artef = np.setdiff1d(list(range(N)), id_noise) else: - id_artef,id_noise = find_blinks(Y,L) + id_artef, id_noise = find_blinks(Y, L) # make sure it's not all noise or artifact - print len(id_artef),len(id_noise) + print(len(id_artef), len(id_noise)) # if isempty(id_artef), # disp(['The component #' num2str(Comp(c)) ' has passed unchanged']); @@ -135,14 +138,14 @@ def _clean_find_thresh(Y,Kthr,wavelet,L): # end if len(id_artef) == 0: #sys.stdout.write("passed unchanged\n") - #sys.stdout.flush() + # sys.stdout.flush() return xn, thld # KK = 100; KK = 100. # LL = floor(log2(length(Y))); LL = np.int32(np.floor(np.log2(len(Y)))) # [xl, xh] = mrdwt(Y, h, LL); - wres = swt(Y,wavelet,level=LL) + wres = swt(Y, wavelet, level=LL) # make it editable wres = [list(wres[i]) for i in range(len(wres))] @@ -155,36 +158,36 @@ def _clean_find_thresh(Y,Kthr,wavelet,L): # while KK > Kthr, # thld = thld + 0.5; # xh = HardTh(xh, thld); % x = (abs(y) > thld).*y; - # xd = mirdwt(xl,xh,h,LL); + # xd = mirdwt(xl,xh,h,LL); # xn = Y - xd; # cn=corrcoef(Y(id_noise),xn(id_noise)); # ca=corrcoef(Y(id_artef),xd(id_artef)); - # KK = ca(1,2)/cn(1,2); + # KK = ca(1,2)/cn(1,2); # end # thld = 3.6; # not sure where this 3.6 number came from, so I'm dropping it down to get # more low-freq cleaning - thld = 1.1 #3.6 + thld = 1.1 # 3.6 while KK > Kthr: # update what's going on - #sys.stdout.write('.') - #sys.stdout.flush() + # sys.stdout.write('.') + # sys.stdout.flush() # bump up the thresh thld += 0.5 # zero out everything below threshold in each wavelet coef - for i in xrange(len(wres)): + for i in range(len(wres)): wres[i][1] = (np.abs(wres[i][1]) > thld) * wres[i][1] # invert the wavelet back xd = iswt(wres, wavelet) # check if clean based on the ratio of correlations for noise # and artifact data - xn = Y-xd + xn = Y - xd # cn measures the correlation between the cleaned and original # data in the non-artifactual regions - cn = np.corrcoef(Y[id_noise],xn[id_noise])[0,1] + cn = np.corrcoef(Y[id_noise], xn[id_noise])[0, 1] # ca measures the corr b/t the signal removed from the # artifacts and the original artifacts - ca = np.corrcoef(Y[id_artef],xd[id_artef])[0,1] + ca = np.corrcoef(Y[id_artef], xd[id_artef])[0, 1] # must not go negative, it should just be a small positive # number if that happens if cn <= 0.0: @@ -194,18 +197,19 @@ def _clean_find_thresh(Y,Kthr,wavelet,L): # we want the ratio of the bad things getting cleaned to the # good things sticking around to be small, ideally both very # close to 1.0 - KK = ca/cn - sys.stdout.write('(%.2f,%.2f,%.2f) '%(ca,cn,KK)) + KK = ca / cn + sys.stdout.write('(%.2f,%.2f,%.2f) ' % (ca, cn, KK)) sys.stdout.flush() # return the cleaned data and the thresh return xn, thld -def _clean_use_thresh(Y,thld,wavelet): + +def _clean_use_thresh(Y, thld, wavelet): LL = np.int32(np.floor(np.log2(len(Y)))) - wres = swt(Y,wavelet,level=LL) + wres = swt(Y, wavelet, level=LL) wres = [list(wres[i]) for i in range(len(wres))] # xh = HardTh(xh, thld); - for i in xrange(len(wres)): + for i in range(len(wres)): wres[i][1] = (np.abs(wres[i][1]) > thld) * wres[i][1] # xd = mirdwt(xl,xh,h,LL); xd = iswt(wres, wavelet) @@ -214,16 +218,17 @@ def _clean_use_thresh(Y,thld,wavelet): # return the cleaned data return xn + def _clean_comp(comp, Kthr, L, thld=None): wavelet = pywt.Wavelet('db3') N = np.int32(2**np.floor(np.log2(len(comp)))) # Y = icaEEG(Comp(c),1:N); Y = comp[:N] - + if thld is None: # opt(c) = thld; - xn,thld = _clean_find_thresh(Y,Kthr,wavelet,L) + xn, thld = _clean_find_thresh(Y, Kthr, wavelet, L) if thld == 0.0: return comp, thld else: @@ -231,22 +236,22 @@ def _clean_comp(comp, Kthr, L, thld=None): if thld == 0.0: # it was skipped, so skip it return comp, thld - xn = _clean_use_thresh(Y,thld,wavelet) + xn = _clean_use_thresh(Y, thld, wavelet) # icaEEG(Comp(c),1:N) = xn; comp[:N] = xn - + # clean the second half # Y = icaEEG(Comp(c),end-N+1:end); Y = comp[-N:] - xn = _clean_use_thresh(Y,thld,wavelet) + xn = _clean_use_thresh(Y, thld, wavelet) # icaEEG(Comp(c),N+1:end) = xn(end-(Nobser-N)+1:end); - comp[N:] = xn[-(len(comp)-N):] + comp[N:] = xn[-(len(comp) - N):] return comp, thld - + def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, Cthr=None, num_mp_procs=0): """ @@ -255,11 +260,11 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, % Ported and enhanced from Matlab code distributed by the authors of: - + N.P. Castellanos, and V.A. Makarov (2006). 'Recovering EEG brain signals: Artifact suppression with wavelet enhanced independent component analysis' J. Neurosci. Methods, 158, 300--312. - + % INPUT: % % icaEEG - matrix of ICA components (Nchanel x Nobservations) @@ -286,13 +291,13 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, #icaEEG = data.copy() # allow to modify to save memory #icaEEG = data - + # L = round(F*0.1); - L = np.int32(np.round(F*0.1)) + L = np.int32(np.round(F * 0.1)) # [Nchan, Nobser] = size(icaEEG); Nchan, Nobser = icaEEG.shape - # if Nchan > Nobser, - # error('Problem with data orientation, try to transpose the matrix!'); + # if Nchan > Nobser, + # error('Problem with data orientation, try to transpose the matrix!'); # end # N = 2^floor(log2(Nobser)); #N = np.int32(2**np.floor(np.log2(Nobser))) @@ -314,13 +319,12 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, opt = Cthr find_thresh = False - if has_mp and num_mp_procs != 0: po = mp.Pool(num_mp_procs) mp_res = [] - + # for c=1:length(Comp), - for c in xrange(len(Comp)): + for c in range(len(Comp)): if find_thresh: thld = None else: @@ -331,15 +335,15 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, (icaEEG[Comp[c]], Kthr, L, thld))) else: - sys.stdout.write("Component #%d: "%(Comp[c])) + sys.stdout.write("Component #%d: " % (Comp[c])) sys.stdout.flush() - comp,thld = _clean_comp(icaEEG[Comp[c]], Kthr, L, thld=thld) + comp, thld = _clean_comp(icaEEG[Comp[c]], Kthr, L, thld=thld) icaEEG[Comp[c]] = comp if find_thresh: opt[c] = thld if opt[c] > 0.0: # disp(['The component #' num2str(Comp(c)) ' has been filtered']); - sys.stdout.write("was filtered at %f\n"%(opt[c])) + sys.stdout.write("was filtered at %f\n" % (opt[c])) sys.stdout.flush() else: sys.stdout.write("passed unchanged\n") @@ -349,16 +353,16 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, # collect results po.close() po.join() - for c in xrange(len(Comp)): - sys.stdout.write("Component #%d: "%(Comp[c])) + for c in range(len(Comp)): + sys.stdout.write("Component #%d: " % (Comp[c])) sys.stdout.flush() - comp,thld = mp_res[c].get() + comp, thld = mp_res[c].get() icaEEG[Comp[c]] = comp if find_thresh: opt[c] = thld if opt[c] > 0.0: # disp(['The component #' num2str(Comp(c)) ' has been filtered']); - sys.stdout.write("was filtered at %f\n"%(opt[c])) + sys.stdout.write("was filtered at %f\n" % (opt[c])) sys.stdout.flush() else: sys.stdout.write("passed unchanged\n") @@ -366,7 +370,8 @@ def remove_strong_artifacts(data, A, icaEEG, Comp, Kthr=1.25, F=256, # end return opt - #return icaEEG, opt + # return icaEEG, opt + class WICA(object): """ @@ -403,28 +408,29 @@ def __init__(self, data, samplerate, pure_range=None): """ # process the pure range if pure_range is None: - pure_range = (None,None) + pure_range = (None, None) self._pure_range = pure_range # run pca sys.stdout.write("Running PCA...") - Wpca,pca_data = pca(data[:,pure_range[0]:pure_range[1]]) #, ncomps, eigratio) + # , ncomps, eigratio) + Wpca, pca_data = pca(data[:, pure_range[0]:pure_range[1]]) # Run iwasobi sys.stdout.write("Running IWASOBI ICA...") sys.stdout.flush() #(W,Winit,ISR,signals) = iwasobi(data[:,pure_range[0]:pure_range[1]]) - (W,Winit,ISR,signals) = iwasobi(pca_data) + (W, Winit, ISR, signals) = iwasobi(pca_data) # combine the iwasobi weights with the pca weights - W = np.dot(W,Wpca) + W = np.dot(W, Wpca) A = np.linalg.pinv(W) # reorder the signals by loading (reordered from high to low) ind = np.argsort(np.abs(A).sum(0))[::-1] - A = A[:,ind] - W = W[ind,:] - signals = signals[ind,:] + A = A[:, ind] + W = W[ind, :] + signals = signals[ind, :] self._ICA_weights = A #A = np.linalg.inv(W) @@ -433,15 +439,15 @@ def __init__(self, data, samplerate, pure_range=None): # expand signals to span the entire dataset if necessary if (not pure_range[0] is None) or (not pure_range[1] is None): - #Xmean=data[:,pure_range[0]:pure_range[1]].mean(1) + # Xmean=data[:,pure_range[0]:pure_range[1]].mean(1) #signals = np.add(np.dot(W,data).T,np.dot(W,Xmean)).T - signals = np.dot(W,data) + signals = np.dot(W, data) self._components = signals self._samplerate = samplerate self._data = data - def pick(self, EOG_elecs=[0,1], std_fact=1.5): + def pick(self, EOG_elecs=[0, 1], std_fact=1.5): # pick which signals to clean (ones that weigh on EOG elecs) # vals = np.sum(np.abs(A[EOG_elecs,:]),0) # std_thresh = std_fact*np.std(vals) @@ -451,19 +457,19 @@ def pick(self, EOG_elecs=[0,1], std_fact=1.5): # loop over EOG elecs for e in EOG_elecs: # get the weights of each component onto that electrode - vals = np.abs(A[e,:]) + vals = np.abs(A[e, :]) # calculate the threshold that the std must exceed for that # component to be considered - std_thresh = std_fact*np.std(vals) - #comp_ind.extend(np.nonzero(vals>=std_thresh)[0].tolist()) + std_thresh = std_fact * np.std(vals) + # comp_ind.extend(np.nonzero(vals>=std_thresh)[0].tolist()) # loop over potential components - for s in np.nonzero(vals>=std_thresh)[0].tolist(): + for s in np.nonzero(vals >= std_thresh)[0].tolist(): # calculate the weights of all electrodes into that component - sweights = np.abs(A[:,s]) + sweights = np.abs(A[:, s]) # get threshold based on the std across those weights - sthresh2 = std_fact*sweights.std() + sthresh2 = std_fact * sweights.std() # see if that component crosses this second threshold - if np.abs(A[e,s]) >= sthresh2: + if np.abs(A[e, s]) >= sthresh2: # yes, so append to the list to clean comp_ind.append(s) # Instead, try and make sure the weights are above the STD thresh @@ -473,10 +479,8 @@ def pick(self, EOG_elecs=[0,1], std_fact=1.5): return comp_ind - def get_loading(self, comp): - return self.ICA_weights[:,comp] - + return self.ICA_weights[:, comp] def clean(self, comp_inds=None, Kthr=2.5, num_mp_procs=0): if comp_inds is None: @@ -487,14 +491,15 @@ def clean(self, comp_inds=None, Kthr=2.5, num_mp_procs=0): # remove strong artifacts if (not self._pure_range[0] is None) or (not self._pure_range[1] is None): # figure out the thresh for the range - Cthr = remove_strong_artifacts(self._data[:,self._pure_range[0]:self._pure_range[1]], self.ICA_weights, - self._components[:,self._pure_range[0]:self._pure_range[1]], + Cthr = remove_strong_artifacts(self._data[:, self._pure_range[0]:self._pure_range[1]], self.ICA_weights, + self._components[:, self._pure_range[0] + :self._pure_range[1]], comp_inds, Kthr, self._samplerate, num_mp_procs=num_mp_procs) else: Cthr = None - Cthr = remove_strong_artifacts(self._data,self.ICA_weights,self._components, + Cthr = remove_strong_artifacts(self._data, self.ICA_weights, self._components, comp_inds, Kthr, self._samplerate, Cthr, num_mp_procs=num_mp_procs) @@ -502,12 +507,11 @@ def clean(self, comp_inds=None, Kthr=2.5, num_mp_procs=0): def get_corrected(self): # return cleaned data back in EEG space - return np.dot(self.ICA_weights,self._components) + return np.dot(self.ICA_weights, self._components) - -def wica_clean(data, samplerate=None, pure_range=(None,None), - EOG_elecs=[0,1], std_fact=1.5, Kthr=2.5,num_mp_procs=0): +def wica_clean(data, samplerate=None, pure_range=(None, None), + EOG_elecs=[0, 1], std_fact=1.5, Kthr=2.5, num_mp_procs=0): """ Clean data with the Wavelet-ICA method described here: @@ -537,14 +541,15 @@ def wica_clean(data, samplerate=None, pure_range=(None,None), """ # run pca sys.stdout.write("Running PCA...") - Wpca,pca_data = pca(data[:,pure_range[0]:pure_range[1]]) #, ncomps, eigratio) - + # , ncomps, eigratio) + Wpca, pca_data = pca(data[:, pure_range[0]:pure_range[1]]) + # Run iwasobi sys.stdout.write("Running IWASOBI ICA...") sys.stdout.flush() #(W,Winit,ISR,signals) = iwasobi(data[:,pure_range[0]:pure_range[1]]) - (W,Winit,ISR,signals) = iwasobi(pca_data) - W = np.dot(W,Wpca) + (W, Winit, ISR, signals) = iwasobi(pca_data) + W = np.dot(W, Wpca) A = np.linalg.pinv(W) #A = np.linalg.inv(W) sys.stdout.write("DONE!\n") @@ -552,9 +557,9 @@ def wica_clean(data, samplerate=None, pure_range=(None,None), # expand signals to span the entire dataset if necessary if (not pure_range[0] is None) or (not pure_range[1] is None): - #Xmean=data[:,pure_range[0]:pure_range[1]].mean(1) + # Xmean=data[:,pure_range[0]:pure_range[1]].mean(1) #signals = np.add(np.dot(W,data).T,np.dot(W,Xmean)).T - signals = np.dot(W,data) + signals = np.dot(W, data) # pick which signals to clean (ones that weigh on EOG elecs) # vals = np.sum(np.abs(A[EOG_elecs,:]),0) @@ -564,19 +569,19 @@ def wica_clean(data, samplerate=None, pure_range=(None,None), # loop over EOG elecs for e in EOG_elecs: # get the weights of each component onto that electrode - vals = np.abs(A[e,:]) + vals = np.abs(A[e, :]) # calculate the threshold that the std must exceed for that # component to be considered - std_thresh = std_fact*np.std(vals) - #comp_ind.extend(np.nonzero(vals>=std_thresh)[0].tolist()) + std_thresh = std_fact * np.std(vals) + # comp_ind.extend(np.nonzero(vals>=std_thresh)[0].tolist()) # loop over potential components - for s in np.nonzero(vals>=std_thresh)[0].tolist(): + for s in np.nonzero(vals >= std_thresh)[0].tolist(): # calculate the weights of all electrodes into that component - sweights = np.abs(A[:,s]) + sweights = np.abs(A[:, s]) # get threshold based on the std across those weights - sthresh2 = std_fact*sweights.std() + sthresh2 = std_fact * sweights.std() # see if that component crosses this second threshold - if np.abs(A[e,s]) >= sthresh2: + if np.abs(A[e, s]) >= sthresh2: # yes, so append to the list to clean comp_ind.append(s) # Instead, try and make sure the weights are above the STD thresh @@ -590,16 +595,15 @@ def wica_clean(data, samplerate=None, pure_range=(None,None), # remove strong artifacts if (not pure_range[0] is None) or (not pure_range[1] is None): # figure out the thresh for the range - Cthr = remove_strong_artifacts(signals[:,pure_range[0]:pure_range[1]], - comp_ind,Kthr, + Cthr = remove_strong_artifacts(signals[:, pure_range[0]:pure_range[1]], + comp_ind, Kthr, samplerate, num_mp_procs=num_mp_procs) else: Cthr = None - Cthr = remove_strong_artifacts(signals,comp_ind,Kthr, - samplerate,Cthr, + Cthr = remove_strong_artifacts(signals, comp_ind, Kthr, + samplerate, Cthr, num_mp_procs=num_mp_procs) - - # return cleaned data back in EEG space - return np.dot(A,signals).astype(data.dtype) + # return cleaned data back in EEG space + return np.dot(A, signals).astype(data.dtype) diff --git a/setup.py b/setup.py index 0d18d1a..5f4192d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ try: import numpy except ImportError: - print 'Numpy is required to build PTSA. Please install Numpy before proceeding' + print('Numpy is required to build PTSA. Please install Numpy before proceeding') import sys sys.exit(1) @@ -14,29 +14,28 @@ import sys # get the version loaded as vstr -execfile('ptsa/versionString.py') +exec(compile(open('ptsa/versionString.py').read(), 'ptsa/versionString.py', 'exec')) # set up extensions ext_modules = [] edf_ext = Extension("ptsa.data.edf.edf", - sources = ["ptsa/data/edf/edf.c", - "ptsa/data/edf/edfwrap.c", - "ptsa/data/edf/edflib.c"], - include_dirs=[numpy.get_include()], - define_macros = [('_LARGEFILE64_SOURCE', None), - ('_LARGEFILE_SOURCE', None)]) + sources=["ptsa/data/edf/edf.c", + "ptsa/data/edf/edfwrap.c", + "ptsa/data/edf/edflib.c"], + include_dirs=[numpy.get_include()], + define_macros=[('_LARGEFILE64_SOURCE', None), + ('_LARGEFILE_SOURCE', None)]) ext_modules.append(edf_ext) # define the setup -setup(name='ptsa', - version=vstr, +setup(name='ptsa', + version=vstr, maintainer=['Per B. Sederberg'], maintainer_email=['psederberg@gmail.com'], url=['http://ptsa.sourceforge.net'], - packages=['ptsa','ptsa.tests','ptsa.data','ptsa.data.tests', - 'ptsa.data.edf','ptsa.plotting','ptsa.plotting.tests', + packages=['ptsa', 'ptsa.tests', 'ptsa.data', 'ptsa.data.tests', + 'ptsa.data.edf', 'ptsa.plotting', 'ptsa.plotting.tests', 'ptsa.stats', - 'dimarray','dimarray.tests'], - ext_modules = ext_modules + 'dimarray', 'dimarray.tests'], + ext_modules=ext_modules ) - diff --git a/tools/gitwash_dumper.py b/tools/gitwash_dumper.py index da468b6..19471e1 100644 --- a/tools/gitwash_dumper.py +++ b/tools/gitwash_dumper.py @@ -72,14 +72,14 @@ def copy_replace(replace_pairs, out_path, cp_globs=('*',), rep_globs=('*',), - renames = ()): + renames=()): out_fnames = cp_files(repo_path, cp_globs, out_path) renames = [(re.compile(in_exp), out_exp) for in_exp, out_exp in renames] fnames = [] for rep_glob in rep_globs: fnames += fnmatch.filter(out_fnames, rep_glob) if verbose: - print '\n'.join(fnames) + print('\n'.join(fnames)) for fname in fnames: filename_search_replace(replace_pairs, fname, False) for in_exp, out_exp in renames: @@ -176,7 +176,7 @@ def main(): metavar="MAIN_GH_USER") parser.add_option("--gitwash-url", dest="gitwash_url", help="URL to gitwash repository - default %s" - % GITWASH_CENTRAL, + % GITWASH_CENTRAL, default=GITWASH_CENTRAL, metavar="GITWASH_URL") parser.add_option("--gitwash-branch", dest="gitwash_branch",