NaN handling

test/test_images.py

@@ -1,7 +1,8 @@
 import pytest
-from numpy import arange, allclose, array, mean, apply_along_axis, float64
+from numpy import arange, allclose, array, mean, apply_along_axis, float64, \
+    nanmean, nan, nansum, nanvar, nanmin, nanmax, nanstd
 
-from thunder.images.readers import fromlist, fromarray
+from thunder.images.readers import fromlist
 from thunder.images.images import Images
 from thunder.series.series import Series
 
@@ -33,6 +34,7 @@ def test_sample(eng):
     assert allclose(data.sample(1).shape, (1, 2, 2))
     assert allclose(data.filter(lambda x: x.max() > 5).sample(1).toarray(), [[1, 10], [1, 10]])
 
+
 def test_labels(eng):
     x = arange(10).reshape(10, 1, 1)
     data = fromlist(x, labels=range(10), engine=eng)
@@ -159,27 +161,131 @@ def test_mean(eng):
     assert allclose(data.mean().toarray(), original.mean(axis=0))
 
 
+def test_nanmean(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmean().shape, (1, 3, 4))
+    assert allclose(data.nanmean().toarray(), nanmean(original, axis=0))
+
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmean().shape, (1, 3, 4))
+    assert allclose(data.nanmean().toarray(), nanmean(original, axis=0))
+
+
+def test_min(eng):
+    original = arange(24).reshape((2, 3, 4))
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.min().shape, (1, 3, 4))
+    assert allclose(data.min().toarray(), original.min(axis=0))
+
+
+def test_nanmin(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmin().shape, (1, 3, 4))
+    assert allclose(data.nanmin().toarray(), original.min(axis=0))
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmin().shape, (1, 3, 4))
+    assert allclose(data.nanmin().toarray(), nanmin(original, axis=0))
+
+
+def test_max(eng):
+    original = arange(24).reshape((2, 3, 4))
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.max().shape, (1, 3, 4))
+    assert allclose(data.max().toarray(), original.max(axis=0))
+
+
+def test_nanmax(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmax().shape, (1, 3, 4))
+    assert allclose(data.nanmax().toarray(), original.max(axis=0))
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanmax().shape, (1, 3, 4))
+    assert allclose(data.nanmax().toarray(), nanmax(original, axis=0))
+
+
 def test_sum(eng):
     original = arange(24).reshape((2, 3, 4))
     data = fromlist(list(original), engine=eng)
     assert allclose(data.sum().shape, (1, 3, 4))
     assert allclose(data.sum().toarray(), original.sum(axis=0))
 
 
+def test_nansum(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nansum().shape, (1, 3, 4))
+    assert allclose(data.nansum().toarray(), nansum(original, axis=0))
+
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nansum().shape, (1, 3, 4))
+    assert allclose(data.nansum().toarray(), nansum(original, axis=0))
+
+
 def test_var(eng):
     original = arange(24).reshape((2, 3, 4))
     data = fromlist(list(original), engine=eng)
     assert allclose(data.var().shape, (1, 3, 4))
     assert allclose(data.var().toarray(), original.var(axis=0))
 
 
+def test_nanvar(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanvar().shape, (1, 3, 4))
+    assert allclose(data.nanvar().toarray(), nanvar(original, axis=0))
+
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanvar().shape, (1, 3, 4))
+    assert allclose(data.nanvar().toarray(), nanvar(original, axis=0))
+
+
+def test_std(eng):
+    original = arange(24).reshape((2, 3, 4))
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.std().shape, (1, 3, 4))
+    assert allclose(data.std().toarray(), original.std(axis=0))
+
+
+def test_nanstd(eng):
+    original = arange(24).reshape((2, 3, 4)).astype(float64)
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanstd().shape, (1, 3, 4))
+    assert allclose(data.nanstd().toarray(), nanstd(original, axis=0))
+
+    original[0, 2, 3] = nan
+    original[1, 0, 2] = nan
+    original[1, 2, 2] = nan
+    data = fromlist(list(original), engine=eng)
+    assert allclose(data.nanstd().shape, (1, 3, 4))
+    assert allclose(data.nanstd().toarray(), nanstd(original, axis=0))
+
+
 def test_subtract(eng):
     original = arange(24).reshape((4, 6))
     data = fromlist([original], engine=eng)
     assert allclose(data.subtract(1).toarray(), original - 1)
     sub = arange(24).reshape((4, 6))
     assert allclose(data.subtract(sub).toarray(), original - sub)
 
+
 def test_map_as_series(eng):
     original = arange(4*4).reshape(4, 4)
     data = fromlist(5*[original], engine=eng)
@@ -202,7 +308,7 @@ def f(x):
     assert allclose(data.map_as_series(f, chunk_size=size, value_size=4).toarray(), result)
 
 def test_reshape_values(eng):
-    original = fromarray(arange(72).reshape(2, 6, 6), engine=eng)
+    original = fromlist(arange(72).reshape(2, 6, 6), engine=eng)
     arr = original.toarray()
 
     assert allclose(arr.reshape(2, 12, 3), original.reshape(2, 12, 3).toarray())

test/test_series.py

@@ -1,5 +1,6 @@
 import pytest
-from numpy import allclose, arange, array, asarray, dot, cov, corrcoef, float64
+from numpy import allclose, arange, array, asarray, dot, cov, corrcoef, float64, \
+     nanmean, nan, nansum, nanvar, nanmin, nanmax, nanstd
 
 from thunder.series.readers import fromlist, fromarray
 from thunder.images.readers import fromlist as img_fromlist
@@ -127,6 +128,23 @@ def test_mean(eng):
     assert str(val.dtype) == 'float64'
 
 
+def test_nanmean(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nanmean().toarray()
+    expected = nanmean(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nanmean().toarray()
+    expected = nanmean(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_sum(eng):
     data = fromlist([arange(8), arange(8)], engine=eng)
     val = data.sum().toarray()
@@ -135,6 +153,23 @@ def test_sum(eng):
     assert str(val.dtype) == 'int64'
 
 
+def test_nansum(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nansum().toarray()
+    expected = nansum(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nansum().toarray()
+    expected = nansum(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_var(eng):
     data = fromlist([arange(8), arange(8)], engine=eng)
     val = data.var().toarray()
@@ -143,6 +178,23 @@ def test_var(eng):
     assert str(val.dtype) == 'float64'
 
 
+def test_nanvar(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nanvar().toarray()
+    expected = nanvar(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nanvar().toarray()
+    expected = nanvar(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_std(eng):
     data = fromlist([arange(8), arange(8)], engine=eng)
     val = data.std().toarray()
@@ -151,19 +203,71 @@ def test_std(eng):
     assert str(val.dtype) == 'float64'
 
 
+def test_nanstd(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nanstd().toarray()
+    expected = nanstd(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nanstd().toarray()
+    expected = nanstd(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_max(eng):
     data = fromlist([arange(8), arange(8)], engine=eng)
     val = data.max().toarray()
     expected = data.toarray().max(axis=0)
     assert allclose(val, expected)
 
 
+def test_nanmax(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nanmax().toarray()
+    expected = nanmax(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nanmax().toarray()
+    expected = nanmax(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_min(eng):
     data = fromlist([arange(8), arange(8)], engine=eng)
     val = data.min().toarray()
     expected = data.toarray().min(axis=0)
     assert allclose(val, expected)
 
+
+def test_nanmin(eng):
+    arr = array([arange(8), arange(8)]).astype(float64)
+    data = fromarray(arr, engine=eng)
+    val = data.nanmin().toarray()
+    expected = nanmin(data.toarray(), axis=0)
+    assert allclose(val, expected)
+    assert str(val.dtype) == 'float64'
+    arr[0, 4] = nan
+    arr[1, 3] = nan
+    arr[1, 4] = nan
+    data = fromarray(arr, engine=eng)
+    val = data.nanmin().toarray()
+    expected = nanmin(data.toarray(), axis=0)
+    assert allclose(val, expected, equal_nan=True)
+    assert str(val.dtype) == 'float64'
+
+
 def test_labels(eng):
     x = [array([0, 1]), array([2, 3]), array([4, 5]), array([6, 7])]
     data = fromlist(x, labels=[0, 1, 2, 3], engine=eng)

thunder/base.py

@@ -254,7 +254,7 @@ def labels(self, value):
     def astype(self, dtype, casting='unsafe'):
         """
         Cast values to the specified type.
-        
+
         Parameters
         ----------
         dtype : str or dtype
@@ -338,6 +338,48 @@ def min(self):
         """
         raise NotImplementedError
 
+    def nanmean(self):
+        """
+        Mean of values computed along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def nansum(self):
+        """
+        Sum of values computed along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def nanvar(self):
+        """
+        Variance of values computed along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def nanstd(self):
+        """
+        Standard deviation computed of values along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def nanmax(self):
+        """
+        Maximum of values computed along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def nanmin(self):
+        """
+        Minimum of values computed along the appropriate dimension ignoring NaNs.
+        """
+        raise NotImplementedError
+
+    def map(self, func, **kwargs):
+        """
+        Map a function over elements.
+        """
+        raise NotImplementedError
+
     def _align(self, axes, key_shape=None):
         """
         Align local arrays so that axes for iteration are in the keys.