Merge branch 'dev' of https://github.com/datajoint/datajoint-python into custom-attribute-type

Author: dimitri-yatsenko

CHANGELOG.md

@@ -19,7 +19,7 @@
 * Support for pandas and order by "KEY" (#459, #537, #538, #541) PR #534
 * Support file attachment datatype and configurable blob storage (#467, #475, #480, #497) PR #532
 * Increase default display rows (#523) PR #526
-* Bugfixes (#521, #205, #279, #570, #581, #597, #596, #618, #633)
+* Bugfixes (#521, #205, #279, #477, #570, #581, #597, #596, #618, #633, #643, #644, #647)
 
 ### 0.11.3 -- Jul 26, 2019
 * Fix incompatibility with pyparsing 2.4.1 (#629) PR #631

datajoint/fetch.py

@@ -177,13 +177,14 @@ def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None,
             else:
                 ret = list(cur.fetchall())
                 record_type = (heading.as_dtype if not ret else np.dtype(
-                    [(name, type(value))
-                        if heading.as_dtype[name] == 'O' and isinstance(
-                            value, numbers.Number)  # value of blob is packed here
+                    [(name, type(value))   # use the first element to determine the type for blobs
+                        if heading[name].is_blob and isinstance(value, numbers.Number)
                         else (name, heading.as_dtype[name])
-                        for value, name
-                        in zip(ret[0], heading.as_dtype.names)]))
-                ret = np.array(ret, dtype=record_type)
+                        for value, name in zip(ret[0], heading.as_dtype.names)]))
+                try:
+                    ret = np.array(ret, dtype=record_type)
+                except Exception as e:
+                    raise
                 for name in heading:
                     ret[name] = list(map(partial(get, heading[name]), ret[name]))
                 if format == "frame":

tests/schema.py

@@ -39,6 +39,17 @@ class TTest3(dj.Manual):
     """
 
 
+@schema
+class NullableNumbers(dj.Manual):
+    definition = """
+    key : int 
+    ---
+    fvalue = null : float
+    dvalue = null : double
+    ivalue = null : int
+    """
+
+
 @schema
 class TTestExtra(dj.Manual):
     """

tests/test_fetch.py

@@ -203,3 +203,15 @@ def test_decimal(self):
         assert_true(len(rel & keys[0]) == 1)
         keys = rel.fetch(dj.key)
         assert_true(len(rel & keys[1]) == 1)
+
+    def test_nullable_numbers(self):
+        """ test mixture of values and nulls in numeric attributes """
+        table = schema.NullableNumbers()
+        table.insert((
+            (k, np.random.randn(), np.random.randint(-1000, 1000), np.random.randn())
+            for k in range(10)))
+        table.insert1((100, None, None, None))
+        f, d, i = table.fetch('fvalue', 'dvalue', 'ivalue')
+        assert_true(None in i)
+        assert_true(any(np.isnan(d)))
+        assert_true(any(np.isnan(f)))

tests/test_fetch_same.py

@@ -35,31 +35,23 @@ class TestFetchSame:
 
     @staticmethod
     def test_object_conversion_one():
-
-        new = ProjData.proj(sub='resp-sim').fetch('sub')
-
+        new = ProjData.proj(sub='resp').fetch('sub')
         assert_equal(new.dtype, np.float64)
 
     @staticmethod
     def test_object_conversion_two():
-
-        [sub, add] = ProjData.proj(sub='resp-sim', add='resp+sim').fetch(
-                'sub', 'add')
-
+        [sub, add] = ProjData.proj(sub='resp', add='sim').fetch('sub', 'add')
         assert_equal(sub.dtype, np.float64)
         assert_equal(add.dtype, np.float64)
 
     @staticmethod
     def test_object_conversion_all():
-
-        new = ProjData.proj(sub='resp-sim', add='resp+sim').fetch()
-
+        new = ProjData.proj(sub='resp', add='sim').fetch()
         assert_equal(new['sub'].dtype, np.float64)
         assert_equal(new['add'].dtype, np.float64)
 
     @staticmethod
     def test_object_no_convert():
-
         new = ProjData.fetch()
         assert_equal(new['big'].dtype, 'object')
         assert_equal(new['blah'].dtype, 'object')