Merge pull request #456 from dimitri-yatsenko/master

Fix  #431,  #451, #458, #463, #466

Author: eywalker

CHANGELOG.md

@@ -1,10 +1,17 @@
 ## Release notes
-### 0.10.0 --  work in progress
+### 0.10.1 -- Work in progress
+* Networkx 2.0 support (#443)
+* Sped up queries (#428)
+
+### 0.10.0 -- January 10, 2018 
+* Deletes are more efficient (#424)
+* ERD shows table definition on tooltip hover in Jupyter (#422) 
 * S3 external storage
 * Garbage collection for external sorage
-* Most operators and methods of tables can be invoked as class methods
+* Most operators and methods of tables can be invoked as class methods rather than instance methods (#407)
 * The schema decorator object no longer requires locals() to specify the context
 * Compatibility with pymysql 0.8.0+
+* More efficient loading of dependencies (#403)
 
 ### 0.9.0 -- November 17, 2017
 * Made graphviz installation optional

datajoint/__init__.py

@@ -24,6 +24,7 @@
            'Connection', 'Heading', 'FreeRelation', 'Not', 'schema',
            'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
            'AndList', 'OrList', 'ERD', 'U',
+           'DataJointError', 'DuplicateError',
            'set_password']
 
 
@@ -34,12 +35,6 @@ class key:
     pass
 
 
-class DataJointError(Exception):
-    """
-    Base class for errors specific to DataJoint internal operation.
-    """
-    pass
-
 # ----------- loads local configuration from file ----------------
 from .settings import Config, LOCALCONFIG, GLOBALCONFIG, logger, log_levels
 config = Config()
@@ -73,6 +68,7 @@ class DataJointError(Exception):
 from .schema import Schema as schema
 from .erd import ERD
 from .admin import set_password, kill
+from .errors import DataJointError, DuplicateError
 
 
 def create_virtual_module(modulename, dbname):

datajoint/autopopulate.py

@@ -6,7 +6,7 @@
 from tqdm import tqdm
 from pymysql import OperationalError
 from .relational_operand import RelationalOperand, AndList, U
-from . import DataJointError
+from .errors import DataJointError
 from .base_relation import FreeRelation
 import signal
 

datajoint/base_relation.py

@@ -7,13 +7,13 @@
 import logging
 import warnings
 from pymysql import OperationalError, InternalError, IntegrityError
-from . import config, DataJointError
+from . import config
 from .declare import declare
 from .relational_operand import RelationalOperand
 from .blob import pack
 from .utils import user_choice
 from .heading import Heading
-from .settings import server_error_codes
+from .errors import server_error_codes, DataJointError, DuplicateError
 from . import __version__ as version
 
 logger = logging.getLogger(__name__)
@@ -42,7 +42,12 @@ def heading(self):
         if self._heading is None:
             self._heading = Heading()  # instance-level heading
         if not self._heading:  # lazy loading of heading
-            self._heading.init_from_database(self.connection, self.database, self.table_name)
+            if self.connection is None:
+                raise DataJointError(
+                    'DataJoint class is missing a database connection. '
+                    'Missing schema decorator on the class? (e.g. @schema)')
+            else:
+                self._heading.init_from_database(self.connection, self.database, self.table_name)
         return self._heading
 
     @property
@@ -172,7 +177,8 @@ def insert(self, rows, replace=False, skip_duplicates=False, ignore_extra_fields
                 fields='`' + '`,`'.join(fields) + '`',
                 table=self.full_table_name,
                 select=rows.make_sql(select_fields=fields),
-                duplicate=(' ON DUPLICATE KEY UPDATE `{pk}`=`{pk}`'.format(pk=self.primary_key[0])
+                duplicate=(' ON DUPLICATE KEY UPDATE `{pk}`={table}.`{pk}`'.format(
+                    table=self.full_table_name, pk=self.primary_key[0])
                            if skip_duplicates else ''))
             self.connection.query(query)
             return
@@ -282,10 +288,12 @@ def check_fields(fields):
                 elif err.args[0] == server_error_codes['unknown column']:
                     # args[1] -> Unknown column 'extra' in 'field list'
                     raise DataJointError(
-                        '{} : To ignore extra fields, set ignore_extra_fields=True in insert.'.format(err.args[1])) from None
+                        '{} : To ignore extra fields, set ignore_extra_fields=True in insert.'.format(err.args[1])
+                    ) from None
                 elif err.args[0] == server_error_codes['duplicate entry']:
-                    raise DataJointError(
-                        '{} : To ignore duplicate entries, set skip_duplicates=True in insert.'.format(err.args[1])) from None
+                    raise DuplicateError(
+                        '{} : To ignore duplicate entries, set skip_duplicates=True in insert.'.format(err.args[1])
+                    ) from None
                 else:
                     raise
 
@@ -434,11 +442,15 @@ def show_definition(self):
         logger.warning('show_definition is deprecated.  Use describe instead.')
         return self.describe()
 
-    def describe(self, printout=True):
+    def describe(self, context=None, printout=True):
         """
         :return:  the definition string for the relation using DataJoint DDL.
             This does not yet work for aliased foreign keys.
         """
+        if context is None:
+            frame = inspect.currentframe().f_back
+            context = dict(frame.f_globals, **frame.f_locals)
+            del frame
         if self.full_table_name not in self.connection.dependencies:
             self.connection.dependencies.load()
         parents = self.parents()
@@ -460,14 +472,14 @@ def describe(self, printout=True):
                         parents.pop(parent_name)
                         if not parent_name.isdigit():
                             definition += '-> {class_name}\n'.format(
-                                class_name=lookup_class_name(parent_name, self.context) or parent_name)
+                                class_name=lookup_class_name(parent_name, context) or parent_name)
                         else:
                             # aliased foreign key
                             parent_name = list(self.connection.dependencies.in_edges(parent_name))[0][0]
                             lst = [(attr, ref) for attr, ref in fk_props['attr_map'].items() if ref != attr]
                             definition += '({attr_list}) -> {class_name}{ref_list}\n'.format(
                                 attr_list=','.join(r[0] for r in lst),
-                                class_name=lookup_class_name(parent_name, self.context) or parent_name,
+                                class_name=lookup_class_name(parent_name, context) or parent_name,
                                 ref_list=('' if len(attributes_thus_far) - len(attributes_declared) == 1
                                           else '(%s)' % ','.join(r[1] for r in lst)))
                             attributes_declared.update(fk_props['attr_map'])
@@ -540,25 +552,26 @@ def lookup_class_name(name, context, depth=3):
     while nodes:
         node = nodes.pop(0)
         for member_name, member in node['context'].items():
-            if inspect.isclass(member) and issubclass(member, BaseRelation):
-                if member.full_table_name == name:   # found it!
-                    return '.'.join([node['context_name'],  member_name]).lstrip('.')
-                try:  # look for part tables
-                    parts = member._ordered_class_members
-                except AttributeError:
-                    pass  # not a UserRelation -- cannot have part tables.
-                else:
-                    for part in (getattr(member, p) for p in parts if p[0].isupper() and hasattr(member, p)):
-                        if inspect.isclass(part) and issubclass(part, BaseRelation) and part.full_table_name == name:
-                            return '.'.join([node['context_name'], member_name, part.__name__]).lstrip('.')
-            elif node['depth'] > 0 and inspect.ismodule(member) and member.__name__ != 'datajoint':
-                try:
-                    nodes.append(
-                        dict(context=dict(inspect.getmembers(member)),
-                             context_name=node['context_name'] + '.' + member_name,
-                             depth=node['depth']-1))
-                except ImportError:
-                    pass  # could not import, so do not attempt
+            if not member_name.startswith('_'):  # skip IPython's implicit variables
+                if inspect.isclass(member) and issubclass(member, BaseRelation):
+                    if member.full_table_name == name:   # found it!
+                        return '.'.join([node['context_name'],  member_name]).lstrip('.')
+                    try:  # look for part tables
+                        parts = member._ordered_class_members
+                    except AttributeError:
+                        pass  # not a UserRelation -- cannot have part tables.
+                    else:
+                        for part in (getattr(member, p) for p in parts if p[0].isupper() and hasattr(member, p)):
+                            if inspect.isclass(part) and issubclass(part, BaseRelation) and part.full_table_name == name:
+                                return '.'.join([node['context_name'], member_name, part.__name__]).lstrip('.')
+                elif node['depth'] > 0 and inspect.ismodule(member) and member.__name__ != 'datajoint':
+                    try:
+                        nodes.append(
+                            dict(context=dict(inspect.getmembers(member)),
+                                 context_name=node['context_name'] + '.' + member_name,
+                                 depth=node['depth']-1))
+                    except ImportError:
+                        pass  # could not import, so do not attempt
     return None
 
 

datajoint/blob.py

@@ -4,8 +4,9 @@
 
 import zlib
 from collections import OrderedDict, Mapping, Iterable
+from decimal import Decimal
 import numpy as np
-from . import DataJointError
+from .errors import DataJointError
 
 mxClassID = OrderedDict((
     # see http://www.mathworks.com/help/techdoc/apiref/mxclassid.html
@@ -223,14 +224,16 @@ def pack_obj(obj):
     blob = b''
     if isinstance(obj, np.ndarray):
         blob += pack_array(obj)
-    elif isinstance(obj, Mapping):  # TODO: check if this is a good inheritance check for dict etc.
+    elif isinstance(obj, Mapping):
         blob += pack_dict(obj)
     elif isinstance(obj, str):
         blob += pack_array(np.array(obj, dtype=np.dtype('c')))
     elif isinstance(obj, Iterable):
         blob += pack_array(np.array(list(obj)))
     elif isinstance(obj, int) or isinstance(obj, float):
         blob += pack_array(np.array(obj))
+    elif isinstance(obj, Decimal):
+        blob += pack_array(np.array(np.float64(obj)))
     else:
         raise DataJointError("Packing object of type %s currently not supported!" % type(obj))
 

datajoint/connection.py

@@ -7,11 +7,12 @@
 import pymysql as client
 import logging
 from getpass import getpass
+from pymysql import err
 
 from . import config
-from . import DataJointError
+from .errors import DataJointError, server_error_codes
 from .dependencies import Dependencies
-from pymysql import err
+
 
 logger = logging.getLogger(__name__)
 
@@ -94,6 +95,7 @@ def connect(self):
             self._conn = client.connect(init_command=self.init_fun,
                                         sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,"
                                                  "STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION",
+                                        charset=config['connection.charset'],
                                         **self.conn_info)
 
     def register(self, schema):
@@ -118,7 +120,7 @@ def query(self, query, args=(), as_dict=False, suppress_warnings=True):
         :param args: additional arguments for the client.cursor
         :param as_dict: If as_dict is set to True, the returned cursor objects returns
                         query results as dictionary.
-        :param suppress_warning: If True, suppress all warnings arising from underlying query library
+        :param suppress_warnings: If True, suppress all warnings arising from underlying query library
         """
 
         cursor = client.cursors.DictCursor if as_dict else client.cursors.Cursor
@@ -144,11 +146,12 @@ def query(self, query, args=(), as_dict=False, suppress_warnings=True):
             else:
                 raise
         except err.ProgrammingError as e:
-            raise DataJointError("\n".join((
-                "Error in query:", query,
-                "Please check spelling, syntax, and existence of tables and attributes.",
-                "When restricting a relation by a condition in a string, enclose attributes in backquotes."
-            )))
+            if e.args[0] == server_error_codes['parse error']:
+                raise DataJointError("\n".join((
+                    "Error in query:", query,
+                    "Please check spelling, syntax, and existence of tables and attributes.",
+                    "When restricting a relation by a condition in a string, enclose attributes in backquotes."
+                ))) from None
         return cur
 
     def get_user(self):

datajoint/declare.py

@@ -6,7 +6,8 @@
 import pyparsing as pp
 import logging
 
-from . import DataJointError, config
+from . import config
+from .errors import DataJointError
 
 STORE_NAME_LENGTH = 8
 STORE_HASH_LENGTH = 43
@@ -176,7 +177,7 @@ def declare(full_table_name, definition, context):
                        ['PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)'] +
                        foreign_key_sql +
                        index_sql) +
-            '\n) ENGINE=InnoDB, CHARACTER SET latin1, COMMENT "%s"' % table_comment), uses_external
+            '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment), uses_external
 
 
 def compile_attribute(line, in_key, foreign_key_sql):

datajoint/dependencies.py

@@ -1,7 +1,7 @@
 import networkx as nx
 import itertools
 from collections import defaultdict
-from . import DataJointError
+from .errors import DataJointError
 
 
 class Dependencies(nx.DiGraph):

datajoint/erd.py

@@ -3,6 +3,7 @@
 import functools
 import io
 import warnings
+import inspect
 from .base_relation import BaseRelation
 
 try:
@@ -12,7 +13,8 @@
 except:
     erd_active = False
 
-from . import Manual, Imported, Computed, Lookup, Part, DataJointError
+from . import Manual, Imported, Computed, Lookup, Part
+from .errors import DataJointError
 from .base_relation import lookup_class_name
 
 
@@ -82,15 +84,13 @@ def __init__(self, source, context=None):
                 super().__init__(source)
                 return
 
-            # get the caller's locals()
+            # get the caller's context
             if context is None:
-                import inspect
-                frame = inspect.currentframe()
-                try:
-                    context = frame.f_back.f_locals
-                finally:
-                    del frame
-            self.context = context
+                frame = inspect.currentframe().f_back
+                self.context = dict(frame.f_globals, **frame.f_locals)
+                del frame
+            else:
+                self.context = context
 
             # find connection in the source
             try:
@@ -209,10 +209,7 @@ def _make_graph(self):
             graph = nx.DiGraph(nx.DiGraph(self).subgraph(nodes))
             nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) for n in graph})
             # relabel nodes to class names
-            clean_context = dict((k, v) for k, v in self.context.items()
-                                 if not k.startswith('_'))  # exclude ipython's implicit variables
-            mapping = {node: (lookup_class_name(node, clean_context) or node)
-                       for node in graph.nodes()}
+            mapping = {node: lookup_class_name(node, self.context) or node for node in graph.nodes()}
             new_names = [mapping.values()]
             if len(new_names) > len(set(new_names)):
                 raise DataJointError('Some classes have identical names. The ERD cannot be plotted.')
@@ -258,7 +255,7 @@ def make_dot(self):
                 if name.split('.')[0] in self.context:
                     cls = eval(name, self.context)
                     assert(issubclass(cls, BaseRelation))
-                    description = cls().describe(printout=False).split('\n')
+                    description = cls().describe(context=self.context, printout=False).split('\n')
                     description = (
                         '-'*30 if q.startswith('---') else q.replace('->', '→') if '->' in q else q.split(':')[0]
                         for q in description if not q.startswith('#'))

datajoint/errors.py

@@ -0,0 +1,22 @@
+server_error_codes = {
+    'unknown column': 1054,
+    'duplicate entry': 1062,
+    'parse error': 1064,
+    'command denied': 1142,
+    'table does not exist': 1146,
+    'syntax error': 1149
+}
+
+
+class DataJointError(Exception):
+    """
+    Base class for errors specific to DataJoint internal operation.
+    """
+    pass
+
+
+class DuplicateError(DataJointError):
+    """
+    Error caused by a violation of a unique constraint when inserting data
+    """
+    pass