Merge pull request #833 from dimitri-yatsenko/master

table.children and table.parents can now return table objects, not just names

Author: guzman-raphael

CHANGELOG.md

@@ -1,5 +1,14 @@
 ## Release notes
 
+### 0.12.8 -- Jan 12, 2021
+* table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833
+* Load dependencies before querying dependencies. (#179) PR #833
+* Fix display of part tables in `schema.save`. (#821) PR #833
+* Add `schema.list_tables`. (#838) PR #844
+* Fix minio new version regression.  PR #847
+* Add more S3 logging for debugging. (#831) PR #832 
+* Convert testing framework from TravisCI to GitHub Actions (#841) PR #840
+ 
 ### 0.12.7 -- Oct 27, 2020
 * Fix case sensitivity issues to adapt to MySQL 8+.  PR #819
 * Fix pymysql regression bug (#814) PR #816

LNX-docker-compose.yml

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.10
+    image: raphaelguzman/nginx:v0.0.13
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

datajoint/autopopulate.py

@@ -32,25 +32,19 @@ def key_source(self):
                 The default value is the join of the parent relations.
                 Users may override to change the granularity or the scope of populate() calls.
         """
-        def parent_gen(self):
-            if self.target.full_table_name not in self.connection.dependencies:
-                self.connection.dependencies.load()
-            for parent_name, fk_props in self.target.parents(primary=True).items():
-                if not parent_name.isdigit():  # simple foreign key
-                    yield FreeTable(self.connection, parent_name).proj()
-                else:
-                    grandparent = list(self.connection.dependencies.in_edges(parent_name))[0][0]
-                    yield FreeTable(self.connection, grandparent).proj(**{
-                        attr: ref for attr, ref in fk_props['attr_map'].items() if ref != attr})
+        def _rename_attributes(table, props):
+            return (table.proj(
+                **{attr: ref for attr, ref in props['attr_map'].items() if attr != ref})
+                if props['aliased'] else table)
 
         if self._key_source is None:
-            parents = parent_gen(self)
-            try:
-                self._key_source = next(parents)
-            except StopIteration:
-                raise DataJointError('A relation must have primary dependencies for auto-populate to work') from None
-            for q in parents:
-                self._key_source *= q
+            parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True)
+            if not parents:
+                raise DataJointError(
+                    'A relation must have primary dependencies for auto-populate to work') from None
+            self._key_source = _rename_attributes(*parents[0])
+            for q in parents[1:]:
+                self._key_source *= _rename_attributes(*q)
         return self._key_source
 
     def make(self, key):

datajoint/connection.py

@@ -166,6 +166,7 @@ def close(self):
 
     def register(self, schema):
         self.schemas[schema.database] = schema
+        self.dependencies.clear()
 
     def ping(self):
         """

datajoint/dependencies.py

@@ -1,9 +1,36 @@
 import networkx as nx
 import itertools
+import re
 from collections import defaultdict, OrderedDict
 from .errors import DataJointError
 
 
+def unite_master_parts(lst):
+    """
+    re-order a list of table names so that part tables immediately follow their master tables without breaking
+    the topological order.
+    Without this correction, a simple topological sort may insert other descendants between master and parts.
+    The input list must be topologically sorted.
+    :example:
+    unite_master_parts(
+        ['`s`.`a`', '`s`.`a__q`', '`s`.`b`', '`s`.`c`', '`s`.`c__q`', '`s`.`b__q`', '`s`.`d`', '`s`.`a__r`']) ->
+        ['`s`.`a`', '`s`.`a__q`', '`s`.`a__r`', '`s`.`b`', '`s`.`b__q`', '`s`.`c`', '`s`.`c__q`', '`s`.`d`']
+    """
+    for i in range(2, len(lst)):
+        name = lst[i]
+        match = re.match(r'(?P<master>`\w+`.`\w+)__\w+`', name)
+        if match:  # name is a part table
+            master = match.group('master')
+            for j in range(i-1, -1, -1):
+                if lst[j] == master + '`' or lst[j].startswith(master + '__'):
+                    # move from the ith position to the (j+1)th position
+                    lst[j+1:i+1] = [name] + lst[j+1:i]
+                    break
+            else:
+                raise DataJointError("Found a part table {name} without its master table.".format(name=name))
+    return lst
+
+
 class Dependencies(nx.DiGraph):
     """
     The graph of dependencies (foreign keys) between loaded tables.
@@ -16,15 +43,22 @@ class Dependencies(nx.DiGraph):
     def __init__(self, connection=None):
         self._conn = connection
         self._node_alias_count = itertools.count()
+        self._loaded = False
         super().__init__(self)
 
-    def load(self):
+    def clear(self):
+        self._loaded = False
+        super().clear()
+
+    def load(self, force=True):
         """
         Load dependencies for all loaded schemas.
         This method gets called before any operation that requires dependencies: delete, drop, populate, progress.
         """
-
         # reload from scratch to prevent duplication of renamed edges
+        if self._loaded and not force:
+            return
+
         self.clear()
 
         # load primary key info
@@ -77,6 +111,7 @@ def load(self):
 
         if not nx.is_directed_acyclic_graph(self):  # pragma: no cover
             raise DataJointError('DataJoint can only work with acyclic dependencies')
+        self._loaded = True
 
     def parents(self, table_name, primary=None):
         """
@@ -86,6 +121,7 @@ def parents(self, table_name, primary=None):
             attribute are considered.
         :return: dict of tables referenced by the foreign keys of table
         """
+        self.load(force=False)
         return {p[0]: p[2] for p in self.in_edges(table_name, data=True)
                 if primary is None or p[2]['primary'] == primary}
 
@@ -97,6 +133,7 @@ def children(self, table_name, primary=None):
             attribute are considered.
         :return: dict of tables referencing the table through foreign keys
         """
+        self.load(force=False)
         return {p[1]: p[2] for p in self.out_edges(table_name, data=True)
                 if primary is None or p[2]['primary'] == primary}
 
@@ -105,17 +142,19 @@ def descendants(self, full_table_name):
         :param full_table_name:  In form `schema`.`table_name`
         :return: all dependent tables sorted in topological order.  Self is included.
         """
+        self.load(force=False)
         nodes = self.subgraph(
             nx.algorithms.dag.descendants(self, full_table_name))
-        return [full_table_name] + list(
-            nx.algorithms.dag.topological_sort(nodes))
+        return unite_master_parts([full_table_name] + list(
+            nx.algorithms.dag.topological_sort(nodes)))
 
     def ancestors(self, full_table_name):
         """
         :param full_table_name:  In form `schema`.`table_name`
         :return: all dependent tables sorted in topological order.  Self is included.
         """
+        self.load(force=False)
         nodes = self.subgraph(
             nx.algorithms.dag.ancestors(self, full_table_name))
-        return [full_table_name] + list(reversed(list(
-            nx.algorithms.dag.topological_sort(nodes))))
+        return list(reversed(unite_master_parts(list(
+            nx.algorithms.dag.topological_sort(nodes)) + [full_table_name])))

datajoint/diagram.py

@@ -5,6 +5,7 @@
 import warnings
 import inspect
 from .table import Table
+from .dependencies import unite_master_parts
 
 try:
     from matplotlib import pyplot as plt
@@ -155,29 +156,8 @@ def is_part(part, master):
             return self
 
         def topological_sort(self):
-            """
-            :return:  list of nodes in topological order
-            """
-
-            def _unite(lst):
-                """
-                reorder list so that parts immediately follow their masters without breaking the topological order.
-                Without this correction, simple topological sort may insert other descendants between master and parts
-                :example:
-                _unite(['a', 'a__q', 'b', 'c', 'c__q', 'b__q', 'd', 'a__r'])
-                -> ['a', 'a__q', 'a__r', 'b', 'b__q', 'c', 'c__q', 'd']
-                """
-                if len(lst) <= 2:
-                    return lst
-                el = lst.pop()
-                lst = _unite(lst)
-                if '__' in el:
-                    master = el.split('__')[0]
-                    if not lst[-1].startswith(master):
-                        return _unite(lst[:-1] + [el, lst[-1]])
-                return lst + [el]
-
-            return _unite(list(nx.algorithms.dag.topological_sort(
+            """ :return:  list of nodes in topological order """
+            return unite_master_parts(list(nx.algorithms.dag.topological_sort(
                 nx.DiGraph(self).subgraph(self.nodes_to_show))))
 
         def __add__(self, arg):

datajoint/hash.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 
-def hash_key_values(mapping):
+def key_hash(mapping):
     """
     32-byte hash of the mapping's key values sorted by the key name.
     This is often used to convert a long primary key value into a shorter hash.

datajoint/jobs.py

@@ -1,5 +1,5 @@
 import os
-from .hash import hash_key_values
+from .hash import key_hash
 import platform
 from .table import Table
 from .settings import config
@@ -69,7 +69,7 @@ def reserve(self, table_name, key):
         """
         job = dict(
             table_name=table_name,
-            key_hash=hash_key_values(key),
+            key_hash=key_hash(key),
             status='reserved',
             host=platform.node(),
             pid=os.getpid(),
@@ -89,7 +89,7 @@ def complete(self, table_name, key):
         :param table_name: `database`.`table_name`
         :param key: the dict of the job's primary key
         """
-        job_key = dict(table_name=table_name, key_hash=hash_key_values(key))
+        job_key = dict(table_name=table_name, key_hash=key_hash(key))
         (self & job_key).delete_quick()
 
     def error(self, table_name, key, error_message, error_stack=None):
@@ -107,7 +107,7 @@ def error(self, table_name, key, error_message, error_stack=None):
             self.insert1(
                 dict(
                     table_name=table_name,
-                    key_hash=hash_key_values(key),
+                    key_hash=key_hash(key),
                     status="error",
                     host=platform.node(),
                     pid=os.getpid(),

datajoint/schemas.py

@@ -247,7 +247,6 @@ def save(self, python_filename=None):
         This method is in preparation for a future release and is not officially supported.
         :return: a string containing the body of a complete Python module defining this schema.
         """
-
         module_count = itertools.count()
         # add virtual modules for referenced modules with names vmod0, vmod1, ...
         module_lookup = collections.defaultdict(lambda: 'vmod' + str(next(module_count)))
@@ -258,15 +257,16 @@ def make_class_definition(table):
             class_name = table.split('.')[1].strip('`')
             indent = ''
             if tier == 'Part':
-                class_name = class_name.split('__')[1]
+                class_name = class_name.split('__')[-1]
                 indent += '    '
             class_name = to_camel_case(class_name)
 
             def repl(s):
-                d, tab = s.group(1), s.group(2)
-                return ('' if d == db else (module_lookup[d]+'.')) + to_camel_case(tab)
+                d, tabs = s.group(1), s.group(2)
+                return ('' if d == db else (module_lookup[d]+'.')) + '.'.join(
+                    to_camel_case(tab) for tab in tabs.lstrip('__').split('__'))
 
-            return ('' if tier == 'Part' else '@schema\n') + \
+            return ('' if tier == 'Part' else '\n@schema\n') + \
                 '{indent}class {class_name}(dj.{tier}):\n{indent}    definition = """\n{indent}    {defi}"""'.format(
                     class_name=class_name,
                     indent=indent,
@@ -276,8 +276,8 @@ def repl(s):
                         FreeTable(self.connection, table).describe(printout=False).replace('\n', '\n    ' + indent)))
 
         diagram = Diagram(self)
-        body = '\n\n\n'.join(make_class_definition(table) for table in diagram.topological_sort())
-        python_code = '\n\n\n'.join((
+        body = '\n\n'.join(make_class_definition(table) for table in diagram.topological_sort())
+        python_code = '\n\n'.join((
             '"""This module was auto-generated by datajoint from an existing schema"""',
             "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
             '\n'.join("{module} = dj.VirtualModule('{module}', '{schema_name}')".format(module=v, schema_name=k)