rename database to schema_name in schema constructor and add the create_schema kwarg.

Author: dimitri-yatsenko

datajoint/__init__.py

@@ -1,8 +1,10 @@
 """
-DataJoint for Python is a high-level programming interface for MySQL databases
-to support data processing chains in science labs. DataJoint is built on the
-foundation of the relational data model and prescribes a consistent method for
-organizing, populating, and querying data.
+DataJoint for Python is a framework for building data piplines using MySQL databases 
+to represent pipeline structure and bulk storage systems for large objects.
+DataJoint is built on the foundation of the relational data model and prescribes a 
+consistent method for organizing, populating, and querying data.
+
+The DataJoint data model is described in https://arxiv.org/abs/1807.11104
 
 DataJoint is free software under the LGPL License. In addition, we request
 that any use of DataJoint leading to a publication be acknowledged in the publication.
@@ -18,7 +20,7 @@
 from .version import __version__
 
 __author__ = "Dimitri Yatsenko, Edgar Y. Walker, and Fabian Sinz at Baylor College of Medicine"
-__date__ = "July 26, 2017"
+__date__ = "August 24, 2018"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill', 'BaseRelation',
            'Connection', 'Heading', 'FreeRelation', 'Not', 'schema',
@@ -71,17 +73,19 @@ class key:
 from .errors import DataJointError, DuplicateError
 
 
-def create_virtual_module(modulename, dbname):
+def create_virtual_module(module_name, schema_name, create_schema=False, create_tables=False):
     """
-    Creates a python module with the given name from a database name in mysql with datajoint tables.
-    Automatically creates the classes of the appropriate tier in the module.
+    Creates a python module with the given name from the name of a schema on the server and
+    automatically adds classes to it corresponding to the tables in the schema.
 
-    :param modulename: desired name of the module
-    :param dbname:     name of the database in mysql
-    :return: the python module
+    :param module_name: displayed module name
+    :param schema_name: name of the database in mysql
+    :param create_schema: if True, create the schema on the database server
+    :param create_tables: if True, module.schema can be used as the decorator for declaring new 
+    :return: the python module containing classes from the schema object and the table classes
     """
-    mod = ModuleType(modulename)
-    s = schema(dbname, create_tables=False)
-    s.spawn_missing_classes(context=mod.__dict__)
-    mod.__dict__['schema'] = s
-    return mod
+    module = ModuleType(module_name)
+    _schema = schema(schema_name, create_schema=create_schema, create_tables=create_tables)
+    _schema.spawn_missing_classes(context=module.__dict__)
+    module.__dict__['schema'] = _schema
+    return module

datajoint/base_relation.py

@@ -21,7 +21,7 @@
 
 class BaseRelation(RelationalOperand):
     """
-    BaseRelation is an abstract class that represents a base relation, i.e. a table in the database.
+    BaseRelation is an abstract class that represents a base relation, i.e. a table in the schema.
     To make it a concrete class, override the abstract properties specifying the connection,
     table name, database, context, and definition.
     A Relation implements insert and delete methods in addition to inherited relational operators.
@@ -56,7 +56,7 @@ def context(self):
 
     def declare(self):
         """
-        Use self.definition to declare the table in the database
+        Use self.definition to declare the table in the schema.
         """
         try:
             sql, uses_external = declare(self.full_table_name, self.definition, self._context)
@@ -108,7 +108,7 @@ def children(self, primary=None):
     @property
     def is_declared(self):
         """
-        :return: True is the table is declared in the database
+        :return: True is the table is declared in the schema.
         """
         return self.connection.query(
             'SHOW TABLES in `{database}` LIKE "{table_name}"'.format(
@@ -117,7 +117,7 @@ def is_declared(self):
     @property
     def full_table_name(self):
         """
-        :return: full table name in the database
+        :return: full table name in the schema
         """
         return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name)
 
@@ -541,8 +541,8 @@ def _update(self, attrname, value=None):
 
 def lookup_class_name(name, context, depth=3):
     """
-    given a table name in the form `database`.`table_name`, find its class in the context.
-    :param name: `database`.`table_name`
+    given a table name in the form `schema_name`.`table_name`, find its class in the context.
+    :param name: `schema_name`.`table_name`
     :param context: dictionary representing the namespace
     :param depth: search depth into imported modules, helps avoid infinite recursion.
     :return: class name found in the context or None if not found
@@ -599,14 +599,14 @@ def __repr__(self):
     @property
     def table_name(self):
         """
-        :return: the table name in the database
+        :return: the table name in the schema
         """
         return self._table_name
 
 
 class Log(BaseRelation):
     """
-    The log table for each database.
+    The log table for each schema.
     Instances are callable.  Calls log the time and identifying information along with the event.
     """
 

datajoint/connection.py

@@ -1,5 +1,5 @@
 """
-This module hosts the Connection class that manages the connection to the mysql database,
+This module hosts the Connection class that manages the connection to the database,
  and the `conn` function that provides access to a persistent connection in datajoint.
 """
 import warnings

datajoint/fetch.py

@@ -19,8 +19,8 @@ def to_dicts(recarray):
 
 class Fetch:
     """
-    A fetch object that handles retrieving elements from the database table.
-    :param relation: relation the fetch object fetches data from
+    A fetch object that handles retrieving elements from the table expression.
+    :param relation: the table expression to fetch from
     """
 
     def __init__(self, relation):

datajoint/schema.py

@@ -40,39 +40,40 @@ class Schema:
     It also specifies the namespace `context` in which other UserRelation classes are defined.
     """
 
-    def __init__(self, database, context=None, connection=None, create_tables=True):
+    def __init__(self, schema_name, context=None, connection=None, create_schema=True, create_tables=True):
         """
-        Associates the specified database with this schema object. If the target database does not exist
-        already, will attempt on creating the database.
+        Associate database schema `schema_name`. If the schema does not exist, attempt to create it on the server.
 
-        :param database: name of the database to associate the decorated class with
-        :param context: dictionary for looking up foreign keys references, leave None to use local context
-        :param connection: Connection object. Defaults to datajoint.conn()
+        :param schema_name: the database schema to associate.
+        :param context: dictionary for looking up foreign key references, leave None to use local context.
+        :param connection: Connection object. Defaults to datajoint.conn().
+        :param create_schema: When False, do not create the schema and raise an error if missing.
+        :param create_tables: When False, do not create tables and raise errors when accessing missing tables.
         """
         if connection is None:
             connection = conn()
         self._log = None
-        self.database = database
+        self.database = schema_name
         self.connection = connection
         self.context = context
         self.create_tables = create_tables
         self._jobs = None
         self._external = None
         if not self.exists:
-            if not self.create_tables:
-                raise DataJointError("Database named `{database}` was not defined. "
-                                     "Set the create_tables flag to create it.".format(database=database))
+            if not create_schema:
+                raise DataJointError(
+                    "Database named `{name}` was not defined. "
+                    "Set argument create_schema=True to create it.".format(name=schema_name))
             else:
                 # create database
-                logger.info("Database `{database}` could not be found. "
-                            "Attempting to create the database.".format(database=database))
+                logger.info("Creating schema `{name}`.".format(name=schema_name))
                 try:
-                    connection.query("CREATE DATABASE `{database}`".format(database=database))
-                    logger.info('Created database `{database}`.'.format(database=database))
+                    connection.query("CREATE DATABASE `{name}`".format(name=schema_name))
+                    logger.info('Creating schema `{name}`.'.format(name=schema_name))
                 except pymysql.OperationalError:
-                    raise DataJointError("Database named `{database}` was not defined, and"
-                                         " an attempt to create has failed. Check"
-                                         " permissions.".format(database=database))
+                    raise DataJointError(
+                        "Schema `{name}` does not exist and could not be created. "
+                        "Check permissions.".format(name=schema_name))
                 else:
                     self.log('created')
         self.log('connect')
@@ -85,12 +86,12 @@ def log(self):
         return self._log
 
     def __repr__(self):
-        return 'Schema database: `{database}`\n'.format(database=self.database)
+        return 'Schema `{name}`\n'.format(name=self.database)
 
     @property
     def size_on_disk(self):
         """
-        :return: size of the database in bytes
+        :return: size of the entire schema in bytes
         """
         return int(self.connection.query(
             """
@@ -106,7 +107,8 @@ def _make_module_code(self):
         """
 
         module_count = itertools.count()
-        module_lookup = collections.defaultdict(lambda: 'vmodule' + str(next(module_count)))
+        # add virtual modules for referenced modules with names vmod0, vmod1, ...
+        module_lookup = collections.defaultdict(lambda: 'vmod' + str(next(module_count)))
         db = self.database
 
         def make_class_definition(table):
@@ -136,13 +138,13 @@ def repl(s):
         return '\n\n\n'.join((
             '"""This module was auto-generated by datajoint from an existing schema"""',
             "import datajoint as dj\n\nschema=dj.schema('{db}')".format(db=db),
-            '\n'.join("{module} = dj.create_virtual_module('{module}', '{database}')".format(module=v, database=k)
+            '\n'.join("{module} = dj.create_virtual_module('{module}', '{schema_name}')".format(module=v, schema_name=k)
                       for k, v in module_lookup.items()),
             body))
 
     def spawn_missing_classes(self, context=None):
         """
-        Creates the appropriate python user relation classes from tables in the database and places them
+        Creates the appropriate python user relation classes from tables in the schema and places them
         in the context.
         :param context: alternative context to place the missing classes into, e.g. locals()
         """
@@ -186,25 +188,25 @@ def spawn_missing_classes(self, context=None):
 
     def drop(self, force=False):
         """
-        Drop the associated database if it exists
+        Drop the associated schema if it exists
         """
         if not self.exists:
-            logger.info("Database named `{database}` does not exist. Doing nothing.".format(database=self.database))
+            logger.info("Schema named `{database}` does not exist. Doing nothing.".format(database=self.database))
         elif (not config['safemode'] or
               force or
               user_choice("Proceed to delete entire schema `%s`?" % self.database, default='no') == 'yes'):
             logger.info("Dropping `{database}`.".format(database=self.database))
             try:
                 self.connection.query("DROP DATABASE `{database}`".format(database=self.database))
-                logger.info("Database `{database}` was dropped successfully.".format(database=self.database))
+                logger.info("Schema `{database}` was dropped successfully.".format(database=self.database))
             except pymysql.OperationalError:
-                raise DataJointError("An attempt to drop database named `{database}` "
+                raise DataJointError("An attempt to drop schema `{database}` "
                                      "has failed. Check permissions.".format(database=self.database))
 
     @property
     def exists(self):
         """
-        :return: true if the associated database exists on the server
+        :return: true if the associated schema exists on the server
         """
         cur = self.connection.query("SHOW DATABASES LIKE '{database}'".format(database=self.database))
         return cur.rowcount > 0
@@ -238,8 +240,8 @@ def process_relation_class(self, relation_class, context, assert_declared=False)
 
     def __call__(self, cls):
         """
-        Binds the passed in class object to a database. This is intended to be used as a decorator.
-        :param cls: class to be decorated
+        Binds the supplied class to a schema. This is intended to be used as a decorator.
+        :param cls: class to decorate.
         """
         context = self.context if self.context is not None else inspect.currentframe().f_back.f_locals
         if issubclass(cls, Part):