Merge branch 'master' of https://github.com/jverswijver/datajoint-python into remove_checksum

Author: jverswijver

.github/workflows/development.yaml

@@ -36,8 +36,6 @@ jobs:
         include:
           - py_ver: "3.7"
             mysql_ver: "5.7"
-          - py_ver: "3.6"
-            mysql_ver: "5.7"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
@@ -106,4 +104,4 @@ jobs:
         with:
           branch: gh-pages
           directory: gh-pages
-          github_token: ${{secrets.GITHUB_TOKEN}}
+          github_token: ${{secrets.GITHUB_TOKEN}}

CHANGELOG.md

@@ -1,15 +1,15 @@
 ## Release notes
 
-### 0.13.6 -- TBD
+### 0.13.6 -- Jun 13, 2022
 * Add - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025
-* Add - unified package level logger for package
-* Update - swap various datajoint messages, warnings, ect. to use the new logger.
+* Add - unified package level logger for package (#667) PR #1031
+* Update - swap various datajoint messages, warnings, etc. to use the new logger. (#667) PR #1031
 
 ### 0.13.5 -- May 19, 2022
 * Update - Import ABC from collections.abc for Python 3.10 compatibility
 * Bugfix - Fix multiprocessing value error (#1013) PR #1026
 
-### 0.13.4 -- March, 28 2022
+### 0.13.4 -- Mar, 28 2022
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
 * Bugfix - Add missing `jobs` argument for multiprocessing PR #997
 * Add - Test for multiprocessing PR #1008

datajoint/autopopulate.py

@@ -9,6 +9,7 @@
 from .errors import DataJointError, LostConnectionError
 import signal
 import multiprocessing as mp
+import contextlib
 
 # noinspection PyExceptionInherit,PyCallingNonCallable
 
@@ -158,7 +159,7 @@ def populate(
         max_calls=None,
         display_progress=False,
         processes=1,
-        make_kwargs=None
+        make_kwargs=None,
     ):
         """
         ``table.populate()`` calls ``table.make(key)`` for every primary key in
@@ -206,14 +207,14 @@ def handler(signum, frame):
         elif order == "random":
             random.shuffle(keys)
 
-        logger.info("Found %d keys to populate" % len(keys))
+        logger.debug("Found %d keys to populate" % len(keys))
 
         keys = keys[:max_calls]
         nkeys = len(keys)
         if not nkeys:
             return
 
-        processes = min(*(_ for _ in (processes, nkeys, mp.cpu_count()) if _))
+        processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
 
         error_list = []
         populate_kwargs = dict(
@@ -235,17 +236,16 @@ def handler(signum, frame):
             del self.connection._conn.ctx  # SSLContext is not pickleable
             with mp.Pool(
                 processes, _initialize_populate, (self, jobs, populate_kwargs)
-            ) as pool:
-                if display_progress:
-                    with tqdm(desc="Processes: ", total=nkeys) as pbar:
-                        for error in pool.imap(_call_populate1, keys, chunksize=1):
-                            if error is not None:
-                                error_list.append(error)
-                            pbar.update()
-                else:
-                    for error in pool.imap(_call_populate1, keys):
-                        if error is not None:
-                            error_list.append(error)
+            ) as pool, (
+                tqdm(desc="Processes: ", total=nkeys)
+                if display_progress
+                else contextlib.nullcontext()
+            ) as progress_bar:
+                for error in pool.imap(_call_populate1, keys, chunksize=1):
+                    if error is not None:
+                        error_list.append(error)
+                    if display_progress:
+                        progress_bar.update()
             self.connection.connect()  # reconnect parent process to MySQL server
 
         # restore original signal handler:
@@ -275,7 +275,7 @@ def _populate1(
                 if jobs is not None:
                     jobs.complete(self.target.table_name, self._job_key(key))
             else:
-                logger.debug("Populating: " + str(key))
+                logger.debug(f"Making {key} -> {self.target.full_table_name}")
                 self.__class__._allow_insert = True
                 try:
                     make(dict(key), **(make_kwargs or {}))
@@ -288,6 +288,9 @@ def _populate1(
                         exception=error.__class__.__name__,
                         msg=": " + str(error) if str(error) else "",
                     )
+                    logger.debug(
+                        f"Error making {key} -> {self.target.full_table_name} - {error_message}"
+                    )
                     if jobs is not None:
                         # show error name and error message (if any)
                         jobs.error(
@@ -303,6 +306,9 @@ def _populate1(
                         return key, error if return_exception_objects else error_message
                 else:
                     self.connection.commit_transaction()
+                    logger.debug(
+                        f"Success making {key} -> {self.target.full_table_name}"
+                    )
                     if jobs is not None:
                         jobs.complete(self.target.table_name, self._job_key(key))
                 finally:

datajoint/connection.py

@@ -21,6 +21,9 @@
 query_log_max_length = 300
 
 
+cache_key = "query_cache"  # the key to lookup the query_cache folder in dj.config
+
+
 def get_host_hook(host_input):
     if "://" in host_input:
         plugin_name = host_input.split("://")[0]
@@ -220,7 +223,7 @@ def connect(self):
                         k: v
                         for k, v in self.conn_info.items()
                         if k not in ["ssl_input", "host_input"]
-                    }
+                    },
                 )
             except client.err.InternalError:
                 self._conn = client.connect(
@@ -236,7 +239,7 @@ def connect(self):
                             or k == "ssl"
                             and self.conn_info["ssl_input"] is None
                         )
-                    }
+                    },
                 )
         self._conn.autocommit(True)
 
@@ -254,13 +257,12 @@ def set_query_cache(self, query_cache=None):
     def purge_query_cache(self):
         """Purges all query cache."""
         if (
-            "query_cache" in config
-            and isinstance(config["query_cache"], str)
-            and pathlib.Path(config["query_cache"]).is_dir()
+            isinstance(config.get(cache_key), str)
+            and pathlib.Path(config[cache_key]).is_dir()
         ):
-            path_iter = pathlib.Path(config["query_cache"]).glob("**/*")
-            for path in path_iter:
-                path.unlink()
+            for path in pathlib.Path(config[cache_key]).iterdir():
+                if not path.is_dir():
+                    path.unlink()
 
     def close(self):
         self._conn.close()
@@ -313,15 +315,15 @@ def query(
                 "Only SELECT queries are allowed when query caching is on."
             )
         if use_query_cache:
-            if not config["query_cache"]:
+            if not config[cache_key]:
                 raise errors.DataJointError(
-                    "Provide filepath dj.config['query_cache'] when using query caching."
+                    f"Provide filepath dj.config['{cache_key}'] when using query caching."
                 )
             hash_ = uuid_from_buffer(
                 (str(self._query_cache) + re.sub(r"`\$\w+`", "", query)).encode()
                 + pack(args)
             )
-            cache_path = pathlib.Path(config["query_cache"]) / str(hash_)
+            cache_path = pathlib.Path(config[cache_key]) / str(hash_)
             try:
                 buffer = cache_path.read_bytes()
             except FileNotFoundError:

datajoint/diagram.py

@@ -10,7 +10,6 @@
 from .errors import DataJointError
 from .table import lookup_class_name
 
-logger = logging.getLogger(__name__.split(".")[0])
 
 try:
     from matplotlib import pyplot as plt
@@ -27,6 +26,7 @@
     diagram_active = False
 
 
+logger = logging.getLogger(__name__.split(".")[0])
 user_table_classes = (Manual, Lookup, Computed, Imported, Part)
 
 

datajoint/logging.py

@@ -5,11 +5,9 @@
 
 logger = logging.getLogger(__name__.split(".")[0])
 
-log_level = os.environ.get("DJ_LOG_LEVEL", "warning").upper()
+log_level = os.getenv("DJ_LOG_LEVEL", "info").upper()
 
-log_format = logging.Formatter(
-    "[%(asctime)s][%(funcName)s][%(levelname)s]: %(message)s"
-)
+log_format = logging.Formatter("[%(asctime)s][%(levelname)s]: %(message)s")
 
 stream_handler = logging.StreamHandler()  # default handler
 stream_handler.setFormatter(log_format)
@@ -32,26 +30,3 @@ def excepthook(exc_type, exc_value, exc_traceback):
 
 
 sys.excepthook = excepthook
-
-
-# https://github.com/tqdm/tqdm/issues/313#issuecomment-267959111
-class TqdmToLogger(io.StringIO):
-    """
-    Output stream for TQDM which will output to logger module instead of
-    the StdOut.
-    """
-
-    logger = None
-    level = None
-    buf = ""
-
-    def __init__(self, logger, level=None):
-        super(TqdmToLogger, self).__init__()
-        self.logger = logger
-        self.level = level or logging.INFO
-
-    def write(self, buf):
-        self.buf = buf.strip("\r\n\t ")
-
-    def flush(self):
-        self.logger.log(self.level, self.buf)

datajoint/schemas.py

@@ -134,7 +134,7 @@ def activate(
                     )
                 )
             # create database
-            logger.info("Creating schema `{name}`.".format(name=schema_name))
+            logger.debug("Creating schema `{name}`.".format(name=schema_name))
             try:
                 self.connection.query(
                     "CREATE DATABASE `{name}`".format(name=schema_name)
@@ -360,12 +360,12 @@ def drop(self, force=False):
             )
             == "yes"
         ):
-            logger.info("Dropping `{database}`.".format(database=self.database))
+            logger.debug("Dropping `{database}`.".format(database=self.database))
             try:
                 self.connection.query(
                     "DROP DATABASE `{database}`".format(database=self.database)
                 )
-                logger.info(
+                logger.debug(
                     "Schema `{database}` was dropped successfully.".format(
                         database=self.database
                     )

datajoint/settings.py

@@ -241,8 +241,8 @@ def __getitem__(self, key):
             return self._conf[key]
 
         def __setitem__(self, key, value):
-            logger.log(
-                logging.INFO, "Setting {0:s} to {1:s}".format(str(key), str(value))
+            logger.debug(
+                logging.DEBUG, "Setting {0:s} to {1:s}".format(str(key), str(value))
             )
             if validators[key](value):
                 self._conf[key] = value

datajoint/table.py

@@ -310,7 +310,7 @@ def update1(self, row):
             raise DataJointError("Update cannot be applied to a restricted table.")
         key = {k: row[k] for k in self.primary_key}
         if len(self & key) != 1:
-            raise DataJointError("Update entry must exist.")
+            raise DataJointError("Update can only be applied to one existing entry.")
         # UPDATE query
         row = [
             self.__make_placeholder(k, v)

docs-parts/intro/Releases_lang1.rst

@@ -1,15 +1,15 @@
-0.13.6 -- TBD
+### 0.13.6 -- Jun 13, 2022
 ----------------------
 * Add - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025
-* Add - unified package level logger for package
-* Update - swap various datajoint messages, warnings, ect. to use the new logger.
+* Add - unified package level logger for package (#667) PR #1031
+* Update - swap various datajoint messages, warnings, etc. to use the new logger. (#667) PR #1031
 
 0.13.5 -- May 19, 2022
 ----------------------
 * Update - Import ABC from collections.abc for Python 3.10 compatibility
 * Bugfix - Fix multiprocessing value error (#1013) PR #1026
 
-0.13.4 -- March 28, 2022
+0.13.4 -- Mar 28, 2022
 ----------------------
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
 * Bugfix - Add missing ``jobs`` argument for multiprocessing PR #997