Merge pull request #628 from dimitri-yatsenko/blob-migrate

blobs can be read from the explicit filepath in the external tables,

Author: guzman-raphael

datajoint/errors.py

@@ -35,3 +35,9 @@ class DuplicateError(DataJointError):
     Error caused by a violation of a unique constraint when inserting data
     """
     pass
+
+
+class MissingExternalFile(DataJointError):
+    """
+    Error raised when an external file managed by DataJoint is no longer accessible
+    """

datajoint/external.py

@@ -2,14 +2,15 @@
 import itertools
 from collections import Mapping
 from .settings import config
-from .errors import DataJointError
+from .errors import DataJointError, MissingExternalFile
 from .hash import uuid_from_buffer, uuid_from_file
 from .table import Table
 from .declare import EXTERNAL_TABLE_ROOT
 from . import s3 
 from .utils import safe_write, safe_copy
 
 CACHE_SUBFOLDING = (2, 2)   # (2, 2) means  "0123456789abcd" will be saved as "01/23/0123456789abcd"
+SUPPORT_MIGRATED_BLOBS = True   # support blobs migrated from datajoint 0.11.*
 
 
 def subfold(name, folds):
@@ -124,11 +125,21 @@ def fput(self, local_filepath):
     def peek(self, blob_hash, bytes_to_peek=120):
         return self.get(blob_hash, size=bytes_to_peek)
 
-    def get(self, blob_hash, size=-1):
+    def get(self, blob_hash, *, size=-1):
         """
         get an object from external store.
         :param size: max number of bytes to retrieve. If size<0, retrieve entire blob
+        :param explicit_path: if given, then use it as relative path rather than the path derived from
         """
+
+        def read_file(filepath, size):
+            try:
+                with open(filepath, 'rb') as f:
+                    blob = f.read(size)
+            except FileNotFoundError:
+                raise MissingExternalFile('Lost access to external blob %s.' % full_path) from None
+            return blob
+
         if blob_hash is None:
             return None
 
@@ -154,18 +165,33 @@ def get(self, blob_hash, size=-1):
                 subfolders = os.path.join(*subfold(blob_hash.hex, self.spec['subfolding']))
                 full_path = os.path.join(self.spec['location'], self.database, subfolders, blob_hash.hex)
                 try:
-                    with open(full_path, 'rb') as f:
-                        blob = f.read(size)
-                except FileNotFoundError:
-                    raise DataJointError('Lost access to external blob %s.' % full_path) from None
+                    blob = read_file(full_path, size)
+                except MissingExternalFile:
+                    if not SUPPORT_MIGRATED_BLOBS:
+                        raise
+                    # migrated blobs from 0.11
+                    relative_filepath, contents_hash = (self & {'hash': blob_hash}).fetch1(
+                        'filepath', 'contents_hash')
+                    if relative_filepath is None:
+                        raise
+                    blob = read_file(os.path.join(self.spec['location'], relative_filepath))
                 else:
                     if size > 0:
                         blob_size = os.path.getsize(full_path)
             elif self.spec['protocol'] == 's3':
                 full_path = '/'.join(
                     (self.database,) + subfold(blob_hash.hex, self.spec['subfolding']) + (blob_hash.hex,))
                 if size < 0:
-                    blob = self.s3.get(full_path)
+                    try:
+                        blob = self.s3.get(full_path)
+                    except MissingExternalFile:
+                        if not SUPPORT_MIGRATED_BLOBS:
+                            raise
+                        relative_filepath, contents_hash = (self & {'hash': blob_hash}).fetch1(
+                            'filepath', 'contents_hash')
+                        if relative_filepath is None:
+                            raise
+                        blob = self.s3.get(relative_filepath)
                 else:
                     blob = self.s3.partial_get(full_path, 0, size)
                     blob_size = self.s3.get_size(full_path)

datajoint/heading.py

@@ -4,7 +4,7 @@
 import re
 import logging
 from .errors import DataJointError
-from .declare import UUID_DATA_TYPE, CUSTOM_TYPES, TYPE_PATTERN, EXTERNAL_TYPES, SERIALIZED_TYPES
+from .declare import UUID_DATA_TYPE, CUSTOM_TYPES, TYPE_PATTERN, EXTERNAL_TYPES
 from .utils import OrderedDict
 
 
@@ -225,11 +225,13 @@ def init_from_database(self, conn, database, table_name):
                 try:
                     category = next(c for c in CUSTOM_TYPES if TYPE_PATTERN[c].match(attr['type']))
                 except StopIteration:
+                    if attr['type'].startswith('external'):
+                        raise DataJointError('Legacy datatype `{type}`.'.format(**attr)) from None
                     raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None
                 attr.update(
                     is_attachment=category in ('INTERNAL_ATTACH', 'EXTERNAL_ATTACH'),
                     is_filepath=category == 'FILEPATH',
-                    is_blob=category in ('INTERNAL_BLOB', 'EXTERNAL_BLOB'), # INTERNAL_BLOB is not a custom type but is included for completeness
+                    is_blob=category in ('INTERNAL_BLOB', 'EXTERNAL_BLOB'),  # INTERNAL_BLOB is not a custom type but is included for completeness
                     uuid=category == 'UUID',
                     is_external=category in EXTERNAL_TYPES,
                     store=attr['type'].split('@')[1] if category in EXTERNAL_TYPES else None)

datajoint/s3.py

@@ -6,7 +6,7 @@
 import warnings
 import uuid
 import os
-
+from . import errors
 
 class Folder:
     """
@@ -29,7 +29,10 @@ def fput(self, relative_name, local_file, **meta):
             self.bucket, '/'.join((self.remote_path, relative_name)), local_file, metadata=meta or None)
 
     def get(self, relative_name):
-        return self.client.get_object(self.bucket, '/'.join((self.remote_path, relative_name))).data
+        try:
+            return self.client.get_object(self.bucket, '/'.join((self.remote_path, relative_name))).data
+        except minio.error.NoSuchKey:
+            raise errors.MissingExternalFile from None
 
     def fget(self, relative_name, local_filepath):
         """get file from object name to local filepath"""
@@ -48,13 +51,13 @@ def partial_get(self, relative_name, offset, size):
             return self.client.get_partial_object(
                 self.bucket, '/'.join((self.remote_path, relative_name)), offset, size).data
         except minio.error.NoSuchKey:
-            return None
+            raise errors.MissingExternalFile from None
 
     def get_size(self, relative_name):
         try:
             return self.client.stat_object(self.bucket, '/'.join((self.remote_path, relative_name))).size
         except minio.error.NoSuchKey:
-            return None
+            raise errors.MissingExternalFile from None
 
     def list_objects(self, folder=''):
         return self.client.list_objects(self.bucket, '/'.join((self.remote_path, folder, '')), recursive=True)