Merge pull request #693 from guzman-raphael/master

Strip root from external S3 storage on insert

Author: dimitri-yatsenko

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## Release notes
 
+### 0.12.2 -- Nov 8, 2019
+* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['<store>']['location']` (#692)
+
 ### 0.12.1 -- Nov 2, 2019
 * Bugfix - AttributeAdapter converts into a string (#684)
 

datajoint/external.py

@@ -1,4 +1,4 @@
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PurePosixPath, PureWindowsPath
 from collections import Mapping
 from tqdm import tqdm
 from .settings import config
@@ -74,7 +74,20 @@ def s3(self):
 
     def _make_external_filepath(self, relative_filepath):
         """resolve the complete external path based on the relative path"""
-        return PurePosixPath(Path(self.spec['location']), relative_filepath)
+        # Strip root
+        if self.spec['protocol'] == 's3':
+            posix_path = PurePosixPath(PureWindowsPath(self.spec['location']))
+            location_path = Path(
+                    *posix_path.parts[1:]) if len(
+                    self.spec['location']) > 0 and any(
+                    case in posix_path.parts[0] for case in (
+                        '\\', ':')) else Path(posix_path)
+            return PurePosixPath(location_path, relative_filepath)
+        # Preserve root
+        elif self.spec['protocol'] == 'file':
+            return PurePosixPath(Path(self.spec['location']), relative_filepath)
+        else:
+            assert False
 
     def _make_uuid_path(self, uuid, suffix=''):
         """create external path based on the uuid hash"""

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.12.1"
+__version__ = "0.12.2"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

docs-parts/intro/Releases_lang1.rst

@@ -1,3 +1,7 @@
+0.12.1 -- Nov 8, 2019
+-------------------------
+* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['<store>']['location']` (#692)
+
 0.12.1 -- Nov 2, 2019
 -------------------------
 * Bugfix - AttributeAdapter converts into a string (#684)

tests/schema_external.py

@@ -55,6 +55,15 @@ class Simple(dj.Manual):
     """
 
 
+@schema
+class SimpleRemote(dj.Manual):
+    definition = """
+    simple  : int
+    ---
+    item  : blob@share
+    """
+
+
 @schema
 class Seed(dj.Lookup):
     definition = """

tests/test_blob_migrate.py

@@ -7,6 +7,7 @@
 from . import S3_CONN_INFO
 from . import CONN_INFO
 from datajoint.migrate import _migrate_dj011_blob
+dj.config['enable_python_native_blobs'] = True
 
 
 class TestBlobMigrate:

tests/test_external.py

@@ -7,12 +7,20 @@
 from .schema_external import schema
 import datajoint as dj
 from .schema_external import stores_config
+from .schema_external import SimpleRemote
+current_location_s3 = dj.config['stores']['share']['location']
+current_location_local = dj.config['stores']['local']['location']
 
 
 def setUp(self):
     dj.config['stores'] = stores_config
 
 
+def tearDown(self):
+    dj.config['stores']['share']['location'] = current_location_s3
+    dj.config['stores']['local']['location'] = current_location_local
+
+
 def test_external_put():
     """
     external storage put and get and remove
@@ -33,3 +41,65 @@ def test_external_put():
 
     output_ = unpack(ext.get(hash1))
     assert_array_equal(input_, output_)
+
+
+def test_s3_leading_slash(index=100, store='share'):
+    """
+    s3 external storage configured with leading slash
+    """
+    value = np.array([1, 2, 3])
+
+    id = index
+    dj.config['stores'][store]['location'] = 'leading/slash/test'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 1
+    dj.config['stores'][store]['location'] = '/leading/slash/test'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 2
+    dj.config['stores'][store]['location'] = 'leading\\slash\\test'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 3
+    dj.config['stores'][store]['location'] = 'f:\\leading\\slash\\test'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 4
+    dj.config['stores'][store]['location'] = 'f:\\leading/slash\\test'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 5
+    dj.config['stores'][store]['location'] = '/'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 6
+    dj.config['stores'][store]['location'] = 'C:\\'
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+    id = index + 7
+    dj.config['stores'][store]['location'] = ''
+    SimpleRemote.insert([{'simple': id, 'item': value}])
+    assert_true(np.array_equal(
+        value, (SimpleRemote & 'simple={}'.format(id)).fetch1('item')))
+
+
+def test_file_leading_slash():
+    """
+    file external storage configured with leading slash
+    """
+    test_s3_leading_slash(index=200, store='local')