Merge pull request #641 from guzman-raphael/ext-migrate-test

Add test cases for external blob migration 0.11 to 0.12

Author: dimitri-yatsenko

.gitignore

@@ -18,3 +18,8 @@ build/
 ./tests/.coverage
 ./tests/dj-store/*
 *.log
+*.env
+LNX-docker-compose.yml
+notebooks/*
+test.dockerfile
+__main__.py

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 dist: xenial
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ENDPOINT="127.0.0.1:9000" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint" S3_BUCKET="datajoint-test"
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint"
 python:
   - "3.4"
   - "3.5"
@@ -16,8 +16,6 @@ before_install:
   - docker pull minio/minio
   - docker run -d -p 9000:9000 -e "MINIO_ACCESS_KEY=$S3_ACCESS_KEY" -e "MINIO_SECRET_KEY=$S3_SECRET_KEY" minio/minio server /data
   - sleep 120
-  - docker pull minio/mc
-  - docker run --network="host" --entrypoint=/bin/sh minio/mc -c "mc config host add dj-s3 http://$S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY;mc mb $S3_BUCKET;mc policy download $S3_BUCKET;exit 0;"
   - sudo apt-get install -y libblas-dev liblapack-dev libatlas-dev gfortran
   - sudo apt-get install -y graphviz graphviz-dev pkg-config
   - mysql --version 

datajoint/utils.py

@@ -95,3 +95,19 @@ def safe_copy(src, dest, overwrite=False):
         temp_file = dest + '.copying'
         shutil.copyfile(src, temp_file)
         os.rename(temp_file, dest)
+
+
+def parse_sql(filepath):
+    DELIMITER = ';'
+    statement = []
+    with open(filepath, 'rt') as f:
+        for line in f:
+            line = line.strip()
+            if not line.startswith('--') and len(line) > 1:
+                if line.startswith('DELIMITER'):
+                    DELIMITER = line.split()[1]
+                else:
+                    statement.append(line)
+                    if line.endswith(DELIMITER):
+                        yield ' '.join(statement)
+                        statement = []

tests/__init__.py

@@ -9,8 +9,15 @@
 from os import environ, remove
 import datajoint as dj
 from distutils.version import LooseVersion
+import os
+from pathlib import Path
+from minio import Minio
+import urllib3
+import certifi
+import shutil
+from datajoint.utils import parse_sql
 
-__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko'
+__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman'
 
 # turn on verbose logging
 logging.basicConfig(level=logging.DEBUG)
@@ -61,7 +68,7 @@
     conn_root.query(
         "GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';")
 else:
-    # grant permissions. For mysql5.6/5.7 this also automatically creates user 
+    # grant permissions. For mysql5.6/5.7 this also automatically creates user
     # if not exists
     conn_root.query("""
         GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%'
@@ -76,6 +83,25 @@
         REQUIRE SSL;
         """)
 
+# Initialize httpClient with relevant timeout.
+httpClient = urllib3.PoolManager(
+            timeout=30,
+            cert_reqs='CERT_REQUIRED',
+            ca_certs=certifi.where(),
+            retries=urllib3.Retry(
+                total=3,
+                backoff_factor=0.2,
+                status_forcelist=[500, 502, 503, 504]
+            )
+        )
+
+# Initialize minioClient with an endpoint and access/secret keys.
+minioClient = Minio(S3_CONN_INFO['endpoint'],
+                    access_key=S3_CONN_INFO['access_key'],
+                    secret_key=S3_CONN_INFO['secret_key'],
+                    secure=False,
+                    http_client=httpClient)
+
 
 def setup_package():
     """
@@ -84,6 +110,44 @@ def setup_package():
     """
     dj.config['safemode'] = False
 
+    # Add old MySQL
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data"
+    db_name = "djtest_blob_migrate"
+    db_file = "v0_11.sql"
+    conn_root.query("""
+        CREATE DATABASE {};
+        """.format(db_name))
+
+    statements = parse_sql('{}/{}'.format(source, db_file))
+    for s in statements:
+        conn_root.query(s)
+
+    # Add old S3
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data/s3"
+    bucket = "migrate-test"
+    region = "us-east-1"
+    minioClient.make_bucket(bucket, location=region)
+
+    pathlist = Path(source).glob('**/*')
+    for path in pathlist:
+        if os.path.isfile(str(path)) and ".sql" not in str(path):
+            minioClient.fput_object(
+                    bucket, os.path.relpath(
+                        str(path),
+                        '{}/{}'.format(source, bucket)
+                        ), str(path))
+
+    # Add S3
+    minioClient.make_bucket("datajoint-test", location=region)
+
+    # Add old File Content
+    shutil.copytree(
+            os.path.dirname(os.path.realpath(__file__)) +
+            "/external-legacy-data/file/temp",
+            os.path.expanduser('~/temp'))
+
 
 def teardown_package():
     """
@@ -99,3 +163,21 @@ def teardown_package():
         conn.query('DROP DATABASE `{}`'.format(db[0]))
     conn.query('SET FOREIGN_KEY_CHECKS=1')
     remove("dj_local_conf.json")
+
+    # Remove old S3
+    bucket = "migrate-test"
+    objs = list(minioClient.list_objects_v2(
+            bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove S3
+    bucket = "datajoint-test"
+    objs = list(minioClient.list_objects_v2(bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove old File Content
+    shutil.rmtree(os.path.expanduser('~/temp'))