Merge pull request #88 from infosiftr/minio

Minio Snapshotter

Author: Justin Reagor

Dockerfile

@@ -19,6 +19,7 @@ RUN set -ex \
     && pip install \
        python-Consul==0.7.0 \
        manta==2.5.0 \
+       minio==2.2.4 \
        mock==2.0.0 \
        json5==0.2.4 \
     # \

bin/manage.py

@@ -14,6 +14,7 @@
 from manager.network import get_ip
 
 from manager.storage.manta_stor import Manta
+from manager.storage.minio_stor import Minio
 from manager.storage.local import Local
 
 from manager.utils import log, debug, \
@@ -402,8 +403,11 @@ def main():
 
     my = MySQL()
 
-    if os.environ.get('SNAPSHOT_BACKEND', 'manta') == 'local':
+    snapshot_backend = os.environ.get('SNAPSHOT_BACKEND', 'manta')
+    if snapshot_backend == 'local':
         snaps = Local()
+    elif snapshot_backend == 'minio':
+        snaps = Minio()
     else:
         snaps = Manta()
 

bin/manager/storage/minio_stor.py

@@ -0,0 +1,53 @@
+""" Module for storing snapshots in shared local disk """
+import logging
+import os
+from shutil import copyfile
+
+from manager.env import env, to_flag
+from manager.utils import debug
+from minio import Minio as pyminio, error as minioerror
+
+logging.getLogger('manta').setLevel(logging.INFO)
+
+class Minio(object):
+    """
+
+    The Minio class wraps access to the Minio object store, where we'll put
+    our MySQL backups.
+    """
+    def __init__(self, envs=os.environ):
+        self.access_key = env('MINIO_ACCESS_KEY', None, envs)
+        self.secret_key = env('MINIO_SECRET_KEY', None, envs)
+        self.bucket = env('MINIO_BUCKET', 'backups', envs)
+        self.location = env('MINIO_LOCATION', 'us-east-1', envs)
+        self.url = env('MINIO_URL', 'minio:9000')
+        is_tls = env('MINIO_TLS_SECURE', False, envs, fn=to_flag)
+
+        self.client = pyminio(
+            self.url,
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            secure=is_tls)
+        try:
+            self.client.make_bucket(self.bucket, location=self.location)
+        except minioerror.BucketAlreadyOwnedByYou:
+            pass
+
+    @debug
+    def get_backup(self, backup_id):
+        """
+        Download file from Minio, allowing exceptions to bubble up.
+        """
+        try:
+            os.mkdir('/tmp/backup', 0770)
+        except OSError:
+            pass
+        outfile = '/tmp/backup/{}'.format(backup_id)
+        self.client.fget_object(self.bucket, backup_id, outfile)
+
+    def put_backup(self, backup_id, infile):
+        """
+        Upload the backup file to the expected path.
+        """
+        self.client.fput_object(self.bucket, backup_id, infile)
+        return backup_id

examples/compose/docker-compose.yml

@@ -17,12 +17,15 @@ services:
       - BACKUP_TTL=120
       - LOG_LEVEL=DEBUG
       - CONSUL=consul
-      - SNAPSHOT_BACKEND=local
+      - SNAPSHOT_BACKEND=minio
+      - MINIO_ACCESS_KEY=supersecretaccesskey
+      - MINIO_SECRET_KEY=supersecretsecretkey
     volumes:
       # shared storage location for snapshots
       - ${WORK_DIR:-../..}/tmp:/tmp/snapshots
     links:
       - consul:consul
+      - minio:minio
 
   consul:
     image: consul:0.8.4
@@ -42,3 +45,14 @@ services:
     network_mode: bridge
     dns:
       - 127.0.0.1
+
+  minio:
+    image: minio/minio
+    command: server /export
+    restart: always
+    expose:
+      - 9000
+    network_mode: bridge
+    environment:
+      - MINIO_ACCESS_KEY=supersecretaccesskey
+      - MINIO_SECRET_KEY=supersecretsecretkey