Add support for s3 canned ACLs for uploaded objects

Author: ivanmp91

docs/aws_s3_setup.md

@@ -51,6 +51,7 @@ Create an IAM Policy called `MedusaStorageStrategy`, with the following definiti
                 "s3:GetReplicationConfiguration",
                 "s3:ListMultipartUploadParts",
                 "s3:PutObject",
+                "s3:PutObjectAcl",
                 "s3:GetObject",
                 "s3:GetObjectTorrent",
                 "s3:PutObjectRetention",

medusa-example.ini

@@ -71,6 +71,9 @@ concurrent_transfers = 1
 ; Size over which S3 uploads will be using the awscli with multi part uploads. Defaults to 100MB.
 multi_part_upload_threshold = 104857600
 
+; Canned ACL for uploaded objects on S3. Defaults to private
+canned_acl = private
+
 [monitoring]
 ;monitoring_provider = <Provider used for sending metrics. Currently either of "ffwd" or "local">
 

medusa/config.py

@@ -29,7 +29,8 @@
     'StorageConfig',
     ['bucket_name', 'key_file', 'prefix', 'fqdn', 'host_file_separator', 'storage_provider',
      'base_path', 'max_backup_age', 'max_backup_count', 'api_profile', 'transfer_max_bandwidth',
-     'concurrent_transfers', 'multi_part_upload_threshold', 'host', 'region', 'port', 'secure', 'aws_cli_path']
+     'concurrent_transfers', 'multi_part_upload_threshold', 'canned_acl', 'host',
+     'region', 'port', 'secure', 'aws_cli_path']
 )
 
 CassandraConfig = collections.namedtuple(
@@ -95,6 +96,7 @@ def load_config(args, config_file):
         'aws_cli_path': 'aws',
         'fqdn': socket.getfqdn(),
         'region': 'default',
+        'canned_acl': 'private',
     }
 
     config['logging'] = {

medusa/storage/aws_s3_storage/awscli.py

@@ -27,6 +27,7 @@ class AwsCli(object):
     def __init__(self, storage):
         self._config = storage.config
         self.storage = storage
+        self.canned_acl = storage.config.canned_acl
 
     @property
     def bucket_name(self):
@@ -73,7 +74,8 @@ def cp_upload(self, *, srcs, bucket_name, dest, max_retries=5):
         awscli_output = "/tmp/awscli_{0}.output".format(job_id)
         objects = []
         for src in srcs:
-            cmd = [self._aws_cli_path, "s3", "cp", str(src), "s3://{}/{}".format(bucket_name, dest)]
+            cmd = [self._aws_cli_path, "s3", "cp", "--acl", self.canned_acl,
+                   str(src), "s3://{}/{}".format(bucket_name, dest)]
             objects.append(self.upload_file(cmd, dest, awscli_output))
 
         return objects

medusa/storage/aws_s3_storage/concurrent.py

@@ -64,7 +64,7 @@ def with_storage(self, iterable):
 
 
 def upload_blobs(
-    storage, src, dest, bucket, max_workers=None, multi_part_upload_threshold=0
+    storage, src, dest, bucket, canned_acl, max_workers=None, multi_part_upload_threshold=0
 ):
     """
     Uploads a list of files from local storage concurrently to the remote storage.
@@ -80,14 +80,14 @@ def upload_blobs(
     job = StorageJob(
         storage,
         lambda storage, connection, src_file: __upload_file(
-            storage, connection, src_file, dest, bucket, multi_part_upload_threshold
+            storage, connection, src_file, dest, bucket, canned_acl, multi_part_upload_threshold
         ),
         max_workers,
     )
     return job.execute(list(src))
 
 
-def __upload_file(storage, connection, src, dest, bucket, multi_part_upload_threshold):
+def __upload_file(storage, connection, src, dest, bucket, canned_acl, multi_part_upload_threshold):
     """
     This function is called by StorageJob. It may be called concurrently by multiple threads.
 
@@ -116,15 +116,16 @@ def __upload_file(storage, connection, src, dest, bucket, multi_part_upload_thre
         obj = _upload_multi_part(storage, connection, src, bucket, full_object_name)
     else:
         logging.debug("Uploading {} as single part".format(full_object_name))
-        obj = _upload_single_part(connection, src, bucket, full_object_name)
+        obj = _upload_single_part(connection, src, bucket, full_object_name, canned_acl)
 
     return medusa.storage.ManifestObject(obj.name, int(obj.size), obj.hash)
 
 
 @retry(stop_max_attempt_number=MAX_UP_DOWN_LOAD_RETRIES, wait_fixed=5000)
-def _upload_single_part(connection, src, bucket, object_name):
+def _upload_single_part(connection, src, bucket, object_name, canned_acl):
+    extra = {'content_type': 'application/octet-stream', 'acl': canned_acl}
     obj = connection.upload_object(
-        str(src), container=bucket, object_name=object_name
+        str(src), container=bucket, object_name=object_name, extra=extra
     )
 
     return obj

medusa/storage/s3_storage.py

@@ -157,6 +157,7 @@ def upload_blobs(self, srcs, dest):
             srcs,
             dest,
             self.bucket,
+            canned_acl=self.config.canned_acl,
             max_workers=self.config.concurrent_transfers,
             multi_part_upload_threshold=int(self.config.multi_part_upload_threshold),
         )

tests/integration/features/steps/integration_steps.py

@@ -426,7 +426,8 @@ def i_am_using_storage_provider_with_grpc_server(context, storage_provider, clie
             "multi_part_upload_threshold": 1 * 1024,
             "concurrent_transfers": 4,
             "prefix": storage_prefix,
-            "aws_cli_path": "aws"
+            "aws_cli_path": "aws",
+            "canned_acl": "private"
         }
 
     config["cassandra"] = {