add life cycle to s3 express to test helper (#494)

Author: TingDaoK

tests/mock_s3_server/mock_s3_server.py

@@ -90,12 +90,12 @@ def resolve_response(self, wrapper, request_type, chunked=False, head_request=Fa
 
         # If request_headers is present, validate that the request contains all required headers
         if 'request_headers' in data:
-                for header in data['request_headers']:
-                    header_bytes = header.encode('utf-8')
-                    if not any(header_bytes == h[0] for h in self.request_headers):
-                        response =  Response(status_code=500, delay=0, headers=headers,
-                                             data=json.dumps({'error': f"Missing required header: {header}"}), chunked=chunked, head_request=head_request)
-                        return response
+            for header in data['request_headers']:
+                header_bytes = header.encode('utf-8')
+                if not any(header_bytes == h[0] for h in self.request_headers):
+                    response = Response(status_code=500, delay=0, headers=headers,
+                                        data=json.dumps({'error': f"Missing required header: {header}"}), chunked=chunked, head_request=head_request)
+                    return response
 
         # if response has delay, then sleep before sending it
         delay = data.get('delay', 0)
@@ -411,7 +411,6 @@ def handle_get_object_modified(start_range, end_range, request):
     else:
         # Check the request header to make sure "If-Match" is set
         etag = get_request_header_value(request, "if-match")
-        print(etag)
         # fetch Etag from the first_part response file
         response_file = os.path.join(
             base_dir, S3Opts.GetObject.name, f"get_object_modified_first_part.json")
@@ -436,7 +435,7 @@ def handle_get_object(wrapper, request, parsed_path, head_request=False):
 
     if (parsed_path.path == "/get_object_invalid_response_missing_content_range" or
         parsed_path.path == "/get_object_invalid_response_missing_etags" or
-        parsed_path.path == "/get_object_long_error"):
+            parsed_path.path == "/get_object_long_error"):
         # Don't generate the body for those requests
         return response_config
 

tests/test_helper/test_helper.py

@@ -3,6 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 import argparse
+import json
 import boto3
 import botocore
 import sys
@@ -114,7 +115,7 @@ def create_bucket(client, **kwargs):
         raise e
 
 
-def create_bucket_with_lifecycle(availability_zone=None, client=s3_client):
+def create_bucket_with_lifecycle(availability_zone=None, client=s3_client, region=REGION):
     if availability_zone is not None:
         bucket_config = {
             'Location': {
@@ -128,37 +129,58 @@ def create_bucket_with_lifecycle(availability_zone=None, client=s3_client):
         }
         bucket_name = BUCKET_NAME_BASE+f"--{availability_zone}--x-s3"
     else:
-        bucket_config = {'LocationConstraint': REGION}
+        bucket_config = {'LocationConstraint': region}
         bucket_name = BUCKET_NAME_BASE
 
     create_bucket(client,
                   Bucket=bucket_name,
                   CreateBucketConfiguration=bucket_config)
-    if availability_zone is None:
-        print(f"s3://{bucket_name} - Configuring bucket...")
-        client.put_bucket_lifecycle_configuration(
-            Bucket=bucket_name,
-            LifecycleConfiguration={
-                'Rules': [
-                    {
-                        'ID': 'clean up non-pre-existing objects',
-                        'Expiration': {
-                            'Days': 1,
-                        },
-                        'Filter': {
-                            'Prefix': 'upload/',
-                        },
-                        'Status': 'Enabled',
-                        'NoncurrentVersionExpiration': {
-                            'NoncurrentDays': 1,
-                        },
-                        'AbortIncompleteMultipartUpload': {
-                            'DaysAfterInitiation': 1,
-                        },
+    print(f"s3://{bucket_name} - Configuring bucket...")
+    if availability_zone is not None:
+        # https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-lifecycle.html#directory-bucket-lifecycle-differences
+        # S3 express requires a bucket policy to allow session-based access to perform lifecycle actions
+        account_id = boto3.client(
+            'sts').get_caller_identity().get('Account')
+        bucket_policy = {
+            "Version": "2008-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Service": "lifecycle.s3.amazonaws.com"
                     },
-                ],
-            },
-        )
+                    "Action": "s3express:CreateSession",
+                    "Condition": {
+                        "StringEquals": {
+                            "s3express:SessionMode": "ReadWrite"
+                        }
+                    },
+                    "Resource": [
+                        f"arn:aws:s3express:{region}:{account_id}:bucket/{bucket_name}"
+                    ]
+                }
+            ]
+        }
+        client.put_bucket_policy(
+            Bucket=bucket_name, Policy=json.dumps(bucket_policy))
+
+    client.put_bucket_lifecycle_configuration(
+        Bucket=bucket_name,
+        LifecycleConfiguration={
+            'Rules': [
+                {
+                    'ID': 'Abort all incomplete multipart uploads after 1 day',
+                    'Status': 'Enabled',
+                    'Filter': {'Prefix': ''},  # blank string means all
+                    'AbortIncompleteMultipartUpload': {'DaysAfterInitiation': 1},
+                },
+                {
+                    'ID': 'Objects under upload directory expire after 1 day',
+                    'Status': 'Enabled',
+                    'Filter': {'Prefix': 'upload/'},
+                    'Expiration': {'Days': 1},
+                },
+            ]})
 
     put_pre_existing_objects(
         10*MB, 'pre-existing-10MB', bucket=bucket_name, client=client)
@@ -231,7 +253,7 @@ def cleanup(bucket_name, availability_zone=None, client=s3_client):
 
 
 if args.action == 'init':
-    create_bucket_with_lifecycle("use1-az4", s3_client_east1)
+    create_bucket_with_lifecycle("use1-az4", s3_client_east1, "us-east-1")
     create_bucket_with_lifecycle("usw2-az1")
     create_bucket_with_lifecycle()
     create_bucket_with_public_object()