Test post object with plain HTTP and large payload

S3Mock must be re-configured to accept large payloads.
Python requests lib sends payloads in full even if it does not know how
large the payload is going to be. From my POV, this is a bug and other
libaries handle this correctly (e.g., Java's HttpClient).

Author: afranken

pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
 # Runtime dependencies (for running tests/examples)
 dependencies = [
   "boto3>=1.40.35",
+  "requests>=2.32.3",
 ]
 
 [dependency-groups]

s3mock_test.py

@@ -33,11 +33,18 @@
 REGION = os.getenv("AWS_REGION", "us-east-1")
 ONE_MB = 1024 * 1024
 PREFIX = "prefix"
+PAYLOAD_MAX_SIZE = "100MB"
 
 container = (
-    DockerContainer("adobe/s3mock:4.9.0")
+    DockerContainer("adobe/s3mock:4.11.0")
     .with_exposed_ports(9090, 9191)
     .with_env("debug", "true")
+    # Increase various limits to allow large payload testing, see test_presigned_urls.py
+    .with_env("SERVER_TOMCAT_MAX_PART_COUNT", "-1")
+    .with_env("SERVER_TOMCAT_MAX_SWALLOW_SIZE", PAYLOAD_MAX_SIZE)
+    .with_env("SERVER_TOMCAT_MAX_HTTP_FORM_POST_SIZE", PAYLOAD_MAX_SIZE)
+    .with_env("SPRING_SERVLET_MULTIPART_MAX_FILE_SIZE", PAYLOAD_MAX_SIZE)
+    .with_env("SPRING_SERVLET_MULTIPART_MAX_REQUEST_SIZE", PAYLOAD_MAX_SIZE)
     .with_env("COM_ADOBE_TESTING_S3MOCK_DOMAIN_INITIAL_BUCKETS", "bucket-a, bucket-b")
     .with_env("COM_ADOBE_TESTING_S3MOCK_DOMAIN_VALID_KMS_KEYS",
               "arn:aws:kms:us-east-1:1234567890:key/valid-test-key-id")

tests/test_presigned_urls.py

@@ -0,0 +1,72 @@
+import os
+import tempfile
+
+import requests
+
+from s3mock_test import (
+    UPLOAD_FILE_NAME,
+    compute_md5_etag,
+    given_bucket,
+    random_name,
+)
+
+# Reimplementation of https://github.com/adobe/S3Mock/blob/main/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlIT.kt
+
+
+def test_presigned_post_object(s3_client, bucket_name: str):
+    given_bucket(s3_client, bucket_name)
+
+    key = f"{random_name()}-{UPLOAD_FILE_NAME}"
+    payload = os.urandom(20 * 1024 * 1024)
+    expected_etag = compute_md5_etag(payload)
+    tagging_xml = (
+        '<Tagging><TagSet><Tag><Key>Tag Name</Key>'
+        '<Value>Tag Value</Value></Tag></TagSet></Tagging>'
+    )
+    tmp_file = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        tmp_file.write(payload)
+        tmp_file.flush()
+        tmp_file_path = tmp_file.name
+    finally:
+        tmp_file.close()
+
+    presigned = s3_client.generate_presigned_post(
+        Bucket=bucket_name,
+        Key=key,
+        Fields={
+            'Content-Type': 'application/octet-stream',
+            'x-amz-storage-class': 'INTELLIGENT_TIERING',
+            'tagging': tagging_xml,
+        },
+        Conditions=[
+            {'Content-Type': 'application/octet-stream'},
+            {'x-amz-storage-class': 'INTELLIGENT_TIERING'},
+            {'tagging': tagging_xml},
+        ],
+        ExpiresIn=60,
+    )
+
+    form_fields = presigned['fields'].copy()
+    try:
+        with open(tmp_file_path, 'rb') as upload_fp:
+            files = {'file': (key, upload_fp, 'application/octet-stream')}
+            resp = requests.post(
+                presigned['url'], data=form_fields, files=files, timeout=30, verify=False
+            )
+    finally:
+        os.unlink(tmp_file_path)
+
+    assert resp.status_code in (200, 201, 204)
+    actual_etag = resp.headers.get('ETag')
+    assert actual_etag == expected_etag
+
+    head = s3_client.head_object(Bucket=bucket_name, Key=key)
+    assert head['ETag'] == expected_etag
+    assert head.get('StorageClass') == 'INTELLIGENT_TIERING'
+
+    tagging = s3_client.get_object_tagging(Bucket=bucket_name, Key=key)
+    tags = tagging.get('TagSet', [])
+    assert len(tags) == 1
+    assert tags[0]['Key'] == 'Tag Name'
+    assert tags[0]['Value'] == 'Tag Value'