add client configuration for aws-chunked encoding buffer size

Author: sbiscigl

src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h

@@ -602,6 +602,18 @@ namespace Aws
            * First available auth scheme will be used for each operation.
            */
           Aws::Vector<Aws::String> authPreferences;
+
+          /**
+           * Buffer size in bytes that will be used to content encode
+           * bodies using aws-chunked. Changing this is useful when you
+           * want to minimize memory use while uploading to S3. Size MUST
+           * be greater than 8KB otherwise S3 will reject the request.
+           *
+           * Defaults to 64KiB.
+           *
+           * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+           */
+          size_t awsChunkedBufferSize = 65536;
         };
 
         /**

src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h

@@ -107,7 +107,8 @@ namespace client
           m_interceptors({
               Aws::MakeShared<ChecksumInterceptor>("AwsSmithyClientBase", *m_clientConfig),
               Aws::MakeShared<features::ChunkingInterceptor>("AwsSmithyClientBase", 
-                  m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode)
+                  m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode,
+                  m_clientConfig->awsChunkedBufferSize)
           })
         {
 

src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h

@@ -152,7 +152,12 @@ class AwsChunkedIOStream : public Aws::IOStream {
 class ChunkingInterceptor : public smithy::interceptor::Interceptor {
 public:
     explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode)
-        : m_httpClientChunkedMode(httpClientChunkedMode) {}
+        : m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(AWS_DATA_BUFFER_SIZE) {}
+    explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode, size_t bufferSize)
+        : m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(bufferSize) {
+      // buffer size MUST be over 8KiB
+      assert(bufferSize >= 8192);
+    }
     ~ChunkingInterceptor() override = default;
 
     ModifyRequestOutcome ModifyBeforeSigning(smithy::interceptor::InterceptorContext& context) override {
@@ -199,8 +204,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor {
             return request;
         }
 
-        auto chunkedBody = Aws::MakeShared<AwsChunkedIOStream>(
-            ALLOCATION_TAG, request.get(), originalBody);
+        auto chunkedBody = Aws::MakeShared<AwsChunkedIOStream>(ALLOCATION_TAG, request.get(), originalBody, m_bufferSize);
 
         request->AddContentBody(chunkedBody);
         return request;
@@ -233,6 +237,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor {
     }
 
     Aws::Client::HttpClientChunkedMode m_httpClientChunkedMode;
+    size_t m_bufferSize;
 };
 
 } // namespace features

src/aws-cpp-sdk-core/source/client/AWSClient.cpp

@@ -141,7 +141,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration,
     m_requestCompressionConfig(configuration.requestCompressionConfig),
     m_userAgentInterceptor{Aws::MakeShared<smithy::client::UserAgentInterceptor>(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)},
     m_interceptors{Aws::MakeShared<smithy::client::ChecksumInterceptor>(AWS_CLIENT_LOG_TAG), Aws::MakeShared<smithy::client::features::ChunkingInterceptor>(AWS_CLIENT_LOG_TAG, 
-        m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor}
+        m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode,
+        configuration.awsChunkedBufferSize),
+    m_userAgentInterceptor}
 {
 }
 
@@ -168,7 +170,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration,
     m_requestCompressionConfig(configuration.requestCompressionConfig),
     m_userAgentInterceptor{Aws::MakeShared<smithy::client::UserAgentInterceptor>(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)},
     m_interceptors{Aws::MakeShared<smithy::client::ChecksumInterceptor>(AWS_CLIENT_LOG_TAG, configuration), Aws::MakeShared<smithy::client::features::ChunkingInterceptor>(AWS_CLIENT_LOG_TAG, 
-        m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor}
+        m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode,
+        configuration.awsChunkedBufferSize),
+    m_userAgentInterceptor}
 {
 }
 

src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp

@@ -107,7 +107,8 @@ void AwsSmithyClientBase::baseCopyAssign(const AwsSmithyClientBase& other,
   m_interceptors = Aws::Vector<std::shared_ptr<interceptor::Interceptor>>{
       Aws::MakeShared<ChecksumInterceptor>("AwsSmithyClientBase", *m_clientConfig),
       Aws::MakeShared<features::ChunkingInterceptor>("AwsSmithyClientBase", 
-          m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode)
+          m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode,
+          m_clientConfig->awsChunkedBufferSize)
   };
 
   baseCopyInit();

tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp

@@ -96,6 +96,7 @@ namespace
     static std::string BASE_CONTENT_ENCODING_BUCKET_NAME = "contentencoding";
     static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion";
     static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride";
+    static std::string BASE_STREAM_SIZE_BUCKET_NAME = "streamsize";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
     static const char* TEST_OBJ_KEY = "TestObjectKey";
     static const char* TEST_NEWLINE_KEY = "TestNewlineKey";
@@ -143,6 +144,7 @@ namespace
           std::ref(BASE_CONTENT_ENCODING_BUCKET_NAME),
           std::ref(BASE_CROSS_REGION_BUCKET_NAME),
           std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME),
+          std::ref(BASE_STREAM_SIZE_BUCKET_NAME)
       };
 
       for (auto& testBucketName : TEST_BUCKETS) {
@@ -2715,4 +2717,29 @@ namespace
         .WithKey(objectKey));
       AWS_EXPECT_SUCCESS(getObjectResponse);
     }
+
+    TEST_F(BucketAndObjectOperationTest, ShouldSuccessfullyUploadObjectForSmallerBufferSize) {
+      const String fullBucketName = CalculateBucketName(BASE_STREAM_SIZE_BUCKET_NAME.c_str());
+      SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
+      CreateBucketRequest createBucketRequest;
+      createBucketRequest.SetBucket(fullBucketName);
+      createBucketRequest.SetACL(BucketCannedACL::private_);
+      CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest);
+      AWS_ASSERT_SUCCESS(createBucketOutcome);
+
+      S3ClientConfiguration configuration{};
+      configuration.region = Aws::Region::US_EAST_1;
+      // Set aws-chunked buffer to 12KB
+      configuration.awsChunkedBufferSize = 1024 * 12;
+      const S3Client shortStreamClient{configuration};
+
+      auto request = PutObjectRequest().WithBucket(fullBucketName).WithKey("test-stream-size");
+
+      // Create a 24KB body
+      const Aws::String body(24L * 1024, 'a');
+      request.SetBody(Aws::MakeShared<StringStream>(ALLOCATION_TAG, body));
+
+      const auto response = shortStreamClient.PutObject(request);
+      AWS_EXPECT_SUCCESS(response);
+    }
 }