Skip to content

Commit 79c2d68

Browse files
sbisciglsbiscigl
committed
add client configuration for aws-chunked encoding buffer size
1 parent 89cca88 commit 79c2d68

File tree

7 files changed

+82
-8
lines changed

7 files changed

+82
-8
lines changed

‎src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,18 @@ namespace Aws
602602
* First available auth scheme will be used for each operation.
603603
*/
604604
Aws::Vector<Aws::String> authPreferences;
605+
606+
/**
607+
* Buffer size in bytes that will be used to content encode
608+
* bodies using aws-chunked. Changing this is useful when you
609+
* want to minimize memory use while uploading to S3. Size MUST
610+
* be greater than 8KB otherwise S3 will reject the request.
611+
*
612+
* Defaults to 64KiB.
613+
*
614+
* https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
615+
*/
616+
size_t awsChunkedBufferSize = 64UL * 1024;
605617
};
606618

607619
/**

‎src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,8 @@ namespace client
107107
m_interceptors({
108108
Aws::MakeShared<ChecksumInterceptor>("AwsSmithyClientBase", *m_clientConfig),
109109
Aws::MakeShared<features::ChunkingInterceptor>("AwsSmithyClientBase",
110-
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode)
110+
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode,
111+
m_clientConfig->awsChunkedBufferSize)
111112
})
112113
{
113114

‎src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,18 @@ class AwsChunkedIOStream : public Aws::IOStream {
152152
class ChunkingInterceptor : public smithy::interceptor::Interceptor {
153153
public:
154154
explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode)
155-
: m_httpClientChunkedMode(httpClientChunkedMode) {}
155+
: m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(AWS_DATA_BUFFER_SIZE) {}
156+
explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode, size_t bufferSize)
157+
: m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(bufferSize) {}
156158
~ChunkingInterceptor() override = default;
157159

158160
ModifyRequestOutcome ModifyBeforeSigning(smithy::interceptor::InterceptorContext& context) override {
161+
if (m_bufferSize < 8 * 1024) {
162+
return Aws::Client::AWSError<Aws::Client::CoreErrors>{Aws::Client::CoreErrors::VALIDATION,
163+
"ValidationErrorException",
164+
"aws-chunked buffer must be over 8KiB to content encode",
165+
false};
166+
}
159167
auto request = context.GetTransmitRequest();
160168

161169
if (!ShouldApplyChunking(request, context)) {
@@ -188,6 +196,12 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor {
188196
}
189197

190198
ModifyRequestOutcome ModifyBeforeTransmit(smithy::interceptor::InterceptorContext& context) override {
199+
if (m_bufferSize < 8 * 1024) {
200+
return Aws::Client::AWSError<Aws::Client::CoreErrors>{Aws::Client::CoreErrors::VALIDATION,
201+
"ValidationErrorException",
202+
"aws-chunked buffer must be over 8KiB to content encode",
203+
false};
204+
}
191205
auto request = context.GetTransmitRequest();
192206

193207
if (!ShouldApplyChunking(request, context)) {
@@ -199,8 +213,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor {
199213
return request;
200214
}
201215

202-
auto chunkedBody = Aws::MakeShared<AwsChunkedIOStream>(
203-
ALLOCATION_TAG, request.get(), originalBody);
216+
auto chunkedBody = Aws::MakeShared<AwsChunkedIOStream>(ALLOCATION_TAG, request.get(), originalBody, m_bufferSize);
204217

205218
request->AddContentBody(chunkedBody);
206219
return request;
@@ -233,6 +246,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor {
233246
}
234247

235248
Aws::Client::HttpClientChunkedMode m_httpClientChunkedMode;
249+
size_t m_bufferSize;
236250
};
237251

238252
} // namespace features

‎src/aws-cpp-sdk-core/source/client/AWSClient.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration,
141141
m_requestCompressionConfig(configuration.requestCompressionConfig),
142142
m_userAgentInterceptor{Aws::MakeShared<smithy::client::UserAgentInterceptor>(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)},
143143
m_interceptors{Aws::MakeShared<smithy::client::ChecksumInterceptor>(AWS_CLIENT_LOG_TAG), Aws::MakeShared<smithy::client::features::ChunkingInterceptor>(AWS_CLIENT_LOG_TAG,
144-
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor}
144+
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode,
145+
configuration.awsChunkedBufferSize),
146+
m_userAgentInterceptor}
145147
{
146148
}
147149

@@ -168,7 +170,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration,
168170
m_requestCompressionConfig(configuration.requestCompressionConfig),
169171
m_userAgentInterceptor{Aws::MakeShared<smithy::client::UserAgentInterceptor>(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)},
170172
m_interceptors{Aws::MakeShared<smithy::client::ChecksumInterceptor>(AWS_CLIENT_LOG_TAG, configuration), Aws::MakeShared<smithy::client::features::ChunkingInterceptor>(AWS_CLIENT_LOG_TAG,
171-
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor}
173+
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode,
174+
configuration.awsChunkedBufferSize),
175+
m_userAgentInterceptor}
172176
{
173177
}
174178

‎src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,8 @@ void AwsSmithyClientBase::baseCopyAssign(const AwsSmithyClientBase& other,
107107
m_interceptors = Aws::Vector<std::shared_ptr<interceptor::Interceptor>>{
108108
Aws::MakeShared<ChecksumInterceptor>("AwsSmithyClientBase", *m_clientConfig),
109109
Aws::MakeShared<features::ChunkingInterceptor>("AwsSmithyClientBase",
110-
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode)
110+
m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode,
111+
m_clientConfig->awsChunkedBufferSize)
111112
};
112113

113114
baseCopyInit();

‎tests/aws-cpp-sdk-core-tests/utils/stream/ChunkingInterceptorTest.cpp‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,4 +159,19 @@ TEST_F(ChunkingInterceptorTest, ShouldNotApplyChunkingForCustomHttpClient) {
159159
EXPECT_EQ(request, result.GetResult());
160160
EXPECT_FALSE(request->HasHeader(Aws::Http::AWS_TRAILER_HEADER));
161161
EXPECT_FALSE(request->HasHeader(Aws::Http::TRANSFER_ENCODING_HEADER));
162-
}
162+
}
163+
164+
TEST_F(ChunkingInterceptorTest, ShouldFailWhenBufferIsTooShort) {
165+
const Aws::Client::ClientConfiguration config;
166+
ChunkingInterceptor interceptor(config.httpClientChunkedMode, 7UL * 1024);
167+
168+
// Create interceptor context with a mock request
169+
const MockRequest mockRequest;
170+
smithy::interceptor::InterceptorContext context(mockRequest);
171+
auto respnse = interceptor.ModifyBeforeSigning(context);
172+
EXPECT_FALSE(respnse.IsSuccess());
173+
EXPECT_STREQ("aws-chunked buffer must be over 8KiB to content encode", respnse.GetError().GetMessage().c_str());
174+
respnse = interceptor.ModifyBeforeTransmit(context);
175+
EXPECT_FALSE(respnse.IsSuccess());
176+
EXPECT_STREQ("aws-chunked buffer must be over 8KiB to content encode", respnse.GetError().GetMessage().c_str());
177+
}

‎tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ namespace
9696
static std::string BASE_CONTENT_ENCODING_BUCKET_NAME = "contentencoding";
9797
static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion";
9898
static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride";
99+
static std::string BASE_STREAM_SIZE_BUCKET_NAME = "streamsize";
99100
static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
100101
static const char* TEST_OBJ_KEY = "TestObjectKey";
101102
static const char* TEST_NEWLINE_KEY = "TestNewlineKey";
@@ -143,6 +144,7 @@ namespace
143144
std::ref(BASE_CONTENT_ENCODING_BUCKET_NAME),
144145
std::ref(BASE_CROSS_REGION_BUCKET_NAME),
145146
std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME),
147+
std::ref(BASE_STREAM_SIZE_BUCKET_NAME)
146148
};
147149

148150
for (auto& testBucketName : TEST_BUCKETS) {
@@ -2715,4 +2717,29 @@ namespace
27152717
.WithKey(objectKey));
27162718
AWS_EXPECT_SUCCESS(getObjectResponse);
27172719
}
2720+
2721+
TEST_F(BucketAndObjectOperationTest, ShouldSuccessfullyUploadObjectForSmallerBufferSize) {
2722+
const String fullBucketName = CalculateBucketName(BASE_STREAM_SIZE_BUCKET_NAME.c_str());
2723+
SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
2724+
CreateBucketRequest createBucketRequest;
2725+
createBucketRequest.SetBucket(fullBucketName);
2726+
createBucketRequest.SetACL(BucketCannedACL::private_);
2727+
CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest);
2728+
AWS_ASSERT_SUCCESS(createBucketOutcome);
2729+
2730+
S3ClientConfiguration configuration{};
2731+
configuration.region = Aws::Region::US_EAST_1;
2732+
// Set aws-chunked buffer to 12KB
2733+
configuration.awsChunkedBufferSize = 1024 * 12;
2734+
const S3Client shortStreamClient{configuration};
2735+
2736+
auto request = PutObjectRequest().WithBucket(fullBucketName).WithKey("the-jack-bros");
2737+
2738+
// Create a 24KB body
2739+
const Aws::String body(24L * 1024, 'a');
2740+
request.SetBody(Aws::MakeShared<StringStream>(ALLOCATION_TAG, body));
2741+
2742+
const auto response = shortStreamClient.PutObject(request);
2743+
AWS_EXPECT_SUCCESS(response);
2744+
}
27182745
}

0 commit comments

Comments
 (0)