Skip to content

Commit 40567cb

Browse files
DmitriyMusatkinDmitriyMusatkin
committed
fix unknown checksum handling
1 parent 9a6fc98 commit 40567cb

10 files changed

Lines changed: 69 additions & 8 deletions

‎include/aws/s3/private/s3_checksum_context.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ struct aws_s3_upload_request_checksum_context {
4040

4141
/* Validation */
4242
size_t encoded_checksum_size;
43+
44+
bool has_review_callback;
4345
};
4446

4547
/**

‎include/aws/s3/private/s3_util.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,14 @@ int aws_s3_calculate_request_optimal_range_size(
385385
AWS_S3_API
386386
int aws_s3_extract_parts_from_etag(struct aws_byte_cursor etag_header_value, uint32_t *out_num_parts);
387387

388+
/**
389+
* Helper to figure out if given header name is one of the checksum value headers.
390+
* ex. returns true for x-amz-checksum-crc32 or x-amz-checksum-sha256, but false for x-amz-checksum-type.
391+
* Note: relies on hardcoded list of non-checksum headers, which needs to be updated if s3 expands list of those.
392+
*/
393+
AWS_S3_API
394+
bool aws_s3_is_checksum_value_header_name(struct aws_byte_cursor header_name);
395+
388396
AWS_EXTERN_C_END
389397

390398
#endif /* AWS_S3_UTIL_H */

‎source/s3_checksum_context.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ static struct aws_s3_upload_request_checksum_context *s_s3_upload_request_checks
4848
context->algorithm = checksum_config->checksum_algorithm;
4949
context->location = checksum_config->location;
5050
context->encoded_checksum_size = aws_get_digest_size_from_checksum_algorithm(context->algorithm);
51+
context->has_review_callback = checksum_config->full_object_checksum_callback;
5152

5253
/* Convert to base64 encoded size */
5354
size_t encoded_size = 0;
@@ -115,7 +116,8 @@ struct aws_s3_upload_request_checksum_context *aws_s3_upload_request_checksum_co
115116
}
116117

117118
bool aws_s3_upload_request_checksum_context_should_calculate(struct aws_s3_upload_request_checksum_context *context) {
118-
if (!context || context->algorithm == AWS_SCA_NONE) {
119+
if (!context || context->algorithm == AWS_SCA_NONE || context->algorithm == AWS_SCA_UNKNOWN ||
120+
!context->has_review_callback) {
119121
return false;
120122
}
121123

‎source/s3_checksums.c‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -453,8 +453,6 @@ int aws_checksum_compute(
453453
}
454454
}
455455

456-
static const struct aws_byte_cursor s_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");
457-
458456
static void s_byte_buf_to_upper(struct aws_byte_buf *buf) {
459457
AWS_PRECONDITION(buf);
460458

@@ -489,7 +487,7 @@ static int s_init_and_verify_checksum_config_from_headers(
489487
return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
490488
}
491489

492-
if (aws_byte_cursor_starts_with_ignore_case(&header.name, &s_checksum_prefix)) {
490+
if (aws_s3_is_checksum_value_header_name(header.name)) {
493491
checksum_header_name = header.name;
494492
has_checksum_value_header = true;
495493
break;
@@ -551,7 +549,7 @@ static int s_init_and_verify_checksum_config_from_headers(
551549
checksum_config->location = AWS_SCL_NONE;
552550

553551
if (header_algo == AWS_SCA_UNKNOWN) {
554-
aws_byte_cursor_advance(&checksum_header_name, s_checksum_prefix.len);
552+
aws_byte_cursor_advance(&checksum_header_name, sizeof("x-amz-checksum-") - 1);
555553
aws_byte_buf_init_copy_from_cursor(
556554
&checksum_config->unknown_checksum_algo, checksum_config->allocator, checksum_header_name);
557555
s_byte_buf_to_upper(&checksum_config->unknown_checksum_algo);

‎source/s3_default_meta_request.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ static void s_s3_default_prepare_request_finish(
347347
struct aws_http_message *message = aws_s3_message_util_copy_http_message_no_body_all_headers(
348348
meta_request->allocator, meta_request->initial_request_message);
349349

350-
bool flexible_checksum = meta_request->checksum_config.location != AWS_SCL_NONE;
350+
bool flexible_checksum = meta_request->checksum_config.checksum_algorithm != AWS_SCA_NONE;
351351
if (!flexible_checksum && meta_request->should_compute_content_md5) {
352352
/* If flexible checksum used, client MUST skip Content-MD5 header computation */
353353
aws_s3_message_util_add_content_md5_header(meta_request->allocator, &request->request_body, message);

‎source/s3_request_messages.c‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,6 @@ const struct aws_byte_cursor g_s3_create_session_allowed_headers[] = {
239239
const size_t g_s3_create_session_allowed_headers_count = AWS_ARRAY_SIZE(g_s3_create_session_allowed_headers);
240240

241241
static const struct aws_byte_cursor s_x_amz_meta_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-meta-");
242-
static const struct aws_byte_cursor s_x_amz_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");
243242

244243
static const struct aws_byte_cursor s_checksum_type_header =
245244
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-type");
@@ -1234,7 +1233,7 @@ void aws_s3_message_util_copy_headers(
12341233
}
12351234

12361235
if (exclude_x_amz_checksum) {
1237-
if (aws_byte_cursor_starts_with_ignore_case(&header.name, &s_x_amz_checksum_prefix)) {
1236+
if (aws_s3_is_checksum_value_header_name(header.name)) {
12381237
continue;
12391238
}
12401239
}

‎source/s3_util.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,3 +1019,12 @@ int aws_s3_extract_parts_from_etag(struct aws_byte_cursor etag_header_value, uin
10191019

10201020
return AWS_OP_SUCCESS;
10211021
}
1022+
1023+
static const struct aws_byte_cursor s_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");
1024+
1025+
bool aws_s3_is_checksum_value_header_name(struct aws_byte_cursor header_name) {
1026+
return aws_byte_cursor_starts_with_ignore_case(&header_name, &s_checksum_prefix) &&
1027+
!aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-type") &&
1028+
!aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-mode") &&
1029+
!aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-algorithm");
1030+
}

‎tests/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ add_net_test_case(test_s3_range_requests_less_than_a_part)
243243
add_net_test_case(test_s3_not_satisfiable_range)
244244
add_net_test_case(test_s3_invalid_start_range_greator_than_end_range)
245245
add_net_test_case(test_s3_empty_file_edge_case)
246+
add_net_test_case(test_s3_default_put_object_checksum)
246247

247248
add_net_test_case(test_s3_bad_endpoint)
248249
add_net_test_case(test_s3_different_endpoints)
@@ -261,6 +262,7 @@ add_test_case(test_s3_calculate_client_optimal_range_size)
261262
add_test_case(test_s3_calculate_request_optimal_range_size)
262263
add_test_case(test_s3_extract_parts_from_etag)
263264
add_test_case(test_add_user_agent_header)
265+
add_test_case(test_s3_checksum_header)
264266

265267
add_test_case(test_get_existing_platform_info)
266268
add_test_case(test_get_nonexistent_platform_info)

‎tests/s3_checksum_context_test.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ static int s_test_upload_request_checksum_context_get_checksum_cursor(struct aws
4949

5050
return AWS_OP_SUCCESS;
5151
}
52+
53+
static struct aws_string *s_test_fn(struct aws_s3_meta_request *, void *) {
54+
return NULL;
55+
}
56+
5257
AWS_TEST_CASE(
5358
test_upload_request_checksum_context_get_checksum_cursor,
5459
s_test_upload_request_checksum_context_get_checksum_cursor)
@@ -80,6 +85,25 @@ static int s_test_upload_request_checksum_context_error_cases(struct aws_allocat
8085
ASSERT_NULL(aws_s3_upload_request_checksum_context_acquire(NULL));
8186
ASSERT_NULL(aws_s3_upload_request_checksum_context_release(NULL));
8287

88+
/* unknown algo */
89+
struct aws_s3_meta_request_checksum_config_storage config2 = {
90+
.allocator = allocator,
91+
.checksum_algorithm = AWS_SCA_UNKNOWN,
92+
.location = AWS_SCL_NONE,
93+
.has_full_object_checksum = false,
94+
};
95+
ASSERT_FALSE(aws_s3_upload_request_checksum_context_should_calculate(&config2));
96+
97+
/* unknown algo */
98+
struct aws_s3_meta_request_checksum_config_storage config2 = {
99+
.allocator = allocator,
100+
.checksum_algorithm = AWS_SCA_CRC32,
101+
.location = AWS_SCL_NONE,
102+
.has_full_object_checksum = false,
103+
.full_object_checksum_callback = s_test_fn,
104+
};
105+
ASSERT_FALSE(aws_s3_upload_request_checksum_context_should_calculate(&config2));
106+
83107
return AWS_OP_SUCCESS;
84108
}
85109
AWS_TEST_CASE(test_upload_request_checksum_context_error_cases, s_test_upload_request_checksum_context_error_cases)

‎tests/s3_util_tests.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,3 +1046,20 @@ static int s_test_s3_extract_parts_from_etag(struct aws_allocator *allocator, vo
10461046
aws_s3_library_clean_up();
10471047
return 0;
10481048
}
1049+
1050+
AWS_TEST_CASE(test_s3_checksum_header, s_test_s3_checksum_header)
1051+
static int s_test_s3_checksum_header(struct aws_allocator *allocator, void *ctx) {
1052+
(void)ctx;
1053+
aws_s3_library_init(allocator);
1054+
1055+
ASSERT_TRUE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-crc32")));
1056+
ASSERT_TRUE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-foo")));
1057+
ASSERT_TRUE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-md5")));
1058+
1059+
ASSERT_FALSE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-type")));
1060+
ASSERT_FALSE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-mode")));
1061+
ASSERT_FALSE(aws_s3_is_checksum_value_header_name(aws_byte_cursor_from_c_str("x-amz-checksum-algorithm ")));
1062+
1063+
aws_s3_library_clean_up();
1064+
return 0;
1065+
}

0 commit comments

Comments
 (0)