Fix recognition of user provided unknown checksums (#624)

DmitriyMusatkin · web-flow · commit a31a657840da · 2026-04-24T13:12:18.000-07:00
diff --git a/include/aws/s3/private/s3_checksums.h b/include/aws/s3/private/s3_checksums.h
@@ -56,6 +56,7 @@ struct aws_s3_meta_request_checksum_config_storage {
 
     enum aws_s3_checksum_location location;
     enum aws_s3_checksum_algorithm checksum_algorithm;
+    struct aws_byte_buf unknown_checksum_algo; /* if checksum algo is unknown this will have checksum name */
     bool validate_response_checksum;
     struct {
         bool crc64nvme;
diff --git a/include/aws/s3/private/s3_request_messages.h b/include/aws/s3/private/s3_request_messages.h
@@ -36,7 +36,8 @@ struct aws_http_message *aws_s3_message_util_copy_http_message_no_body_filter_he
     struct aws_http_message *message,
     const struct aws_byte_cursor *excluded_headers_arrays,
     size_t excluded_headers_size,
-    bool exclude_x_amz_meta);
+    bool exclude_x_amz_meta,
+    bool exclude_x_checksum);
 
 /* Copy headers from one message to the other and exclude specific headers.
  * exclude_x_amz_meta controls whether S3 user metadata headers (prefixed with "x-amz-meta) are excluded.*/
@@ -46,7 +47,8 @@ void aws_s3_message_util_copy_headers(
     struct aws_http_message *dest_message,
     const struct aws_byte_cursor *excluded_headers_arrays,
     size_t excluded_headers_size,
-    bool exclude_x_amz_meta);
+    bool exclude_x_amz_meta,
+    bool exclude_x_checksum);
 
 AWS_S3_API
 struct aws_input_stream *aws_s3_message_util_assign_body(
diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h
@@ -288,6 +288,7 @@ enum aws_s3_checksum_algorithm {
     AWS_SCA_XXHASH3_64,
     AWS_SCA_XXHASH3_128,
     AWS_SCA_END = AWS_SCA_XXHASH3_128,
+    AWS_SCA_UNKNOWN, /* special value for forwards compat to indicate checksum type that crt is not aware of */
 };
 
 enum aws_s3_checksum_location {
diff --git a/source/s3_auto_ranged_put.c b/source/s3_auto_ranged_put.c
@@ -957,13 +957,15 @@ static struct aws_future_http_message *s_s3_prepare_list_parts(struct aws_s3_req
             message,
             g_s3_list_parts_excluded_headers,
             g_s3_list_parts_excluded_headers_count,
+            true,
             true);
     } else {
         aws_s3_message_util_copy_headers(
             meta_request->initial_request_message,
             message,
             g_s3_list_parts_with_checksum_excluded_headers,
             g_s3_list_parts_with_checksum_excluded_headers_count,
+            true,
             true);
     }
     AWS_ASSERT(message);
diff --git a/source/s3_checksums.c b/source/s3_checksums.c
@@ -453,6 +453,18 @@ int aws_checksum_compute(
     }
 }
 
+static const struct aws_byte_cursor s_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");
+
+static void s_byte_buf_to_upper(struct aws_byte_buf *buf) {
+    AWS_PRECONDITION(buf);
+
+    for (size_t i = 0; i < buf->len; ++i) {
+        if (buf->buffer[i] >= 'a' && buf->buffer[i] <= 'z') {
+            buf->buffer[i] = buf->buffer[i] + ('A' - 'a');
+        }
+    }
+}
+
 static int s_init_and_verify_checksum_config_from_headers(
     struct aws_s3_meta_request_checksum_config_storage *checksum_config,
     const struct aws_http_message *message,
@@ -463,34 +475,63 @@ static int s_init_and_verify_checksum_config_from_headers(
     struct aws_byte_cursor header_value;
     AWS_ZERO_STRUCT(header_value);
 
-    for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) {
-        enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i];
-        const struct aws_byte_cursor algorithm_header_name =
-            aws_get_http_header_name_from_checksum_algorithm(algorithm);
-        if (aws_http_headers_get(headers, algorithm_header_name, &header_value) == AWS_OP_SUCCESS) {
-            if (header_algo == AWS_SCA_NONE) {
-                header_algo = algorithm;
-            } else {
-                /* If there are multiple checksum headers set, it's malformed request */
-                AWS_LOGF_ERROR(
-                    AWS_LS_S3_META_REQUEST,
-                    "id=%p Could not create auto-ranged-put meta request; multiple checksum headers has been set",
-                    log_id);
-                return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+    /*
+     * We want to detect if there are any checksum headers, whether known or unknown.
+     * Current approach first looks for any header that starts with x-amz-checksum and then
+     * if it finds one, it checks whether it maps to any known checksum headers.
+     * if not then we mark checksum as unknown.
+     */
+    bool has_checksum_value_header = false;
+    struct aws_byte_cursor checksum_header_name;
+    for (size_t i = 0; i < aws_http_headers_count(headers); ++i) {
+        struct aws_http_header header;
+        if (aws_http_headers_get_index(headers, i, &header)) {
+            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+        }
+
+        if (aws_byte_cursor_starts_with_ignore_case(&header.name, &s_checksum_prefix)) {
+            checksum_header_name = header.name;
+            has_checksum_value_header = true;
+            break;
+        }
+    }
+
+    if (has_checksum_value_header) {
+        for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) {
+            enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i];
+            const struct aws_byte_cursor algorithm_header_name =
+                aws_get_http_header_name_from_checksum_algorithm(algorithm);
+            if (aws_http_headers_get(headers, algorithm_header_name, &header_value) == AWS_OP_SUCCESS) {
+                if (header_algo == AWS_SCA_NONE) {
+                    header_algo = algorithm;
+                } else {
+                    /* If there are multiple checksum headers set, it's malformed request */
+                    AWS_LOGF_ERROR(
+                        AWS_LS_S3_META_REQUEST,
+                        "id=%p Could not create auto-ranged-put meta request; multiple checksum headers has been set",
+                        log_id);
+                    return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+                }
             }
         }
     }
-    if (header_algo == AWS_SCA_NONE) {
+
+    if (!has_checksum_value_header) {
         /* No checksum header found, done */
         return AWS_OP_SUCCESS;
     }
 
+    if (header_algo == AWS_SCA_NONE) {
+        header_algo = AWS_SCA_UNKNOWN;
+    }
+
     if (checksum_config->has_full_object_checksum) {
         /* If the full object checksum has been set, it's malformed request */
         AWS_LOGF_ERROR(
             AWS_LS_S3_META_REQUEST,
             "id=%p: Could not create auto-ranged-put meta request; full object checksum is set from multiple ways.",
             log_id);
+
         return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
     }
 
@@ -509,6 +550,13 @@ static int s_init_and_verify_checksum_config_from_headers(
      **/
     checksum_config->location = AWS_SCL_NONE;
 
+    if (header_algo == AWS_SCA_UNKNOWN) {
+        aws_byte_cursor_advance(&checksum_header_name, s_checksum_prefix.len);
+        aws_byte_buf_init_copy_from_cursor(
+            &checksum_config->unknown_checksum_algo, checksum_config->allocator, checksum_header_name);
+        s_byte_buf_to_upper(&checksum_config->unknown_checksum_algo);
+    }
+
     /* Set full object checksum from the header value. */
     aws_byte_buf_init_copy_from_cursor(
         &checksum_config->full_object_checksum, checksum_config->allocator, header_value);
@@ -526,6 +574,8 @@ int aws_s3_meta_request_checksum_config_storage_init(
     /* Zero out the struct and set the allocator regardless. */
     internal_config->allocator = allocator;
 
+    /* Potential improvement here is that right now unless you configure checksum (which sdks seem to do),
+      the checksum value in the message is not detected and does not get propagated to create/complete */
     if (!config) {
         return AWS_OP_SUCCESS;
     }
@@ -631,4 +681,5 @@ void aws_s3_meta_request_checksum_config_storage_cleanup(
     if (internal_config->has_full_object_checksum) {
         aws_byte_buf_clean_up(&internal_config->full_object_checksum);
     }
+    aws_byte_buf_clean_up(&internal_config->unknown_checksum_algo);
 }
diff --git a/source/s3_request_messages.c b/source/s3_request_messages.c
@@ -22,13 +22,9 @@ const struct aws_byte_cursor g_s3_create_multipart_upload_excluded_headers[] = {
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-MD5"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc64nvme"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
+    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-sdk-checksum-algorithm"),
 };
 
 const size_t g_s3_create_multipart_upload_excluded_headers_count =
@@ -58,11 +54,6 @@ const struct aws_byte_cursor g_s3_upload_part_excluded_headers[] = {
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-mode"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-retain-until-date"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc64nvme"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"),
-    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
 };
@@ -248,6 +239,7 @@ const struct aws_byte_cursor g_s3_create_session_allowed_headers[] = {
 const size_t g_s3_create_session_allowed_headers_count = AWS_ARRAY_SIZE(g_s3_create_session_allowed_headers);
 
 static const struct aws_byte_cursor s_x_amz_meta_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-meta-");
+static const struct aws_byte_cursor s_x_amz_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");
 
 static const struct aws_byte_cursor s_checksum_type_header =
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-type");
@@ -299,7 +291,8 @@ struct aws_http_message *aws_s3_create_multipart_upload_message_new(
         base_message,
         g_s3_create_multipart_upload_excluded_headers,
         AWS_ARRAY_SIZE(g_s3_create_multipart_upload_excluded_headers),
-        false /*exclude_x_amz_meta*/);
+        false /*exclude_x_amz_meta*/,
+        true /*exclude_x_checksum_meta*/);
 
     if (message == NULL) {
         return NULL;
@@ -322,7 +315,12 @@ struct aws_http_message *aws_s3_create_multipart_upload_message_new(
     }
 
     if (checksum_config && (checksum_config->location != AWS_SCL_NONE || checksum_config->has_full_object_checksum)) {
-        if (checksum_config->checksum_algorithm) {
+        if (checksum_config->checksum_algorithm == AWS_SCA_UNKNOWN) {
+            struct aws_byte_cursor checksum_name = aws_byte_cursor_from_buf(&checksum_config->unknown_checksum_algo);
+            if (aws_http_headers_set(headers, g_checksum_algorithm_header_name, checksum_name)) {
+                goto error_clean_up;
+            }
+        } else if (checksum_config->checksum_algorithm) {
             if (aws_http_headers_set(
                     headers,
                     g_checksum_algorithm_header_name,
@@ -374,7 +372,8 @@ struct aws_http_message *aws_s3_upload_part_message_new_streaming(
         base_message,
         g_s3_upload_part_excluded_headers,
         AWS_ARRAY_SIZE(g_s3_upload_part_excluded_headers),
-        true /*exclude_x_amz_meta*/);
+        true /*exclude_x_amz_meta*/,
+        true /*exclude_x_checksum_meta*/);
 
     if (message == NULL) {
         return NULL;
@@ -429,7 +428,8 @@ struct aws_http_message *aws_s3_upload_part_message_new(
         base_message,
         g_s3_upload_part_excluded_headers,
         AWS_ARRAY_SIZE(g_s3_upload_part_excluded_headers),
-        true /*exclude_x_amz_meta*/);
+        true /*exclude_x_amz_meta*/,
+        true /*exclude_x_checksum_meta*/);
 
     if (message == NULL) {
         return NULL;
@@ -477,7 +477,8 @@ struct aws_http_message *aws_s3_upload_part_copy_message_new(
         base_message,
         g_s3_upload_part_excluded_headers,
         AWS_ARRAY_SIZE(g_s3_upload_part_excluded_headers),
-        true /*exclude_x_amz_meta*/);
+        true /*exclude_x_amz_meta*/,
+        true /*exclude_x_checksum_meta*/);
 
     if (message == NULL) {
         goto error_clean_up;
@@ -706,15 +707,17 @@ struct aws_http_message *aws_s3_complete_multipart_message_new(
             base_message,
             g_s3_complete_multipart_upload_with_checksum_excluded_headers,
             AWS_ARRAY_SIZE(g_s3_complete_multipart_upload_with_checksum_excluded_headers),
-            true /*exclude_x_amz_meta*/);
+            true /*exclude_x_amz_meta*/,
+            false /*exclude_x_checksum_meta*/);
 
     } else {
         message = aws_s3_message_util_copy_http_message_no_body_filter_headers(
             allocator,
             base_message,
             g_s3_complete_multipart_upload_excluded_headers,
             AWS_ARRAY_SIZE(g_s3_complete_multipart_upload_excluded_headers),
-            true /*exclude_x_amz_meta*/);
+            true /*exclude_x_amz_meta*/,
+            false /*exclude_x_checksum_meta*/);
     }
 
     struct aws_http_headers *headers = NULL;
@@ -863,7 +866,8 @@ struct aws_http_message *aws_s3_abort_multipart_upload_message_new(
         base_message,
         g_s3_abort_multipart_upload_excluded_headers,
         AWS_ARRAY_SIZE(g_s3_abort_multipart_upload_excluded_headers),
-        true /*exclude_x_amz_meta*/);
+        true /*exclude_x_amz_meta*/,
+        true /*exclude_x_checksum_meta*/);
 
     if (aws_s3_message_util_set_multipart_request_path(allocator, upload_id, 0, false, message)) {
         goto error_clean_up;
@@ -1141,15 +1145,16 @@ struct aws_http_message *aws_s3_message_util_copy_http_message_no_body_all_heade
     struct aws_allocator *allocator,
     struct aws_http_message *base_message) {
 
-    return aws_s3_message_util_copy_http_message_no_body_filter_headers(allocator, base_message, NULL, 0, false);
+    return aws_s3_message_util_copy_http_message_no_body_filter_headers(allocator, base_message, NULL, 0, false, false);
 }
 
 struct aws_http_message *aws_s3_message_util_copy_http_message_no_body_filter_headers(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message,
     const struct aws_byte_cursor *excluded_header_array,
     size_t excluded_header_array_size,
-    bool exclude_x_amz_meta) {
+    bool exclude_x_amz_meta,
+    bool exclude_x_amz_checksum) {
 
     AWS_PRECONDITION(allocator);
     AWS_PRECONDITION(base_message);
@@ -1178,7 +1183,12 @@ struct aws_http_message *aws_s3_message_util_copy_http_message_no_body_filter_he
     }
 
     aws_s3_message_util_copy_headers(
-        base_message, message, excluded_header_array, excluded_header_array_size, exclude_x_amz_meta);
+        base_message,
+        message,
+        excluded_header_array,
+        excluded_header_array_size,
+        exclude_x_amz_meta,
+        exclude_x_amz_checksum);
 
     return message;
 
@@ -1192,7 +1202,8 @@ void aws_s3_message_util_copy_headers(
     struct aws_http_message *dest_message,
     const struct aws_byte_cursor *excluded_header_array,
     size_t excluded_header_array_size,
-    bool exclude_x_amz_meta) {
+    bool exclude_x_amz_meta,
+    bool exclude_x_amz_checksum) {
 
     size_t num_headers = aws_http_message_get_header_count(source_message);
 
@@ -1222,6 +1233,12 @@ void aws_s3_message_util_copy_headers(
             }
         }
 
+        if (exclude_x_amz_checksum) {
+            if (aws_byte_cursor_starts_with_ignore_case(&header.name, &s_x_amz_checksum_prefix)) {
+                continue;
+            }
+        }
+
         error |= aws_http_message_add_header(dest_message, header);
         (void)error;
         AWS_ASSERT(!error);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -25,6 +25,8 @@ add_test_case(test_s3_upload_part_message_fail)
 add_test_case(test_s3_complete_multipart_message_new)
 add_test_case(test_s3_abort_multipart_upload_message_new)
 
+add_net_test_case(test_s3_put_object_custom_md5)
+
 add_net_test_case(test_s3_client_create_destroy)
 add_net_test_case(test_s3_client_create_error)
 add_net_test_case(test_s3_client_create_error_with_invalid_memory_limit_config)
diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c
diff --git a/tests/s3_request_messages_tests.c b/tests/s3_request_messages_tests.c
