support if-none-match for upload (#462)

Author: TingDaoK

.github/workflows/ci.yml

@@ -5,6 +5,11 @@ on:
     branches-ignore:
       - 'main'
 
+# cancel in-progress builds after a new commit
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   BUILDER_VERSION: v0.9.64
   BUILDER_SOURCE: releases

source/s3_request_messages.c

@@ -20,6 +20,7 @@ const struct aws_byte_cursor g_s3_create_multipart_upload_excluded_headers[] = {
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-MD5"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
+    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
 };
 
 const size_t g_s3_create_multipart_upload_excluded_headers_count =
@@ -49,6 +50,7 @@ const struct aws_byte_cursor g_s3_upload_part_excluded_headers[] = {
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-mode"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-retain-until-date"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"),
+    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
 };
 
 const size_t g_s3_upload_part_excluded_headers_count = AWS_ARRAY_SIZE(g_s3_upload_part_excluded_headers);
@@ -211,6 +213,7 @@ const struct aws_byte_cursor g_s3_abort_multipart_upload_excluded_headers[] = {
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
     AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
+    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
 };
 
 static const struct aws_byte_cursor s_x_amz_meta_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-meta-");

tests/CMakeLists.txt

@@ -140,6 +140,8 @@ add_net_test_case(test_s3_put_object_async_no_content_length_1part)
 add_net_test_case(test_s3_put_object_async_no_content_length_empty_part2)
 add_net_test_case(test_s3_put_object_async_no_content_length_2parts)
 add_net_test_case(test_s3_put_object_async_fail_reading)
+add_net_test_case(test_s3_put_object_if_none_match)
+add_net_test_case(test_s3_put_object_mpu_if_none_match)
 add_net_test_case(test_s3_many_async_uploads_without_data)
 add_net_test_case(test_s3_download_empty_file_with_checksum)
 add_net_test_case(test_s3_download_empty_file_with_checksum_header)

tests/s3_data_plane_tests.c

@@ -3010,6 +3010,93 @@ static int s_test_s3_put_object_async_fail_reading(struct aws_allocator *allocat
     return 0;
 }
 
+static int s_test_validate_if_none_match_failure_response(struct aws_s3_meta_request_test_results *test_results) {
+
+    /**
+     * response body should be like:
+     * <Error>
+     * <Code>PreconditionFailed</Code>
+     * <Message>At least one of the pre-conditions you specified did not hold</Message>
+     * <Condition>If-None-Match</Condition>
+     * <RequestId></RequestId>
+     * <HostId></HostId>
+     * </Error>
+     */
+
+    struct aws_byte_cursor xml_doc = aws_byte_cursor_from_buf(&test_results->error_response_body);
+    struct aws_byte_cursor error_code_string = {0};
+    struct aws_byte_cursor condition_string = {0};
+
+    const char *error_code_path[] = {"Error", "Code", NULL};
+    ASSERT_SUCCESS(aws_xml_get_body_at_path(test_results->allocator, xml_doc, error_code_path, &error_code_string));
+    ASSERT_TRUE(aws_byte_cursor_eq_c_str(&error_code_string, "PreconditionFailed"));
+
+    const char *condition_path[] = {"Error", "Condition", NULL};
+    ASSERT_SUCCESS(aws_xml_get_body_at_path(test_results->allocator, xml_doc, condition_path, &condition_string));
+    ASSERT_TRUE(aws_byte_cursor_eq_c_str(&condition_string, "If-None-Match"));
+
+    return AWS_OP_SUCCESS;
+}
+
+AWS_TEST_CASE(test_s3_put_object_if_none_match, s_test_s3_put_object_if_none_match)
+static int s_test_s3_put_object_if_none_match(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    struct aws_s3_meta_request_test_results test_results;
+    aws_s3_meta_request_test_results_init(&test_results, allocator);
+    struct aws_byte_cursor if_none_match_all = aws_byte_cursor_from_c_str("*");
+    struct aws_s3_tester_meta_request_options put_options = {
+        .allocator = allocator,
+        .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
+        .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_FAILURE,
+        .put_options =
+            {
+                /* Use pre_exist object so that the request should fail with the expected failure message. */
+                .object_path_override = g_pre_existing_object_1MB,
+                .object_size_mb = 1,
+                .if_none_match_header = if_none_match_all,
+            },
+    };
+    ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(NULL, &put_options, &test_results));
+
+    ASSERT_UINT_EQUALS(AWS_HTTP_STATUS_CODE_412_PRECONDITION_FAILED, test_results.finished_response_status);
+    ASSERT_SUCCESS(s_test_validate_if_none_match_failure_response(&test_results));
+
+    aws_s3_meta_request_test_results_clean_up(&test_results);
+    return 0;
+}
+
+AWS_TEST_CASE(test_s3_put_object_mpu_if_none_match, s_test_s3_put_object_mpu_if_none_match)
+static int s_test_s3_put_object_mpu_if_none_match(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    struct aws_s3_meta_request_test_results test_results;
+    aws_s3_meta_request_test_results_init(&test_results, allocator);
+    struct aws_byte_cursor if_none_match_all = aws_byte_cursor_from_c_str("*");
+    struct aws_s3_tester_meta_request_options put_options = {
+        .allocator = allocator,
+        .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
+        .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_FAILURE,
+        .put_options =
+            {
+                /* Use pre_exist object so that the request should fail with the expected failure message. */
+                .object_path_override = g_pre_existing_object_10MB,
+                .object_size_mb = 10,
+                .if_none_match_header = if_none_match_all,
+            },
+    };
+    ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(NULL, &put_options, &test_results));
+
+    /** Complete MPU can fail with 200 error */
+    ASSERT_TRUE(
+        AWS_HTTP_STATUS_CODE_412_PRECONDITION_FAILED == test_results.finished_response_status ||
+        AWS_HTTP_STATUS_CODE_200_OK == test_results.finished_response_status);
+    ASSERT_SUCCESS(s_test_validate_if_none_match_failure_response(&test_results));
+
+    aws_s3_meta_request_test_results_clean_up(&test_results);
+    return 0;
+}
+
 AWS_TEST_CASE(test_s3_put_object_sse_kms, s_test_s3_put_object_sse_kms)
 static int s_test_s3_put_object_sse_kms(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;

tests/s3_tester.c

@@ -1696,6 +1696,13 @@ int aws_s3_tester_send_meta_request_with_options(
                 aws_http_message_add_header(message, content_encoding_header);
             }
 
+            if (options->put_options.if_none_match_header.ptr != NULL) {
+                struct aws_http_header if_none_match_header = {
+                    .name = aws_byte_cursor_from_c_str("if-none-match"),
+                    .value = options->put_options.if_none_match_header,
+                };
+                aws_http_message_add_header(message, if_none_match_header);
+            }
             meta_request_options.message = message;
             aws_byte_buf_clean_up(&object_path_buffer);
         }

tests/s3_tester.h

@@ -219,6 +219,7 @@ struct aws_s3_tester_meta_request_options {
         size_t content_length;
         bool skip_content_length;
         struct aws_byte_cursor content_encoding;
+        struct aws_byte_cursor if_none_match_header;
     } put_options;
 
     enum aws_s3_tester_sse_type sse_type;