Retry RequestTimeout error by S3. (#457)

Author: waahm7

include/aws/s3/s3.h

@@ -48,6 +48,7 @@ enum aws_s3_errors {
     AWS_ERROR_S3_REQUEST_HAS_COMPLETED,
     AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS,
     AWS_ERROR_S3_RECV_FILE_NOT_FOUND,
+    AWS_ERROR_S3_REQUEST_TIMEOUT,
 
     AWS_ERROR_S3_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };

source/s3.c

@@ -51,6 +51,7 @@ static struct aws_error_info s_errors[] = {
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_REQUEST_HAS_COMPLETED, "Request has already completed, action cannot be performed."),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS, "File already exists, cannot create as new."),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RECV_FILE_NOT_FOUND, "The receive file doesn't exist, cannot create as configuration required."),
+    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_REQUEST_TIMEOUT, "RequestTimeout error received from S3."),
 };
 /* clang-format on */
 

source/s3_util.c

@@ -687,6 +687,11 @@ int aws_s3_crt_error_code_from_recoverable_server_error_code_string(struct aws_b
     if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "RequestTimeTooSkewed")) {
         return AWS_ERROR_S3_REQUEST_TIME_TOO_SKEWED;
     }
+
+    if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "RequestTimeout")) {
+        return AWS_ERROR_S3_REQUEST_TIMEOUT;
+    }
+
     return AWS_ERROR_UNKNOWN;
 }
 

tests/CMakeLists.txt

@@ -339,6 +339,7 @@ if(ENABLE_MOCK_SERVER_TESTS)
     add_net_test_case(s3express_client_sanity_test_mock_server)
     add_net_test_case(s3express_client_sanity_override_test_mock_server)
     add_net_test_case(request_time_too_skewed_mock_server)
+    add_net_test_case(request_timeout_error_mock_server)
 endif()
 
 add_net_test_case(s3express_provider_long_running_session_refresh)

tests/mock_s3_server/CreateMultipartUpload/request_timeout.json

@@ -0,0 +1,15 @@
+
+{
+    "status": 400,
+    "headers": {"x-amz-request-id": "12345"},
+    "body": [
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
+        "",
+        "<Error>",
+         "<Code>RequestTimeout</Code>",
+         "<Message>Your socket connection to the server was not read from or written to within the timeout period. Idle connections will be closed.</Message>",
+         "<RequestId>1234</RequestId>",
+         "<HostId>asdf</HostId>",
+         "</Error>"
+    ]
+  }

tests/s3_mock_server_tests.c

@@ -1078,3 +1078,50 @@ TEST_CASE(request_time_too_skewed_mock_server) {
 
     return AWS_OP_SUCCESS;
 }
+
+TEST_CASE(request_timeout_error_mock_server) {
+    (void)ctx;
+
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+    struct aws_s3_tester_client_options client_options = {
+        .part_size = MB_TO_BYTES(5),
+        .tls_usage = AWS_S3_TLS_DISABLED,
+    };
+
+    struct aws_s3_client *client = NULL;
+    ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
+
+    struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/request_timeout");
+    struct aws_s3_meta_request_test_results out_results;
+    aws_s3_meta_request_test_results_init(&out_results, allocator);
+
+    struct aws_s3_tester_meta_request_options put_options = {
+        .allocator = allocator,
+        .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
+        .client = client,
+        .checksum_algorithm = AWS_SCA_CRC32,
+        .validate_get_response_checksum = false,
+        .put_options =
+            {
+                .object_size_mb = 10,
+                .object_path_override = object_path,
+            },
+        .mock_server = true,
+        .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_FAILURE,
+    };
+
+    ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results));
+
+    ASSERT_UINT_EQUALS(AWS_ERROR_S3_REQUEST_TIMEOUT, out_results.finished_error_code);
+
+    /* The default retry will max out after 5 times. So, in total, it will be 6 requests, first one and 5 retries. */
+    size_t result_num = aws_array_list_length(&out_results.synced_data.metrics);
+    ASSERT_UINT_EQUALS(6, result_num);
+
+    aws_s3_meta_request_test_results_clean_up(&out_results);
+    aws_s3_client_release(client);
+    aws_s3_tester_clean_up(&tester);
+
+    return AWS_OP_SUCCESS;
+}