Fix the deadlock for pause/cancel (#596)

Author: TingDaoK

include/aws/s3/private/s3_meta_request_impl.h

@@ -195,6 +195,9 @@ struct aws_s3_meta_request {
     /* If the buffer pool optimized for the specific size or not. */
     bool buffer_pool_optimized;
 
+    /* Track the number of requests being prepared for this meta request. */
+    struct aws_atomic_var num_request_being_prepared;
+
     struct {
         struct aws_mutex lock;
 
@@ -278,9 +281,6 @@ struct aws_s3_meta_request {
         /* True if this meta request is currently in the client's list. */
         bool scheduled;
 
-        /* Track the number of requests being prepared for this meta request. */
-        size_t num_request_being_prepared;
-
     } client_process_work_threaded_data;
 
     /* Anything in this structure should only ever be accessed by the meta-request from its io_event_loop thread. */

source/s3_client.c

@@ -1906,8 +1906,9 @@ static bool s_s3_client_should_update_meta_request(
         return false;
     }
 
-    if (meta_request->client_process_work_threaded_data.num_request_being_prepared >=
-        aws_s3_client_get_max_active_connections(client, meta_request)) {
+    /* This is not 100% thread safe, but prepare a bit more for the meta request level won't actually hurt. */
+    size_t specific_request_being_prepared = aws_atomic_load_int(&meta_request->num_request_being_prepared);
+    if (specific_request_being_prepared >= aws_s3_client_get_max_active_connections(client, meta_request)) {
         /* Don't prepare more than it's allowed for the meta request */
         return false;
     }
@@ -2125,7 +2126,7 @@ void aws_s3_client_update_meta_requests_threaded(struct aws_s3_client *client) {
                     request->tracked_by_client = true;
 
                     ++client->threaded_data.num_requests_being_prepared;
-                    ++meta_request->client_process_work_threaded_data.num_request_being_prepared;
+                    aws_atomic_fetch_add(&meta_request->num_request_being_prepared, 1);
 
                     num_requests_in_flight =
                         (uint32_t)aws_atomic_fetch_add(&client->stats.num_requests_in_flight, 1) + 1;
@@ -2192,6 +2193,7 @@ static void s_s3_client_prepare_callback_queue_request(
         request = aws_s3_request_release(request);
     }
 
+    aws_atomic_fetch_sub(&meta_request->num_request_being_prepared, 1);
     /* BEGIN CRITICAL SECTION */
     {
         aws_s3_client_lock_synced_data(client);
@@ -2223,8 +2225,6 @@ void aws_s3_client_update_connections_threaded(struct aws_s3_client *client) {
         struct aws_s3_request *request = aws_s3_client_dequeue_request_threaded(client);
         struct aws_s3_meta_request *meta_request = request->meta_request;
         const uint32_t max_active_connections = aws_s3_client_get_max_active_connections(client, meta_request);
-        /* As the request removed from the queue. Decrement the preparing track */
-        --meta_request->client_process_work_threaded_data.num_request_being_prepared;
         if (request->is_noop) {
             /* If request is no-op, finishes and cleans up the request */
             s_s3_client_meta_request_finished_request(client, meta_request, request, AWS_ERROR_SUCCESS);
@@ -2250,8 +2250,6 @@ void aws_s3_client_update_connections_threaded(struct aws_s3_client *client) {
         } else {
             /* Push the request into the left-over list to be used in a future call of this function. */
             aws_linked_list_push_back(&left_over_requests, &request->node);
-            /* Increment the count as we put it back to the queue. */
-            ++meta_request->client_process_work_threaded_data.num_request_being_prepared;
         }
         client_max_active_connections = aws_s3_client_get_max_active_connections(client, NULL);
         num_requests_network_io = s_s3_client_get_num_requests_network_io(client, AWS_S3_META_REQUEST_TYPE_MAX);

source/s3_meta_request.c

@@ -200,6 +200,7 @@ int aws_s3_meta_request_init_base(
     /* Set up reference count. */
     aws_ref_count_init(&meta_request->ref_count, meta_request, s_s3_meta_request_destroy);
     aws_atomic_init_int(&meta_request->num_requests_network, 0);
+    aws_atomic_init_int(&meta_request->num_request_being_prepared, 0);
     aws_linked_list_init(&meta_request->synced_data.cancellable_http_streams_list);
     aws_linked_list_init(&meta_request->synced_data.pending_buffer_futures);