register on buffer allocated async instead

Author: DmitriyMusatkin

include/aws/s3/private/s3_request.h

@@ -139,8 +139,14 @@ struct aws_s3_request {
      * retried.*/
     struct aws_byte_buf request_body;
 
+    /**
+     * Ticket to acquire the buffer.
+     */
     struct aws_s3_buffer_ticket *ticket;
 
+    /* Result of memory acquisition */
+    int buffer_acquire_result;
+
     /* Beginning range of this part. */
     /* TODO currently only used by auto_range_get, could be hooked up to auto_range_put as well. */
     uint64_t part_range_start;

source/s3_client.c

@@ -1909,7 +1909,10 @@ void s_acquire_mem_and_prepare_request(
         }
         /* END CRITICAL SECTION */
 
-        aws_future_s3_buffer_ticket_register_callback(payload->buffer_future, s_on_pool_buffer_reserved, payload);
+        /* Note: run callback async on event loop. this is done to avoid any race conditions between cancelling
+           which requires a lock on meta request and buffer acquire callback which also requires a lock. */
+        aws_future_s3_buffer_ticket_register_event_loop_callback(
+            payload->buffer_future, meta_request->io_event_loop, s_on_pool_buffer_reserved, payload);
         return;
     }
 

source/s3_meta_request.c

@@ -1832,13 +1832,12 @@ void aws_s3_meta_request_cancel_pending_buffer_futures_synced(
     struct aws_s3_meta_request *meta_request,
     int error_code) {
     ASSERT_SYNCED_DATA_LOCK_HELD(meta_request);
-    while (!aws_linked_list_empty(&meta_request->synced_data.pending_buffer_futures)) {
 
-        struct aws_linked_list_node *request_node =
-            aws_linked_list_pop_front(&meta_request->synced_data.pending_buffer_futures);
+    for (struct aws_linked_list_node *node = aws_linked_list_begin(&meta_request->synced_data.pending_buffer_futures);
+         node != aws_linked_list_end(&meta_request->synced_data.pending_buffer_futures);
+         node = aws_linked_list_next(node)) {
 
-        struct aws_s3_request *request =
-            AWS_CONTAINER_OF(request_node, struct aws_s3_request, pending_buffer_future_list_node);
+        struct aws_s3_request *request = AWS_CONTAINER_OF(node, struct aws_s3_request, pending_buffer_future_list_node);
 
         aws_future_s3_buffer_ticket_set_error(request->synced_data.buffer_future, error_code);
     }