Merge branch 'main' into update-rule-set

Author: TingDaoK

include/aws/s3/private/s3_meta_request_impl.h

@@ -66,6 +66,7 @@ struct aws_s3_meta_request_event {
         /* data for AWS_S3_META_REQUEST_EVENT_RESPONSE_BODY */
         struct {
             struct aws_s3_request *completed_request;
+            size_t bytes_delivered;
         } response_body;
 
         /* data for AWS_S3_META_REQUEST_EVENT_PROGRESS */
@@ -224,12 +225,15 @@ struct aws_s3_meta_request {
 
         /* Task for delivering events on the meta-request's io_event_loop thread.
          * We do this to ensure a meta-request's callbacks are fired sequentially and non-overlapping.
-         * If `event_delivery_array` has items in it, then this task is scheduled.
+         * If `event_delivery_task_scheduled` is true, then this task is scheduled.
          * If `event_delivery_active` is true, then this task is actively running.
          * Delivery is not 100% complete until `event_delivery_array` is empty AND `event_delivery_active` is false
          * (use aws_s3_meta_request_are_events_out_for_delivery_synced()  to check) */
         struct aws_task event_delivery_task;
 
+        /* Whether or not event delivery is currently scheduled. */
+        uint32_t event_delivery_task_scheduled : 1;
+
         /* Array of `struct aws_s3_meta_request_event` to deliver when the `event_delivery_task` runs. */
         struct aws_array_list event_delivery_array;
 
@@ -292,6 +296,10 @@ struct aws_s3_meta_request {
 
         /* The range start for the next response body delivery */
         uint64_t next_deliver_range_start;
+
+        /* Total number of bytes that have been attempted to be delivered. (Will equal the sum of succeeded and
+         * failed.)*/
+        uint64_t num_bytes_delivery_completed;
     } io_threaded_data;
 
     const bool should_compute_content_md5;

include/aws/s3/s3_client.h

@@ -606,7 +606,8 @@ struct aws_s3_client_config {
      *
      * WARNING: This feature is experimental.
      * Currently, backpressure is only applied to GetObject requests which are split into multiple parts,
-     * and you may still receive some data after the window reaches 0.
+     * - If you set body_callback, no more data will be delivered once the window reaches 0.
+     * - If you set body_callback_ex, you may still receive some data after the window reaches 0. TODO: fix it.
      */
     bool enable_read_backpressure;
 
@@ -1220,7 +1221,8 @@ struct aws_future_void *aws_s3_meta_request_write(
  *
  * WARNING: This feature is experimental.
  * Currently, backpressure is only applied to GetObject requests which are split into multiple parts,
- * and you may still receive some data after the window reaches 0.
+ * - If you set body_callback, no more data will be delivered once the window reaches 0.
+ * - If you set body_callback_ex, you may still receive some data after the window reaches 0. TODO: fix it.
  */
 AWS_S3_API
 void aws_s3_meta_request_increment_read_window(struct aws_s3_meta_request *meta_request, uint64_t bytes);

source/s3_auto_ranged_put.c

@@ -464,8 +464,11 @@ static void s_s3_auto_ranged_put_send_request_finish(
     struct aws_http_stream *stream,
     int error_code) {
     struct aws_s3_request *request = connection->request;
-    if (request->request_tag == AWS_S3_AUTO_RANGED_PUT_REQUEST_TAG_PART) {
+    if (request->request_tag == AWS_S3_AUTO_RANGED_PUT_REQUEST_TAG_PART && !request->meta_request->is_express) {
         /* TODO: the single part upload may also be improved from a timeout as multipart. */
+        /* Note: For S3express, server responses random ETag. Killing the connection in the middle may cause server
+         * receive two same part and result in confusing during complete MPU with etag doesn't match.
+         * Disable the first byte timeout for s3 express.  */
         aws_s3_client_update_upload_part_timeout(request->meta_request->client, request, error_code);
     }
     aws_s3_meta_request_send_request_finish_default(connection, stream, error_code);
@@ -1216,11 +1219,13 @@ static void s_s3_prepare_upload_part_finish(struct aws_s3_prepare_upload_part_jo
             aws_s3_meta_request_lock_synced_data(meta_request);
             struct aws_s3_mpu_part_info *part = NULL;
             aws_array_list_get_at(&auto_ranged_put->synced_data.part_list, &part, request->part_number - 1);
+            /**
+             * 1. If request retries, get the checksum context from the previous attempt to reuse it. In case of the the
+             *      request body in memory mangled
+             * 2. If this is the first attempt, the context is initialized from the previous prepare step.
+             **/
             AWS_ASSERT(part != NULL && part->checksum_context != NULL);
-            /* Use checksum context if available, otherwise NULL for new parts */
             checksum_context = part->checksum_context;
-            /* If checksum already calculated, it means either the part being retried or the part resumed from list
-             * parts. Keep reusing the old checksum in case of the request body in memory mangled */
             aws_s3_meta_request_unlock_synced_data(meta_request);
         }
         /* END CRITICAL SECTION */

source/s3_checksum_context.c

@@ -182,7 +182,6 @@ struct aws_byte_cursor aws_s3_upload_request_checksum_context_get_checksum_curso
         return checksum_cursor;
     }
     s_lock_synced_data(context);
-    /* If not previous calculated */
     if (context->synced_data.checksum_calculated) {
         checksum_cursor = aws_byte_cursor_from_buf(&context->synced_data.base64_checksum);
     }