apply the dynamic size

Author: TingDaoK

include/aws/s3/private/s3_auto_ranged_get.h

@@ -23,6 +23,8 @@ struct aws_s3_auto_ranged_get {
 
     /* Estimated object stored part size based on ETag analysis */
     uint64_t estimated_object_stored_part_size;
+    /* Part size was set or not from user for this meta request. */
+    bool part_size_set;
 
     bool initial_message_has_start_range;
     bool initial_message_has_end_range;
@@ -77,6 +79,7 @@ AWS_S3_API struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
     struct aws_allocator *allocator,
     struct aws_s3_client *client,
     size_t part_size,
+    bool part_size_set,
     const struct aws_s3_meta_request_options *options);
 
 AWS_EXTERN_C_END

include/aws/s3/private/s3_client_impl.h

@@ -234,6 +234,8 @@ struct aws_s3_client {
      * to meta requests for use. */
     const size_t part_size;
 
+    bool part_size_set;
+
     /* Size of parts for files when doing gets or puts.  This exists on the client as configurable option that is passed
      * to meta requests for use. */
     const uint64_t max_part_size;

source/s3_auto_ranged_get.c

@@ -62,6 +62,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
     struct aws_allocator *allocator,
     struct aws_s3_client *client,
     size_t part_size,
+    bool part_size_set,
     const struct aws_s3_meta_request_options *options) {
     AWS_PRECONDITION(allocator);
     AWS_PRECONDITION(client);
@@ -91,6 +92,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
         return NULL;
     }
 
+    auto_ranged_get->part_size_set = part_size_set;
     struct aws_http_headers *headers = aws_http_message_get_headers(auto_ranged_get->base.initial_request_message);
     AWS_ASSERT(headers != NULL);
 
@@ -319,8 +321,12 @@ static bool s_s3_auto_ranged_get_update(
                      * we could end up stuck in a situation where the user is
                      * waiting for more bytes before they'll open the window,
                      * and this implementation is waiting for more window before it will send more parts. */
-                    uint64_t read_data_requested =
-                        auto_ranged_get->synced_data.num_parts_requested * meta_request->part_size;
+                    uint64_t read_data_requested = 0;
+                    if (auto_ranged_get->synced_data.num_parts_requested > 0) {
+                        read_data_requested =
+                            (auto_ranged_get->synced_data.num_parts_requested - 1) * meta_request->part_size +
+                            auto_ranged_get->synced_data.first_part_size;
+                    }
                     if (read_data_requested >= meta_request->synced_data.read_window_running_total) {
 
                         /* Avoid spamming users with this DEBUG message */
@@ -805,6 +811,20 @@ static void s_s3_auto_ranged_get_request_finished(
         error_code = AWS_ERROR_SUCCESS;
         found_object_size = true;
 
+        if (!auto_ranged_get->part_size_set && !meta_request->client->part_size_set) {
+            /* No part size has been set from user. Now we use the optimal part size based on the throughput and memory
+             * limit */
+            uint64_t out_request_optimal_range_size = 0;
+            int error = aws_s3_calculate_request_optimal_range_size(
+                meta_request->client->optimal_range_size,
+                auto_ranged_get->estimated_object_stored_part_size,
+                &out_request_optimal_range_size);
+            if (!error) {
+                /* Override the part size to be  */
+                *((size_t *)&meta_request->part_size) = (size_t)out_request_optimal_range_size;
+            }
+        }
+
         /* Check for checksums if requested to */
         if (meta_request->checksum_config.validate_response_checksum) {
             if (aws_s3_check_headers_for_checksum(

source/s3_client.c

@@ -403,6 +403,7 @@ struct aws_s3_client *aws_s3_client_new(
         } else {
             part_size = (size_t)client_config->part_size;
         }
+        client->part_size_set = true;
     }
 
     /* default max part size is the smallest of either half of mem limit or default max part size */
@@ -1279,12 +1280,14 @@ static struct aws_s3_meta_request *s_s3_client_meta_request_factory_default(
         return NULL;
     }
     size_t part_size = client->part_size;
+    bool part_size_set = false;
     if (options->part_size != 0) {
         if (options->part_size > SIZE_MAX) {
             part_size = SIZE_MAX;
         } else {
             part_size = (size_t)options->part_size;
         }
+        part_size_set = true;
     }
 
     /* Call the appropriate meta-request new function. */
@@ -1318,7 +1321,8 @@ static struct aws_s3_meta_request *s_s3_client_meta_request_factory_default(
                     }
                 }
             }
-            return aws_s3_meta_request_auto_ranged_get_new(client->allocator, client, part_size, options);
+            return aws_s3_meta_request_auto_ranged_get_new(
+                client->allocator, client, part_size, part_size_set, options);
         }
         case AWS_S3_META_REQUEST_TYPE_PUT_OBJECT: {
             if (body_source_count == 0) {

tests/s3_client_test.c

@@ -489,7 +489,7 @@ TEST_CASE(s3_calculate_client_optimal_range_size) {
         /* Verify optimal range size is calculated and within expected bounds */
         ASSERT_TRUE(client->optimal_range_size >= MB_TO_BYTES(8)); /* At least 8MiB minimum */
         ASSERT_TRUE(client->optimal_range_size <= GB_TO_BYTES(1)); /* At most 1GB maximum */
-        
+
         /* Verify it's properly initialized (not zero) */
         ASSERT_TRUE(client->optimal_range_size > 0);
 
@@ -521,7 +521,7 @@ TEST_CASE(s3_calculate_client_optimal_range_size) {
     {
         struct aws_s3_client_config client_config = {
             .part_size = MB_TO_BYTES(8),
-            .throughput_target_gbps = 100.0, /* High throughput = more connections */
+            .throughput_target_gbps = 100.0,           /* High throughput = more connections */
             .memory_limit_in_bytes = MB_TO_BYTES(512), /* Low memory */
         };
 

tests/s3_meta_request_test.c

@@ -43,7 +43,7 @@ TEST_CASE(meta_request_auto_ranged_get_new_error_handling) {
         .type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT,
     };
     struct aws_s3_meta_request *meta_request =
-        aws_s3_meta_request_auto_ranged_get_new(allocator, client, SIZE_MAX, &options);
+        aws_s3_meta_request_auto_ranged_get_new(allocator, client, SIZE_MAX, false, &options);
 
     ASSERT_NULL(meta_request);
     aws_http_message_release(message);