add a default file streaming options (#566)

Author: TingDaoK

include/aws/s3/private/s3_client_impl.h

@@ -239,6 +239,7 @@ struct aws_s3_client {
     const uint64_t max_part_size;
 
     /* File I/O options. */
+    bool fio_options_set;
     struct aws_s3_file_io_options fio_opts;
 
     /* The size threshold in bytes for when to use multipart uploads for a AWS_S3_META_REQUEST_TYPE_PUT_OBJECT meta

include/aws/s3/private/s3_meta_request_impl.h

@@ -310,6 +310,7 @@ int aws_s3_meta_request_init_base(
     struct aws_s3_client *client,
     size_t part_size,
     bool should_compute_content_md5,
+    bool should_default_streaming,
     const struct aws_s3_meta_request_options *options,
     void *impl,
     struct aws_s3_meta_request_vtable *vtable,

include/aws/s3/private/s3_util.h

@@ -24,7 +24,8 @@
 #endif
 #define KB_TO_BYTES(kb) ((kb) * 1024)
 #define MB_TO_BYTES(mb) ((mb) * 1024 * 1024)
-#define GB_TO_BYTES(gb) ((gb) * 1024 * 1024 * 1024ULL)
+#define GB_TO_BYTES(gb) ((uint64_t)(gb) * 1024 * 1024 * 1024ULL)
+#define TB_TO_BYTES(tb) ((uint64_t)(tb) * 1024 * 1024 * 1024 * 1024ULL)
 
 #define MS_TO_NS(ms) ((uint64_t)(ms) * 1000000)
 #define SEC_TO_NS(ms) ((uint64_t)(ms) * 1000000000)
@@ -154,6 +155,11 @@ AWS_S3_API
 /* TODO: now this is hard-coded as 8MB, but maybe something else is better. */
 extern const size_t g_streaming_buffer_size;
 
+AWS_S3_API
+extern const double g_default_throughput_target_gbps;
+
+AWS_S3_API
+extern const uint64_t g_streaming_object_size_threshold;
 /**
  * Returns AWS_S3_REQUEST_TYPE_UNKNOWN if name doesn't map to an enum value.
  */

include/aws/s3/s3_client.h

@@ -321,6 +321,7 @@ struct aws_s3_file_io_options {
      * Skip buffering the part in memory before sending the request.
      * If set, set the `disk_throughput_gbps` to be reasonable align with the available disk throughput.
      * Otherwise, the transfer may fail with connection starvation.
+     *
      * Default to false.
      **/
     bool should_stream;
@@ -333,6 +334,8 @@ struct aws_s3_file_io_options {
      * Notes: There are possibilities that cannot reach the all available disk throughput:
      * 1. Disk is busy with other applications
      * 2. OS Cache may cap the throughput, use `direct_io` to get around this.
+     *
+     * Note: When `streaming_upload` is true, this default to 10 Gbps.
      **/
     double disk_throughput_gbps;
 
@@ -475,7 +478,7 @@ struct aws_s3_client_config {
      * Optional.
      * If set, this controls how the client interact with file I/O.
      * Read `aws_s3_file_io_options` for details.
-     *  Notes: Only applies when AWS_S3_META_REQUEST_TYPE_PUT_OBJECT is set.
+     *  Notes: Only applies to meta requests with `send_filepath` set.
      *  TODO: adapt it to `recv_filepath`.
      *
      * eg:
@@ -821,6 +824,9 @@ struct aws_s3_meta_request_options {
      *  Notes: Only applies when `send_filepath` is set.
      *  TODO: adapt it to `recv_filepath`.
      *
+     * Note: if both client and meta request don't set this, for objects larger than 2TiB, this will be set to a default
+     * options with `should_stream` to be True and others follow the default to avoid memory issues.
+     *
      * eg:
      * - When the file is too large to fit in the buffer, set `should_stream` to avoid buffering the whole parts in
      *  memory.

source/s3_auto_ranged_get.c

@@ -77,6 +77,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
             client,
             part_size,
             false,
+            false,
             options,
             auto_ranged_get,
             &s_s3_auto_ranged_get_vtable,

source/s3_auto_ranged_put.c

@@ -352,6 +352,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_put_new(
             part_size,
             client->compute_content_md5 == AWS_MR_CONTENT_MD5_ENABLED ||
                 aws_http_headers_has(aws_http_message_get_headers(options->message), g_content_md5_header_name),
+            content_length > g_streaming_object_size_threshold,
             options,
             auto_ranged_put,
             &s_s3_auto_ranged_put_vtable,

source/s3_client.c

@@ -79,7 +79,6 @@ const uint32_t g_max_num_connections = 10000;
  */
 static const size_t s_default_part_size = 8 * 1024 * 1024;
 static const uint64_t s_default_max_part_size = 5368709120ULL;
-static const double s_default_throughput_target_gbps = 10.0;
 static const uint32_t s_default_max_retries = 5;
 static size_t s_dns_host_address_ttl_seconds = 5 * 60;
 
@@ -386,6 +385,7 @@ struct aws_s3_client *aws_s3_client_new(
 
     if (client_config->fio_opts) {
         client->fio_opts = *client_config->fio_opts;
+        client->fio_options_set = true;
     }
 
     struct aws_s3_buffer_pool_config buffer_pool_config = {
@@ -590,7 +590,7 @@ struct aws_s3_client *aws_s3_client_new(
     if (client_config->throughput_target_gbps > 0.0) {
         *((double *)&client->throughput_target_gbps) = client_config->throughput_target_gbps;
     } else {
-        *((double *)&client->throughput_target_gbps) = s_default_throughput_target_gbps;
+        *((double *)&client->throughput_target_gbps) = g_default_throughput_target_gbps;
     }
 
     *((enum aws_s3_meta_request_compute_content_md5 *)&client->compute_content_md5) =
@@ -1206,7 +1206,6 @@ static struct aws_s3_meta_request *s_s3_client_meta_request_factory_default(
                     "Could not create meta request."
                     "the send file path %s is invalid",
                     aws_string_c_str(file_path));
-                aws_raise_error(AWS_ERROR_FILE_INVALID_PATH);
             } else if (aws_file_get_length(file, (int64_t *)&content_length)) {
                 AWS_LOGF_ERROR(
                     AWS_LS_S3_META_REQUEST,

source/s3_copy_object.c

@@ -79,6 +79,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_copy_object_new(
             client,
             UNKNOWN_PART_SIZE,
             false,
+            false,
             options,
             copy_object,
             &s_s3_copy_object_vtable,

source/s3_default_meta_request.c

@@ -92,6 +92,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_default_new(
             client,
             0,
             should_compute_content_md5,
+            false,
             options,
             meta_request_default,
             &s_s3_meta_request_default_vtable,

source/s3_meta_request.c

@@ -157,6 +157,7 @@ int aws_s3_meta_request_init_base(
     struct aws_s3_client *client,
     size_t part_size,
     bool should_compute_content_md5,
+    bool should_default_streaming,
     const struct aws_s3_meta_request_options *options,
     void *impl,
     struct aws_s3_meta_request_vtable *vtable,
@@ -184,8 +185,16 @@ int aws_s3_meta_request_init_base(
     /* Deep copy the file io options. */
     if (options->fio_opts) {
         meta_request->fio_opts = *options->fio_opts;
-    } else if (client != NULL) {
+    } else if (client != NULL && client->fio_options_set) {
         meta_request->fio_opts = client->fio_opts;
+    } else {
+        /* Take a default options. */
+        meta_request->fio_opts.should_stream = should_default_streaming;
+    }
+
+    if (meta_request->fio_opts.should_stream && meta_request->fio_opts.disk_throughput_gbps == 0) {
+        /* If disk throughput is not set, set it to the default. */
+        meta_request->fio_opts.disk_throughput_gbps = g_default_throughput_target_gbps;
     }
 
     /* Set up reference count. */
@@ -333,7 +342,6 @@ int aws_s3_meta_request_init_base(
             goto error;
         }
         if (meta_request->fio_opts.direct_io && !meta_request->fio_opts.should_stream) {
-            /* TODO: aws_system_info_page_size */
             size_t page_size = aws_system_info_page_size();
             if (part_size % page_size != 0) {
                 AWS_LOGF_ERROR(