add more metrics

Author: TingDaoK

include/aws/s3/private/s3_parallel_input_stream.h

@@ -16,6 +16,7 @@ struct aws_byte_buf;
 struct aws_future_bool;
 struct aws_input_stream;
 struct aws_s3_meta_request;
+struct aws_s3_request;
 
 struct aws_event_loop_group;
 
@@ -126,6 +127,7 @@ struct aws_input_stream *aws_input_stream_new_from_parallel_stream(
     struct aws_allocator *allocator,
     struct aws_parallel_input_stream *stream,
     struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
     uint64_t offset,
     size_t request_body_size);
 

include/aws/s3/private/s3_request.h

@@ -70,6 +70,11 @@ struct aws_s3_request_metrics {
         /* The time duration for the request from start signing to finish signing (sign_end_timestamp_ns -
          * sign_start_timestamp_ns). When sign_end_timestamp_ns is 0, means data not available. */
         int64_t signing_duration_ns;
+
+        int64_t body_read_start_timestamp_ns;
+        int64_t body_read_end_timestamp_ns;
+        int64_t body_read_duration_ns;
+        int64_t body_read_total_ns;
     } time_metrics;
 
     struct {

include/aws/s3/s3_client.h

@@ -1462,6 +1462,18 @@ AWS_S3_API
 void aws_s3_request_metrics_get_request_type(
     const struct aws_s3_request_metrics *metrics,
     enum aws_s3_request_type *out_request_type);
+int aws_s3_request_metrics_get_body_read_start_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_start_timestamp_ns);
+int aws_s3_request_metrics_get_body_read_end_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_end_timestamp_ns);
+int aws_s3_request_metrics_get_body_read_duration_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_duration_ns);
+int aws_s3_request_metrics_get_body_read_total_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_total_ns);
 
 /* Get the AWS CRT error code from request metrics. */
 AWS_S3_API

source/s3_auto_ranged_put.c

@@ -982,7 +982,12 @@ struct aws_future_http_message *s_s3_prepare_upload_part(struct aws_s3_request *
             uint64_t offset = 0;
             size_t request_body_size = s_compute_request_body_size(meta_request, request->part_number, &offset);
             request->request_stream = aws_input_stream_new_from_parallel_stream(
-                allocator, meta_request->request_body_parallel_stream, meta_request, offset, request_body_size);
+                allocator,
+                meta_request->request_body_parallel_stream,
+                meta_request,
+                request,
+                offset,
+                request_body_size);
             request->content_length = request_body_size;
             struct aws_s3_auto_ranged_put *auto_ranged_put = meta_request->impl;
 
@@ -1025,7 +1030,11 @@ struct aws_future_http_message *s_s3_prepare_upload_part(struct aws_s3_request *
             /* BEGIN CRITICAL SECTION */
             aws_s3_meta_request_lock_synced_data(meta_request);
             meta_request->count++;
-            snprintf(filename, sizeof(filename), "/tmp/s3_read_metrics_%d.csv", meta_request->count);
+            snprintf(
+                filename,
+                sizeof(filename),
+                "/data/aws-crt-s3-benchmarks/metrics/s3_read_metrics_%d.csv",
+                meta_request->count);
 
             FILE *metrics_file = fopen(filename, "w");
             /* write every read metric to a file */
@@ -1074,7 +1083,12 @@ struct aws_future_http_message *s_s3_prepare_upload_part(struct aws_s3_request *
             uint64_t offset = 0;
             size_t request_body_size = s_compute_request_body_size(meta_request, request->part_number, &offset);
             request->request_stream = aws_input_stream_new_from_parallel_stream(
-                allocator, meta_request->request_body_parallel_stream, meta_request, offset, request_body_size);
+                allocator,
+                meta_request->request_body_parallel_stream,
+                meta_request,
+                request,
+                offset,
+                request_body_size);
             s_s3_prepare_upload_part_finish(part_prep, AWS_ERROR_SUCCESS);
         }
     } else if (request->num_times_prepared == 0) {

source/s3_meta_request.c

@@ -513,7 +513,8 @@ static void s_s3_meta_request_destroy(void *user_data) {
     aws_mutex_clean_up(&meta_request->synced_data.lock);
     char filename[256];
     meta_request->count++;
-    snprintf(filename, sizeof(filename), "/tmp/s3_read_metrics_%d.csv", meta_request->count);
+    snprintf(
+        filename, sizeof(filename), "/data/aws-crt-s3-benchmarks/metrics/s3_read_metrics_%d.csv", meta_request->count);
     FILE *metrics_file = fopen(filename, "w");
     /* write every read metric to a file */
     size_t metric_length = aws_array_list_length(&meta_request->read_metrics_list);

source/s3_parallel_input_stream.c

@@ -265,6 +265,7 @@ struct aws_s3_mmap_part_streaming_input_stream_impl {
     struct s3_data_read_metrics metrics;
 
     struct aws_s3_meta_request *meta_request;
+    struct aws_s3_request *request;
 };
 
 static int s_aws_s3_mmap_part_streaming_input_stream_seek(
@@ -282,6 +283,11 @@ static int s_aws_s3_mmap_part_streaming_input_stream_read(struct aws_input_strea
         AWS_CONTAINER_OF(stream, struct aws_s3_mmap_part_streaming_input_stream_impl, base);
     /* Map the content */
     size_t read_length = aws_min_size(dest->capacity - dest->len, impl->total_length - impl->total_length_read);
+    uint64_t time = 0;
+    if (impl->request->send_data.metrics->time_metrics.body_read_start_timestamp_ns == -1) {
+        aws_high_res_clock_get_ticks(&time);
+        impl->request->send_data.metrics->time_metrics.body_read_start_timestamp_ns = time;
+    }
     aws_high_res_clock_get_ticks(&impl->metrics.start_timestamp);
     impl->metrics.thread_id = aws_thread_current_thread_id();
 
@@ -326,6 +332,12 @@ static int s_aws_s3_mmap_part_streaming_input_stream_read(struct aws_input_strea
     impl->in_chunk_offset += read_length;
     impl->total_length_read += read_length;
     aws_high_res_clock_get_ticks(&impl->metrics.end_timestamp);
+    int64_t duration = impl->metrics.end_timestamp - impl->metrics.start_timestamp;
+    if (impl->request->send_data.metrics->time_metrics.body_read_total_ns == -1) {
+        impl->request->send_data.metrics->time_metrics.body_read_total_ns = duration;
+    } else {
+        impl->request->send_data.metrics->time_metrics.body_read_total_ns += duration;
+    }
 
     /* BEGIN CRITICAL SECTION */
     aws_s3_meta_request_lock_synced_data(impl->meta_request);
@@ -340,6 +352,13 @@ static int s_aws_s3_mmap_part_streaming_input_stream_read(struct aws_input_strea
             /* We reached the end of the stream. */
             impl->eos_reached = true;
             AWS_ASSERT(impl->total_length_read == impl->total_length);
+            if (impl->request->send_data.metrics->time_metrics.body_read_end_timestamp_ns == -1) {
+                aws_high_res_clock_get_ticks(&time);
+                impl->request->send_data.metrics->time_metrics.body_read_end_timestamp_ns = time;
+            }
+            impl->request->send_data.metrics->time_metrics.body_read_duration_ns =
+                impl->request->send_data.metrics->time_metrics.body_read_end_timestamp_ns -
+                impl->request->send_data.metrics->time_metrics.body_read_start_timestamp_ns;
         }
     }
 
@@ -409,6 +428,7 @@ struct aws_input_stream *aws_input_stream_new_from_parallel_stream(
     struct aws_allocator *allocator,
     struct aws_parallel_input_stream *stream,
     struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
     uint64_t offset,
     size_t request_body_size) {
 
@@ -431,6 +451,7 @@ struct aws_input_stream *aws_input_stream_new_from_parallel_stream(
     impl->loading_chunk_buf = &impl->chunk_buf_1;
     impl->reading_chunk_buf = &impl->chunk_buf_2;
     impl->meta_request = meta_request;
+    impl->request = request;
 
     /* Reset the input stream to start */
     aws_streaming_input_stream_reset(&impl->base);

source/s3_request.c

@@ -193,6 +193,10 @@ struct aws_s3_request_metrics *aws_s3_request_metrics_new(
     metrics->time_metrics.sign_start_timestamp_ns = -1;
     metrics->time_metrics.sign_end_timestamp_ns = -1;
     metrics->time_metrics.signing_duration_ns = -1;
+    metrics->time_metrics.body_read_start_timestamp_ns = -1;
+    metrics->time_metrics.body_read_end_timestamp_ns = -1;
+    metrics->time_metrics.body_read_duration_ns = -1;
+    metrics->time_metrics.body_read_total_ns = -1;
 
     metrics->req_resp_info_metrics.response_status = -1;
 
@@ -426,3 +430,24 @@ int aws_s3_request_metrics_get_error_code(const struct aws_s3_request_metrics *m
     AWS_PRECONDITION(metrics);
     return metrics->crt_info_metrics.error_code;
 }
+
+int aws_s3_request_metrics_get_body_read_start_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_start_timestamp_ns) {
+    *body_read_start_timestamp_ns = metrics->time_metrics.body_read_start_timestamp_ns;
+}
+int aws_s3_request_metrics_get_body_read_end_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_end_timestamp_ns) {
+    *body_read_end_timestamp_ns = metrics->time_metrics.body_read_end_timestamp_ns;
+}
+int aws_s3_request_metrics_get_body_read_duration_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_duration_ns) {
+    *body_read_duration_ns = metrics->time_metrics.body_read_duration_ns;
+}
+int aws_s3_request_metrics_get_body_read_total_ns(
+    const struct aws_s3_request_metrics *metrics,
+    int64_t *body_read_total_ns) {
+    *body_read_total_ns = metrics->time_metrics.body_read_total_ns;
+}