Skip to content

Commit ab2120f

Browse files
TingDaoKTingDaoK
authored
empty download (#86)
1 parent 977815f commit ab2120f

6 files changed

Lines changed: 293 additions & 76 deletions

File tree

‎include/aws/s3/private/s3_auto_ranged_get.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
enum aws_s3_auto_ranged_get_request_type {
1212
AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART,
13+
AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE,
1314
};
1415

1516
struct aws_s3_auto_ranged_get {
@@ -26,6 +27,9 @@ struct aws_s3_auto_ranged_get {
2627

2728
size_t total_object_size;
2829

30+
uint32_t get_without_range : 1;
31+
uint32_t get_without_range_sent : 1;
32+
uint32_t get_without_range_completed : 1;
2933
} synced_data;
3034
};
3135

‎include/aws/s3/private/s3_request_messages.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88

99
#include <inttypes.h>
10+
#include <stdbool.h>
1011
#include <stddef.h>
1112

1213
struct aws_allocator;
@@ -30,7 +31,8 @@ struct aws_http_message *aws_s3_get_object_message_new(
3031
struct aws_allocator *allocator,
3132
struct aws_http_message *base_message,
3233
uint32_t part_number,
33-
size_t part_size);
34+
size_t part_size,
35+
bool has_range);
3436

3537
/* Create an HTTP request for an S3 Put Object request, using the original request as a basis. Creates and assigns a
3638
* body stream using the passed in buffer. If multipart is not needed, part number and upload_id can be 0 and NULL,

‎source/s3_auto_ranged_get.c‎

Lines changed: 150 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#endif
1818

1919
const uint32_t s_conservative_max_requests_in_flight = 8;
20+
const struct aws_byte_cursor g_application_xml_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("application/xml");
21+
const struct aws_byte_cursor g_object_size_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ActualObjectSize");
2022

2123
static void s_s3_meta_request_auto_ranged_get_destroy(struct aws_s3_meta_request *meta_request);
2224

@@ -100,6 +102,31 @@ static void s_s3_meta_request_auto_ranged_get_destroy(struct aws_s3_meta_request
100102
aws_mem_release(meta_request->allocator, auto_ranged_get);
101103
}
102104

105+
/* Check the finish result of meta request, in case of the request failed because of downloading an empty file */
106+
static bool s_check_empty_file_download_error(struct aws_s3_request *failed_request) {
107+
struct aws_http_headers *failed_headers = failed_request->send_data.response_headers;
108+
struct aws_byte_buf failed_body = failed_request->send_data.response_body;
109+
if (failed_headers && failed_body.capacity > 0) {
110+
struct aws_byte_cursor content_type;
111+
AWS_ZERO_STRUCT(content_type);
112+
if (!aws_http_headers_get(failed_headers, g_content_type_header_name, &content_type)) {
113+
/* Content type found */
114+
if (aws_byte_cursor_eq_ignore_case(&content_type, &g_application_xml_value)) {
115+
/* XML response */
116+
struct aws_byte_cursor body_cursor = aws_byte_cursor_from_buf(&failed_body);
117+
struct aws_string *size =
118+
get_top_level_xml_tag_value(failed_request->allocator, &g_object_size_value, &body_cursor);
119+
bool check_size = aws_string_eq_c_str(size, "0");
120+
aws_string_destroy(size);
121+
if (check_size) {
122+
return true;
123+
}
124+
}
125+
}
126+
}
127+
return false;
128+
}
129+
103130
static bool s_s3_auto_ranged_get_update(
104131
struct aws_s3_meta_request *meta_request,
105132
uint32_t flags,
@@ -161,6 +188,28 @@ static bool s_s3_auto_ranged_get_update(
161188
goto has_work_remaining;
162189
}
163190

191+
if (auto_ranged_get->synced_data.get_without_range) {
192+
if (auto_ranged_get->synced_data.get_without_range_sent) {
193+
if (auto_ranged_get->synced_data.get_without_range_completed) {
194+
goto no_work_remaining;
195+
} else {
196+
goto has_work_remaining;
197+
}
198+
}
199+
if (out_request == NULL) {
200+
goto has_work_remaining;
201+
}
202+
203+
request = aws_s3_request_new(
204+
meta_request,
205+
AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE,
206+
1,
207+
AWS_S3_REQUEST_DESC_RECORD_RESPONSE_HEADERS);
208+
209+
auto_ranged_get->synced_data.get_without_range_sent = true;
210+
goto has_work_remaining;
211+
}
212+
164213
/* If we have gotten a response for the first request, then the total number of parts for the object is now
165214
* known. Continue sending parts until the total number of parts is reached.*/
166215
if (auto_ranged_get->synced_data.num_parts_requested < auto_ranged_get->synced_data.total_num_parts) {
@@ -190,6 +239,15 @@ static bool s_s3_auto_ranged_get_update(
190239
goto has_work_remaining;
191240
}
192241

242+
if (auto_ranged_get->synced_data.get_without_range) {
243+
if (auto_ranged_get->synced_data.get_without_range_sent &&
244+
!auto_ranged_get->synced_data.get_without_range_completed) {
245+
goto has_work_remaining;
246+
} else {
247+
goto no_work_remaining;
248+
}
249+
}
250+
193251
/* If some parts are still being delivered to the caller, then wait for those to finish. */
194252
if (meta_request->synced_data.num_parts_delivery_completed <
195253
meta_request->synced_data.num_parts_delivery_sent) {
@@ -249,11 +307,13 @@ static int s_s3_auto_ranged_get_prepare_request(
249307

250308
struct aws_http_message *message = NULL;
251309

252-
AWS_ASSERT(request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART);
253-
254310
/* Generate a new ranged get request based on the original message. */
255311
message = aws_s3_get_object_message_new(
256-
meta_request->allocator, meta_request->initial_request_message, request->part_number, meta_request->part_size);
312+
meta_request->allocator,
313+
meta_request->initial_request_message,
314+
request->part_number,
315+
meta_request->part_size,
316+
request->request_tag != AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE);
257317

258318
if (message == NULL) {
259319
AWS_LOGF_ERROR(
@@ -294,66 +354,67 @@ static void s_s3_auto_ranged_get_request_finished(
294354

295355
struct aws_s3_auto_ranged_get *auto_ranged_get = meta_request->impl;
296356

297-
AWS_ASSERT(request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART);
298-
299357
uint32_t num_parts = 0;
300358

301359
if (error_code == AWS_ERROR_SUCCESS && request->part_number == 1) {
302-
struct aws_byte_cursor content_range_header_value;
303-
304-
if (aws_http_headers_get(
305-
request->send_data.response_headers, g_content_range_header_name, &content_range_header_value)) {
306-
AWS_LOGF_ERROR(
307-
AWS_LS_S3_META_REQUEST,
308-
"id=%p Could not find content range header for request %p",
309-
(void *)meta_request,
310-
(void *)request);
311-
312-
error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER;
313-
goto error_encountered;
314-
}
315-
316-
uint64_t range_start = 0;
317-
uint64_t range_end = 0;
318360
uint64_t total_object_size = 0;
361+
if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART) {
362+
struct aws_byte_cursor content_range_header_value;
363+
364+
if (aws_http_headers_get(
365+
request->send_data.response_headers, g_content_range_header_name, &content_range_header_value)) {
366+
AWS_LOGF_ERROR(
367+
AWS_LS_S3_META_REQUEST,
368+
"id=%p Could not find content range header for request %p",
369+
(void *)meta_request,
370+
(void *)request);
371+
372+
error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER;
373+
goto error_encountered;
374+
}
319375

320-
/* The memory the byte cursor refers to should be valid, but if it's referring to a buffer that was
321-
* previously used, the null terminating character may not be where we expect. We copy to a string to
322-
* ensure that our null terminating character placement corresponds with the length. */
323-
struct aws_string *content_range_header_value_str =
324-
aws_string_new_from_cursor(meta_request->allocator, &content_range_header_value);
325-
326-
/* Format of header is: "bytes StartByte-EndByte/TotalObjectSize" */
327-
sscanf(
328-
(const char *)content_range_header_value_str->bytes,
329-
"bytes %" PRIu64 "-%" PRIu64 "/%" PRIu64,
330-
&range_start,
331-
&range_end,
332-
&total_object_size);
333-
334-
aws_string_destroy(content_range_header_value_str);
335-
content_range_header_value_str = NULL;
336-
337-
if (total_object_size == 0) {
338-
AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p Get Object has invalid content range.", (void *)meta_request);
339-
error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER;
340-
goto error_encountered;
341-
}
376+
uint64_t range_start = 0;
377+
uint64_t range_end = 0;
378+
379+
/* The memory the byte cursor refers to should be valid, but if it's referring to a buffer that was
380+
* previously used, the null terminating character may not be where we expect. We copy to a string to
381+
* ensure that our null terminating character placement corresponds with the length. */
382+
struct aws_string *content_range_header_value_str =
383+
aws_string_new_from_cursor(meta_request->allocator, &content_range_header_value);
384+
385+
/* Format of header is: "bytes StartByte-EndByte/TotalObjectSize" */
386+
sscanf(
387+
(const char *)content_range_header_value_str->bytes,
388+
"bytes %" PRIu64 "-%" PRIu64 "/%" PRIu64,
389+
&range_start,
390+
&range_end,
391+
&total_object_size);
392+
393+
aws_string_destroy(content_range_header_value_str);
394+
content_range_header_value_str = NULL;
395+
396+
if (total_object_size == 0) {
397+
AWS_LOGF_ERROR(
398+
AWS_LS_S3_META_REQUEST, "id=%p Get Object has invalid content range.", (void *)meta_request);
399+
error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER;
400+
goto error_encountered;
401+
}
342402

343-
num_parts = (uint32_t)(total_object_size / meta_request->part_size);
403+
num_parts = (uint32_t)(total_object_size / meta_request->part_size);
344404

345-
if (total_object_size % meta_request->part_size) {
346-
++num_parts;
347-
}
405+
if (total_object_size % meta_request->part_size) {
406+
++num_parts;
407+
}
348408

349-
AWS_LOGF_DEBUG(
350-
AWS_LS_S3_META_REQUEST,
351-
"id=%p Object being requested is %" PRIu64 " bytes which will have %d parts based off of a %" PRIu64
352-
" part size.",
353-
(void *)meta_request,
354-
total_object_size,
355-
num_parts,
356-
(uint64_t)meta_request->part_size);
409+
AWS_LOGF_DEBUG(
410+
AWS_LS_S3_META_REQUEST,
411+
"id=%p Object being requested is %" PRIu64 " bytes which will have %d parts based off of a %" PRIu64
412+
" part size.",
413+
(void *)meta_request,
414+
total_object_size,
415+
num_parts,
416+
(uint64_t)meta_request->part_size);
417+
}
357418

358419
if (meta_request->headers_callback != NULL) {
359420
struct aws_http_headers *response_headers = aws_http_headers_new(meta_request->allocator);
@@ -381,28 +442,44 @@ static void s_s3_auto_ranged_get_request_finished(
381442

382443
aws_s3_meta_request_lock_synced_data(meta_request);
383444

384-
++auto_ranged_get->synced_data.num_parts_completed;
445+
if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART) {
446+
++auto_ranged_get->synced_data.num_parts_completed;
385447

386-
if (error_code == AWS_ERROR_SUCCESS) {
387-
++auto_ranged_get->synced_data.num_parts_successful;
448+
if (error_code == AWS_ERROR_SUCCESS) {
449+
++auto_ranged_get->synced_data.num_parts_successful;
388450

389-
if (request->part_number == 1) {
390-
AWS_ASSERT(num_parts > 0);
391-
auto_ranged_get->synced_data.total_num_parts = num_parts;
392-
}
451+
if (request->part_number == 1) {
452+
AWS_ASSERT(num_parts > 0);
453+
auto_ranged_get->synced_data.total_num_parts = num_parts;
454+
}
393455

394-
aws_s3_meta_request_stream_response_body_synced(meta_request, request);
456+
aws_s3_meta_request_stream_response_body_synced(meta_request, request);
395457

396-
AWS_LOGF_DEBUG(
397-
AWS_LS_S3_META_REQUEST,
398-
"id=%p: %d out of %d parts have completed.",
399-
(void *)meta_request,
400-
(auto_ranged_get->synced_data.num_parts_successful + auto_ranged_get->synced_data.num_parts_failed),
401-
auto_ranged_get->synced_data.total_num_parts);
402-
} else {
403-
++auto_ranged_get->synced_data.num_parts_failed;
404-
405-
aws_s3_meta_request_set_fail_synced(meta_request, request, error_code);
458+
AWS_LOGF_DEBUG(
459+
AWS_LS_S3_META_REQUEST,
460+
"id=%p: %d out of %d parts have completed.",
461+
(void *)meta_request,
462+
(auto_ranged_get->synced_data.num_parts_successful + auto_ranged_get->synced_data.num_parts_failed),
463+
auto_ranged_get->synced_data.total_num_parts);
464+
} else {
465+
++auto_ranged_get->synced_data.num_parts_failed;
466+
if (s_check_empty_file_download_error(request)) {
467+
AWS_LOGF_DEBUG(
468+
AWS_LS_S3_META_REQUEST,
469+
"id=%p Getting an empty file, create a new request without range header to fetch the empty "
470+
"file",
471+
(void *)meta_request);
472+
auto_ranged_get->synced_data.get_without_range = true;
473+
} else {
474+
aws_s3_meta_request_set_fail_synced(meta_request, request, error_code);
475+
}
476+
}
477+
} else if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE) {
478+
AWS_LOGF_DEBUG(AWS_LS_S3_META_REQUEST, "id=%p Get empty file completed", (void *)meta_request);
479+
auto_ranged_get->synced_data.get_without_range_completed = true;
480+
if (error_code != AWS_ERROR_SUCCESS) {
481+
aws_s3_meta_request_set_fail_synced(meta_request, request, error_code);
482+
}
406483
}
407484

408485
aws_s3_meta_request_unlock_synced_data(meta_request);

‎source/s3_request_messages.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ struct aws_http_message *aws_s3_get_object_message_new(
3333
struct aws_allocator *allocator,
3434
struct aws_http_message *base_message,
3535
uint32_t part_number,
36-
size_t part_size) {
36+
size_t part_size,
37+
bool has_range) {
3738
AWS_PRECONDITION(allocator);
3839
AWS_PRECONDITION(base_message);
3940

@@ -44,7 +45,7 @@ struct aws_http_message *aws_s3_get_object_message_new(
4445
return NULL;
4546
}
4647

47-
if (part_number > 0) {
48+
if (part_number > 0 && has_range) {
4849
if (s_s3_message_util_add_content_range_header(part_number - 1, part_size, message)) {
4950
goto error_clean_up;
5051
}

‎tests/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ add_net_test_case(test_s3_get_object_tls_disabled)
3030
add_net_test_case(test_s3_get_object_tls_enabled)
3131
add_net_test_case(test_s3_get_object_tls_default)
3232
add_net_test_case(test_s3_get_object_less_than_part_size)
33+
add_net_test_case(test_s3_get_object_empty_object)
3334
add_net_test_case(test_s3_get_object_multiple)
3435
add_net_test_case(test_s3_get_object_sse_kms)
3536
add_net_test_case(test_s3_get_object_sse_aes256)
@@ -39,7 +40,9 @@ add_net_test_case(test_s3_put_object_tls_disabled)
3940
add_net_test_case(test_s3_put_object_tls_enabled)
4041
add_net_test_case(test_s3_put_object_tls_default)
4142
add_net_test_case(test_s3_multipart_put_object_with_acl)
43+
add_net_test_case(test_s3_put_object_multiple)
4244
add_net_test_case(test_s3_put_object_less_than_part_size)
45+
add_net_test_case(test_s3_put_object_empty_object)
4346
add_net_test_case(test_s3_put_object_with_part_remainder)
4447
add_net_test_case(test_s3_put_object_sse_kms)
4548
add_net_test_case(test_s3_put_object_sse_kms_multipart)

0 commit comments

Comments
 (0)