Skip to content

Commit e0b2c8d

Browse files
committed
tests: runtime: cover restored S3 uploader deadlines
Signed-off-by: Hiroshi Hatake <hiroshi@chronosphere.io>
1 parent 3ae5086 commit e0b2c8d

1 file changed

Lines changed: 225 additions & 11 deletions

File tree

‎tests/runtime/out_s3.c‎

Lines changed: 225 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
22
#include <fluent-bit.h>
3+
#include <fluent-bit/aws/flb_aws_compress.h>
34
#include <fluent-bit/flb_time.h>
45
#include <cfl/cfl_atomic.h>
56
#include "../../plugins/out_s3/s3.h"
@@ -2032,7 +2033,7 @@ void flb_test_s3_empty_upload_queue_file_deleted(void)
20322033
flb_free(store_dir);
20332034
}
20342035

2035-
void flb_test_s3_near_full_buffer_append_succeeds(void)
2036+
static void check_near_full_buffer(int uploader)
20362037
{
20372038
int ret;
20382039
int in_ffd;
@@ -2070,30 +2071,37 @@ void flb_test_s3_near_full_buffer_append_succeeds(void)
20702071
flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL);
20712072
flb_output_set(ctx, out_ffd, "store_dir_limit_size", "100", NULL);
20722073

2074+
flb_output_set(ctx, out_ffd, "throttle", uploader ? "true" : "false", NULL);
20732075
ret = flb_start(ctx);
20742076
TEST_CHECK(ret == 0);
20752077

20762078
s3_ctx = get_s3_context(ctx);
20772079
TEST_CHECK(s3_ctx != NULL);
2080+
pthread_mutex_lock(&s3_ctx->files_mutex);
20782081
ret = s3_store_buffer_put(s3_ctx, NULL, "test", 4, payload,
2079-
sizeof(payload), time(NULL));
2082+
sizeof(payload) - 1, time(NULL));
20802083
TEST_CHECK_(ret == 0,
20812084
"Expected a committed near-full append to return success, got %d",
20822085
ret);
2083-
TEST_CHECK(cfl_atomic_load(&s3_ctx->current_buffer_size) == sizeof(payload));
2084-
20852086
s3_file = s3_store_file_get(s3_ctx, "test", 4);
20862087
TEST_CHECK(s3_file != NULL);
2088+
TEST_CHECK(cio_chunk_down(s3_file->fsf->chunk) == CIO_OK);
2089+
ret = s3_store_buffer_put(s3_ctx, s3_file, "test", 4, payload, 1, time(NULL));
2090+
TEST_CHECK(ret == 0);
2091+
TEST_CHECK(cio_chunk_is_up(s3_file->fsf->chunk) == CIO_FALSE);
20872092
TEST_CHECK(s3_store_file_size_get(s3_file) == sizeof(payload));
20882093

20892094
ret = s3_store_buffer_put(s3_ctx, s3_file, "test", 4, excess_payload,
20902095
sizeof(excess_payload), time(NULL));
2091-
TEST_CHECK_(ret == -1,
2096+
TEST_CHECK_(ret == (uploader ? S3_STORE_BUFFER_FULL : -1),
20922097
"Expected an over-limit append to fail, got %d",
20932098
ret);
20942099
TEST_CHECK(cfl_atomic_load(&s3_ctx->current_buffer_size) == sizeof(payload));
20952100
TEST_CHECK(s3_store_file_size_get(s3_file) == sizeof(payload));
20962101

2102+
TEST_CHECK(s3_file->sealed == uploader);
2103+
pthread_mutex_unlock(&s3_ctx->files_mutex);
2104+
20972105
flb_stop(ctx);
20982106
flb_destroy(ctx);
20992107

@@ -2102,6 +2110,16 @@ void flb_test_s3_near_full_buffer_append_succeeds(void)
21022110
flb_free(store_dir);
21032111
}
21042112

2113+
void flb_test_s3_near_full_buffer_append_succeeds(void)
2114+
{
2115+
check_near_full_buffer(FLB_FALSE);
2116+
}
2117+
2118+
static void flb_test_s3_uploader_seals_full_buffer(void)
2119+
{
2120+
check_near_full_buffer(FLB_TRUE);
2121+
}
2122+
21052123
void flb_test_s3_startup_index_order_after_failure(void)
21062124
{
21072125
int ret;
@@ -2494,17 +2512,11 @@ static struct flb_http_client *s3_test_concurrent_request(struct flb_aws_client
24942512

24952513
static void *s3_test_request_worker(void *data)
24962514
{
2497-
#ifdef S3_UPLOAD_DEFERRED
24982515
int deferred;
2499-
#endif
25002516
struct flb_s3 *ctx = data;
25012517

25022518
pthread_mutex_lock(&ctx->files_mutex);
2503-
#ifdef S3_UPLOAD_DEFERRED
25042519
s3_request(ctx, FLB_HTTP_PUT, "/test", NULL, 0, NULL, 0, &deferred);
2505-
#else
2506-
s3_request(ctx, FLB_HTTP_PUT, "/test", NULL, 0, NULL, 0);
2507-
#endif
25082520
pthread_mutex_unlock(&ctx->files_mutex);
25092521
return NULL;
25102522
}
@@ -2556,8 +2568,210 @@ static void flb_test_s3_credentials_serialized(void)
25562568
pthread_mutex_destroy(&credentials_test.mutex);
25572569
}
25582570

2571+
static struct flb_http_client *check_last_part_request(struct flb_s3 *ctx, int method,
2572+
const char *uri, const char *body, size_t size,
2573+
struct flb_aws_header *headers, size_t count,
2574+
int *deferred)
2575+
{
2576+
TEST_CHECK(strstr(body, "<Part><ETag>last-part</ETag><PartNumber>10000</PartNumber></Part>") != NULL);
2577+
return mock_s3_call("TEST_COMPLETE_MULTIPART_UPLOAD_ERROR", "CompleteMultipartUpload");
2578+
}
2579+
2580+
static void flb_test_s3_restore_last_multipart_part(void)
2581+
{
2582+
flb_ctx_t *ctx;
2583+
struct flb_s3 *s3;
2584+
struct multipart_upload *upload;
2585+
struct flb_fstore_file *manifest;
2586+
flb_sds_t key;
2587+
flb_sds_t part;
2588+
char *dir;
2589+
int out;
2590+
2591+
dir = create_test_store_directory("/flb-s3-manifest-XXXXXX");
2592+
TEST_ASSERT(dir != NULL);
2593+
flb_s3_test_env_set("FLB_S3_PLUGIN_UNDER_TEST", "true", 1);
2594+
ctx = flb_create();
2595+
flb_input(ctx, "lib", NULL);
2596+
out = flb_output(ctx, "s3", NULL);
2597+
flb_output_set(ctx, out, "match", "*", "region", "us-west-2", "bucket", "fluent",
2598+
"throttle", "true", "store_dir", dir, NULL);
2599+
TEST_ASSERT(flb_start(ctx) == 0);
2600+
s3 = get_s3_context(ctx);
2601+
pthread_mutex_lock(&s3->files_mutex);
2602+
key = flb_sds_create("/test-key\ntest-id");
2603+
part = flb_sds_create("part_number=10000\tetag=last-part\n");
2604+
TEST_ASSERT(s3_store_file_upload_put(s3, NULL, key, part, NULL) == 0);
2605+
manifest = s3_store_file_upload_get(s3, key, flb_sds_len(key));
2606+
TEST_ASSERT(manifest != NULL);
2607+
TEST_CHECK(cio_chunk_down(manifest->chunk) == CIO_OK);
2608+
TEST_CHECK(s3_store_file_upload_put(s3, manifest, key, part, NULL) == 0);
2609+
TEST_CHECK(cio_chunk_is_up(manifest->chunk) == CIO_FALSE);
2610+
2611+
TEST_ASSERT(sqlite3_exec(s3->uploader_db, "ALTER TABLE retry RENAME TO unavailable",
2612+
NULL, NULL, NULL) == SQLITE_OK);
2613+
TEST_CHECK(multipart_read_uploads_from_fs(s3) < 0);
2614+
TEST_CHECK(mk_list_size(&s3->uploads) == 0);
2615+
TEST_ASSERT(sqlite3_exec(s3->uploader_db, "ALTER TABLE unavailable RENAME TO retry",
2616+
NULL, NULL, NULL) == SQLITE_OK);
2617+
TEST_CHECK(s3_uploader_storage_recover(s3) == 0);
2618+
TEST_ASSERT(multipart_read_uploads_from_fs(s3) == 0);
2619+
upload = mk_list_entry_first(&s3->uploads, struct multipart_upload, _head);
2620+
TEST_CHECK(upload->part_number == S3_MAX_PARTS + 1);
2621+
TEST_CHECK(strcmp(upload->etags[S3_MAX_PARTS - 1], "last-part") == 0);
2622+
upload->request = check_last_part_request;
2623+
flb_s3_test_env_unset("FLB_S3_PLUGIN_UNDER_TEST");
2624+
TEST_CHECK(complete_multipart_upload(s3, upload, NULL) == 0);
2625+
flb_s3_test_env_set("FLB_S3_PLUGIN_UNDER_TEST", "true", 1);
2626+
mk_list_del(&upload->_head);
2627+
multipart_upload_destroy(upload);
2628+
flb_sds_destroy(part);
2629+
part = flb_sds_create("part_number=2\tetag=\n");
2630+
TEST_ASSERT(s3_store_file_upload_put(s3, NULL, key, part, NULL) == 0);
2631+
TEST_CHECK(multipart_read_uploads_from_fs(s3) < 0);
2632+
TEST_CHECK(mk_list_size(&s3->uploads) == 0);
2633+
flb_sds_destroy(part);
2634+
flb_sds_destroy(key);
2635+
pthread_mutex_unlock(&s3->files_mutex);
2636+
flb_stop(ctx);
2637+
flb_destroy(ctx);
2638+
flb_s3_test_env_unset("FLB_S3_PLUGIN_UNDER_TEST");
2639+
flb_free(dir);
2640+
}
2641+
2642+
static void flb_test_s3_retry_accepted_part_checkpoint(void)
2643+
{
2644+
flb_ctx_t *ctx;
2645+
struct flb_s3 *s3;
2646+
struct multipart_upload *upload;
2647+
struct flb_fstore_file *manifest;
2648+
char held_dir[PATH_MAX];
2649+
flb_sds_t key;
2650+
flb_sds_t part;
2651+
char *dir;
2652+
int calls;
2653+
int out;
2654+
2655+
dir = create_test_store_directory("/flb-s3-part-checkpoint-XXXXXX");
2656+
TEST_ASSERT(dir != NULL);
2657+
flb_s3_test_env_set("FLB_S3_PLUGIN_UNDER_TEST", "true", 1);
2658+
ctx = flb_create();
2659+
flb_input(ctx, "lib", NULL);
2660+
out = flb_output(ctx, "s3", NULL);
2661+
flb_output_set(ctx, out, "match", "*", "region", "us-west-2", "bucket", "fluent",
2662+
"throttle", "true", "retry_limit", "0", "store_dir", dir, NULL);
2663+
TEST_ASSERT(flb_start(ctx) == 0);
2664+
s3 = get_s3_context(ctx);
2665+
pthread_mutex_lock(&s3->files_mutex);
2666+
key = flb_sds_create("/test-key\ntest-id");
2667+
part = flb_sds_create("part_number=1\tetag=first-part\n");
2668+
TEST_ASSERT(s3_store_file_upload_put(s3, NULL, key, part, NULL) == 0);
2669+
TEST_ASSERT(multipart_read_uploads_from_fs(s3) == 0);
2670+
upload = mk_list_entry_first(&s3->uploads, struct multipart_upload, _head);
2671+
manifest = s3_store_file_upload_get(s3, key, flb_sds_len(key));
2672+
TEST_ASSERT(manifest != NULL);
2673+
if (cio_chunk_is_up(manifest->chunk)) {
2674+
TEST_ASSERT(cio_chunk_down(manifest->chunk) == CIO_OK);
2675+
}
2676+
snprintf(held_dir, sizeof(held_dir), "%s.held", s3->stream_upload->path);
2677+
TEST_ASSERT(rename(s3->stream_upload->path, held_dir) == 0);
2678+
calls = get_s3_call_count("UploadPart");
2679+
TEST_CHECK(upload_part(s3, upload, "body", 4, NULL) == S3_UPLOAD_DEFERRED);
2680+
TEST_CHECK(s3_uploader_storage_recover(s3) < 0);
2681+
TEST_CHECK(upload->part_number == 2);
2682+
TEST_CHECK(upload->etags[1] == NULL);
2683+
TEST_ASSERT(rename(held_dir, s3->stream_upload->path) == 0);
2684+
TEST_CHECK(s3_uploader_storage_recover(s3) == 0);
2685+
TEST_CHECK(upload->part_number == 3);
2686+
TEST_CHECK(upload->etags[1] != NULL);
2687+
TEST_CHECK(upload->bytes == 4);
2688+
TEST_CHECK(get_s3_call_count("UploadPart") == calls + 1);
2689+
mk_list_del(&upload->_head);
2690+
multipart_upload_destroy(upload);
2691+
TEST_ASSERT(multipart_read_uploads_from_fs(s3) == 0);
2692+
upload = mk_list_entry_first(&s3->uploads, struct multipart_upload, _head);
2693+
TEST_CHECK(upload->part_number == 3);
2694+
TEST_CHECK(upload->etags[1] != NULL);
2695+
flb_sds_destroy(part);
2696+
flb_sds_destroy(key);
2697+
pthread_mutex_unlock(&s3->files_mutex);
2698+
flb_stop(ctx);
2699+
flb_destroy(ctx);
2700+
flb_s3_test_env_unset("FLB_S3_PLUGIN_UNDER_TEST");
2701+
flb_free(dir);
2702+
}
2703+
2704+
static void flb_test_s3_uploader_spool_format_limit(void)
2705+
{
2706+
struct flb_s3 ctx = {0};
2707+
2708+
ctx.compression = FLB_AWS_COMPRESS_GZIP;
2709+
ctx.upload_chunk_size = 4294000000ULL;
2710+
TEST_CHECK(!s3_uploader_should_seal(&ctx, 4293000000ULL, UINT32_MAX - 4293000000ULL));
2711+
TEST_CHECK(s3_uploader_should_seal(&ctx, 4293000000ULL, UINT32_MAX - 4293000000ULL + 1));
2712+
TEST_CHECK(s3_uploader_should_seal(&ctx, 4293000000ULL, 3000000));
2713+
TEST_CHECK(s3_uploader_should_seal(&ctx, ctx.upload_chunk_size, 1));
2714+
}
2715+
2716+
static void flb_test_s3_uploader_restored_deadline(void)
2717+
{
2718+
struct flb_s3 ctx = {0};
2719+
struct flb_output_instance ins = {0};
2720+
struct s3_retry_state state = {0};
2721+
char *dir;
2722+
uint64_t before;
2723+
int ret;
2724+
2725+
dir = create_test_store_directory("/flb-s3-gate-XXXXXX");
2726+
TEST_CHECK(dir != NULL);
2727+
if (dir == NULL) {
2728+
return;
2729+
}
2730+
ctx.buffer_dir = dir;
2731+
ctx.ins = &ins;
2732+
ins.throttle.enabled = FLB_TRUE;
2733+
ins.throttle.base_ms = 100;
2734+
ins.throttle.cap_ms = 1000;
2735+
mk_list_init(&ctx.pending_checkpoints);
2736+
ret = s3_uploader_init(&ctx);
2737+
TEST_CHECK(ret == 0);
2738+
if (ret == 0) {
2739+
state.until_ms = INT64_MAX;
2740+
TEST_CHECK(s3_uploader_state_write(&ctx, "gate", "", &state) == 0);
2741+
s3_uploader_destroy(&ctx);
2742+
before = s3_uploader_wall_ms();
2743+
ret = s3_uploader_init(&ctx);
2744+
TEST_CHECK(ret == 0);
2745+
if (ret == 0) {
2746+
TEST_CHECK(ctx.uploader_gate.until_ms - ctx.uploader_gate.started_ms == 1000);
2747+
TEST_CHECK(s3_uploader_state_read(&ctx, "gate", "", &state) == 0);
2748+
TEST_CHECK(state.until_ms >= before + 1000);
2749+
TEST_CHECK(state.until_ms <= s3_uploader_wall_ms() + 1000);
2750+
state.until_ms = INT64_MAX;
2751+
state.failures = 2;
2752+
TEST_CHECK(s3_uploader_state_write(&ctx, "complete", "old", &state) == 0);
2753+
TEST_CHECK(s3_uploader_state_write(&ctx, "chunk", "old", &state) == 0);
2754+
before = flb_output_throttle_now_ms();
2755+
TEST_CHECK(s3_uploader_state_read(&ctx, "complete", "old", &state) == 0);
2756+
TEST_CHECK(state.retry_at >= before + 4000);
2757+
TEST_CHECK(state.retry_at <= flb_output_throttle_now_ms() + 4000);
2758+
state.retry_at = 0;
2759+
TEST_CHECK(s3_uploader_state_read(&ctx, "chunk", "old", &state) == 0);
2760+
TEST_CHECK(state.retry_at >= before + 4000);
2761+
TEST_CHECK(state.retry_at <= flb_output_throttle_now_ms() + 4000);
2762+
}
2763+
}
2764+
s3_uploader_destroy(&ctx);
2765+
flb_free(dir);
2766+
}
2767+
25592768
/* Test list */
25602769
TEST_LIST = {
2770+
{"uploader_spool_format_limit", flb_test_s3_uploader_spool_format_limit},
2771+
{"retry_accepted_part_checkpoint", flb_test_s3_retry_accepted_part_checkpoint},
2772+
{"restore_last_multipart_part", flb_test_s3_restore_last_multipart_part},
2773+
{"uploader_seals_full_buffer", flb_test_s3_uploader_seals_full_buffer},
2774+
{"uploader_restored_deadline", flb_test_s3_uploader_restored_deadline},
25612775
{"hostname_verification_propagates", flb_test_s3_hostname_verification_propagates },
25622776
{"credentials_serialized", flb_test_s3_credentials_serialized },
25632777
{"multipart_success", flb_test_s3_multipart_success },

0 commit comments

Comments
 (0)