11/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
22#include <fluent-bit.h>
3+ #include <fluent-bit/aws/flb_aws_compress.h>
34#include <fluent-bit/flb_time.h>
45#include <cfl/cfl_atomic.h>
56#include "../../plugins/out_s3/s3.h"
@@ -2032,7 +2033,7 @@ void flb_test_s3_empty_upload_queue_file_deleted(void)
20322033 flb_free (store_dir );
20332034}
20342035
2035- void flb_test_s3_near_full_buffer_append_succeeds ( void )
2036+ static void check_near_full_buffer ( int uploader )
20362037{
20372038 int ret ;
20382039 int in_ffd ;
@@ -2070,30 +2071,37 @@ void flb_test_s3_near_full_buffer_append_succeeds(void)
20702071 flb_output_set (ctx , out_ffd , "store_dir" , store_dir , NULL );
20712072 flb_output_set (ctx , out_ffd , "store_dir_limit_size" , "100" , NULL );
20722073
2074+ flb_output_set (ctx , out_ffd , "throttle" , uploader ? "true" : "false" , NULL );
20732075 ret = flb_start (ctx );
20742076 TEST_CHECK (ret == 0 );
20752077
20762078 s3_ctx = get_s3_context (ctx );
20772079 TEST_CHECK (s3_ctx != NULL );
2080+ pthread_mutex_lock (& s3_ctx -> files_mutex );
20782081 ret = s3_store_buffer_put (s3_ctx , NULL , "test" , 4 , payload ,
2079- sizeof (payload ), time (NULL ));
2082+ sizeof (payload ) - 1 , time (NULL ));
20802083 TEST_CHECK_ (ret == 0 ,
20812084 "Expected a committed near-full append to return success, got %d" ,
20822085 ret );
2083- TEST_CHECK (cfl_atomic_load (& s3_ctx -> current_buffer_size ) == sizeof (payload ));
2084-
20852086 s3_file = s3_store_file_get (s3_ctx , "test" , 4 );
20862087 TEST_CHECK (s3_file != NULL );
2088+ TEST_CHECK (cio_chunk_down (s3_file -> fsf -> chunk ) == CIO_OK );
2089+ ret = s3_store_buffer_put (s3_ctx , s3_file , "test" , 4 , payload , 1 , time (NULL ));
2090+ TEST_CHECK (ret == 0 );
2091+ TEST_CHECK (cio_chunk_is_up (s3_file -> fsf -> chunk ) == CIO_FALSE );
20872092 TEST_CHECK (s3_store_file_size_get (s3_file ) == sizeof (payload ));
20882093
20892094 ret = s3_store_buffer_put (s3_ctx , s3_file , "test" , 4 , excess_payload ,
20902095 sizeof (excess_payload ), time (NULL ));
2091- TEST_CHECK_ (ret == -1 ,
2096+ TEST_CHECK_ (ret == ( uploader ? S3_STORE_BUFFER_FULL : -1 ) ,
20922097 "Expected an over-limit append to fail, got %d" ,
20932098 ret );
20942099 TEST_CHECK (cfl_atomic_load (& s3_ctx -> current_buffer_size ) == sizeof (payload ));
20952100 TEST_CHECK (s3_store_file_size_get (s3_file ) == sizeof (payload ));
20962101
2102+ TEST_CHECK (s3_file -> sealed == uploader );
2103+ pthread_mutex_unlock (& s3_ctx -> files_mutex );
2104+
20972105 flb_stop (ctx );
20982106 flb_destroy (ctx );
20992107
@@ -2102,6 +2110,16 @@ void flb_test_s3_near_full_buffer_append_succeeds(void)
21022110 flb_free (store_dir );
21032111}
21042112
2113+ void flb_test_s3_near_full_buffer_append_succeeds (void )
2114+ {
2115+ check_near_full_buffer (FLB_FALSE );
2116+ }
2117+
2118+ static void flb_test_s3_uploader_seals_full_buffer (void )
2119+ {
2120+ check_near_full_buffer (FLB_TRUE );
2121+ }
2122+
21052123void flb_test_s3_startup_index_order_after_failure (void )
21062124{
21072125 int ret ;
@@ -2494,17 +2512,11 @@ static struct flb_http_client *s3_test_concurrent_request(struct flb_aws_client
24942512
24952513static void * s3_test_request_worker (void * data )
24962514{
2497- #ifdef S3_UPLOAD_DEFERRED
24982515 int deferred ;
2499- #endif
25002516 struct flb_s3 * ctx = data ;
25012517
25022518 pthread_mutex_lock (& ctx -> files_mutex );
2503- #ifdef S3_UPLOAD_DEFERRED
25042519 s3_request (ctx , FLB_HTTP_PUT , "/test" , NULL , 0 , NULL , 0 , & deferred );
2505- #else
2506- s3_request (ctx , FLB_HTTP_PUT , "/test" , NULL , 0 , NULL , 0 );
2507- #endif
25082520 pthread_mutex_unlock (& ctx -> files_mutex );
25092521 return NULL ;
25102522}
@@ -2556,8 +2568,210 @@ static void flb_test_s3_credentials_serialized(void)
25562568 pthread_mutex_destroy (& credentials_test .mutex );
25572569}
25582570
2571+ static struct flb_http_client * check_last_part_request (struct flb_s3 * ctx , int method ,
2572+ const char * uri , const char * body , size_t size ,
2573+ struct flb_aws_header * headers , size_t count ,
2574+ int * deferred )
2575+ {
2576+ TEST_CHECK (strstr (body , "<Part><ETag>last-part</ETag><PartNumber>10000</PartNumber></Part>" ) != NULL );
2577+ return mock_s3_call ("TEST_COMPLETE_MULTIPART_UPLOAD_ERROR" , "CompleteMultipartUpload" );
2578+ }
2579+
2580+ static void flb_test_s3_restore_last_multipart_part (void )
2581+ {
2582+ flb_ctx_t * ctx ;
2583+ struct flb_s3 * s3 ;
2584+ struct multipart_upload * upload ;
2585+ struct flb_fstore_file * manifest ;
2586+ flb_sds_t key ;
2587+ flb_sds_t part ;
2588+ char * dir ;
2589+ int out ;
2590+
2591+ dir = create_test_store_directory ("/flb-s3-manifest-XXXXXX" );
2592+ TEST_ASSERT (dir != NULL );
2593+ flb_s3_test_env_set ("FLB_S3_PLUGIN_UNDER_TEST" , "true" , 1 );
2594+ ctx = flb_create ();
2595+ flb_input (ctx , "lib" , NULL );
2596+ out = flb_output (ctx , "s3" , NULL );
2597+ flb_output_set (ctx , out , "match" , "*" , "region" , "us-west-2" , "bucket" , "fluent" ,
2598+ "throttle" , "true" , "store_dir" , dir , NULL );
2599+ TEST_ASSERT (flb_start (ctx ) == 0 );
2600+ s3 = get_s3_context (ctx );
2601+ pthread_mutex_lock (& s3 -> files_mutex );
2602+ key = flb_sds_create ("/test-key\ntest-id" );
2603+ part = flb_sds_create ("part_number=10000\tetag=last-part\n" );
2604+ TEST_ASSERT (s3_store_file_upload_put (s3 , NULL , key , part , NULL ) == 0 );
2605+ manifest = s3_store_file_upload_get (s3 , key , flb_sds_len (key ));
2606+ TEST_ASSERT (manifest != NULL );
2607+ TEST_CHECK (cio_chunk_down (manifest -> chunk ) == CIO_OK );
2608+ TEST_CHECK (s3_store_file_upload_put (s3 , manifest , key , part , NULL ) == 0 );
2609+ TEST_CHECK (cio_chunk_is_up (manifest -> chunk ) == CIO_FALSE );
2610+
2611+ TEST_ASSERT (sqlite3_exec (s3 -> uploader_db , "ALTER TABLE retry RENAME TO unavailable" ,
2612+ NULL , NULL , NULL ) == SQLITE_OK );
2613+ TEST_CHECK (multipart_read_uploads_from_fs (s3 ) < 0 );
2614+ TEST_CHECK (mk_list_size (& s3 -> uploads ) == 0 );
2615+ TEST_ASSERT (sqlite3_exec (s3 -> uploader_db , "ALTER TABLE unavailable RENAME TO retry" ,
2616+ NULL , NULL , NULL ) == SQLITE_OK );
2617+ TEST_CHECK (s3_uploader_storage_recover (s3 ) == 0 );
2618+ TEST_ASSERT (multipart_read_uploads_from_fs (s3 ) == 0 );
2619+ upload = mk_list_entry_first (& s3 -> uploads , struct multipart_upload , _head );
2620+ TEST_CHECK (upload -> part_number == S3_MAX_PARTS + 1 );
2621+ TEST_CHECK (strcmp (upload -> etags [S3_MAX_PARTS - 1 ], "last-part" ) == 0 );
2622+ upload -> request = check_last_part_request ;
2623+ flb_s3_test_env_unset ("FLB_S3_PLUGIN_UNDER_TEST" );
2624+ TEST_CHECK (complete_multipart_upload (s3 , upload , NULL ) == 0 );
2625+ flb_s3_test_env_set ("FLB_S3_PLUGIN_UNDER_TEST" , "true" , 1 );
2626+ mk_list_del (& upload -> _head );
2627+ multipart_upload_destroy (upload );
2628+ flb_sds_destroy (part );
2629+ part = flb_sds_create ("part_number=2\tetag=\n" );
2630+ TEST_ASSERT (s3_store_file_upload_put (s3 , NULL , key , part , NULL ) == 0 );
2631+ TEST_CHECK (multipart_read_uploads_from_fs (s3 ) < 0 );
2632+ TEST_CHECK (mk_list_size (& s3 -> uploads ) == 0 );
2633+ flb_sds_destroy (part );
2634+ flb_sds_destroy (key );
2635+ pthread_mutex_unlock (& s3 -> files_mutex );
2636+ flb_stop (ctx );
2637+ flb_destroy (ctx );
2638+ flb_s3_test_env_unset ("FLB_S3_PLUGIN_UNDER_TEST" );
2639+ flb_free (dir );
2640+ }
2641+
2642+ static void flb_test_s3_retry_accepted_part_checkpoint (void )
2643+ {
2644+ flb_ctx_t * ctx ;
2645+ struct flb_s3 * s3 ;
2646+ struct multipart_upload * upload ;
2647+ struct flb_fstore_file * manifest ;
2648+ char held_dir [PATH_MAX ];
2649+ flb_sds_t key ;
2650+ flb_sds_t part ;
2651+ char * dir ;
2652+ int calls ;
2653+ int out ;
2654+
2655+ dir = create_test_store_directory ("/flb-s3-part-checkpoint-XXXXXX" );
2656+ TEST_ASSERT (dir != NULL );
2657+ flb_s3_test_env_set ("FLB_S3_PLUGIN_UNDER_TEST" , "true" , 1 );
2658+ ctx = flb_create ();
2659+ flb_input (ctx , "lib" , NULL );
2660+ out = flb_output (ctx , "s3" , NULL );
2661+ flb_output_set (ctx , out , "match" , "*" , "region" , "us-west-2" , "bucket" , "fluent" ,
2662+ "throttle" , "true" , "retry_limit" , "0" , "store_dir" , dir , NULL );
2663+ TEST_ASSERT (flb_start (ctx ) == 0 );
2664+ s3 = get_s3_context (ctx );
2665+ pthread_mutex_lock (& s3 -> files_mutex );
2666+ key = flb_sds_create ("/test-key\ntest-id" );
2667+ part = flb_sds_create ("part_number=1\tetag=first-part\n" );
2668+ TEST_ASSERT (s3_store_file_upload_put (s3 , NULL , key , part , NULL ) == 0 );
2669+ TEST_ASSERT (multipart_read_uploads_from_fs (s3 ) == 0 );
2670+ upload = mk_list_entry_first (& s3 -> uploads , struct multipart_upload , _head );
2671+ manifest = s3_store_file_upload_get (s3 , key , flb_sds_len (key ));
2672+ TEST_ASSERT (manifest != NULL );
2673+ if (cio_chunk_is_up (manifest -> chunk )) {
2674+ TEST_ASSERT (cio_chunk_down (manifest -> chunk ) == CIO_OK );
2675+ }
2676+ snprintf (held_dir , sizeof (held_dir ), "%s.held" , s3 -> stream_upload -> path );
2677+ TEST_ASSERT (rename (s3 -> stream_upload -> path , held_dir ) == 0 );
2678+ calls = get_s3_call_count ("UploadPart" );
2679+ TEST_CHECK (upload_part (s3 , upload , "body" , 4 , NULL ) == S3_UPLOAD_DEFERRED );
2680+ TEST_CHECK (s3_uploader_storage_recover (s3 ) < 0 );
2681+ TEST_CHECK (upload -> part_number == 2 );
2682+ TEST_CHECK (upload -> etags [1 ] == NULL );
2683+ TEST_ASSERT (rename (held_dir , s3 -> stream_upload -> path ) == 0 );
2684+ TEST_CHECK (s3_uploader_storage_recover (s3 ) == 0 );
2685+ TEST_CHECK (upload -> part_number == 3 );
2686+ TEST_CHECK (upload -> etags [1 ] != NULL );
2687+ TEST_CHECK (upload -> bytes == 4 );
2688+ TEST_CHECK (get_s3_call_count ("UploadPart" ) == calls + 1 );
2689+ mk_list_del (& upload -> _head );
2690+ multipart_upload_destroy (upload );
2691+ TEST_ASSERT (multipart_read_uploads_from_fs (s3 ) == 0 );
2692+ upload = mk_list_entry_first (& s3 -> uploads , struct multipart_upload , _head );
2693+ TEST_CHECK (upload -> part_number == 3 );
2694+ TEST_CHECK (upload -> etags [1 ] != NULL );
2695+ flb_sds_destroy (part );
2696+ flb_sds_destroy (key );
2697+ pthread_mutex_unlock (& s3 -> files_mutex );
2698+ flb_stop (ctx );
2699+ flb_destroy (ctx );
2700+ flb_s3_test_env_unset ("FLB_S3_PLUGIN_UNDER_TEST" );
2701+ flb_free (dir );
2702+ }
2703+
2704+ static void flb_test_s3_uploader_spool_format_limit (void )
2705+ {
2706+ struct flb_s3 ctx = {0 };
2707+
2708+ ctx .compression = FLB_AWS_COMPRESS_GZIP ;
2709+ ctx .upload_chunk_size = 4294000000ULL ;
2710+ TEST_CHECK (!s3_uploader_should_seal (& ctx , 4293000000ULL , UINT32_MAX - 4293000000ULL ));
2711+ TEST_CHECK (s3_uploader_should_seal (& ctx , 4293000000ULL , UINT32_MAX - 4293000000ULL + 1 ));
2712+ TEST_CHECK (s3_uploader_should_seal (& ctx , 4293000000ULL , 3000000 ));
2713+ TEST_CHECK (s3_uploader_should_seal (& ctx , ctx .upload_chunk_size , 1 ));
2714+ }
2715+
2716+ static void flb_test_s3_uploader_restored_deadline (void )
2717+ {
2718+ struct flb_s3 ctx = {0 };
2719+ struct flb_output_instance ins = {0 };
2720+ struct s3_retry_state state = {0 };
2721+ char * dir ;
2722+ uint64_t before ;
2723+ int ret ;
2724+
2725+ dir = create_test_store_directory ("/flb-s3-gate-XXXXXX" );
2726+ TEST_CHECK (dir != NULL );
2727+ if (dir == NULL ) {
2728+ return ;
2729+ }
2730+ ctx .buffer_dir = dir ;
2731+ ctx .ins = & ins ;
2732+ ins .throttle .enabled = FLB_TRUE ;
2733+ ins .throttle .base_ms = 100 ;
2734+ ins .throttle .cap_ms = 1000 ;
2735+ mk_list_init (& ctx .pending_checkpoints );
2736+ ret = s3_uploader_init (& ctx );
2737+ TEST_CHECK (ret == 0 );
2738+ if (ret == 0 ) {
2739+ state .until_ms = INT64_MAX ;
2740+ TEST_CHECK (s3_uploader_state_write (& ctx , "gate" , "" , & state ) == 0 );
2741+ s3_uploader_destroy (& ctx );
2742+ before = s3_uploader_wall_ms ();
2743+ ret = s3_uploader_init (& ctx );
2744+ TEST_CHECK (ret == 0 );
2745+ if (ret == 0 ) {
2746+ TEST_CHECK (ctx .uploader_gate .until_ms - ctx .uploader_gate .started_ms == 1000 );
2747+ TEST_CHECK (s3_uploader_state_read (& ctx , "gate" , "" , & state ) == 0 );
2748+ TEST_CHECK (state .until_ms >= before + 1000 );
2749+ TEST_CHECK (state .until_ms <= s3_uploader_wall_ms () + 1000 );
2750+ state .until_ms = INT64_MAX ;
2751+ state .failures = 2 ;
2752+ TEST_CHECK (s3_uploader_state_write (& ctx , "complete" , "old" , & state ) == 0 );
2753+ TEST_CHECK (s3_uploader_state_write (& ctx , "chunk" , "old" , & state ) == 0 );
2754+ before = flb_output_throttle_now_ms ();
2755+ TEST_CHECK (s3_uploader_state_read (& ctx , "complete" , "old" , & state ) == 0 );
2756+ TEST_CHECK (state .retry_at >= before + 4000 );
2757+ TEST_CHECK (state .retry_at <= flb_output_throttle_now_ms () + 4000 );
2758+ state .retry_at = 0 ;
2759+ TEST_CHECK (s3_uploader_state_read (& ctx , "chunk" , "old" , & state ) == 0 );
2760+ TEST_CHECK (state .retry_at >= before + 4000 );
2761+ TEST_CHECK (state .retry_at <= flb_output_throttle_now_ms () + 4000 );
2762+ }
2763+ }
2764+ s3_uploader_destroy (& ctx );
2765+ flb_free (dir );
2766+ }
2767+
25592768/* Test list */
25602769TEST_LIST = {
2770+ {"uploader_spool_format_limit" , flb_test_s3_uploader_spool_format_limit },
2771+ {"retry_accepted_part_checkpoint" , flb_test_s3_retry_accepted_part_checkpoint },
2772+ {"restore_last_multipart_part" , flb_test_s3_restore_last_multipart_part },
2773+ {"uploader_seals_full_buffer" , flb_test_s3_uploader_seals_full_buffer },
2774+ {"uploader_restored_deadline" , flb_test_s3_uploader_restored_deadline },
25612775 {"hostname_verification_propagates" , flb_test_s3_hostname_verification_propagates },
25622776 {"credentials_serialized" , flb_test_s3_credentials_serialized },
25632777 {"multipart_success" , flb_test_s3_multipart_success },
0 commit comments