Skip to content

Flaky TestDistributor_Push_ShouldSupportIngestStorage test #10693

Open
@56quarters

Description

@56quarters
 --- FAIL: TestDistributor_Push_ShouldSupportIngestStorage (0.00s)
    --- FAIL: TestDistributor_Push_ShouldSupportIngestStorage/should_return_gRPC_error_if_writing_to_1_out_of_N_partitions_fail_with_a_non-retryable_error (1.31s)
        distributor_ingest_storage_test.go:194: 
            	Error Trace:	/__w/mimir/mimir/pkg/distributor/distributor_ingest_storage_test.go:194
            	Error:      	Not equal: 
            	            	expected: map[int32][]string{0:[]string{"series_four", "series_one", "series_three"}, 2:[]string{"series_five"}}
            	            	actual  : map[int32][]string{}
            	            	
            	            	Diff:
            	            	--- Expected
            	            	+++ Actual
            	            	@@ -1,10 +1,2 @@
            	            	-(map[int32][]string) (len=2) {
            	            	- (int32) 0: ([]string) (len=3) {
            	            	-  (string) (len=11) "series_four",
            	            	-  (string) (len=10) "series_one",
            	            	-  (string) (len=12) "series_three"
            	            	- },
            	            	- (int32) 2: ([]string) (len=1) {
            	            	-  (string) (len=11) "series_five"
            	            	- }
            	            	+(map[int32][]string) {
            	            	 }
            	Test:       	TestDistributor_Push_ShouldSupportIngestStorage/should_return_gRPC_error_if_writing_to_1_out_of_N_partitions_fail_with_a_non-retryable_error
level=debug msg=Get key=prefixuser/cluster wait_index=0
level=debug msg="Get - not found" key=prefixuser/cluster
level=debug msg=CAS key=prefixuser/cluster modify_index=0 value="\"\\x15P\\n\\x05first\\x10\\x94\\xa4\\xe5\\xfa\\xd12 \\x94\\xa4\\xe5\\xfa\\xd12\""
level=debug msg=Get key=prefixuser/cluster wait_index=0
level=debug msg=Get key=prefixuser/cluster modify_index=2 value="\"\\x15P\\n\\x05first\\x10\\x94\\xa4\\xe5\\xfa\\xd12 \\x94\\xa4\\xe5\\xfa\\xd12\""
level=debug msg=CAS key=prefixuser/cluster modify_index=2 value="\"\\x18\\\\\\n\\x06second\\x10\\xa4\\xf2\\xe5\\xfa\\xd12 \\xa4\\xf2\\xe5\\xfa\\xd12(\\x01\""
level=debug msg=Get key=prefixuser/cluster wait_index=0
level=debug msg=Get key=prefixuser/cluster modify_index=3 value="\"\\x18\\\\\\n\\x06second\\x10\\xa4\\xf2\\xe5\\xfa\\xd12 \\xa4\\xf2\\xe5\\xfa\\xd12(\\x01\""
level=debug msg=CAS key=prefixuser/cluster modify_index=3 value="\"\\x17X\\n\\x05first\\x10̸\\xe6\\xfa\\xd12 \\xb4\\xc0\\xe6\\xfa\\xd12(\\x02\""
2025/02/19 17:55:02 label __name__ is overwritten. Check if Prometheus reserved labels are used.
2025/02/19 17:55:02 label __name__ is overwritten. Check if Prometheus reserved labels are used.
2025/02/19 17:55:02 label __name__ is overwritten. Check if Prometheus reserved labels are used.
2025/02/19 17:55:02 label __name__ is overwritten. Check if Prometheus reserved labels are used.
2025/02/19 17:55:02 label __name__ is overwritten. Check if Prometheus reserved labels are used.
level=info msg="server listening on addresses" http=127.0.0.1:40401 grpc=127.0.0.1:42765
level=warn method=/httpgrpc.HTTP/Handle duration=951.413µs request="&HTTPRequest{Method:POST,Url:/otlp,Headers:[]*Header{&Header{Key:Content-Type,Values:[application/json],},},Body:[105 110 118 97 108 105 100],}" msg=gRPC err="rpc error: code = Code(400) desc = ReadObjectCB: expect { or n, but found i, error found in #1 byte of ...|invalid|..., bigger context ...|invalid|..."
level=warn method=/httpgrpc.HTTP/Handle duration=561.847µs request="&HTTPRequest{Method:POST,Url:/otlp,Headers:[]*Header{&Header{Key:Content-Type,Values:[application/json],},},Body:[10 246 22 10 211 2 10 29 10 17 99 111 110 116 97 105 110 101 114 46 114 117 110 116 105 109 101 18 8 10 6 100 111 99 107 101 114 10 39 10 18 99 111 110 116 97 105 110 101 114 46 104],}" msg=gRPC err="rpc error: code = Code(400) desc = ReadObjectCB: expect { or n, but found \ufffd, error found in #2 byte of ...|\n\ufffd\u0016\n\ufffd\u0002\n\u001d\n\u0011co|..., bigger context ...|\n\ufffd\u0016\n\ufffd\u0002\n\u001d\n\u0011container.runtime\u0012\u0008\n\u0006docker\n'\n\u0012container.h|..."
level=warn method=/httpgrpc.HTTP/Handle duration=609.144µs request="&HTTPRequest{Method:POST,Url:/otlp,Headers:[]*Header{&Header{Key:Content-Type,Values:[application/json],},},Body:[123 34 114 101 115 111 117 114 99 101 77 101 116 114 105 99 115 34 58 32 91 123 34 115 99 111 112 101 77 101 116 114 105 99 115 34 58 32 91 123 34 109 101 116 114 105 99 115 34 58 32 91 123 34 110 97 109 101 34 58 32 34 114 101 112 111 114 116 95 115 101 114 118 101 114 95 101 114 114 111 114 34 44 32 34 103 97 117 103 101 34 58 32 123 34 100 97 116 97 80 111 105 110 116 115 34 58 32 91 123 34 116 105 109 101 85 110 105 120 78 97 110 111 34 58 32 34 49 54 55 57 57 49 50 52 54 51 51 52 48 48 48 48 48 48 48 34 44 32 34 97 115 68 111 117 98 108 101 34 58 32 49 48 46 54 54 125 93 125 125 93 125 93 125 93 125],}" msg=gRPC err="rpc error: code = Code(503) desc = some random push error"
level=warn method=/httpgrpc.HTTP/Handle duration=395.677µs request="&HTTPRequest{Method:POST,Url:/otlp,Headers:[]*Header{},Body:[104 101 108 108 111],}" msg=gRPC err="rpc error: code = Code(415) desc = unsupported content type: , supported: [application/json, application/x-protobuf]"
level=info msg="=== Handler.Stop()'d ==="

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions