Skip to content

Commit f5ab738

Browse files
joe-elliottbpfoster
joe-elliott
and
bpfoster
authored
Handle prefixes when listing blocks from S3 and GCS (#3466) (#3502)
* Handle prefixes when listing blocks from S3 fixes #3465 * Handle prefixes when listing blocks from GCS * Add test for prefixes when listing blocks from Azure * Update unit tests to check for actual block IDs instead of just length of the slices Cleanup unit tests * Further refine S3/GCS backend for ListBlocks Brings logic more in line with Azure object parsing. Also has the benefit of handling prefixes without a trailing slash. * Update poller integration test to exercise prefixes * Update e2e test to exercise prefixes * Fix format check error * Fix failing e2e tests * Remove unnecessary prefix permutations from e2e test * Remove unnecessary test config file copy * Ignore lint --------- Co-authored-by: Zach Leslie <[email protected]> (cherry picked from commit 8e6e7fe) Co-authored-by: Ben Foster <[email protected]>
1 parent ca6818a commit f5ab738

12 files changed

+646
-194
lines changed

‎CHANGELOG.md

+9
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
## main / unreleased
22

33
* [BUGFIX] Fix some instances where spanmetrics histograms could be inconsistent [#3412](https://github.com/grafana/tempo/pull/3412) (@mdisibio)
4+
* [ENHANCEMENT] Add string interning to TraceQL queries [#3411](https://github.com/grafana/tempo/pull/3411) (@mapno)
5+
* [ENHANCEMENT] Add new (unsafe) query hints for metrics queries [#3396](https://github.com/grafana/tempo/pull/3396) (@mdisibio)
6+
* [BUGFIX] Fix metrics query results when filtering and rating on the same attribute [#3428](https://github.com/grafana/tempo/issues/3428) (@mdisibio)
7+
* [BUGFIX] Fix metrics query results when series contain empty strings or nil values [#3429](https://github.com/grafana/tempo/issues/3429) (@mdisibio)
8+
* [BUGFIX] Fix metrics query duration check, add per-tenant override for max metrics query duration [#3479](https://github.com/grafana/tempo/issues/3479) (@mdisibio)
9+
* [BUGFIX] Return unfiltered results when a bad TraceQL query is provided in autocomplete. [#3426](https://github.com/grafana/tempo/pull/3426) (@mapno)
10+
* [BUGFIX] Correctly handle 429s in GRPC search streaming. [#3469](https://github.com/grafana/tempo/pull/3469) (@joe-ellitot)
11+
* [BUGFIX] Correctly cancel GRPC and HTTP contexts in the frontend to prevent having to rely on http write timeout. [#3443](https://github.com/grafana/tempo/pull/3443) (@joe-elliott)
12+
* [BUGFIX] Fix compaction/retention in AWS S3 and GCS when a prefix is configured. [#3465](https://github.com/grafana/tempo/issues/3465) (@bpfoster)
413

514
## v2.4.0-rc.0
615

‎integration/e2e/config-all-in-one-azurite.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ storage:
3737
endpoint_suffix: tempo_e2e-azurite:10000
3838
storage_account_name: "devstoreaccount1"
3939
storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
40+
prefix: {{ .Prefix }}
4041
pool:
4142
max_workers: 10
4243
queue_depth: 100

‎integration/e2e/config-all-in-one-gcs.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ storage:
3636
bucket_name: tempo
3737
endpoint: https://tempo_e2e-gcs:4443/storage/v1/
3838
insecure: true
39+
prefix: {{ .Prefix }}
3940
pool:
4041
max_workers: 10
4142
queue_depth: 1000

‎integration/e2e/config-all-in-one-s3.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ storage:
3838
access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
3939
secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
4040
insecure: true
41+
prefix: {{ .Prefix }}
4142
pool:
4243
max_workers: 10
4344
queue_depth: 100

‎integration/e2e/e2e_test.go

+82-58
Original file line numberDiff line numberDiff line change
@@ -58,83 +58,103 @@ func TestAllInOne(t *testing.T) {
5858
},
5959
}
6060

61+
storageBackendTestPermutations := []struct {
62+
name string
63+
prefix string
64+
}{
65+
{
66+
name: "no-prefix",
67+
},
68+
{
69+
name: "prefix",
70+
prefix: "a/b/c/",
71+
},
72+
}
73+
6174
for _, tc := range testBackends {
62-
t.Run(tc.name, func(t *testing.T) {
63-
s, err := e2e.NewScenario("tempo_e2e")
64-
require.NoError(t, err)
65-
defer s.Close()
75+
for _, pc := range storageBackendTestPermutations {
76+
t.Run(tc.name+"-"+pc.name, func(t *testing.T) {
77+
s, err := e2e.NewScenario("tempo_e2e")
78+
require.NoError(t, err)
79+
defer s.Close()
6680

67-
// set up the backend
68-
cfg := app.Config{}
69-
buff, err := os.ReadFile(tc.configFile)
70-
require.NoError(t, err)
71-
err = yaml.UnmarshalStrict(buff, &cfg)
72-
require.NoError(t, err)
73-
_, err = backend.New(s, cfg)
74-
require.NoError(t, err)
81+
// copy config template to shared directory and expand template variables
82+
tmplConfig := map[string]any{"Prefix": pc.prefix}
83+
configFile, err := util.CopyTemplateToSharedDir(s, tc.configFile, "config.yaml", tmplConfig)
84+
require.NoError(t, err)
7585

76-
require.NoError(t, util.CopyFileToSharedDir(s, tc.configFile, "config.yaml"))
77-
tempo := util.NewTempoAllInOne()
78-
require.NoError(t, s.StartAndWaitReady(tempo))
86+
// set up the backend
87+
cfg := app.Config{}
88+
buff, err := os.ReadFile(configFile)
89+
require.NoError(t, err)
90+
err = yaml.UnmarshalStrict(buff, &cfg)
91+
require.NoError(t, err)
92+
_, err = backend.New(s, cfg)
93+
require.NoError(t, err)
7994

80-
// Get port for the Jaeger gRPC receiver endpoint
81-
c, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))
82-
require.NoError(t, err)
83-
require.NotNil(t, c)
95+
tempo := util.NewTempoAllInOne()
96+
require.NoError(t, s.StartAndWaitReady(tempo))
8497

85-
info := tempoUtil.NewTraceInfo(time.Now(), "")
86-
require.NoError(t, info.EmitAllBatches(c))
98+
// Get port for the Jaeger gRPC receiver endpoint
99+
c, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250))
100+
require.NoError(t, err)
101+
require.NotNil(t, c)
87102

88-
expected, err := info.ConstructTraceFromEpoch()
89-
require.NoError(t, err)
103+
info := tempoUtil.NewTraceInfo(time.Now(), "")
104+
require.NoError(t, info.EmitAllBatches(c))
90105

91-
// test metrics
92-
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(spanCount(expected)), "tempo_distributor_spans_received_total"))
106+
expected, err := info.ConstructTraceFromEpoch()
107+
require.NoError(t, err)
93108

94-
// test echo
95-
assertEcho(t, "http://"+tempo.Endpoint(3200)+"/api/echo")
109+
// test metrics
110+
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(spanCount(expected)), "tempo_distributor_spans_received_total"))
96111

97-
apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "")
112+
// test echo
113+
// nolint:goconst
114+
assertEcho(t, "http://"+tempo.Endpoint(3200)+"/api/echo")
98115

99-
// query an in-memory trace
100-
queryAndAssertTrace(t, apiClient, info)
116+
apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "")
101117

102-
// wait trace_idle_time and ensure trace is created in ingester
103-
require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
118+
// query an in-memory trace
119+
queryAndAssertTrace(t, apiClient, info)
104120

105-
// flush trace to backend
106-
callFlush(t, tempo)
121+
// wait trace_idle_time and ensure trace is created in ingester
122+
require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
107123

108-
// search for trace in backend
109-
util.SearchAndAssertTrace(t, apiClient, info)
110-
util.SearchTraceQLAndAssertTrace(t, apiClient, info)
124+
// flush trace to backend
125+
callFlush(t, tempo)
111126

112-
// sleep
113-
time.Sleep(10 * time.Second)
127+
// search for trace in backend
128+
util.SearchAndAssertTrace(t, apiClient, info)
129+
util.SearchTraceQLAndAssertTrace(t, apiClient, info)
114130

115-
// force clear completed block
116-
callFlush(t, tempo)
131+
// sleep
132+
time.Sleep(10 * time.Second)
117133

118-
// test metrics
119-
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
120-
require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics))
121-
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(3), "tempo_query_frontend_queries_total"))
134+
// force clear completed block
135+
callFlush(t, tempo)
122136

123-
// query trace - should fetch from backend
124-
queryAndAssertTrace(t, apiClient, info)
137+
// test metrics
138+
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
139+
require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics))
140+
require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(3), "tempo_query_frontend_queries_total"))
125141

126-
// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
127-
now := time.Now()
128-
util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
142+
// query trace - should fetch from backend
143+
queryAndAssertTrace(t, apiClient, info)
129144

130-
util.SearchAndAsserTagsBackend(t, apiClient, now.Add(-20*time.Minute).Unix(), now.Unix())
145+
// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
146+
now := time.Now()
147+
util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
131148

132-
// find the trace with streaming. using the http server b/c that's what Grafana will do
133-
grpcClient, err := util.NewSearchGRPCClient(context.Background(), tempo.Endpoint(3200))
134-
require.NoError(t, err)
149+
util.SearchAndAsserTagsBackend(t, apiClient, now.Add(-20*time.Minute).Unix(), now.Unix())
135150

136-
util.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
137-
})
151+
// find the trace with streaming. using the http server b/c that's what Grafana will do
152+
grpcClient, err := util.NewSearchGRPCClient(context.Background(), tempo.Endpoint(3200))
153+
require.NoError(t, err)
154+
155+
util.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
156+
})
157+
}
138158
}
139159
}
140160

@@ -317,16 +337,20 @@ func TestShutdownDelay(t *testing.T) {
317337
require.NoError(t, err)
318338
defer s.Close()
319339

340+
// copy config template to shared directory and expand template variables
341+
tmplConfig := map[string]any{"Prefix": ""}
342+
configFile, err := util.CopyTemplateToSharedDir(s, configAllInOneS3, "config.yaml", tmplConfig)
343+
require.NoError(t, err)
344+
320345
// set up the backend
321346
cfg := app.Config{}
322-
buff, err := os.ReadFile(configAllInOneS3)
347+
buff, err := os.ReadFile(configFile)
323348
require.NoError(t, err)
324349
err = yaml.UnmarshalStrict(buff, &cfg)
325350
require.NoError(t, err)
326351
_, err = backend.New(s, cfg)
327352
require.NoError(t, err)
328353

329-
require.NoError(t, util.CopyFileToSharedDir(s, configAllInOneS3, "config.yaml"))
330354
tempo := util.NewTempoAllInOne("-shutdown-delay=5s")
331355

332356
// this line tests confirms that the readiness flag is up

‎integration/e2e/overrides_test.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -49,16 +49,20 @@ func TestOverrides(t *testing.T) {
4949
require.NoError(t, err)
5050
defer s.Close()
5151

52+
// copy config template to shared directory and expand template variables
53+
tmplConfig := map[string]any{"Prefix": ""}
54+
configFile, err := util.CopyTemplateToSharedDir(s, tc.configFile, "config.yaml", tmplConfig)
55+
require.NoError(t, err)
56+
5257
// set up the backend
5358
cfg := app.Config{}
54-
buff, err := os.ReadFile(tc.configFile)
59+
buff, err := os.ReadFile(configFile)
5560
require.NoError(t, err)
5661
err = yaml.UnmarshalStrict(buff, &cfg)
5762
require.NoError(t, err)
5863
_, err = backend.New(s, cfg)
5964
require.NoError(t, err)
6065

61-
require.NoError(t, util.CopyFileToSharedDir(s, tc.configFile, "config.yaml"))
6266
tempo := util.NewTempoAllInOne()
6367
require.NoError(t, s.StartAndWaitReady(tempo))
6468

0 commit comments

Comments
 (0)