Skip to content

Commit 80c5af2

Browse files
bborehambboreham
committed
Add global label length limits
 1. config/config.go — Default limits of 1 MiB - Added LabelNameLengthLimit: 1 << 20 and LabelValueLengthLimit: 1 << 20 to DefaultGlobalConfig - Added fallback logic in GlobalConfig.UnmarshalYAML so a partial global config (one that doesn't set these fields) still gets the defaults, matching the pattern already used for ScrapeInterval, EvaluationInterval, etc. 2. scrape/scrape.go — Pre-construction panic prevention Added a check before p.Labels(&lset) that rejects metrics whose raw text exceeds 16 MiB. Since the encoding format panics for any individual string > 16 MiB, and no single label value can be longer than the total metric string, len(met) > 1<<24 is a safe guard. The configured 1 MiB limit then catches the normal cases in verifyLabelLimits. 3. storage/remote/write_handler.go — Configurable limits in RW receiver - Added labelNameLengthLimit and labelValueLengthLimit fields to writeHandler - Added two int parameters to NewWriteHandler - Added checkLabelLengths() helper that enforces the limits - Wired the check into both v1 write() and v2 appendV2() paths 4. web/api/v1/api.go — Wires global config limits to RW handler Passes cfg.GlobalConfig.LabelNameLengthLimit and LabelValueLengthLimit from the config to NewWriteHandler at construction. 5. promql/functions.go — Guards in label_replace and label_join Added explicit checks for the 16 MiB encoding limit before the lb.Labels() call. The evaluator's existing defer ev.recover() already converts panics to errors, but these checks give a clear, explicit error message instead. Tests - Updated config/config_test.go to expect the new 1 MiB defaults in TestGetScrapeConfigs - Updated all ~16 NewWriteHandler call sites in write_handler_test.go to pass 0, 0 (no limit) for the new parameters
1 parent 5b96e61 commit 80c5af2

File tree

7 files changed

+75
-17
lines changed

7 files changed

+75
-17
lines changed

‎config/config.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ var (
189189
ExtraScrapeMetrics: boolPtr(false),
190190
MetricNameValidationScheme: model.UTF8Validation,
191191
MetricNameEscapingScheme: model.AllowUTF8,
192+
// Default to 1 MiB to avoid crashes from the 16 MiB encoding limit.
193+
LabelNameLengthLimit: 1 << 20,
194+
LabelValueLengthLimit: 1 << 21,
192195
}
193196

194197
DefaultRuntimeConfig = RuntimeConfig{
@@ -698,6 +701,9 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(any) error) error {
698701
return fmt.Errorf("%w for global config", err)
699702
}
700703
}
704+
if gc.LabelNameLengthLimit == 0 {
705+
gc.LabelNameLengthLimit = DefaultGlobalConfig.LabelNameLengthLimit
706+
}
701707

702708
*c = *gc
703709
return nil

‎config/config_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2843,6 +2843,8 @@ func TestGetScrapeConfigs(t *testing.T) {
28432843
AlwaysScrapeClassicHistograms: boolPtr(opts.AlwaysScrapeClassicHistograms),
28442844
ConvertClassicHistogramsToNHCB: boolPtr(opts.ConvertClassicHistToNHCB),
28452845
ExtraScrapeMetrics: boolPtr(opts.ExtraScrapeMetrics),
2846+
LabelNameLengthLimit: DefaultGlobalConfig.LabelNameLengthLimit,
2847+
LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit,
28462848
}
28472849
if opts.ScrapeProtocols == nil {
28482850
sc.ScrapeProtocols = DefaultScrapeProtocols
@@ -2927,6 +2929,8 @@ func TestGetScrapeConfigs(t *testing.T) {
29272929
AlwaysScrapeClassicHistograms: boolPtr(false),
29282930
ConvertClassicHistogramsToNHCB: boolPtr(false),
29292931
ExtraScrapeMetrics: boolPtr(false),
2932+
LabelNameLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit,
2933+
LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit,
29302934

29312935
MetricsPath: DefaultScrapeConfig.MetricsPath,
29322936
Scheme: DefaultScrapeConfig.Scheme,
@@ -2966,6 +2970,8 @@ func TestGetScrapeConfigs(t *testing.T) {
29662970
AlwaysScrapeClassicHistograms: boolPtr(false),
29672971
ConvertClassicHistogramsToNHCB: boolPtr(false),
29682972
ExtraScrapeMetrics: boolPtr(false),
2973+
LabelNameLengthLimit: DefaultGlobalConfig.LabelNameLengthLimit,
2974+
LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit,
29692975

29702976
HTTPClientConfig: config.HTTPClientConfig{
29712977
TLSConfig: config.TLSConfig{

‎promql/functions.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2000,6 +2000,9 @@ func (ev *evaluator) evalLabelReplace(ctx context.Context, args parser.Expressio
20002000
indexes := regex.FindStringSubmatchIndex(srcVal)
20012001
if indexes != nil { // Only replace when regexp matches.
20022002
res := regex.ExpandString([]byte{}, repl, srcVal, indexes)
2003+
if len(res) > 1<<24 {
2004+
ev.errorf("label_replace: replacement value too long (%d bytes)", len(res))
2005+
}
20032006
lb.Reset(el.Metric)
20042007
lb.Set(dst, string(res))
20052008
matrix[i].Metric = lb.Labels()
@@ -2051,6 +2054,9 @@ func (ev *evaluator) evalLabelJoin(ctx context.Context, args parser.Expressions)
20512054
srcVals[i] = el.Metric.Get(src)
20522055
}
20532056
strval := strings.Join(srcVals, sep)
2057+
if len(strval) > 1<<24 {
2058+
ev.errorf("label_join: joined value too long (%d bytes)", len(strval))
2059+
}
20542060
lb.Reset(el.Metric)
20552061
lb.Set(dst, strval)
20562062
matrix[i].Metric = lb.Labels()

‎scrape/scrape.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1713,6 +1713,12 @@ loop:
17131713
lset = ce.lset
17141714
hash = ce.hash
17151715
} else {
1716+
// The label encoding format cannot represent strings longer than
1717+
// 2^24-1 bytes. Reject before constructing Labels to avoid a panic.
1718+
if len(met) > 1<<24 {
1719+
err = fmt.Errorf("metric string too long to encode (%d bytes)", len(met))
1720+
break loop
1721+
}
17161722
p.Labels(&lset)
17171723
hash = lset.Hash()
17181724

‎storage/remote/write_handler.go‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ type writeHandler struct {
4848
ingestSTZeroSample bool
4949
enableTypeAndUnitLabels bool
5050
appendMetadata bool
51+
52+
labelNameLengthLimit int
53+
labelValueLengthLimit int
5154
}
5255

5356
const maxAheadTime = 10 * time.Minute
@@ -57,7 +60,7 @@ const maxAheadTime = 10 * time.Minute
5760
//
5861
// NOTE(bwplotka): When accepting v2 proto and spec, partial writes are possible
5962
// as per https://prometheus.io/docs/specs/remote_write_spec_2_0/#partial-write.
60-
func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedMsgs remoteapi.MessageTypes, ingestSTZeroSample, enableTypeAndUnitLabels, appendMetadata bool) http.Handler {
63+
func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedMsgs remoteapi.MessageTypes, ingestSTZeroSample, enableTypeAndUnitLabels, appendMetadata bool, labelNameLengthLimit, labelValueLengthLimit int) http.Handler {
6164
h := &writeHandler{
6265
logger: logger,
6366
appendable: appendable,
@@ -77,6 +80,9 @@ func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable
7780
ingestSTZeroSample: ingestSTZeroSample,
7881
enableTypeAndUnitLabels: enableTypeAndUnitLabels,
7982
appendMetadata: appendMetadata,
83+
84+
labelNameLengthLimit: labelNameLengthLimit,
85+
labelValueLengthLimit: labelValueLengthLimit,
8086
}
8187
return remoteapi.NewWriteHandler(h, acceptedMsgs, remoteapi.WithWriteHandlerLogger(logger))
8288
}
@@ -146,6 +152,23 @@ func (h *writeHandler) Store(r *http.Request, msgType remoteapi.WriteMessageType
146152
return wr, nil
147153
}
148154

155+
// checkLabelLengths returns an error if any label name or value in lset exceeds
156+
// the configured limits. A limit of 0 means no limit.
157+
func (h *writeHandler) checkLabelLengths(lset labels.Labels) error {
158+
if h.labelNameLengthLimit == 0 && h.labelValueLengthLimit == 0 {
159+
return nil
160+
}
161+
return lset.Validate(func(l labels.Label) error {
162+
if h.labelNameLengthLimit > 0 && len(l.Name) > h.labelNameLengthLimit {
163+
return fmt.Errorf("label name too long (label: %.50s, length: %d, limit: %d)", l.Name, len(l.Name), h.labelNameLengthLimit)
164+
}
165+
if h.labelValueLengthLimit > 0 && len(l.Value) > h.labelValueLengthLimit {
166+
return fmt.Errorf("label value too long (label: %.50s, length: %d, limit: %d)", l.Name, len(l.Value), h.labelValueLengthLimit)
167+
}
168+
return nil
169+
})
170+
}
171+
149172
func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err error) {
150173
outOfOrderExemplarErrs := 0
151174
samplesWithInvalidLabels := 0
@@ -181,6 +204,10 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err
181204
h.logger.Warn("Invalid labels for series.", "labels", ls.String(), "duplicated_label", duplicateLabel)
182205
samplesWithInvalidLabels++
183206
continue
207+
} else if err := h.checkLabelLengths(ls); err != nil {
208+
h.logger.Warn("Label length limit exceeded", "err", err)
209+
samplesWithInvalidLabels++
210+
continue
184211
}
185212

186213
if err := h.appendV1Samples(app, ts.Samples, ls); err != nil {
@@ -345,6 +372,10 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *
345372
badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid labels for series, labels %v, duplicated label %s", ls.String(), duplicateLabel))
346373
samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms)
347374
continue
375+
} else if err := h.checkLabelLengths(ls); err != nil {
376+
badRequestErrs = append(badRequestErrs, fmt.Errorf("label length limit exceeded for series %v: %w", ls.String(), err))
377+
samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms)
378+
continue
348379
}
349380

350381
// Validate that the TimeSeries has at least one sample or histogram.

‎storage/remote/write_handler_test.go‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ func TestRemoteWriteHandlerHeadersHandling_V1Message(t *testing.T) {
130130
}
131131

132132
appendable := &mockAppendable{}
133-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
133+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
134134

135135
recorder := httptest.NewRecorder()
136136
handler.ServeHTTP(recorder, req)
@@ -237,7 +237,7 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) {
237237
}
238238

239239
appendable := &mockAppendable{}
240-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false)
240+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0)
241241

242242
recorder := httptest.NewRecorder()
243243
handler.ServeHTTP(recorder, req)
@@ -272,7 +272,7 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) {
272272
}
273273

274274
appendable := &mockAppendable{}
275-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false)
275+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0)
276276

277277
recorder := httptest.NewRecorder()
278278
handler.ServeHTTP(recorder, req)
@@ -301,7 +301,7 @@ func TestRemoteWriteHandler_V1Message(t *testing.T) {
301301
// in Prometheus, so keeping like this to not break existing 1.0 clients.
302302

303303
appendable := &mockAppendable{}
304-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
304+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
305305

306306
recorder := httptest.NewRecorder()
307307
handler.ServeHTTP(recorder, req)
@@ -706,7 +706,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
706706
appendExemplarErr: tc.appendExemplarErr,
707707
updateMetadataErr: tc.updateMetadataErr,
708708
}
709-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.ingestSTZeroSample, tc.enableTypeAndUnitLabels, tc.appendMetadata)
709+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.ingestSTZeroSample, tc.enableTypeAndUnitLabels, tc.appendMetadata, 0, 0)
710710

711711
recorder := httptest.NewRecorder()
712712
handler.ServeHTTP(recorder, req)
@@ -880,7 +880,7 @@ func TestRemoteWriteHandler_V2Message_NoDuplicateTypeAndUnitLabels(t *testing.T)
880880
req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue)
881881

882882
appendable := &mockAppendable{}
883-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, true, false)
883+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, true, false, 0, 0)
884884

885885
recorder := httptest.NewRecorder()
886886
handler.ServeHTTP(recorder, req)
@@ -929,7 +929,7 @@ func TestOutOfOrderSample_V1Message(t *testing.T) {
929929
require.NoError(t, err)
930930

931931
appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
932-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
932+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
933933

934934
recorder := httptest.NewRecorder()
935935
handler.ServeHTTP(recorder, req)
@@ -971,7 +971,7 @@ func TestOutOfOrderExemplar_V1Message(t *testing.T) {
971971
require.NoError(t, err)
972972

973973
appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
974-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
974+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
975975

976976
recorder := httptest.NewRecorder()
977977
handler.ServeHTTP(recorder, req)
@@ -1009,7 +1009,7 @@ func TestOutOfOrderHistogram_V1Message(t *testing.T) {
10091009
require.NoError(t, err)
10101010

10111011
appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}}
1012-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
1012+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
10131013

10141014
recorder := httptest.NewRecorder()
10151015
handler.ServeHTTP(recorder, req)
@@ -1059,7 +1059,7 @@ func BenchmarkRemoteWriteHandler(b *testing.B) {
10591059
for _, tc := range testCases {
10601060
b.Run(tc.name, func(b *testing.B) {
10611061
appendable := &mockAppendable{}
1062-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false, false, false)
1062+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false, false, false, 0, 0)
10631063
b.ResetTimer()
10641064
for b.Loop() {
10651065
b.StopTimer()
@@ -1084,7 +1084,7 @@ func TestCommitErr_V1Message(t *testing.T) {
10841084
require.NoError(t, err)
10851085

10861086
appendable := &mockAppendable{commitErr: errors.New("commit error")}
1087-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
1087+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
10881088

10891089
recorder := httptest.NewRecorder()
10901090
handler.ServeHTTP(recorder, req)
@@ -1150,7 +1150,7 @@ func TestHistogramValidationErrorHandling(t *testing.T) {
11501150
require.NoError(t, err)
11511151
t.Cleanup(func() { require.NoError(t, db.Close()) })
11521152

1153-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false, false, false)
1153+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false, false, false, 0, 0)
11541154
recorder := httptest.NewRecorder()
11551155

11561156
var buf []byte
@@ -1195,7 +1195,7 @@ func TestCommitErr_V2Message(t *testing.T) {
11951195
req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue)
11961196

11971197
appendable := &mockAppendable{commitErr: errors.New("commit error")}
1198-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false)
1198+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0)
11991199

12001200
recorder := httptest.NewRecorder()
12011201
handler.ServeHTTP(recorder, req)
@@ -1222,7 +1222,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) {
12221222
require.NoError(b, db.Close())
12231223
})
12241224
// TODO: test with other proto format(s)
1225-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false)
1225+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0)
12261226

12271227
buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil, "snappy")
12281228
require.NoError(b, err)
@@ -1554,7 +1554,7 @@ func TestHistogramsReduction(t *testing.T) {
15541554
for _, protoMsg := range []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType} {
15551555
t.Run(string(protoMsg), func(t *testing.T) {
15561556
appendable := &mockAppendable{}
1557-
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false, false, false)
1557+
handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false, false, false, 0, 0)
15581558

15591559
var (
15601560
err error
@@ -1650,6 +1650,8 @@ func TestRemoteWriteHandler_ResponseStats(t *testing.T) {
16501650
false,
16511651
false,
16521652
false,
1653+
0,
1654+
0,
16531655
)
16541656

16551657
if tt.forceInjectHeaders {

‎web/api/v1/api.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,8 @@ func NewAPI(
355355
}
356356

357357
if rwEnabled {
358-
a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, stZeroIngestionEnabled, enableTypeAndUnitLabels, appendMetadata)
358+
cfg := configFunc()
359+
a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, stZeroIngestionEnabled, enableTypeAndUnitLabels, appendMetadata, int(cfg.GlobalConfig.LabelNameLengthLimit), int(cfg.GlobalConfig.LabelValueLengthLimit))
359360
}
360361
if otlpEnabled {
361362
a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, registerer, apV2, configFunc, remote.OTLPOptions{

0 commit comments

Comments
 (0)