From 80c5af27fa046c9ff49c379fd5eda12acf258bea Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 23 Mar 2026 12:29:19 +0100 Subject: [PATCH] Add global label length limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. config/config.go — Default limits of 1 MiB - Added LabelNameLengthLimit: 1 << 20 and LabelValueLengthLimit: 1 << 20 to DefaultGlobalConfig - Added fallback logic in GlobalConfig.UnmarshalYAML so a partial global config (one that doesn't set these fields) still gets the defaults, matching the pattern already used for ScrapeInterval, EvaluationInterval, etc. 2. scrape/scrape.go — Pre-construction panic prevention Added a check before p.Labels(&lset) that rejects metrics whose raw text exceeds 16 MiB. Since the encoding format panics for any individual string > 16 MiB, and no single label value can be longer than the total metric string, len(met) > 1<<24 is a safe guard. The configured 1 MiB limit then catches the normal cases in verifyLabelLimits. 3. storage/remote/write_handler.go — Configurable limits in RW receiver - Added labelNameLengthLimit and labelValueLengthLimit fields to writeHandler - Added two int parameters to NewWriteHandler - Added checkLabelLengths() helper that enforces the limits - Wired the check into both v1 write() and v2 appendV2() paths 4. web/api/v1/api.go — Wires global config limits to RW handler Passes cfg.GlobalConfig.LabelNameLengthLimit and LabelValueLengthLimit from the config to NewWriteHandler at construction. 5. promql/functions.go — Guards in label_replace and label_join Added explicit checks for the 16 MiB encoding limit before the lb.Labels() call. The evaluator's existing defer ev.recover() already converts panics to errors, but these checks give a clear, explicit error message instead. Tests - Updated config/config_test.go to expect the new 1 MiB defaults in TestGetScrapeConfigs - Updated all ~16 NewWriteHandler call sites in write_handler_test.go to pass 0, 0 (no limit) for the new parameters --- config/config.go | 6 +++++ config/config_test.go | 6 +++++ promql/functions.go | 6 +++++ scrape/scrape.go | 6 +++++ storage/remote/write_handler.go | 33 +++++++++++++++++++++++++++- storage/remote/write_handler_test.go | 32 ++++++++++++++------------- web/api/v1/api.go | 3 ++- 7 files changed, 75 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 2082743b0d7..5658fdd1efa 100644 --- a/config/config.go +++ b/config/config.go @@ -189,6 +189,9 @@ var ( ExtraScrapeMetrics: boolPtr(false), MetricNameValidationScheme: model.UTF8Validation, MetricNameEscapingScheme: model.AllowUTF8, + // Default to 1 MiB to avoid crashes from the 16 MiB encoding limit. + LabelNameLengthLimit: 1 << 20, + LabelValueLengthLimit: 1 << 21, } DefaultRuntimeConfig = RuntimeConfig{ @@ -698,6 +701,9 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(any) error) error { return fmt.Errorf("%w for global config", err) } } + if gc.LabelNameLengthLimit == 0 { + gc.LabelNameLengthLimit = DefaultGlobalConfig.LabelNameLengthLimit + } *c = *gc return nil diff --git a/config/config_test.go b/config/config_test.go index 8d4df86be67..102eafe488e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2843,6 +2843,8 @@ func TestGetScrapeConfigs(t *testing.T) { AlwaysScrapeClassicHistograms: boolPtr(opts.AlwaysScrapeClassicHistograms), ConvertClassicHistogramsToNHCB: boolPtr(opts.ConvertClassicHistToNHCB), ExtraScrapeMetrics: boolPtr(opts.ExtraScrapeMetrics), + LabelNameLengthLimit: DefaultGlobalConfig.LabelNameLengthLimit, + LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit, } if opts.ScrapeProtocols == nil { sc.ScrapeProtocols = DefaultScrapeProtocols @@ -2927,6 +2929,8 @@ func TestGetScrapeConfigs(t *testing.T) { AlwaysScrapeClassicHistograms: boolPtr(false), ConvertClassicHistogramsToNHCB: boolPtr(false), ExtraScrapeMetrics: boolPtr(false), + LabelNameLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit, + LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit, MetricsPath: DefaultScrapeConfig.MetricsPath, Scheme: DefaultScrapeConfig.Scheme, @@ -2966,6 +2970,8 @@ func TestGetScrapeConfigs(t *testing.T) { AlwaysScrapeClassicHistograms: boolPtr(false), ConvertClassicHistogramsToNHCB: boolPtr(false), ExtraScrapeMetrics: boolPtr(false), + LabelNameLengthLimit: DefaultGlobalConfig.LabelNameLengthLimit, + LabelValueLengthLimit: DefaultGlobalConfig.LabelValueLengthLimit, HTTPClientConfig: config.HTTPClientConfig{ TLSConfig: config.TLSConfig{ diff --git a/promql/functions.go b/promql/functions.go index 546f94df122..7cda86efb9a 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -2000,6 +2000,9 @@ func (ev *evaluator) evalLabelReplace(ctx context.Context, args parser.Expressio indexes := regex.FindStringSubmatchIndex(srcVal) if indexes != nil { // Only replace when regexp matches. res := regex.ExpandString([]byte{}, repl, srcVal, indexes) + if len(res) > 1<<24 { + ev.errorf("label_replace: replacement value too long (%d bytes)", len(res)) + } lb.Reset(el.Metric) lb.Set(dst, string(res)) matrix[i].Metric = lb.Labels() @@ -2051,6 +2054,9 @@ func (ev *evaluator) evalLabelJoin(ctx context.Context, args parser.Expressions) srcVals[i] = el.Metric.Get(src) } strval := strings.Join(srcVals, sep) + if len(strval) > 1<<24 { + ev.errorf("label_join: joined value too long (%d bytes)", len(strval)) + } lb.Reset(el.Metric) lb.Set(dst, strval) matrix[i].Metric = lb.Labels() diff --git a/scrape/scrape.go b/scrape/scrape.go index 9b37a356cf7..24086c5c77f 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1713,6 +1713,12 @@ loop: lset = ce.lset hash = ce.hash } else { + // The label encoding format cannot represent strings longer than + // 2^24-1 bytes. Reject before constructing Labels to avoid a panic. + if len(met) > 1<<24 { + err = fmt.Errorf("metric string too long to encode (%d bytes)", len(met)) + break loop + } p.Labels(&lset) hash = lset.Hash() diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index 9fdd7506924..eea4647418e 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -48,6 +48,9 @@ type writeHandler struct { ingestSTZeroSample bool enableTypeAndUnitLabels bool appendMetadata bool + + labelNameLengthLimit int + labelValueLengthLimit int } const maxAheadTime = 10 * time.Minute @@ -57,7 +60,7 @@ const maxAheadTime = 10 * time.Minute // // NOTE(bwplotka): When accepting v2 proto and spec, partial writes are possible // as per https://prometheus.io/docs/specs/remote_write_spec_2_0/#partial-write. -func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedMsgs remoteapi.MessageTypes, ingestSTZeroSample, enableTypeAndUnitLabels, appendMetadata bool) http.Handler { +func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedMsgs remoteapi.MessageTypes, ingestSTZeroSample, enableTypeAndUnitLabels, appendMetadata bool, labelNameLengthLimit, labelValueLengthLimit int) http.Handler { h := &writeHandler{ logger: logger, appendable: appendable, @@ -77,6 +80,9 @@ func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable ingestSTZeroSample: ingestSTZeroSample, enableTypeAndUnitLabels: enableTypeAndUnitLabels, appendMetadata: appendMetadata, + + labelNameLengthLimit: labelNameLengthLimit, + labelValueLengthLimit: labelValueLengthLimit, } return remoteapi.NewWriteHandler(h, acceptedMsgs, remoteapi.WithWriteHandlerLogger(logger)) } @@ -146,6 +152,23 @@ func (h *writeHandler) Store(r *http.Request, msgType remoteapi.WriteMessageType return wr, nil } +// checkLabelLengths returns an error if any label name or value in lset exceeds +// the configured limits. A limit of 0 means no limit. +func (h *writeHandler) checkLabelLengths(lset labels.Labels) error { + if h.labelNameLengthLimit == 0 && h.labelValueLengthLimit == 0 { + return nil + } + return lset.Validate(func(l labels.Label) error { + if h.labelNameLengthLimit > 0 && len(l.Name) > h.labelNameLengthLimit { + return fmt.Errorf("label name too long (label: %.50s, length: %d, limit: %d)", l.Name, len(l.Name), h.labelNameLengthLimit) + } + if h.labelValueLengthLimit > 0 && len(l.Value) > h.labelValueLengthLimit { + return fmt.Errorf("label value too long (label: %.50s, length: %d, limit: %d)", l.Name, len(l.Value), h.labelValueLengthLimit) + } + return nil + }) +} + func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err error) { outOfOrderExemplarErrs := 0 samplesWithInvalidLabels := 0 @@ -181,6 +204,10 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err h.logger.Warn("Invalid labels for series.", "labels", ls.String(), "duplicated_label", duplicateLabel) samplesWithInvalidLabels++ continue + } else if err := h.checkLabelLengths(ls); err != nil { + h.logger.Warn("Label length limit exceeded", "err", err) + samplesWithInvalidLabels++ + continue } if err := h.appendV1Samples(app, ts.Samples, ls); err != nil { @@ -345,6 +372,10 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid labels for series, labels %v, duplicated label %s", ls.String(), duplicateLabel)) samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms) continue + } else if err := h.checkLabelLengths(ls); err != nil { + badRequestErrs = append(badRequestErrs, fmt.Errorf("label length limit exceeded for series %v: %w", ls.String(), err)) + samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms) + continue } // Validate that the TimeSeries has at least one sample or histogram. diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index 2cf12179337..062063e136d 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -130,7 +130,7 @@ func TestRemoteWriteHandlerHeadersHandling_V1Message(t *testing.T) { } appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -237,7 +237,7 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) { } appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -272,7 +272,7 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) { } appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -301,7 +301,7 @@ func TestRemoteWriteHandler_V1Message(t *testing.T) { // in Prometheus, so keeping like this to not break existing 1.0 clients. appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -706,7 +706,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { appendExemplarErr: tc.appendExemplarErr, updateMetadataErr: tc.updateMetadataErr, } - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.ingestSTZeroSample, tc.enableTypeAndUnitLabels, tc.appendMetadata) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.ingestSTZeroSample, tc.enableTypeAndUnitLabels, tc.appendMetadata, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -880,7 +880,7 @@ func TestRemoteWriteHandler_V2Message_NoDuplicateTypeAndUnitLabels(t *testing.T) req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue) appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, true, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, true, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -929,7 +929,7 @@ func TestOutOfOrderSample_V1Message(t *testing.T) { require.NoError(t, err) appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -971,7 +971,7 @@ func TestOutOfOrderExemplar_V1Message(t *testing.T) { require.NoError(t, err) appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1009,7 +1009,7 @@ func TestOutOfOrderHistogram_V1Message(t *testing.T) { require.NoError(t, err) appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1059,7 +1059,7 @@ func BenchmarkRemoteWriteHandler(b *testing.B) { for _, tc := range testCases { b.Run(tc.name, func(b *testing.B) { appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false, false, false, 0, 0) b.ResetTimer() for b.Loop() { b.StopTimer() @@ -1084,7 +1084,7 @@ func TestCommitErr_V1Message(t *testing.T) { require.NoError(t, err) appendable := &mockAppendable{commitErr: errors.New("commit error")} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1150,7 +1150,7 @@ func TestHistogramValidationErrorHandling(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) - handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false, false, false, 0, 0) recorder := httptest.NewRecorder() var buf []byte @@ -1195,7 +1195,7 @@ func TestCommitErr_V2Message(t *testing.T) { req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue) appendable := &mockAppendable{commitErr: errors.New("commit error")} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false, 0, 0) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1222,7 +1222,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) { require.NoError(b, db.Close()) }) // TODO: test with other proto format(s) - handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false, 0, 0) buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil, "snappy") require.NoError(b, err) @@ -1554,7 +1554,7 @@ func TestHistogramsReduction(t *testing.T) { for _, protoMsg := range []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType} { t.Run(string(protoMsg), func(t *testing.T) { appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false, false, false, 0, 0) var ( err error @@ -1650,6 +1650,8 @@ func TestRemoteWriteHandler_ResponseStats(t *testing.T) { false, false, false, + 0, + 0, ) if tt.forceInjectHeaders { diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 6e61fd19c6a..7184abd6fdf 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -355,7 +355,8 @@ func NewAPI( } if rwEnabled { - a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, stZeroIngestionEnabled, enableTypeAndUnitLabels, appendMetadata) + cfg := configFunc() + a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, stZeroIngestionEnabled, enableTypeAndUnitLabels, appendMetadata, int(cfg.GlobalConfig.LabelNameLengthLimit), int(cfg.GlobalConfig.LabelValueLengthLimit)) } if otlpEnabled { a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, registerer, apV2, configFunc, remote.OTLPOptions{