MQE: subset selector elimination

CHANGELOG.md

@@ -128,6 +128,7 @@
 * [ENHANCEMENT] Ingester: Add experimental file based Kafka consumer group offset tracking via flag `-ingest-storage.kafka.consumer-group-offset-commit-file-enforced`. #14110
 * [ENHANCEMENT] Store-gateway: Add "OOO" column to the tenant blocks page to indicate whether each block was created from out-of-order samples. #14283
 * [ENHANCEMENT] Ingester: Optimize ingestion from Kafka in clusters with mixed size tenants. #13924 #13961 #14302
+* [ENHANCEMENT] MQE: Add experimental support for eliminating selectors that are a subset of another selector. Enable with `-querier.mimir-query-engine.enable-subset-selector-elimination=true`. #14377
 * [BUGFIX] API: Fixed web UI links not respecting `-server.path-prefix` configuration. #14090
 * [BUGFIX] Distributor: Fix issue where distributors didn't send custom values of native histograms. #13849
 * [BUGFIX] Compactor: Fix potential concurrent map writes. #13053

cmd/mimir/config-descriptor.json

@@ -2537,6 +2537,17 @@
               "fieldType": "boolean",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "enable_subset_selector_elimination",
+              "required": false,
+              "desc": "Enable subset selector elimination when evaluating queries.",
+              "fieldValue": null,
+              "fieldDefaultValue": true,
+              "fieldFlag": "querier.mimir-query-engine.enable-subset-selector-elimination",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "field",
               "name": "enable_narrow_binary_selectors",

cmd/mimir/help-all.txt.tmpl

@@ -2179,6 +2179,8 @@ Usage of ./cmd/mimir/mimir:
     	[experimental] Enable pruning query expressions that are toggled off with constants. (default true)
   -querier.mimir-query-engine.enable-reduce-matchers
     	[experimental] Enable eliminating duplicate or redundant matchers that are part of selector expressions. (default true)
+  -querier.mimir-query-engine.enable-subset-selector-elimination
+    	[experimental] Enable subset selector elimination when evaluating queries. (default true)
   -querier.minimize-ingester-requests
     	If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path. (default true)
   -querier.minimize-ingester-requests-hedging-delay duration

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -1921,6 +1921,10 @@ mimir_query_engine:
   # CLI flag: -querier.mimir-query-engine.enable-common-subexpression-elimination
   [enable_common_subexpression_elimination: <boolean> | default = true]
 
+  # (experimental) Enable subset selector elimination when evaluating queries.
+  # CLI flag: -querier.mimir-query-engine.enable-subset-selector-elimination
+  [enable_subset_selector_elimination: <boolean> | default = true]
+
   # (experimental) Enable generating selectors for one side of a binary
   # expression based on results from the other side.
   # CLI flag: -querier.mimir-query-engine.enable-narrow-binary-selectors

operations/mimir/mimir-flags-defaults.json

@@ -171,6 +171,7 @@
   "querier.lookback-delta": 300000000000,
   "querier.mimir-query-engine.enable-prune-toggles": true,
   "querier.mimir-query-engine.enable-common-subexpression-elimination": true,
+  "querier.mimir-query-engine.enable-subset-selector-elimination": true,
   "querier.mimir-query-engine.enable-eliminate-deduplicate-and-merge": true,
   "querier.mimir-query-engine.enable-reduce-matchers": true,
   "querier.mimir-query-engine.enable-multi-aggregation": true,

pkg/querier/dispatcher_test.go

@@ -565,12 +565,12 @@ func TestDispatcher_HandleProtobuf(t *testing.T) {
 				types.NewRangeQueryTimeRange(startT, startT.Add(20*time.Second), 10*time.Second),
 				enableDelayedNameRemoval,
 				2,
-				[]string{"BinaryExpression: LHS + RHS", `LHS: VectorSelector: {__name__="my_three_item_series", idx=~"(0|1|2)"}`},
-				[]string{"BinaryExpression: LHS + RHS", `RHS: VectorSelector: {__name__="my_three_item_series"}`}, // Note that the wildcard selector has been removed by the "reduce matchers" pass.
+				[]string{"BinaryExpression: LHS + RHS", `LHS: DuplicateFilter: {idx=~"(0|1|2)"}`},
+				[]string{"BinaryExpression: LHS + RHS", `RHS: Duplicate`}, // Note that the wildcard selector has been removed by the "reduce matchers" pass.
 			),
 			expectedResponseMessages: []*frontendv2pb.QueryResultStreamRequest{
 				newSeriesMetadataMessage(
-					0,
+					2,
 					querierpb.SeriesMetadata{Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, "my_three_item_series", "idx", "0"))},
 					querierpb.SeriesMetadata{Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, "my_three_item_series", "idx", "1"))},
 					querierpb.SeriesMetadata{Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, "my_three_item_series", "idx", "2"))},
@@ -582,7 +582,7 @@ func TestDispatcher_HandleProtobuf(t *testing.T) {
 					querierpb.SeriesMetadata{Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, "my_three_item_series", "idx", "2"))},
 				),
 				newInstantVectorSeriesDataMessage(
-					0,
+					2,
 					querierpb.InstantVectorSeriesData{
 						Floats: []mimirpb.Sample{
 							{TimestampMs: 0, Value: 3},
@@ -616,7 +616,7 @@ func TestDispatcher_HandleProtobuf(t *testing.T) {
 					},
 				),
 				newInstantVectorSeriesDataMessage(
-					0,
+					2,
 					querierpb.InstantVectorSeriesData{
 						Floats: []mimirpb.Sample{
 							{TimestampMs: 0, Value: 5},
@@ -636,7 +636,7 @@ func TestDispatcher_HandleProtobuf(t *testing.T) {
 					},
 				),
 				newEvaluationCompletedMessage(stats.Stats{
-					SamplesProcessed:   18,
+					SamplesProcessed:   9,
 					QueueTime:          3 * time.Second,
 					WallTime:           expectedQueryWallTime,
 					FetchedSeriesCount: 123,

pkg/streamingpromql/benchmarks/benchmarks.go

@@ -25,9 +25,10 @@ import (
 var MetricSizes = []int{1, 100, 2000}
 
 type BenchCase struct {
-	Expr             string
-	Steps            int
-	InstantQueryOnly bool
+	Expr                        string
+	Steps                       int
+	InstantQueryOnly            bool
+	IgnoreAnnotationDifferences bool
 }
 
 func (c BenchCase) Name() string {
@@ -418,6 +419,29 @@ func TestCases(metricSizes []int) []BenchCase {
 		{
 			Expr: "sum(sum_over_time(a_X[1d])) / sum(count_over_time(a_X[1d]))",
 		},
+		// Subset selector elimination cases
+		{
+			Expr: `a_X{l=~"[13579].*"} / a_X`,
+		},
+		{
+			Expr:                        `rate(a_X{l=~"[13579].*"}[1m]) / rate(a_X[1m])`,
+			IgnoreAnnotationDifferences: true, // The a_1 metric has no 'l' label, so this query selects no data, short-circuits and doesn't generate "metric might not be a counter" annotations.
+		},
+		{
+			Expr: `sum(a_X{l=~"[13579].*"}) / sum(a_X)`,
+		},
+		{
+			Expr:                        `sum(rate(a_X{l=~"[13579].*"}[1m])) / sum(rate(a_X[1m]))`,
+			IgnoreAnnotationDifferences: true, // The a_1 metric has no 'l' label, so this query selects no data, short-circuits and doesn't generate "metric might not be a counter" annotations.
+		},
+		{
+			Expr: `sum(a_X{l=~"[13].*"}) / sum(a_X)`,
+		},
+		{
+			Expr:                        `sum(rate(a_X{l=~"[13].*"}[1m])) / sum(rate(a_X[1m]))`,
+			IgnoreAnnotationDifferences: true, // The a_1 metric has no 'l' label, so this query selects no data, short-circuits and doesn't generate "metric might not be a counter" annotations.
+		},
+		// info()
 		{
 			Expr: `info(info_sparse_100, {__name__="target_info_X"})`,
 		},
@@ -439,7 +463,7 @@ func TestCases(metricSizes []int) []BenchCase {
 			tmp = append(tmp, c)
 		} else {
 			for _, count := range metricSizes {
-				tmp = append(tmp, BenchCase{Expr: strings.ReplaceAll(c.Expr, "X", strconv.Itoa(count)), Steps: c.Steps, InstantQueryOnly: c.InstantQueryOnly})
+				tmp = append(tmp, BenchCase{Expr: strings.ReplaceAll(c.Expr, "X", strconv.Itoa(count)), Steps: c.Steps, InstantQueryOnly: c.InstantQueryOnly, IgnoreAnnotationDifferences: c.IgnoreAnnotationDifferences})
 			}
 		}
 	}
@@ -460,9 +484,9 @@ func TestCases(metricSizes []int) []BenchCase {
 
 			tmp = append(tmp, c)
 		} else {
-			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 0})
-			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 100})
-			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 1000})
+			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 0, IgnoreAnnotationDifferences: c.IgnoreAnnotationDifferences})
+			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 100, IgnoreAnnotationDifferences: c.IgnoreAnnotationDifferences})
+			tmp = append(tmp, BenchCase{Expr: c.Expr, Steps: 1000, IgnoreAnnotationDifferences: c.IgnoreAnnotationDifferences})
 			// Important: if adding test cases with larger numbers of steps, make sure to adjust NumIntervals as well.
 		}
 	}

pkg/streamingpromql/benchmarks/comparison_test.go

@@ -134,7 +134,7 @@ func TestBothEnginesReturnSameResultsForBenchmarkQueries(t *testing.T) {
 			prometheusResult, prometheusClose := c.Run(ctx, t, start, end, interval, prometheusEngine, q)
 			mimirResult, mimirClose := c.Run(ctx, t, start, end, interval, mimirEngine, q)
 
-			testutils.RequireEqualResults(t, c.Expr, prometheusResult, mimirResult, false)
+			testutils.RequireEqualResults(t, c.Expr, prometheusResult, mimirResult, c.IgnoreAnnotationDifferences)
 
 			prometheusClose()
 			mimirClose()

pkg/streamingpromql/config.go

@@ -31,6 +31,7 @@ type EngineOpts struct {
 
 	EnablePruneToggles                   bool `yaml:"enable_prune_toggles" category:"experimental"`
 	EnableCommonSubexpressionElimination bool `yaml:"enable_common_subexpression_elimination" category:"experimental"`
+	EnableSubsetSelectorElimination      bool `yaml:"enable_subset_selector_elimination" category:"experimental"`
 	EnableNarrowBinarySelectors          bool `yaml:"enable_narrow_binary_selectors" category:"experimental"`
 	EnableEliminateDeduplicateAndMerge   bool `yaml:"enable_eliminate_deduplicate_and_merge" category:"experimental"`
 	EnableReduceMatchers                 bool `yaml:"enable_reduce_matchers" category:"experimental"`
@@ -41,6 +42,7 @@ type EngineOpts struct {
 func (o *EngineOpts) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&o.EnablePruneToggles, "querier.mimir-query-engine.enable-prune-toggles", true, "Enable pruning query expressions that are toggled off with constants.")
 	f.BoolVar(&o.EnableCommonSubexpressionElimination, "querier.mimir-query-engine.enable-common-subexpression-elimination", true, "Enable common subexpression elimination when evaluating queries.")
+	f.BoolVar(&o.EnableSubsetSelectorElimination, "querier.mimir-query-engine.enable-subset-selector-elimination", true, "Enable subset selector elimination when evaluating queries.")
 	f.BoolVar(&o.EnableNarrowBinarySelectors, "querier.mimir-query-engine.enable-narrow-binary-selectors", false, "Enable generating selectors for one side of a binary expression based on results from the other side.")
 	f.BoolVar(&o.EnableEliminateDeduplicateAndMerge, "querier.mimir-query-engine.enable-eliminate-deduplicate-and-merge", true, "Enable eliminating redundant DeduplicateAndMerge nodes from the query plan when it can be proven that each input series produces a unique output series.")
 	f.BoolVar(&o.EnableReduceMatchers, "querier.mimir-query-engine.enable-reduce-matchers", true, "Enable eliminating duplicate or redundant matchers that are part of selector expressions.")
@@ -65,6 +67,7 @@ func NewTestEngineOpts() EngineOpts {
 
 		EnablePruneToggles:                   true,
 		EnableCommonSubexpressionElimination: true,
+		EnableSubsetSelectorElimination:      true,
 		EnableNarrowBinarySelectors:          true,
 		EnableEliminateDeduplicateAndMerge:   true,
 		EnableReduceMatchers:                 true,

pkg/streamingpromql/engine.go

@@ -83,6 +83,7 @@ func NewEngine(opts EngineOpts, limitsProvider QueryLimitsProvider, metrics *sta
 		planning.NODE_TYPE_DROP_NAME:             planning.NodeMaterializerFunc[*core.DropName](core.MaterializeDropName),
 
 		planning.NODE_TYPE_DUPLICATE:                  planning.NodeMaterializerFunc[*commonsubexpressionelimination.Duplicate](commonsubexpressionelimination.MaterializeDuplicate),
+		planning.NODE_TYPE_DUPLICATE_FILTER:           planning.NodeMaterializerFunc[*commonsubexpressionelimination.DuplicateFilter](commonsubexpressionelimination.MaterializeDuplicateFilter),
 		planning.NODE_TYPE_STEP_INVARIANT_EXPRESSION:  planning.NodeMaterializerFunc[*core.StepInvariantExpression](core.MaterializeStepInvariantExpression),
 		planning.NODE_TYPE_MULTI_AGGREGATION_GROUP:    planning.NodeMaterializerFunc[*multiaggregation.MultiAggregationGroup](multiaggregation.MaterializeMultiAggregationGroup),
 		planning.NODE_TYPE_MULTI_AGGREGATION_INSTANCE: planning.NodeMaterializerFunc[*multiaggregation.MultiAggregationInstance](multiaggregation.MaterializeMultiAggregationInstance),

pkg/streamingpromql/optimize/ast/sort_labels_and_matchers.go

@@ -6,10 +6,11 @@ import (
 	"context"
 	"fmt"
 	"slices"
-	"strings"
 
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql/parser"
+
+	"github.com/grafana/mimir/pkg/streamingpromql/planning/core"
 )
 
 // SortLabelsAndMatchers is an optimization pass that ensures that all label names and matchers are sorted.
@@ -54,13 +55,5 @@ func (s *SortLabelsAndMatchers) Apply(_ context.Context, expr parser.Expr) (pars
 }
 
 func compareMatchers(a, b *labels.Matcher) int {
-	if a.Name != b.Name {
-		return strings.Compare(a.Name, b.Name)
-	}
-
-	if a.Type != b.Type {
-		return int(a.Type - b.Type)
-	}
-
-	return strings.Compare(a.Value, b.Value)
+	return core.CompareMatchers(a.Name, b.Name, a.Type, b.Type, a.Value, b.Value)
 }