wip: splitting and caching instant queries for functions over range operators

development/mimir-ingest-storage/config/mimir.yaml

@@ -39,14 +39,18 @@ block_builder_scheduler:
   startup_observe_time: 5s
 
 blocks_storage:
-  s3:
-    bucket_name:       mimir-blocks
+  backend: filesystem
+  filesystem:
+    dir: /blocks-data
   tsdb:
     dir: /data
     head_postings_for_matchers_cache_force: true
     block_postings_for_matchers_cache_force: true
 
   bucket_store:
+    bucket_index:
+      max_stale_period: 168h
+
     index_cache:
       backend: memcached
       memcached:
@@ -85,6 +89,16 @@ ruler_storage:
     memcached:
       addresses: dns+memcached:11211
 
+querier:
+  mimir_query_engine:
+    instant_query_splitting:
+      enabled: true
+      split_interval: 2h
+      intermediate_results_cache:
+        backend: memcached
+        memcached:
+          addresses: dns+memcached:11211
+
 query_scheduler:
   service_discovery_mode: "ring"
 

development/mimir-ingest-storage/docker-compose.jsonnet

@@ -379,7 +379,8 @@ std.manifestYamlDoc({
       extraArguments: [],
       debug: false,
       debugPort: self.publishedHttpPort + 3000,
-      extraVolumes: [],
+      // Temp: add volume for filesystem while testing query splitting
+      extraVolumes: ['.blocks-data:/blocks-data:delegated'],
       memberlistBindPort: self.publishedHttpPort + 2000,
     },
 

development/mimir-ingest-storage/docker-compose.yml

@@ -24,6 +24,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "block-builder-0":
     "build":
       "context": "."
@@ -49,6 +50,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "block-builder-scheduler-0":
     "build":
       "context": "."
@@ -74,6 +76,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "compactor":
     "build":
       "context": "."
@@ -99,6 +102,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "distributor-1":
     "build":
       "context": "."
@@ -124,6 +128,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "grafana":
     "environment":
       - "GF_AUTH_ANONYMOUS_ENABLED=true"
@@ -335,6 +340,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "query-frontend":
     "build":
       "context": "."
@@ -360,6 +366,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "query-scheduler":
     "build":
       "context": "."
@@ -385,6 +392,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "redpanda_console":
     "environment":
       - "KAFKA_BROKERS=kafka_1:9092,kafka_2:9092,kafka_3:9092"
@@ -416,6 +424,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "store-gateway-zone-a-1":
     "build":
       "context": "."
@@ -441,6 +450,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "store-gateway-zone-b-1":
     "build":
       "context": "."
@@ -466,6 +476,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "tempo":
     "command":
       - "-config.file=/etc/config/tempo.yaml"
@@ -502,6 +513,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "usage-tracker-zone-a-1":
     "build":
       "context": "."
@@ -527,6 +539,7 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"
   "usage-tracker-zone-b-0":
     "build":
       "context": "."
@@ -552,3 +565,4 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+      - ".blocks-data:/blocks-data:delegated"

pkg/querier/querier.go

@@ -39,10 +39,12 @@ import (
 	"github.com/grafana/mimir/pkg/storage/lazyquery"
 	"github.com/grafana/mimir/pkg/storage/series"
 	"github.com/grafana/mimir/pkg/streamingpromql"
+	"github.com/grafana/mimir/pkg/streamingpromql/cache"
 	"github.com/grafana/mimir/pkg/streamingpromql/compat"
 	"github.com/grafana/mimir/pkg/util"
 	"github.com/grafana/mimir/pkg/util/activitytracker"
 	"github.com/grafana/mimir/pkg/util/limiter"
+	util_log "github.com/grafana/mimir/pkg/util/log"
 	"github.com/grafana/mimir/pkg/util/spanlogger"
 	"github.com/grafana/mimir/pkg/util/validation"
 )
@@ -129,6 +131,10 @@ func (cfg *Config) Validate() error {
 		return errStreamingStoreGatewayBufferSize
 	}
 
+	if err := cfg.EngineConfig.MimirQueryEngine.InstantQuerySplitting.Validate(); err != nil {
+		return fmt.Errorf("invalid instant query splitting config: %w", err)
+	}
+
 	return nil
 }
 
@@ -190,6 +196,16 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, quer
 
 	opts, mqeOpts := engine.NewPromQLEngineOptions(cfg.EngineConfig, tracker, logger, reg)
 
+	if cfg.EngineConfig.MimirQueryEngine.InstantQuerySplitting.Enabled {
+		intermediateCache, err := cache.NewResultsCache(cfg.EngineConfig.MimirQueryEngine.InstantQuerySplitting.IntermediateResultsCache, logger, reg)
+		if err != nil {
+			return nil, nil, nil, nil, fmt.Errorf("failed to init query splitting cache, err: %w", err)
+		} else {
+			mqeOpts.IntermediateResultCache = intermediateCache
+			util_log.Logger.Log("msg", "intermediate results cache enabled", "backend", cfg.EngineConfig.MimirQueryEngine.InstantQuerySplitting.IntermediateResultsCache.Backend)
+		}
+	}
+
 	// Experimental functions are always enabled globally for all engines. Access to them
 	// is controlled by an experimental functions middleware that reads per-tenant settings.
 	parser.EnableExperimentalFunctions = true