tsdb: update vm to v1.76.1; expose more params (#213) (#215)

close #212

Author: ti-chi-bot

config/config.go

@@ -29,6 +29,8 @@ const (
 	DefProfileSeconds                = 10
 	DefProfilingTimeoutSeconds       = 120
 	DefProfilingDataRetentionSeconds = 3 * 24 * 60 * 60 // 3 days
+	DefTSDBRetentionPeriod           = "1"              // 1 month
+	DefTSDBSearchMaxUniqueTimeseries = 300000
 )
 
 type Config struct {
@@ -39,6 +41,7 @@ type Config struct {
 	Storage           Storage                 `toml:"storage" json:"storage"`
 	ContinueProfiling ContinueProfilingConfig `toml:"-" json:"continuous_profiling"`
 	Security          Security                `toml:"security" json:"security"`
+	TSDB              TSDB                    `toml:"tsdb" json:"tsdb"`
 }
 
 var defaultConfig = Config{
@@ -60,6 +63,10 @@ var defaultConfig = Config{
 		TimeoutSeconds:       DefProfilingTimeoutSeconds,
 		DataRetentionSeconds: DefProfilingDataRetentionSeconds,
 	},
+	TSDB: TSDB{
+		RetentionPeriod:           DefTSDBRetentionPeriod,
+		SearchMaxUniqueTimeseries: DefTSDBSearchMaxUniqueTimeseries,
+	},
 }
 
 func GetDefaultConfig() Config {
@@ -388,6 +395,11 @@ func buildTLSConfig(caPath, keyPath, certPath string) *tls.Config {
 	return tlsConfig
 }
 
+type TSDB struct {
+	RetentionPeriod           string `toml:"retention-period" json:"retention_period"`
+	SearchMaxUniqueTimeseries int64  `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
+}
+
 type ContinueProfilingConfig struct {
 	Enable               bool `json:"enable"`
 	ProfileSeconds       int  `json:"profile_seconds"`

config/config.toml.example

@@ -24,3 +24,14 @@ path = "data"
 ca-path = ""
 cert-path = ""
 key-path = ""
+
+[tsdb]
+# Data with timestamps outside the retentionPeriod is automatically deleted
+# The following optional suffixes are supported: h (hour), d (day), w (week), y (year).
+# If suffix isn't set, then the duration is counted in months.
+retention-period = "1"
+# `search-max-unique-timeseries` limits the number of unique time series a single query can find and process.
+# VictoriaMetrics(tsdb) keeps in memory some metainformation about the time series located by each query
+# and spends some CPU time for processing the found time series. This means that the maximum memory usage
+# and CPU usage a single query can use is proportional to `search-max-unique-timeseries`.
+search-max-unique-timeseries = 300000

config/config_test.go

@@ -86,7 +86,7 @@ key-path = "ngm.key"`
 	data, err := json.Marshal(cfg)
 	require.NoError(t, err)
 	// TODO(mornyx): Rollback when tiflash#5687 is fixed.
-	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
+	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -114,7 +114,7 @@ path = "data1"`
 	data, err = json.Marshal(globalCfg)
 	require.NoError(t, err)
 	// TODO(mornyx): Rollback when tiflash#5687 is fixed.
-	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
+	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
 
 	cfgData = ``
 	err = ioutil.WriteFile(cfgFileName, []byte(cfgData), 0666)
@@ -125,7 +125,7 @@ path = "data1"`
 	data, err = json.Marshal(GetGlobalConfig())
 	require.NoError(t, err)
 	// TODO(mornyx): Rollback when tiflash#5687 is fixed.
-	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
+	require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
 }
 
 func TestConfigValid(t *testing.T) {

database/timeseries/vm.go

@@ -2,6 +2,7 @@ package timeseries
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"path"
 	"time"
@@ -14,30 +15,24 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/pingcap/log"
-	"github.com/spf13/pflag"
 	"go.uber.org/zap"
 )
 
-var (
-	retentionPeriod = pflag.String("retention-period", "1", "Data with timestamps outside the retentionPeriod is automatically deleted\nThe following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months")
-)
-
 func Init(cfg *config.Config) {
 	if err := initLogger(cfg); err != nil {
 		log.Fatal("Failed to open log file", zap.Error(err))
 	}
 	initDataDir(path.Join(cfg.Storage.Path, "tsdb"))
 
-	_ = flag.Set("retentionPeriod", *retentionPeriod)
+	_ = flag.Set("retentionPeriod", cfg.TSDB.RetentionPeriod)
 	_ = flag.Set("search.maxStepForPointsAdjustment", "1s")
+	_ = flag.Set("search.maxUniqueTimeseries", fmt.Sprintf("%d", cfg.TSDB.SearchMaxUniqueTimeseries))
 
 	// Some components in VictoriaMetrics want parsed arguments, i.e. assert `flag.Parsed()`. Make them happy.
 	_ = flag.CommandLine.Parse(nil)
 
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(0)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/BurntSushi/toml v1.2.1
-	github.com/VictoriaMetrics/VictoriaMetrics v1.65.0
+	github.com/VictoriaMetrics/VictoriaMetrics v1.76.1
 	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/genjidb/genji v0.13.0
 	github.com/genjidb/genji/engine/badgerengine v0.13.0
@@ -24,8 +24,8 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/common v0.39.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.1
-	github.com/valyala/gozstd v1.14.2
+	github.com/stretchr/testify v1.8.3
+	github.com/valyala/gozstd v1.16.0
 	github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f
 	go.etcd.io/etcd/api/v3 v3.5.7
 	go.etcd.io/etcd/client/pkg/v3 v3.5.7
@@ -42,10 +42,10 @@ require (
 	cloud.google.com/go/compute v1.13.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect
-	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
-	github.com/VictoriaMetrics/fasthttp v1.0.16 // indirect
-	github.com/VictoriaMetrics/metrics v1.17.3 // indirect
-	github.com/VictoriaMetrics/metricsql v0.21.0 // indirect
+	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
+	github.com/VictoriaMetrics/fasthttp v1.1.0 // indirect
+	github.com/VictoriaMetrics/metrics v1.18.1 // indirect
+	github.com/VictoriaMetrics/metricsql v0.41.0 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
@@ -130,10 +130,10 @@ require (
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fastjson v1.6.3 // indirect
-	github.com/valyala/fastrand v1.0.0 // indirect
+	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
-	github.com/valyala/histogram v1.1.2 // indirect
-	github.com/valyala/quicktemplate v1.6.3 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/valyala/quicktemplate v1.7.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect