Skip to content

Commit c0cf08c

Browse files
ti-chi-botti-chi-bot
authored
tsdb: update vm to v1.76.1; expose more params (#213) (#253)
close #212
1 parent 95dde7a commit c0cf08c

File tree

7 files changed

+120
-826
lines changed

7 files changed

+120
-826
lines changed

‎config/config.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ const (
2929
DefProfileSeconds = 10
3030
DefProfilingTimeoutSeconds = 120
3131
DefProfilingDataRetentionSeconds = 3 * 24 * 60 * 60 // 3 days
32+
DefTSDBRetentionPeriod = "1" // 1 month
33+
DefTSDBSearchMaxUniqueTimeseries = 300000
3234
)
3335

3436
type Config struct {
@@ -39,6 +41,7 @@ type Config struct {
3941
Storage Storage `toml:"storage" json:"storage"`
4042
ContinueProfiling ContinueProfilingConfig `toml:"-" json:"continuous_profiling"`
4143
Security Security `toml:"security" json:"security"`
44+
TSDB TSDB `toml:"tsdb" json:"tsdb"`
4245
}
4346

4447
var defaultConfig = Config{
@@ -60,6 +63,10 @@ var defaultConfig = Config{
6063
TimeoutSeconds: DefProfilingTimeoutSeconds,
6164
DataRetentionSeconds: DefProfilingDataRetentionSeconds,
6265
},
66+
TSDB: TSDB{
67+
RetentionPeriod: DefTSDBRetentionPeriod,
68+
SearchMaxUniqueTimeseries: DefTSDBSearchMaxUniqueTimeseries,
69+
},
6370
}
6471

6572
func GetDefaultConfig() Config {
@@ -388,6 +395,11 @@ func buildTLSConfig(caPath, keyPath, certPath string) *tls.Config {
388395
return tlsConfig
389396
}
390397

398+
type TSDB struct {
399+
RetentionPeriod string `toml:"retention-period" json:"retention_period"`
400+
SearchMaxUniqueTimeseries int64 `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
401+
}
402+
391403
type ContinueProfilingConfig struct {
392404
Enable bool `json:"enable"`
393405
ProfileSeconds int `json:"profile_seconds"`

‎config/config.toml.example‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,14 @@ path = "data"
2424
ca-path = ""
2525
cert-path = ""
2626
key-path = ""
27+
28+
[tsdb]
29+
# Data with timestamps outside the retentionPeriod is automatically deleted
30+
# The following optional suffixes are supported: h (hour), d (day), w (week), y (year).
31+
# If suffix isn't set, then the duration is counted in months.
32+
retention-period = "1"
33+
# `search-max-unique-timeseries` limits the number of unique time series a single query can find and process.
34+
# VictoriaMetrics(tsdb) keeps in memory some metainformation about the time series located by each query
35+
# and spends some CPU time for processing the found time series. This means that the maximum memory usage
36+
# and CPU usage a single query can use is proportional to `search-max-unique-timeseries`.
37+
search-max-unique-timeseries = 300000

‎config/config_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ key-path = "ngm.key"`
8585
require.NotNil(t, cfg)
8686
data, err := json.Marshal(cfg)
8787
require.NoError(t, err)
88-
// TODO(mornyx): Rollback when tiflash#5285 is fixed.
89-
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
88+
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
89+
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
9090

9191
ctx, cancel := context.WithCancel(context.Background())
9292
defer cancel()
@@ -113,8 +113,8 @@ path = "data1"`
113113
require.Equal(t, getCfg(), globalCfg)
114114
data, err = json.Marshal(globalCfg)
115115
require.NoError(t, err)
116-
// TODO(mornyx): Rollback when tiflash#5285 is fixed.
117-
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
116+
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
117+
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
118118

119119
cfgData = ``
120120
err = ioutil.WriteFile(cfgFileName, []byte(cfgData), 0666)
@@ -124,8 +124,8 @@ path = "data1"`
124124
time.Sleep(time.Millisecond * 10)
125125
data, err = json.Marshal(GetGlobalConfig())
126126
require.NoError(t, err)
127-
// TODO(mornyx): Rollback when tiflash#5285 is fixed.
128-
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
127+
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
128+
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
129129
}
130130

131131
func TestConfigValid(t *testing.T) {

‎database/timeseries/vm.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package timeseries
22

33
import (
44
"flag"
5+
"fmt"
56
"os"
67
"path"
78
"time"
@@ -14,30 +15,24 @@ import (
1415
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
1516
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
1617
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
17-
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
1818
"github.com/pingcap/log"
19-
"github.com/spf13/pflag"
2019
"go.uber.org/zap"
2120
)
2221

23-
var (
24-
retentionPeriod = pflag.String("retention-period", "1", "Data with timestamps outside the retentionPeriod is automatically deleted\nThe following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months")
25-
)
26-
2722
func Init(cfg *config.Config) {
2823
if err := initLogger(cfg); err != nil {
2924
log.Fatal("Failed to open log file", zap.Error(err))
3025
}
3126
initDataDir(path.Join(cfg.Storage.Path, "tsdb"))
3227

33-
_ = flag.Set("retentionPeriod", *retentionPeriod)
28+
_ = flag.Set("retentionPeriod", cfg.TSDB.RetentionPeriod)
3429
_ = flag.Set("search.maxStepForPointsAdjustment", "1s")
30+
_ = flag.Set("search.maxUniqueTimeseries", fmt.Sprintf("%d", cfg.TSDB.SearchMaxUniqueTimeseries))
3531

3632
// Some components in VictoriaMetrics want parsed arguments, i.e. assert `flag.Parsed()`. Make them happy.
3733
_ = flag.CommandLine.Parse(nil)
3834

3935
startTime := time.Now()
40-
storage.SetMinScrapeIntervalForDeduplication(0)
4136
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
4237
vmselect.Init()
4338
vminsert.Init()

‎go.mod‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
require (
66
github.com/BurntSushi/toml v1.2.1
7-
github.com/VictoriaMetrics/VictoriaMetrics v1.65.0
7+
github.com/VictoriaMetrics/VictoriaMetrics v1.76.1
88
github.com/dgraph-io/badger/v3 v3.2103.1
99
github.com/genjidb/genji v0.13.0
1010
github.com/genjidb/genji/engine/badgerengine v0.13.0
@@ -24,8 +24,8 @@ require (
2424
github.com/prometheus/client_golang v1.14.0
2525
github.com/prometheus/common v0.39.0
2626
github.com/spf13/pflag v1.0.5
27-
github.com/stretchr/testify v1.8.1
28-
github.com/valyala/gozstd v1.14.2
27+
github.com/stretchr/testify v1.8.3
28+
github.com/valyala/gozstd v1.16.0
2929
github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f
3030
go.etcd.io/etcd/api/v3 v3.5.7
3131
go.etcd.io/etcd/client/pkg/v3 v3.5.7
@@ -42,10 +42,10 @@ require (
4242
cloud.google.com/go/compute v1.13.0 // indirect
4343
cloud.google.com/go/compute/metadata v0.2.1 // indirect
4444
github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect
45-
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
46-
github.com/VictoriaMetrics/fasthttp v1.0.16 // indirect
47-
github.com/VictoriaMetrics/metrics v1.17.3 // indirect
48-
github.com/VictoriaMetrics/metricsql v0.21.0 // indirect
45+
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
46+
github.com/VictoriaMetrics/fasthttp v1.1.0 // indirect
47+
github.com/VictoriaMetrics/metrics v1.18.1 // indirect
48+
github.com/VictoriaMetrics/metricsql v0.41.0 // indirect
4949
github.com/benbjohnson/clock v1.3.0 // indirect
5050
github.com/beorn7/perks v1.0.1 // indirect
5151
github.com/buger/jsonparser v1.1.1 // indirect
@@ -130,10 +130,10 @@ require (
130130
github.com/ugorji/go/codec v1.2.7 // indirect
131131
github.com/valyala/bytebufferpool v1.0.0 // indirect
132132
github.com/valyala/fastjson v1.6.3 // indirect
133-
github.com/valyala/fastrand v1.0.0 // indirect
133+
github.com/valyala/fastrand v1.1.0 // indirect
134134
github.com/valyala/fasttemplate v1.2.1 // indirect
135-
github.com/valyala/histogram v1.1.2 // indirect
136-
github.com/valyala/quicktemplate v1.6.3 // indirect
135+
github.com/valyala/histogram v1.2.0 // indirect
136+
github.com/valyala/quicktemplate v1.7.0 // indirect
137137
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
138138
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
139139
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect

0 commit comments

Comments
 (0)