Skip to content

Commit b3f04b6

Browse files
ti-chi-botti-chi-bot
authored
*: upgrade vm; expose tsdb parameters for tuning; optimize memory usage (#296) (#297)
close #295
1 parent e33af79 commit b3f04b6

File tree

14 files changed

+237
-180
lines changed

14 files changed

+237
-180
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
- uses: actions/checkout@v3
1818
- uses: actions/setup-go@v3
1919
with:
20-
go-version: '1.23'
20+
go-version: '1.24.2'
2121
- name: Format
2222
run: make fmt
2323
- name: Lint

‎component/conprof/http/api.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,8 +341,7 @@ func queryAndDownload(c *gin.Context) error {
341341
}
342342

343343
c.Writer.Header().
344-
Set("Content-Disposition",
345-
fmt.Sprintf(`attachment; filename="profile"`+time.Unix(param.Begin, 0).Format("2006-01-02_15-04-05")+".zip"))
344+
Set("Content-Disposition", `attachment; filename="profile"`+time.Unix(param.Begin, 0).Format("2006-01-02_15-04-05")+".zip")
346345
zw := zip.NewWriter(c.Writer)
347346
fn := func(pt meta.ProfileTarget, ts int64, data []byte) error {
348347
fileName := fmt.Sprintf("%v_%v_%v_%v", pt.Kind, pt.Component, pt.Address, ts)

‎component/conprof/scrape/manager_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ func TestMain(m *testing.M) {
2424
opts := []goleak.Option{
2525
goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
2626
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
27+
goleak.IgnoreTopFunction("github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime.init.0.func1"),
2728
}
2829

2930
goleak.VerifyTestMain(m, opts...)

‎component/domain/domain_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ func TestMain(m *testing.M) {
1616
opts := []goleak.Option{
1717
goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
1818
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
19+
goleak.IgnoreTopFunction("github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime.init.0.func1"),
1920
}
2021

2122
goleak.VerifyTestMain(m, opts...)

‎component/subscriber/main_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
func TestMain(m *testing.M) {
1010
opts := []goleak.Option{
1111
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
12+
goleak.IgnoreTopFunction("github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime.init.0.func1"),
1213
}
1314

1415
goleak.VerifyTestMain(m, opts...)

‎component/topsql/subscriber/main_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ func TestMain(m *testing.M) {
1010
opts := []goleak.Option{
1111
goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
1212
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
13+
goleak.IgnoreTopFunction("github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime.init.0.func1"),
1314
}
1415

1516
goleak.VerifyTestMain(m, opts...)

‎config/config.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
type Config struct {
2828
Address string `toml:"address" json:"address"`
2929
AdvertiseAddress string `toml:"advertise-address" json:"advertise_address"`
30+
Go Go `toml:"go" json:"go"`
3031
PD PD `toml:"pd" json:"pd"`
3132
Log Log `toml:"log" json:"log"`
3233
Storage Storage `toml:"storage" json:"storage"`
@@ -46,7 +47,8 @@ var defaultConfig = Config{
4647
Level: "INFO",
4748
},
4849
Storage: Storage{
49-
Path: "data",
50+
Path: "data",
51+
SQLiteUseWAL: true,
5052
},
5153
ContinueProfiling: ContinueProfilingConfig{
5254
Enable: false, // TODO(mornyx): Enable when tiflash#5285 is fixed.
@@ -248,6 +250,11 @@ func validateAddress(address, name string) error {
248250
return nil
249251
}
250252

253+
type Go struct {
254+
GCPercent int `toml:"gc-percent" json:"gc_percent"`
255+
MemoryLimit int64 `toml:"memory-limit" json:"memory_limit"`
256+
}
257+
251258
type PD struct {
252259
Endpoints []string `toml:"endpoints" json:"endpoints"`
253260
}
@@ -278,6 +285,7 @@ func (p *PD) Equal(other PD) bool {
278285
type Storage struct {
279286
Path string `toml:"path" json:"path"`
280287
DocDBBackend string `toml:"docdb-backend" json:"docdb_backend"`
288+
SQLiteUseWAL bool `toml:"sqlite-use-wal" json:"sqlite_use_wal"`
281289
MetaRetentionSecs int64 `toml:"meta-retention-secs" json:"meta_retention_secs"`
282290
}
283291

@@ -414,8 +422,16 @@ func buildTLSConfig(caPath, keyPath, certPath string) *tls.Config {
414422
}
415423

416424
type TSDB struct {
417-
RetentionPeriod string `toml:"retention-period" json:"retention_period"`
418-
SearchMaxUniqueTimeseries int64 `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
425+
RetentionPeriod string `toml:"retention-period" json:"retention_period"`
426+
SearchMaxUniqueTimeseries int64 `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
427+
MemoryAllowedBytes int64 `toml:"memory-allowed-bytes" json:"memory_allowed_bytes"`
428+
MemoryAllowedPercent float64 `toml:"memory-allowed-percent" json:"memory_allowed_percent"`
429+
CacheSizeIndexDBDataBlocks string `toml:"cache-size-indexdb-data-blocks" json:"cache_size_indexdb_data_blocks"`
430+
CacheSizeIndexDBDataBlocksSparse string `toml:"cache-size-indexdb-data-blocks-sparse" json:"cache_size_indexdb_data_blocks_sparse"`
431+
CacheSizeIndexDBIndexBlocks string `toml:"cache-size-indexdb-index-blocks" json:"cache_size_indexdb_index_blocks"`
432+
CacheSizeIndexDBTagFilters string `toml:"cache-size-indexdb-tag-filters" json:"cache_size_indexdb_tag_filters"`
433+
CacheSizeMetricNamesStats string `toml:"cache-size-metric-names-stats" json:"cache_size_metric_names_stats"`
434+
CacheSizeStorageTSID string `toml:"cache-size-storage-tsid" json:"cache_size_storage_tsid"`
419435
}
420436

421437
type ContinueProfilingConfig struct {

‎database/docdb/sqlite.go‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,37 @@ import (
1717

1818
type sqliteDB struct {
1919
db *sql.DB
20+
21+
writeSQLMetaStmt *sql.Stmt
22+
writePlanMetaStmt *sql.Stmt
2023
}
2124

22-
func NewSQLiteDB(dbPath string) (DocDB, error) {
25+
func NewSQLiteDB(dbPath string, useWAL bool) (DocDB, error) {
2326
dbPath = path.Join(dbPath, "ng-sqlite.db")
2427
db, err := sql.Open("sqlite3", dbPath)
2528
if err != nil {
2629
log.Fatal("failed to open sqlite db", zap.String("path", dbPath), zap.Error(err))
2730
}
31+
if useWAL {
32+
_, err := db.Exec("PRAGMA journal_mode=WAL")
33+
if err != nil {
34+
return nil, err
35+
}
36+
}
2837
d := &sqliteDB{db: db}
2938
if err := d.tryInitTables(); err != nil {
3039
return nil, err
3140
}
41+
writeSQLMetaSQL := `INSERT OR REPLACE INTO sql_digest (digest, sql_text, is_internal, created_at_ts) VALUES (?, ?, ?, ?)`
42+
d.writeSQLMetaStmt, err = d.db.Prepare(writeSQLMetaSQL)
43+
if err != nil {
44+
return nil, err
45+
}
46+
writePlanMetaSQL := `INSERT OR REPLACE INTO plan_digest (digest, plan_text, encoded_plan, created_at_ts) VALUES (?, ?, ?, ?)`
47+
d.writePlanMetaStmt, err = d.db.Prepare(writePlanMetaSQL)
48+
if err != nil {
49+
return nil, err
50+
}
3251
return d, nil
3352
}
3453

@@ -89,13 +108,8 @@ func (d *sqliteDB) LoadConfig(ctx context.Context) (map[string]string, error) {
89108
}
90109

91110
func (d *sqliteDB) WriteSQLMeta(ctx context.Context, meta *tipb.SQLMeta) error {
92-
sql := `INSERT OR REPLACE INTO sql_digest (digest, sql_text, is_internal, created_at_ts) VALUES (?, ?, ?, ?)`
93-
stmt, err := d.db.PrepareContext(ctx, sql)
94-
if err != nil {
95-
return err
96-
}
97111
now := time.Now().Unix()
98-
_, err = stmt.ExecContext(ctx, hex.EncodeToString(meta.SqlDigest), meta.NormalizedSql, meta.IsInternalSql, now)
112+
_, err := d.writeSQLMetaStmt.ExecContext(ctx, hex.EncodeToString(meta.SqlDigest), meta.NormalizedSql, meta.IsInternalSql, now)
99113
return err
100114
}
101115

@@ -117,13 +131,8 @@ func (d *sqliteDB) DeleteSQLMetaBeforeTs(ctx context.Context, ts int64) error {
117131
}
118132

119133
func (d *sqliteDB) WritePlanMeta(ctx context.Context, meta *tipb.PlanMeta) error {
120-
sql := `INSERT OR REPLACE INTO plan_digest (digest, plan_text, encoded_plan, created_at_ts) VALUES (?, ?, ?, ?)`
121-
stmt, err := d.db.PrepareContext(ctx, sql)
122-
if err != nil {
123-
return err
124-
}
125134
now := time.Now().Unix()
126-
_, err = stmt.ExecContext(ctx, hex.EncodeToString(meta.PlanDigest), meta.NormalizedPlan, meta.EncodedNormalizedPlan, now)
135+
_, err := d.writePlanMetaStmt.ExecContext(ctx, hex.EncodeToString(meta.PlanDigest), meta.NormalizedPlan, meta.EncodedNormalizedPlan, now)
127136
return err
128137
}
129138

‎database/docdb/sqlite_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
func TestSQLite(t *testing.T) {
1111
dir, err := os.MkdirTemp(os.TempDir(), "ngm-test-.*")
1212
require.NoError(t, err)
13-
db, err := NewSQLiteDB(dir)
13+
db, err := NewSQLiteDB(dir, true)
1414
require.NoError(t, err)
1515
testDocDB(t, db)
1616
}

‎database/timeseries/vm.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,30 @@ func Init(cfg *config.Config) {
2828
_ = flag.Set("retentionPeriod", cfg.TSDB.RetentionPeriod)
2929
_ = flag.Set("search.maxStepForPointsAdjustment", "1s")
3030
_ = flag.Set("search.maxUniqueTimeseries", fmt.Sprintf("%d", cfg.TSDB.SearchMaxUniqueTimeseries))
31+
if cfg.TSDB.MemoryAllowedBytes > 0 {
32+
_ = flag.Set("memory.allowedBytes", fmt.Sprintf("%d", cfg.TSDB.MemoryAllowedBytes))
33+
}
34+
if cfg.TSDB.MemoryAllowedPercent > 0 {
35+
_ = flag.Set("memory.allowedPercent", fmt.Sprintf("%f", cfg.TSDB.MemoryAllowedPercent))
36+
}
37+
if cfg.TSDB.CacheSizeIndexDBDataBlocks != "" {
38+
_ = flag.Set("storage.cacheSizeIndexDBDataBlocks", cfg.TSDB.CacheSizeIndexDBDataBlocks)
39+
}
40+
if cfg.TSDB.CacheSizeIndexDBDataBlocksSparse != "" {
41+
_ = flag.Set("storage.cacheSizeIndexDBDataBlocksSparse", cfg.TSDB.CacheSizeIndexDBDataBlocksSparse)
42+
}
43+
if cfg.TSDB.CacheSizeIndexDBIndexBlocks != "" {
44+
_ = flag.Set("storage.cacheSizeIndexDBIndexBlocks", cfg.TSDB.CacheSizeIndexDBIndexBlocks)
45+
}
46+
if cfg.TSDB.CacheSizeIndexDBTagFilters != "" {
47+
_ = flag.Set("storage.cacheSizeIndexDBTagFilters", cfg.TSDB.CacheSizeIndexDBTagFilters)
48+
}
49+
if cfg.TSDB.CacheSizeMetricNamesStats != "" {
50+
_ = flag.Set("storage.cacheSizeMetricNamesStats", cfg.TSDB.CacheSizeMetricNamesStats)
51+
}
52+
if cfg.TSDB.CacheSizeStorageTSID != "" {
53+
_ = flag.Set("storage.cacheSizeStorageTSID", cfg.TSDB.CacheSizeStorageTSID)
54+
}
3155

3256
// Some components in VictoriaMetrics want parsed arguments, i.e. assert `flag.Parsed()`. Make them happy.
3357
_ = flag.CommandLine.Parse(nil)

0 commit comments

Comments
 (0)