Skip to content

Commit d58e892

Browse files
ti-chi-botmornyx
ti-chi-bot
and
mornyx
authored
docdb: introduce sqlite backend (#287) (#293)
* docdb: introduce sqlite backend Signed-off-by: mornyx <mornyx.z@gmail.com> * comment linter Signed-off-by: mornyx <mornyx.z@gmail.com> * fix lint Signed-off-by: mornyx <mornyx.z@gmail.com> * fix tests Signed-off-by: mornyx <mornyx.z@gmail.com> --------- Signed-off-by: mornyx <mornyx.z@gmail.com> Co-authored-by: mornyx <mornyx.z@gmail.com>
1 parent a9690a9 commit d58e892

36 files changed

Lines changed: 1147 additions & 718 deletions

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
- uses: actions/checkout@v3
1818
- uses: actions/setup-go@v3
1919
with:
20-
go-version: '1.21'
20+
go-version: '1.23'
2121
- name: Format
2222
run: make fmt
2323
- name: Lint

‎.golangci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ linters:
1919
- asciicheck
2020
- bodyclose
2121
# - exportloopref
22-
- copyloopvar
22+
# - copyloopvar
2323
- rowserrcheck
2424
- unconvert
2525
- makezero

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,4 @@ lint: tools/bin/golangci-lint
4848
GO111MODULE=on tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES)) --config .golangci.yml
4949

5050
tools/bin/golangci-lint:
51-
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.61.0
51+
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.64.5

‎component/conprof/contprof.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,15 @@ import (
44
"github.com/pingcap/ng-monitoring/component/conprof/scrape"
55
"github.com/pingcap/ng-monitoring/component/conprof/store"
66
"github.com/pingcap/ng-monitoring/component/topology"
7-
8-
"github.com/genjidb/genji"
7+
"github.com/pingcap/ng-monitoring/database/docdb"
98
)
109

1110
var (
1211
storage *store.ProfileStorage
1312
manager *scrape.Manager
1413
)
1514

16-
func Init(db *genji.DB, subscriber topology.Subscriber) error {
15+
func Init(db docdb.DocDB, subscriber topology.Subscriber) error {
1716
var err error
1817
storage, err = store.NewProfileStorage(db)
1918
if err != nil {

‎component/conprof/http/api_test.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"encoding/json"
77
"errors"
88
"fmt"
9-
"io/ioutil"
9+
"io"
1010
"net"
1111
"net/http"
1212
"os"
@@ -19,23 +19,24 @@ import (
1919
"github.com/pingcap/ng-monitoring/component/conprof/meta"
2020
"github.com/pingcap/ng-monitoring/component/topology"
2121
"github.com/pingcap/ng-monitoring/config"
22+
"github.com/pingcap/ng-monitoring/database/docdb"
2223
"github.com/pingcap/ng-monitoring/utils/testutil"
2324

24-
"github.com/genjidb/genji"
2525
"github.com/gin-gonic/gin"
2626
"github.com/stretchr/testify/require"
2727
)
2828

2929
type testSuite struct {
3030
tmpDir string
31-
db *genji.DB
31+
db docdb.DocDB
3232
}
3333

3434
func (ts *testSuite) setup(t *testing.T) {
3535
var err error
36-
ts.tmpDir, err = ioutil.TempDir(os.TempDir(), "ngm-test-.*")
36+
ts.tmpDir, err = os.MkdirTemp(os.TempDir(), "ngm-test-.*")
37+
require.NoError(t, err)
38+
ts.db, err = docdb.NewGenjiDBFromGenji(testutil.NewGenjiDB(t, ts.tmpDir))
3739
require.NoError(t, err)
38-
ts.db = testutil.NewGenjiDB(t, ts.tmpDir)
3940
cfg := config.GetDefaultConfig()
4041
cfg.ContinueProfiling.Enable = true
4142
cfg.ContinueProfiling.ProfileSeconds = 1
@@ -96,7 +97,7 @@ func testAPIGroupProfiles(t *testing.T, httpAddr string) int64 {
9697
resp, err := http.Get(fmt.Sprintf("http://%v/continuous_profiling/group_profiles?begin_time=%v&end_time=%v", httpAddr, ts-2*60*60, ts))
9798
require.NoError(t, err)
9899
require.Equal(t, 200, resp.StatusCode)
99-
body, err := ioutil.ReadAll(resp.Body)
100+
body, err := io.ReadAll(resp.Body)
100101
require.NoError(t, err)
101102
err = resp.Body.Close()
102103
require.NoError(t, err)
@@ -116,7 +117,7 @@ func testAPIGroupProfileDetail(t *testing.T, httpAddr string, ts int64, componen
116117
resp, err := http.Get("http://" + httpAddr + "/continuous_profiling/group_profile/detail?ts=" + strconv.Itoa(int(ts)))
117118
require.NoError(t, err)
118119
require.Equal(t, 200, resp.StatusCode)
119-
body, err := ioutil.ReadAll(resp.Body)
120+
body, err := io.ReadAll(resp.Body)
120121
require.NoError(t, err)
121122
err = resp.Body.Close()
122123
require.NoError(t, err)
@@ -146,7 +147,7 @@ func testAPISingleProfileView(t *testing.T, httpAddr string, ts int64, component
146147
resp, err := http.Get(url)
147148
require.NoError(t, err)
148149
require.Equal(t, 200, resp.StatusCode)
149-
body, err := ioutil.ReadAll(resp.Body)
150+
body, err := io.ReadAll(resp.Body)
150151
require.NoError(t, err)
151152
require.Equal(t, "profile", string(body))
152153
err = resp.Body.Close()
@@ -165,7 +166,7 @@ func testAPIDownload(t *testing.T, httpAddr string, ts int64, components []topol
165166
resp, err := http.Get(url)
166167
require.NoError(t, err)
167168
require.Equal(t, 200, resp.StatusCode)
168-
body, err := ioutil.ReadAll(resp.Body)
169+
body, err := io.ReadAll(resp.Body)
169170
require.NoError(t, err)
170171
err = resp.Body.Close()
171172
require.NoError(t, err)
@@ -221,7 +222,7 @@ func testAPIComponent(t *testing.T, httpAddr string, components []topology.Compo
221222
resp, err := http.Get("http://" + httpAddr + "/continuous_profiling/components")
222223
require.NoError(t, err)
223224
require.Equal(t, 200, resp.StatusCode)
224-
body, err := ioutil.ReadAll(resp.Body)
225+
body, err := io.ReadAll(resp.Body)
225226
require.NoError(t, err)
226227
err = resp.Body.Close()
227228
require.NoError(t, err)
@@ -246,7 +247,7 @@ func testAPIEstimateSize(t *testing.T, httpAddr string, components []topology.Co
246247
resp, err := http.Get("http://" + httpAddr + "/continuous_profiling/estimate_size")
247248
require.NoError(t, err)
248249
require.Equal(t, 200, resp.StatusCode)
249-
body, err := ioutil.ReadAll(resp.Body)
250+
body, err := io.ReadAll(resp.Body)
250251
require.NoError(t, err)
251252
err = resp.Body.Close()
252253
require.NoError(t, err)
@@ -294,7 +295,7 @@ func testErrorRequest(t *testing.T, httpAddr string) {
294295
resp, err := http.Get("http://" + httpAddr + "/continuous_profiling" + ca.api)
295296
require.NoError(t, err)
296297
require.Equal(t, 503, resp.StatusCode, ca.api)
297-
body, err := ioutil.ReadAll(resp.Body)
298+
body, err := io.ReadAll(resp.Body)
298299
require.NoError(t, err)
299300
require.Equal(t, ca.body, string(body), ca.api)
300301
err = resp.Body.Close()
@@ -371,7 +372,7 @@ func TestQueryStatus(t *testing.T) {
371372
resp, err := http.Get(fmt.Sprintf("http://%v/continuous_profiling/group_profiles?begin_time=%v&end_time=%v", httpAddr, t0.Unix(), t2.Unix()))
372373
require.NoError(t, err)
373374
require.Equal(t, 200, resp.StatusCode)
374-
body, err := ioutil.ReadAll(resp.Body)
375+
body, err := io.ReadAll(resp.Body)
375376
require.NoError(t, err)
376377
err = resp.Body.Close()
377378
require.NoError(t, err)
@@ -393,7 +394,7 @@ func TestQueryStatus(t *testing.T) {
393394
resp, err = http.Get("http://" + httpAddr + "/continuous_profiling/group_profile/detail?ts=" + strconv.Itoa(int(ts)))
394395
require.NoError(t, err)
395396
require.Equal(t, 200, resp.StatusCode)
396-
body, err = ioutil.ReadAll(resp.Body)
397+
body, err = io.ReadAll(resp.Body)
397398
require.NoError(t, err)
398399
err = resp.Body.Close()
399400
require.NoError(t, err)

‎component/conprof/scrape/manager_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package scrape
22

33
import (
44
"fmt"
5-
"io/ioutil"
65
"os"
76
"strings"
87
"testing"
@@ -12,6 +11,7 @@ import (
1211
"github.com/pingcap/ng-monitoring/component/conprof/store"
1312
"github.com/pingcap/ng-monitoring/component/topology"
1413
"github.com/pingcap/ng-monitoring/config"
14+
"github.com/pingcap/ng-monitoring/database/docdb"
1515
"github.com/pingcap/ng-monitoring/utils/testutil"
1616

1717
"github.com/pingcap/log"
@@ -30,7 +30,7 @@ func TestMain(m *testing.M) {
3030
}
3131

3232
func TestManager(t *testing.T) {
33-
tmpDir, err := ioutil.TempDir(os.TempDir(), "ngm-test-.*")
33+
tmpDir, err := os.MkdirTemp(os.TempDir(), "ngm-test-.*")
3434
require.NoError(t, err)
3535
defer func() {
3636
err := os.RemoveAll(tmpDir)
@@ -43,7 +43,8 @@ func TestManager(t *testing.T) {
4343
cfg.ContinueProfiling.IntervalSeconds = 1
4444
config.StoreGlobalConfig(cfg)
4545

46-
db := testutil.NewGenjiDB(t, tmpDir)
46+
db, err := docdb.NewGenjiDBFromGenji(testutil.NewGenjiDB(t, tmpDir))
47+
require.NoError(t, err)
4748
defer db.Close()
4849
storage, err := store.NewProfileStorage(db)
4950
require.NoError(t, err)

‎component/conprof/scrape/scrape.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"context"
66
"fmt"
77
"io"
8-
"io/ioutil"
98
"net/http"
109
"net/url"
1110
"strconv"
@@ -172,7 +171,7 @@ func (s *Scraper) scrape(ctx context.Context, w io.Writer) error {
172171
return fmt.Errorf("server returned HTTP status %s", resp.Status)
173172
}
174173

175-
b, err := ioutil.ReadAll(resp.Body)
174+
b, err := io.ReadAll(resp.Body)
176175
if err != nil {
177176
return errors.Wrap(err, "failed to read body")
178177
}

‎component/conprof/store/gc.go‎

Lines changed: 9 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
package store
22

33
import (
4-
"fmt"
54
"time"
65

76
"github.com/pingcap/ng-monitoring/component/conprof/meta"
87
"github.com/pingcap/ng-monitoring/config"
98

10-
"github.com/genjidb/genji/document"
11-
"github.com/genjidb/genji/types"
129
"github.com/pingcap/log"
1310
"go.uber.org/zap"
1411
)
@@ -42,15 +39,13 @@ func (s *ProfileStorage) runGC() {
4239
safePointTs := s.getLastSafePointTs()
4340
for i, target := range allTargets {
4441
info := allInfos[i]
45-
sql := fmt.Sprintf("DELETE FROM %v WHERE ts <= ?", s.getProfileDataTableName(&info))
46-
err := s.db.Exec(sql, safePointTs)
42+
err := s.db.ConprofDeleteProfileDataBeforeTs(s.ctx, info.ID, safePointTs)
4743
if err != nil {
4844
log.Error("gc delete target data failed", zap.Error(err))
4945
}
50-
sql = fmt.Sprintf("DELETE FROM %v WHERE ts <= ?", s.getProfileMetaTableName(&info))
51-
err = s.db.Exec(sql, safePointTs)
46+
err = s.db.ConprofDeleteProfileMetaBeforeTs(s.ctx, info.ID, safePointTs)
5247
if err != nil {
53-
log.Error("gc delete target data failed", zap.Error(err))
48+
log.Error("gc delete target meta failed", zap.Error(err))
5449
}
5550
err = s.dropProfileTableIfStaled(target, info, safePointTs)
5651
if err != nil {
@@ -64,38 +59,18 @@ func (s *ProfileStorage) runGC() {
6459
}
6560

6661
func (s *ProfileStorage) loadAllTargetsFromTable() ([]meta.ProfileTarget, []meta.TargetInfo, error) {
67-
query := fmt.Sprintf("SELECT id, kind, component, address, last_scrape_ts FROM %v", metaTableName)
68-
res, err := s.db.Query(query)
69-
if err != nil {
70-
return nil, nil, err
71-
}
72-
defer res.Close()
73-
7462
targets := make([]meta.ProfileTarget, 0, 16)
7563
infos := make([]meta.TargetInfo, 0, 16)
76-
err = res.Iterate(func(d types.Document) error {
77-
var id, ts int64
78-
var kind, component, address string
79-
err = document.Scan(d, &id, &kind, &component, &address, &ts)
80-
if err != nil {
81-
return err
82-
}
83-
s.rebaseID(id)
84-
target := meta.ProfileTarget{
85-
Kind: kind,
86-
Component: component,
87-
Address: address,
88-
}
89-
info := meta.TargetInfo{
90-
ID: id,
91-
LastScrapeTs: ts,
92-
}
64+
err := s.db.ConprofQueryAllProfileTargets(s.ctx, func(target meta.ProfileTarget, info meta.TargetInfo) error {
65+
s.rebaseID(info.ID)
9366
targets = append(targets, target)
9467
infos = append(infos, info)
9568
return nil
9669
})
97-
log.Info("gc load all target info from meta table",
98-
zap.Int("all-target-count", len(targets)))
70+
if err != nil {
71+
return nil, nil, err
72+
}
73+
log.Info("gc load all target info from meta table", zap.Int("all-target-count", len(targets)))
9974
return targets, infos, nil
10075
}
10176

0 commit comments

Comments
 (0)