Skip to content

Commit e122acd

Browse files
you06you06
authored
Merge branch 'master' into store-level-commit-ts-check
2 parents 000fc5f + f2266d6 commit e122acd

File tree

8 files changed

+103
-83
lines changed

8 files changed

+103
-83
lines changed

‎.github/workflows/compatibility.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ jobs:
2424
- name: Checkout TiDB
2525
uses: actions/checkout@v2
2626
with:
27-
repository: pingcap/tidb
27+
repository: crazycs520/tidb
28+
ref: opt-stats
2829
path: tidb
2930

3031
- name: Check build

‎integration_tests/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ require (
114114

115115
replace (
116116
github.com/go-ldap/ldap/v3 => github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117
117+
github.com/pingcap/tidb => github.com/crazycs520/tidb v1.1.0-beta.0.20241225031701-9554991d22f6
117118

118119
github.com/tikv/client-go/v2 => ../
119120
)

‎integration_tests/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -946,6 +946,8 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V
946946
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
947947
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
948948
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
949+
github.com/crazycs520/tidb v1.1.0-beta.0.20241225031701-9554991d22f6 h1:yz4Q18nLRc/8R4/zFsG6Lw/q/wT6Yn60QuOC6DpWDXc=
950+
github.com/crazycs520/tidb v1.1.0-beta.0.20241225031701-9554991d22f6/go.mod h1:9slOs9kUMdP48FtRb7K/njHPSKcVNiJqWYVsCxtPvQE=
949951
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
950952
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
951953
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=

‎integration_tests/snapshot_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,13 @@ func (s *testSnapshotSuite) TestSnapshotRuntimeStats() {
347347
"scan_detail: {total_process_keys: 20, total_process_keys_size: 20, total_keys: 30, get_snapshot_time: 1µs, " +
348348
"rocksdb: {delete_skipped_count: 10, key_skipped_count: 2, block: {cache_hit_count: 20, read_count: 40, read_byte: 30 Bytes}}}"
349349
s.Equal(expect, snapshot.FormatStats())
350+
snapshot.GetResolveLockDetail().ResolveLockTime = int64(time.Second)
351+
expect = "Get:{num_rpc:4, total_time:2s},txnLockFast_backoff:{num:2, total_time:10ms}, " +
352+
"time_detail: {total_process_time: 200ms, total_wait_time: 200ms}, " +
353+
"resolve_lock_time:1s, " +
354+
"scan_detail: {total_process_keys: 20, total_process_keys_size: 20, total_keys: 30, get_snapshot_time: 1µs, " +
355+
"rocksdb: {delete_skipped_count: 10, key_skipped_count: 2, block: {cache_hit_count: 20, read_count: 40, read_byte: 30 Bytes}}}"
356+
s.Equal(expect, snapshot.FormatStats())
350357
}
351358

352359
func (s *testSnapshotSuite) TestRCRead() {

‎internal/locate/region_request.go

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ func (s *RegionRequestSender) String() string {
127127

128128
// RegionRequestRuntimeStats records the runtime stats of send region requests.
129129
type RegionRequestRuntimeStats struct {
130-
RPCStats map[tikvrpc.CmdType]*RPCRuntimeStats
130+
// RPCStatsList uses to record RPC requests stats, since in most cases, only one kind of rpc request is sent at a time, use slice instead of map for performance.
131+
RPCStatsList []RPCRuntimeStats
131132
RequestErrorStats
132133
}
133134

@@ -142,29 +143,47 @@ type RequestErrorStats struct {
142143
// NewRegionRequestRuntimeStats returns a new RegionRequestRuntimeStats.
143144
func NewRegionRequestRuntimeStats() *RegionRequestRuntimeStats {
144145
return &RegionRequestRuntimeStats{
145-
RPCStats: make(map[tikvrpc.CmdType]*RPCRuntimeStats),
146+
RPCStatsList: make([]RPCRuntimeStats, 0, 1),
146147
}
147148
}
148149

149150
// RPCRuntimeStats indicates the RPC request count and consume time.
150151
type RPCRuntimeStats struct {
151-
Count int64
152+
Cmd tikvrpc.CmdType
153+
Count uint32
152154
// Send region request consume time.
153-
Consume int64
155+
Consume time.Duration
154156
}
155157

156158
// RecordRPCRuntimeStats uses to record the rpc count and duration stats.
157159
func (r *RegionRequestRuntimeStats) RecordRPCRuntimeStats(cmd tikvrpc.CmdType, d time.Duration) {
158-
stat, ok := r.RPCStats[cmd]
159-
if !ok {
160-
r.RPCStats[cmd] = &RPCRuntimeStats{
161-
Count: 1,
162-
Consume: int64(d),
160+
for i := range r.RPCStatsList {
161+
if r.RPCStatsList[i].Cmd == cmd {
162+
r.RPCStatsList[i].Count++
163+
r.RPCStatsList[i].Consume += d
164+
return
165+
}
166+
}
167+
r.RPCStatsList = append(r.RPCStatsList, RPCRuntimeStats{
168+
Cmd: cmd,
169+
Count: 1,
170+
Consume: d,
171+
})
172+
}
173+
174+
// GetRPCStatsCount returns the total rpc types count.
175+
func (r *RegionRequestRuntimeStats) GetRPCStatsCount() int {
176+
return len(r.RPCStatsList)
177+
}
178+
179+
// GetCmdRPCCount returns the rpc count of the specified cmd type.
180+
func (r *RegionRequestRuntimeStats) GetCmdRPCCount(cmd tikvrpc.CmdType) uint32 {
181+
for i := range r.RPCStatsList {
182+
if r.RPCStatsList[i].Cmd == cmd {
183+
return r.RPCStatsList[i].Count
163184
}
164-
return
165185
}
166-
stat.Count++
167-
stat.Consume += int64(d)
186+
return 0
168187
}
169188

170189
// RecordRPCErrorStats uses to record the request error(region error label and rpc error) info and count.
@@ -198,15 +217,15 @@ func (r *RegionRequestRuntimeStats) String() string {
198217
return ""
199218
}
200219
var builder strings.Builder
201-
for k, v := range r.RPCStats {
220+
for _, v := range r.RPCStatsList {
202221
if builder.Len() > 0 {
203222
builder.WriteByte(',')
204223
}
205-
builder.WriteString(k.String())
224+
builder.WriteString(v.Cmd.String())
206225
builder.WriteString(":{num_rpc:")
207-
builder.WriteString(strconv.FormatInt(v.Count, 10))
226+
builder.WriteString(strconv.FormatUint(uint64(v.Count), 10))
208227
builder.WriteString(", total_time:")
209-
builder.WriteString(util.FormatDuration(time.Duration(v.Consume)))
228+
builder.WriteString(util.FormatDuration(v.Consume))
210229
builder.WriteString("}")
211230
}
212231
if errStatsStr := r.RequestErrorStats.String(); errStatsStr != "" {
@@ -242,7 +261,8 @@ func (r *RequestErrorStats) String() string {
242261
// Clone returns a copy of itself.
243262
func (r *RegionRequestRuntimeStats) Clone() *RegionRequestRuntimeStats {
244263
newRs := NewRegionRequestRuntimeStats()
245-
maps.Copy(newRs.RPCStats, r.RPCStats)
264+
newRs.RPCStatsList = make([]RPCRuntimeStats, 0, len(r.RPCStatsList))
265+
newRs.RPCStatsList = append(newRs.RPCStatsList, r.RPCStatsList...)
246266
if len(r.ErrStats) > 0 {
247267
newRs.ErrStats = make(map[string]int)
248268
maps.Copy(newRs.ErrStats, r.ErrStats)
@@ -256,17 +276,8 @@ func (r *RegionRequestRuntimeStats) Merge(rs *RegionRequestRuntimeStats) {
256276
if rs == nil {
257277
return
258278
}
259-
for cmd, v := range rs.RPCStats {
260-
stat, ok := r.RPCStats[cmd]
261-
if !ok {
262-
r.RPCStats[cmd] = &RPCRuntimeStats{
263-
Count: v.Count,
264-
Consume: v.Consume,
265-
}
266-
continue
267-
}
268-
stat.Count += v.Count
269-
stat.Consume += v.Consume
279+
for i := range rs.RPCStatsList {
280+
r.mergeRPCRuntimeStats(rs.RPCStatsList[i])
270281
}
271282
if len(rs.ErrStats) > 0 {
272283
if r.ErrStats == nil {
@@ -279,6 +290,17 @@ func (r *RegionRequestRuntimeStats) Merge(rs *RegionRequestRuntimeStats) {
279290
}
280291
}
281292

293+
func (r *RegionRequestRuntimeStats) mergeRPCRuntimeStats(rs RPCRuntimeStats) {
294+
for i := range r.RPCStatsList {
295+
if r.RPCStatsList[i].Cmd == rs.Cmd {
296+
r.RPCStatsList[i].Count += rs.Count
297+
r.RPCStatsList[i].Consume += rs.Consume
298+
return
299+
}
300+
}
301+
r.RPCStatsList = append(r.RPCStatsList, rs)
302+
}
303+
282304
// ReplicaAccessStats records the replica access info.
283305
type ReplicaAccessStats struct {
284306
// AccessInfos records the access info

‎internal/locate/region_request3_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1168,10 +1168,10 @@ func (s *testRegionRequestToThreeStoresSuite) TestSendReqFirstTimeout() {
11681168
regionErr, err := resp.GetRegionError()
11691169
s.Nil(err)
11701170
s.True(IsFakeRegionError(regionErr))
1171-
s.Equal(1, len(s.regionRequestSender.Stats.RPCStats))
1172-
s.Equal(int64(3), s.regionRequestSender.Stats.RPCStats[tikvrpc.CmdGet].Count) // 3 rpc
1173-
s.Equal(3, len(reqTargetAddrs)) // each rpc to a different store.
1174-
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
1171+
s.Equal(1, s.regionRequestSender.Stats.GetRPCStatsCount())
1172+
s.Equal(uint32(3), s.regionRequestSender.Stats.GetCmdRPCCount(tikvrpc.CmdGet)) // 3 rpc
1173+
s.Equal(3, len(reqTargetAddrs)) // each rpc to a different store.
1174+
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
11751175
// warn: must rest MaxExecutionDurationMs before retry.
11761176
resetStats()
11771177
if staleRead {
@@ -1187,9 +1187,9 @@ func (s *testRegionRequestToThreeStoresSuite) TestSendReqFirstTimeout() {
11871187
s.Nil(err)
11881188
s.Nil(regionErr)
11891189
s.Equal([]byte("value"), resp.Resp.(*kvrpcpb.GetResponse).Value)
1190-
s.Equal(1, len(s.regionRequestSender.Stats.RPCStats))
1191-
s.Equal(int64(1), s.regionRequestSender.Stats.RPCStats[tikvrpc.CmdGet].Count) // 1 rpc
1192-
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
1190+
s.Equal(1, s.regionRequestSender.Stats.GetRPCStatsCount())
1191+
s.Equal(uint32(1), s.regionRequestSender.Stats.GetCmdRPCCount(tikvrpc.CmdGet)) // 1 rpc
1192+
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
11931193
}
11941194
}
11951195

@@ -1398,9 +1398,9 @@ func (s *testRegionRequestToThreeStoresSuite) TestStaleReadTryFollowerAfterTimeo
13981398
s.Nil(err)
13991399
s.Nil(regionErr)
14001400
s.Equal([]byte("value"), resp.Resp.(*kvrpcpb.GetResponse).Value)
1401-
s.Equal(1, len(s.regionRequestSender.Stats.RPCStats))
1402-
s.Equal(int64(2), s.regionRequestSender.Stats.RPCStats[tikvrpc.CmdGet].Count) // 2 rpc
1403-
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
1401+
s.Equal(1, s.regionRequestSender.Stats.GetRPCStatsCount())
1402+
s.Equal(uint32(2), s.regionRequestSender.Stats.GetCmdRPCCount(tikvrpc.CmdGet)) // 2 rpc
1403+
s.Equal(0, bo.GetTotalBackoffTimes()) // no backoff since fast retry.
14041404
}
14051405

14061406
func (s *testRegionRequestToThreeStoresSuite) TestDoNotTryUnreachableLeader() {

‎txnkv/txnsnapshot/snapshot.go

Lines changed: 31 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ package txnsnapshot
3737
import (
3838
"bytes"
3939
"context"
40-
"fmt"
4140
"math"
41+
"strconv"
4242
"sync"
4343
"sync/atomic"
4444
"time"
@@ -564,7 +564,7 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *retry.Backoffer, batch batchKeys,
564564
resolveLocksOpts := txnlock.ResolveLocksOptions{
565565
CallerStartTS: s.version,
566566
Locks: locks,
567-
Detail: s.getResolveLockDetail(),
567+
Detail: s.GetResolveLockDetail(),
568568
}
569569
resolveLocksRes, err := cli.ResolveLocksWithOpts(bo, resolveLocksOpts)
570570
msBeforeExpired := resolveLocksRes.TTL
@@ -785,7 +785,7 @@ func (s *KVSnapshot) get(ctx context.Context, bo *retry.Backoffer, k []byte) ([]
785785
resolveLocksOpts := txnlock.ResolveLocksOptions{
786786
CallerStartTS: s.version,
787787
Locks: locks,
788-
Detail: s.getResolveLockDetail(),
788+
Detail: s.GetResolveLockDetail(),
789789
}
790790
resolveLocksRes, err := cli.ResolveLocksWithOpts(bo, resolveLocksOpts)
791791
if err != nil {
@@ -810,17 +810,6 @@ func (s *KVSnapshot) mergeExecDetail(detail *kvrpcpb.ExecDetailsV2) {
810810
if detail == nil || s.mu.stats == nil {
811811
return
812812
}
813-
if s.mu.stats.resolveLockDetail == nil {
814-
s.mu.stats.resolveLockDetail = &util.ResolveLockDetail{}
815-
}
816-
if s.mu.stats.scanDetail == nil {
817-
s.mu.stats.scanDetail = &util.ScanDetail{
818-
ResolveLock: s.mu.stats.resolveLockDetail,
819-
}
820-
}
821-
if s.mu.stats.timeDetail == nil {
822-
s.mu.stats.timeDetail = &util.TimeDetail{}
823-
}
824813
s.mu.stats.scanDetail.MergeFromScanDetailV2(detail.ScanDetailV2)
825814
s.mu.stats.timeDetail.MergeFromTimeDetail(detail.TimeDetailV2, detail.TimeDetail)
826815
}
@@ -1103,13 +1092,14 @@ func (s *KVSnapshot) GetKVReadTimeout() time.Duration {
11031092
return s.readTimeout
11041093
}
11051094

1106-
func (s *KVSnapshot) getResolveLockDetail() *util.ResolveLockDetail {
1095+
// GetResolveLockDetail returns ResolveLockDetail, exports for testing.
1096+
func (s *KVSnapshot) GetResolveLockDetail() *util.ResolveLockDetail {
11071097
s.mu.RLock()
11081098
defer s.mu.RUnlock()
11091099
if s.mu.stats == nil {
11101100
return nil
11111101
}
1112-
return s.mu.stats.resolveLockDetail
1102+
return &s.mu.stats.resolveLockDetail
11131103
}
11141104

11151105
// SetPipelined sets the snapshot to pipelined mode.
@@ -1128,14 +1118,18 @@ type SnapshotRuntimeStats struct {
11281118
rpcStats *locate.RegionRequestRuntimeStats
11291119
backoffSleepMS map[string]int
11301120
backoffTimes map[string]int
1131-
scanDetail *util.ScanDetail
1132-
timeDetail *util.TimeDetail
1133-
resolveLockDetail *util.ResolveLockDetail
1121+
scanDetail util.ScanDetail
1122+
timeDetail util.TimeDetail
1123+
resolveLockDetail util.ResolveLockDetail
11341124
}
11351125

11361126
// Clone implements the RuntimeStats interface.
11371127
func (rs *SnapshotRuntimeStats) Clone() *SnapshotRuntimeStats {
1138-
newRs := SnapshotRuntimeStats{}
1128+
newRs := SnapshotRuntimeStats{
1129+
scanDetail: rs.scanDetail,
1130+
timeDetail: rs.timeDetail,
1131+
resolveLockDetail: rs.resolveLockDetail,
1132+
}
11391133
if rs.rpcStats != nil {
11401134
newRs.rpcStats = rs.rpcStats.Clone()
11411135
}
@@ -1149,19 +1143,6 @@ func (rs *SnapshotRuntimeStats) Clone() *SnapshotRuntimeStats {
11491143
newRs.backoffTimes[k] += v
11501144
}
11511145
}
1152-
1153-
if rs.scanDetail != nil {
1154-
newRs.scanDetail = rs.scanDetail
1155-
}
1156-
1157-
if rs.timeDetail != nil {
1158-
newRs.timeDetail = rs.timeDetail
1159-
}
1160-
1161-
if rs.resolveLockDetail != nil {
1162-
newRs.resolveLockDetail = rs.resolveLockDetail
1163-
}
1164-
11651146
return &newRs
11661147
}
11671148

@@ -1187,6 +1168,9 @@ func (rs *SnapshotRuntimeStats) Merge(other *SnapshotRuntimeStats) {
11871168
rs.backoffTimes[k] += v
11881169
}
11891170
}
1171+
rs.scanDetail.Merge(&other.scanDetail)
1172+
rs.timeDetail.Merge(&other.timeDetail)
1173+
rs.resolveLockDetail.Merge(&other.resolveLockDetail)
11901174
}
11911175

11921176
// String implements fmt.Stringer interface.
@@ -1201,13 +1185,23 @@ func (rs *SnapshotRuntimeStats) String() string {
12011185
}
12021186
ms := rs.backoffSleepMS[k]
12031187
d := time.Duration(ms) * time.Millisecond
1204-
buf.WriteString(fmt.Sprintf("%s_backoff:{num:%d, total_time:%s}", k, v, util.FormatDuration(d)))
1188+
buf.WriteString(k)
1189+
buf.WriteString("_backoff:{num:")
1190+
buf.WriteString(strconv.Itoa(v))
1191+
buf.WriteString(", total_time:")
1192+
buf.WriteString(util.FormatDuration(d))
1193+
buf.WriteString("}")
12051194
}
12061195
timeDetail := rs.timeDetail.String()
12071196
if timeDetail != "" {
12081197
buf.WriteString(", ")
12091198
buf.WriteString(timeDetail)
12101199
}
1200+
if rs.resolveLockDetail.ResolveLockTime > 0 {
1201+
buf.WriteString(", ")
1202+
buf.WriteString("resolve_lock_time:")
1203+
buf.WriteString(util.FormatDuration(time.Duration(rs.resolveLockDetail.ResolveLockTime)))
1204+
}
12111205
scanDetail := rs.scanDetail.String()
12121206
if scanDetail != "" {
12131207
buf.WriteString(", ")
@@ -1218,19 +1212,14 @@ func (rs *SnapshotRuntimeStats) String() string {
12181212

12191213
// GetTimeDetail returns the timeDetail
12201214
func (rs *SnapshotRuntimeStats) GetTimeDetail() *util.TimeDetail {
1221-
return rs.timeDetail
1215+
return &rs.timeDetail
12221216
}
12231217

12241218
// GetCmdRPCCount returns the count of the corresponding kind of rpc requests
12251219
func (rs *SnapshotRuntimeStats) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 {
1226-
if rs.rpcStats == nil || len(rs.rpcStats.RPCStats) == 0 {
1227-
return 0
1228-
}
1229-
1230-
stats, ok := rs.rpcStats.RPCStats[cmd]
1231-
if !ok {
1220+
if rs.rpcStats == nil {
12321221
return 0
12331222
}
12341223

1235-
return stats.Count
1224+
return int64(rs.rpcStats.GetCmdRPCCount(cmd))
12361225
}

‎util/execdetails.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,8 +417,6 @@ type ScanDetail struct {
417417
RocksdbBlockReadDuration time.Duration
418418
// GetSnapshotDuration is the time spent getting an engine snapshot.
419419
GetSnapshotDuration time.Duration
420-
421-
ResolveLock *ResolveLockDetail
422420
}
423421

424422
// Merge merges scan detail execution details into self.

0 commit comments

Comments
 (0)