Skip to content

Commit afabcff

Browse files
freehere107freehere107
committed
Optimize extrinsic count retrieval with caching and jittered TTL
1 parent c6bd06c commit afabcff

5 files changed

Lines changed: 133 additions & 7 deletions

File tree

‎internal/dao/chainExtrinsic.go‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,23 @@ package dao
22

33
import (
44
"context"
5+
"crypto/sha1"
6+
"encoding/hex"
7+
"fmt"
58
"github.com/itering/subscan/model"
69
"github.com/itering/subscan/util"
710
"github.com/itering/subscan/util/address"
811
"github.com/itering/substrate-api-rpc"
12+
"gorm.io/gorm"
13+
"math/rand"
914
"strings"
1015
)
1116

17+
const (
18+
extrinsicCountCacheTTL = 24 * 3600
19+
extrinsicCountCacheTTLJitter = 3600
20+
)
21+
1222
func (d *Dao) CreateExtrinsic(c context.Context, txn *GormDB, extrinsic []model.ChainExtrinsic, signedExtrinsicCount int) error {
1323
if len(extrinsic) == 0 {
1424
return nil
@@ -24,17 +34,51 @@ func (d *Dao) CreateExtrinsic(c context.Context, txn *GormDB, extrinsic []model.
2434
func (d *Dao) GetExtrinsicCount(ctx context.Context, queryWhere ...model.Option) int64 {
2535
var count int64
2636
blockNum, _ := d.GetFillBestBlockNum(context.TODO())
27-
for index := blockNum / int(model.SplitTableBlockNum); index >= 0; index-- {
28-
var tableDataCount int64
29-
q := d.db.WithContext(ctx).Scopes(d.TableNameFunc(&model.ChainExtrinsic{BlockNum: uint(index) * model.SplitTableBlockNum}))
30-
q = q.Scopes(queryWhere...)
31-
q.Model(&model.ChainExtrinsic{}).Count(&tableDataCount)
37+
latestTableIndex := blockNum / int(model.SplitTableBlockNum)
38+
for index := latestTableIndex; index >= 0; index-- {
39+
if index == latestTableIndex || d.redis == nil {
40+
count += d.getExtrinsicTableCount(ctx, index, queryWhere...)
41+
continue
42+
}
43+
key := d.extrinsicCountCacheKey(index, queryWhere...)
44+
if cache := d.redis.GetCacheString(ctx, key); cache != "" {
45+
count += util.Int64FromInterface(cache)
46+
continue
47+
}
48+
tableDataCount := d.getExtrinsicTableCount(ctx, index, queryWhere...)
49+
_ = d.redis.SetCache(ctx, key, tableDataCount, extrinsicCountCacheTTL+rand.Intn(extrinsicCountCacheTTLJitter+1))
3250
count += tableDataCount
3351
}
3452
return count
3553

3654
}
3755

56+
func (d *Dao) getExtrinsicTableCount(ctx context.Context, tableIndex int, queryWhere ...model.Option) int64 {
57+
var tableDataCount int64
58+
q := d.db.WithContext(ctx).Scopes(d.TableNameFunc(&model.ChainExtrinsic{BlockNum: uint(tableIndex) * model.SplitTableBlockNum}))
59+
q = q.Scopes(queryWhere...)
60+
q.Model(&model.ChainExtrinsic{}).Count(&tableDataCount)
61+
return tableDataCount
62+
}
63+
64+
func (d *Dao) extrinsicCountCacheKey(tableIndex int, queryWhere ...model.Option) string {
65+
return fmt.Sprintf("%s:table:%d:%s", RedisExtrinsicCountKey, tableIndex, d.extrinsicCountQuerySignature(queryWhere...))
66+
}
67+
68+
func (d *Dao) extrinsicCountQuerySignature(queryWhere ...model.Option) string {
69+
if len(queryWhere) == 0 {
70+
return "all"
71+
}
72+
tx := d.db.Session(&gorm.Session{DryRun: true}).Model(&model.ChainExtrinsic{})
73+
tx = tx.Scopes(queryWhere...).Count(new(int64))
74+
signature := tx.Statement.SQL.String() + "|" + util.ToString(tx.Statement.Vars)
75+
if signature == "|" {
76+
return "all"
77+
}
78+
sum := sha1.Sum([]byte(signature))
79+
return hex.EncodeToString(sum[:])
80+
}
81+
3882
func (d *Dao) GetAccountExtrinsicMapping(ctx context.Context, accountId string) []int {
3983
var mapping model.AccountExtrinsicMapping
4084
query := d.db.WithContext(ctx).Where("account_id = ?", accountId).First(&mapping)

‎internal/dao/chainExtrinsic_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package dao
22

33
import (
44
"context"
5+
"fmt"
6+
"github.com/gomodule/redigo/redis"
57
"github.com/itering/subscan/model"
68
"github.com/itering/subscan/util"
79
"github.com/shopspring/decimal"
@@ -40,3 +42,79 @@ func TestDao_ExtrinsicsAsJson(t *testing.T) {
4042
extrinsics := testDao.GetExtrinsicsByHash(ctx, "0x368f61800f8645f67d59baf0602b236ff47952097dcaef3aa026b50ddc8dcea0")
4143
assert.Equal(t, `[{"name":"dest","type":"Address","value":"563d11af91b3a166d07110bb49e84094f38364ef39c43a26066ca123a8b9532b"},{"name":"value","type":"Compact\u003cBalance\u003e","value":"1000000000000000000"}]`, util.ToString(testDao.ExtrinsicsAsJson(extrinsics).Params))
4244
}
45+
46+
func TestDao_GetExtrinsicCountUsesHistoricalCacheButLatestDB(t *testing.T) {
47+
ctx := context.TODO()
48+
seedLatestExtrinsicFixtures(t)
49+
50+
err := testDao.SaveFillAlreadyBlockNum(ctx, int(model.SplitTableBlockNum+1))
51+
assert.NoError(t, err)
52+
53+
conn, _ := testDao.redis.Redis().GetContext(ctx)
54+
defer conn.Close()
55+
56+
historyAllKey := testDao.extrinsicCountCacheKey(0)
57+
historySignedKey := testDao.extrinsicCountCacheKey(0, model.Where("is_signed = ?", true))
58+
latestAllKey := testDao.extrinsicCountCacheKey(1)
59+
latestSignedKey := testDao.extrinsicCountCacheKey(1, model.Where("is_signed = ?", true))
60+
61+
_, err = conn.Do("SET", historyAllKey, 100)
62+
assert.NoError(t, err)
63+
_, err = conn.Do("SET", historySignedKey, 7)
64+
assert.NoError(t, err)
65+
_, err = conn.Do("SET", latestAllKey, 200)
66+
assert.NoError(t, err)
67+
_, err = conn.Do("SET", latestSignedKey, 300)
68+
assert.NoError(t, err)
69+
70+
assert.EqualValues(t, 101, testDao.GetExtrinsicCount(ctx))
71+
assert.EqualValues(t, 8, testDao.GetExtrinsicCount(ctx, model.Where("is_signed = ?", true)))
72+
}
73+
74+
func TestDao_GetExtrinsicCountBackfillsHistoricalCacheWithJitteredTTL(t *testing.T) {
75+
ctx := context.TODO()
76+
seedLatestExtrinsicFixtures(t)
77+
78+
err := testDao.SaveFillAlreadyBlockNum(ctx, int(model.SplitTableBlockNum+1))
79+
assert.NoError(t, err)
80+
81+
conn, _ := testDao.redis.Redis().GetContext(ctx)
82+
defer conn.Close()
83+
84+
historyAllKey := testDao.extrinsicCountCacheKey(0)
85+
_, err = conn.Do("DEL", historyAllKey)
86+
assert.NoError(t, err)
87+
88+
assert.EqualValues(t, 3, testDao.GetExtrinsicCount(ctx))
89+
90+
cachedCount, err := redis.Int64(conn.Do("GET", historyAllKey))
91+
assert.NoError(t, err)
92+
assert.EqualValues(t, 2, cachedCount)
93+
94+
ttl, err := redis.Int(conn.Do("TTL", historyAllKey))
95+
assert.NoError(t, err)
96+
assert.GreaterOrEqual(t, ttl, extrinsicCountCacheTTL-1)
97+
assert.LessOrEqual(t, ttl, extrinsicCountCacheTTL+extrinsicCountCacheTTLJitter)
98+
}
99+
100+
func seedLatestExtrinsicFixtures(t *testing.T) {
101+
t.Helper()
102+
103+
testDao.AddIndex(model.SplitTableBlockNum)
104+
105+
ctx := context.TODO()
106+
txn := testDao.DbBegin()
107+
err := testDao.CreateExtrinsic(ctx, txn, []model.ChainExtrinsic{{
108+
ID: model.SplitTableBlockNum*model.IdGenerateCoefficient + 1,
109+
ExtrinsicIndex: fmt.Sprintf("%d-1", model.SplitTableBlockNum+1),
110+
BlockNum: model.SplitTableBlockNum + 1,
111+
BlockTimestamp: 1594791901,
112+
CallModuleFunction: "remark",
113+
CallModule: "system",
114+
Success: true,
115+
ExtrinsicHash: "0xsplit-table-extrinsic",
116+
IsSigned: true,
117+
}}, 1)
118+
assert.NoError(t, err)
119+
assert.NoError(t, txn.Commit().Error)
120+
}

‎internal/dao/const.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ var (
66
RedisMetadataKey = model.RedisKeyPrefix() + "metadata"
77
RedisFillAlreadyBlockNum = model.RedisKeyPrefix() + "FillAlreadyBlockNum"
88
RedisFillFinalizedBlockNum = model.RedisKeyPrefix() + "FillFinalizedBlockNum"
9+
RedisExtrinsicCountKey = model.RedisKeyPrefix() + "extrinsic_count"
910
)
1011

1112
// local cache value

‎internal/dao/db.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"database/sql"
66
"errors"
7+
78
"gorm.io/gorm/logger"
89
"gorm.io/gorm/schema"
910

@@ -235,6 +236,7 @@ func newDb() (db *gorm.DB) {
235236
}
236237
if dbDriver == "mysql" {
237238
conf.NamingStrategy = NamingStrategy{}
239+
util.Logger().Debug(fmt.Sprintf("mysql dsn %s", configs.Boot.Database.Mysql.DSN))
238240
db, err = gorm.Open(mysql.Open(configs.Boot.Database.Mysql.DSN), conf)
239241
} else {
240242
db, err = gorm.Open(postgres.Open(configs.Boot.Database.Postgres.DSN), conf)

‎internal/observer/observer.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@ package observer
33
import (
44
"context"
55
"fmt"
6-
"github.com/itering/subscan/internal/script"
76
"os"
87
"os/signal"
98
"sync"
109
"syscall"
1110
"time"
1211

12+
"github.com/itering/subscan/internal/script"
13+
1314
"github.com/itering/subscan/internal/service"
1415
"github.com/itering/subscan/util"
1516
"github.com/robfig/cron/v3"
@@ -63,7 +64,7 @@ func enableTermSignalHandler(cancel func()) {
6364
func RunCron() {
6465
// or use cron.DefaultLogger
6566
c := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger)))
66-
if _, err := c.AddFunc("@every 3m", func() {
67+
if _, err := c.AddFunc("@every 20m", func() {
6768
script.RefreshMetadata()
6869
}); err != nil {
6970
util.Logger().Error(fmt.Errorf("failed to register cron job: %v", err))

0 commit comments

Comments
 (0)