Skip to content

Commit 3d1fe0a

Browse files
libotonylibotony
authored
Merge pull request #243 from qianbin/optimize-log-db
Optimize log db
2 parents 3422da7 + bdb2ba1 commit 3d1fe0a

30 files changed

Lines changed: 416 additions & 158 deletions

‎Gopkg.lock‎

Lines changed: 19 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,7 @@ ignored = []
8181
[[constraint]]
8282
name = "gopkg.in/yaml.v2"
8383
version = "2.2.2"
84+
85+
[[constraint]]
86+
name = "gopkg.in/cheggaaa/pb.v1"
87+
version = "1.0.28"

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ bin/thor -h
122122
- `--p2p-port value` P2P network listening port (default: 11235)
123123
- `--nat value` port mapping mechanism (any|none|upnp|pmp|extip:<IP>) (default: "none")
124124
- `--bootnode value` comma separated list of bootnode IDs
125+
- `--skip-logs` skip writing event|transfer logs (/logs API will be disabled)
125126
- `--pprof` turn on go-pprof
126127
- `--help, -h` show help
127128
- `--version, -v` print the version

‎api/api.go‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,18 @@ import (
3131
)
3232

3333
//New return api router
34-
func New(chain *chain.Chain, stateCreator *state.Creator, txPool *txpool.TxPool, logDB *logdb.LogDB, nw node.Network, allowedOrigins string, backtraceLimit uint32, callGasLimit uint64, pprofOn bool) (http.HandlerFunc, func()) {
34+
func New(
35+
chain *chain.Chain,
36+
stateCreator *state.Creator,
37+
txPool *txpool.TxPool,
38+
logDB *logdb.LogDB,
39+
nw node.Network,
40+
allowedOrigins string,
41+
backtraceLimit uint32,
42+
callGasLimit uint64,
43+
pprofOn bool,
44+
skipLogs bool) (http.HandlerFunc, func()) {
45+
3546
origins := strings.Split(strings.TrimSpace(allowedOrigins), ",")
3647
for i, o := range origins {
3748
origins[i] = strings.ToLower(strings.TrimSpace(o))
@@ -55,18 +66,21 @@ func New(chain *chain.Chain, stateCreator *state.Creator, txPool *txpool.TxPool,
5566

5667
accounts.New(chain, stateCreator, callGasLimit).
5768
Mount(router, "/accounts")
58-
eventslegacy.New(logDB).
59-
Mount(router, "/events")
60-
transferslegacy.New(logDB).
61-
Mount(router, "/transfers")
62-
eventslegacy.New(logDB).
63-
Mount(router, "/logs/events")
64-
events.New(logDB).
65-
Mount(router, "/logs/event")
66-
transferslegacy.New(logDB).
67-
Mount(router, "/logs/transfers")
68-
transfers.New(logDB).
69-
Mount(router, "/logs/transfer")
69+
70+
if !skipLogs {
71+
eventslegacy.New(logDB).
72+
Mount(router, "/events")
73+
transferslegacy.New(logDB).
74+
Mount(router, "/transfers")
75+
eventslegacy.New(logDB).
76+
Mount(router, "/logs/events")
77+
events.New(logDB).
78+
Mount(router, "/logs/event")
79+
transferslegacy.New(logDB).
80+
Mount(router, "/logs/transfers")
81+
transfers.New(logDB).
82+
Mount(router, "/logs/transfer")
83+
}
7084
blocks.New(chain).
7185
Mount(router, "/blocks")
7286
transactions.New(chain, txPool).

‎api/doc/bindata.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/doc/thor.yaml‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ info:
88
license:
99
name: LGPL 3.0
1010
url: 'https://www.gnu.org/licenses/lgpl-3.0.en.html'
11-
version: 1.1.0
11+
version: 1.1.1
1212

1313
servers:
1414
- url: '/'
@@ -663,6 +663,32 @@ components:
663663
description: block unix timestamp
664664
example: 1533267900
665665

666+
ReceiptMeta:
667+
description: tx receipt meta info
668+
properties:
669+
blockID:
670+
type: string
671+
description: block identifier (bytes32)
672+
example: '0x0004f6cc88bb4626a92907718e82f255b8fa511453a78e8797eb8cea3393b215'
673+
blockNumber:
674+
type: integer
675+
format: uint32
676+
description: block number (height)
677+
example: 325324
678+
blockTimestamp:
679+
type: integer
680+
format: uint64
681+
description: block unix timestamp
682+
example: 1533267900
683+
txID:
684+
type: string
685+
description: transaction identifier
686+
example: '0x284bba50ef777889ff1a367ed0b38d5e5626714477c40de38d71cedd6f9fa477'
687+
txOrigin:
688+
type: string
689+
description: transaction origin (signer)
690+
example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d'
691+
666692
LogMeta:
667693
description: event or transfer log meta info
668694
properties:
@@ -688,6 +714,11 @@ components:
688714
type: string
689715
description: transaction origin (signer)
690716
example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d'
717+
clauseIndex:
718+
type: integer
719+
format: uint32
720+
description: index of clause which generates this log
721+
example: 0
691722

692723
Block:
693724
properties:
@@ -937,7 +968,7 @@ components:
937968
items:
938969
$ref: '#/components/schemas/Transfer'
939970
meta:
940-
$ref: '#/components/schemas/LogMeta'
971+
$ref: '#/components/schemas/ReceiptMeta'
941972

942973
CallData:
943974
properties:
@@ -1155,14 +1186,15 @@ components:
11551186
criteria to filter out event. All fields are joined with `and` operator. `null` field are ignored. e.g.
11561187
```
11571188
{
1158-
"address": "0xe59d475abe695c7f67a8a2321f33a856b0b4c71d",
1189+
"address": "0x0000000000000000000000000000456E65726779",
11591190
"topic0": "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
11601191
}
11611192
```
11621193
matches events emitted by `0xe59d475abe695c7f67a8a2321f33a856b0b4c71d` and with `topic0` equals `0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef`.
11631194
example:
1164-
address: "0xe59d475abe695c7f67a8a2321f33a856b0b4c71d"
1195+
address: "0x0000000000000000000000000000456E65726779"
11651196
topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
1197+
topic1: '0x0000000000000000000000005034aa590125b64023a0262112b98d72e3c8e40e'
11661198

11671199
EventFilter:
11681200
properties:

‎api/events/events_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ func initEventServer(t *testing.T) {
8282
header := new(block.Builder).Build().Header()
8383
for i := 0; i < 100; i++ {
8484
if err := db.Prepare(header).ForTransaction(thor.BytesToBytes32([]byte("txID")), thor.BytesToAddress([]byte("txOrigin"))).
85-
Insert(tx.Events{txEv}, nil).Commit(); err != nil {
85+
Insert(tx.Events{txEv}, nil, 0).Commit(); err != nil {
8686
if err != nil {
8787
t.Fatal(err)
8888
}

‎api/events/types.go‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,19 @@ import (
99
"fmt"
1010

1111
"github.com/ethereum/go-ethereum/common/hexutil"
12-
"github.com/vechain/thor/api/transactions"
1312
"github.com/vechain/thor/logdb"
1413
"github.com/vechain/thor/thor"
1514
)
1615

16+
type LogMeta struct {
17+
BlockID thor.Bytes32 `json:"blockID"`
18+
BlockNumber uint32 `json:"blockNumber"`
19+
BlockTimestamp uint64 `json:"blockTimestamp"`
20+
TxID thor.Bytes32 `json:"txID"`
21+
TxOrigin thor.Address `json:"txOrigin"`
22+
ClauseIndex uint32 `json:"clauseIndex"`
23+
}
24+
1725
type TopicSet struct {
1826
Topic0 *thor.Bytes32 `json:"topic0"`
1927
Topic1 *thor.Bytes32 `json:"topic1"`
@@ -24,23 +32,24 @@ type TopicSet struct {
2432

2533
// FilteredEvent only comes from one contract
2634
type FilteredEvent struct {
27-
Address thor.Address `json:"address"`
28-
Topics []*thor.Bytes32 `json:"topics"`
29-
Data string `json:"data"`
30-
Meta transactions.LogMeta `json:"meta"`
35+
Address thor.Address `json:"address"`
36+
Topics []*thor.Bytes32 `json:"topics"`
37+
Data string `json:"data"`
38+
Meta LogMeta `json:"meta"`
3139
}
3240

3341
//convert a logdb.Event into a json format Event
3442
func convertEvent(event *logdb.Event) *FilteredEvent {
3543
fe := FilteredEvent{
3644
Address: event.Address,
3745
Data: hexutil.Encode(event.Data),
38-
Meta: transactions.LogMeta{
46+
Meta: LogMeta{
3947
BlockID: event.BlockID,
4048
BlockNumber: event.BlockNumber,
4149
BlockTimestamp: event.BlockTime,
4250
TxID: event.TxID,
4351
TxOrigin: event.TxOrigin,
52+
ClauseIndex: event.ClauseIndex,
4453
},
4554
}
4655
fe.Topics = make([]*thor.Bytes32, 0)
@@ -62,7 +71,8 @@ func (e *FilteredEvent) String() string {
6271
blockNumber %v,
6372
blockTimestamp %v),
6473
txID %v,
65-
txOrigin %v)
74+
txOrigin %v,
75+
clauseIndex %v)
6676
)`,
6777
e.Address,
6878
e.Topics,
@@ -72,6 +82,7 @@ func (e *FilteredEvent) String() string {
7282
e.Meta.BlockTimestamp,
7383
e.Meta.TxID,
7484
e.Meta.TxOrigin,
85+
e.Meta.ClauseIndex,
7586
)
7687
}
7788

‎api/eventslegacy/eventslegacy_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func initEventServer(t *testing.T) {
7979
header := new(block.Builder).Build().Header()
8080
for i := 0; i < 100; i++ {
8181
if err := db.Prepare(header).ForTransaction(thor.BytesToBytes32([]byte("txID")), thor.BytesToAddress([]byte("txOrigin"))).
82-
Insert(tx.Events{txEv}, nil).Commit(); err != nil {
82+
Insert(tx.Events{txEv}, nil, 0).Commit(); err != nil {
8383
if err != nil {
8484
t.Fatal(err)
8585
}

‎api/eventslegacy/types.go‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,19 @@ import (
99
"fmt"
1010

1111
"github.com/ethereum/go-ethereum/common/hexutil"
12-
"github.com/vechain/thor/api/transactions"
1312
"github.com/vechain/thor/logdb"
1413
"github.com/vechain/thor/thor"
1514
)
1615

16+
type LogMeta struct {
17+
BlockID thor.Bytes32 `json:"blockID"`
18+
BlockNumber uint32 `json:"blockNumber"`
19+
BlockTimestamp uint64 `json:"blockTimestamp"`
20+
TxID thor.Bytes32 `json:"txID"`
21+
TxOrigin thor.Address `json:"txOrigin"`
22+
ClauseIndex uint32 `json:"clauseIndex"`
23+
}
24+
1725
type TopicSet struct {
1826
Topic0 *thor.Bytes32 `json:"topic0"`
1927
Topic1 *thor.Bytes32 `json:"topic1"`
@@ -63,23 +71,24 @@ func convertFilter(filter *FilterLegacy) *logdb.EventFilter {
6371

6472
// FilteredEvent only comes from one contract
6573
type FilteredEvent struct {
66-
Address thor.Address `json:"address"`
67-
Topics []*thor.Bytes32 `json:"topics"`
68-
Data string `json:"data"`
69-
Meta transactions.LogMeta `json:"meta"`
74+
Address thor.Address `json:"address"`
75+
Topics []*thor.Bytes32 `json:"topics"`
76+
Data string `json:"data"`
77+
Meta LogMeta `json:"meta"`
7078
}
7179

7280
//convert a logdb.Event into a json format Event
7381
func convertEvent(event *logdb.Event) *FilteredEvent {
7482
fe := FilteredEvent{
7583
Address: event.Address,
7684
Data: hexutil.Encode(event.Data),
77-
Meta: transactions.LogMeta{
85+
Meta: LogMeta{
7886
BlockID: event.BlockID,
7987
BlockNumber: event.BlockNumber,
8088
BlockTimestamp: event.BlockTime,
8189
TxID: event.TxID,
8290
TxOrigin: event.TxOrigin,
91+
ClauseIndex: event.ClauseIndex,
8392
},
8493
}
8594
fe.Topics = make([]*thor.Bytes32, 0)
@@ -101,7 +110,8 @@ func (e *FilteredEvent) String() string {
101110
blockNumber %v,
102111
blockTimestamp %v),
103112
txID %v,
104-
txOrigin %v)
113+
txOrigin %v,
114+
clauseIndex %v)
105115
)`,
106116
e.Address,
107117
e.Topics,
@@ -111,5 +121,6 @@ func (e *FilteredEvent) String() string {
111121
e.Meta.BlockTimestamp,
112122
e.Meta.TxID,
113123
e.Meta.TxOrigin,
124+
e.Meta.ClauseIndex,
114125
)
115126
}

0 commit comments

Comments
 (0)