Skip to content

Commit e2e3081

Browse files
moglu2017libotony
moglu2017
and
libotony
authored
feat: add offset limit to logs pagination and db metrics (#1632)
Co-authored-by: libotony <tony.li@vechain.org>
1 parent 2eed8ef commit e2e3081

23 files changed

Lines changed: 659 additions & 117 deletions

‎.github/workflows/test-e2e.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
GITHUB_REF_NAME: ${{ github.ref_name }}
2525
run: |
2626
# Default fallback commit
27-
DEFAULT_COMMIT="a51bdc9833d1a4641c2d7969414dbe23a4fa345e"
27+
DEFAULT_COMMIT="c87b8c55ace0d48112a65e8a0659d3af4ef2f1cf"
2828
REF="$DEFAULT_COMMIT"
2929
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
3030
# For pull_request events, we only look at the PR's base branch

‎api/events/events.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,17 @@ import (
2222
type Events struct {
2323
repo *chain.Repository
2424
db *logdb.LogDB
25-
limit uint64
25+
maxLimit uint64
26+
maxOffset uint64
2627
maxCriteriaCount int
2728
}
2829

29-
func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64, maxCriteriaCount int) *Events {
30+
func New(repo *chain.Repository, db *logdb.LogDB, maxLimit uint64, maxOffset uint64, maxCriteriaCount int) *Events {
3031
return &Events{
3132
repo,
3233
db,
33-
logsLimit,
34+
maxLimit,
35+
maxOffset,
3436
maxCriteriaCount,
3537
}
3638
}
@@ -58,7 +60,7 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
5860
if err := restutil.ParseJSON(req.Body, &filter); err != nil {
5961
return restutil.BadRequest(errors.WithMessage(err, "body"))
6062
}
61-
if err := filter.Options.Validate(e.limit); err != nil {
63+
if err := filter.Options.Validate(e.maxLimit, e.maxOffset); err != nil {
6264
return restutil.Forbidden(err)
6365
}
6466
if err := filter.Range.Validate(); err != nil {
@@ -83,7 +85,7 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
8385
if filter.Options.Limit == nil {
8486
// if filter.Options.Limit is nil, set to the default limit +1
8587
// to detect whether there are more logs than the default limit
86-
limit := e.limit + 1
88+
limit := e.maxLimit + 1
8789
filter.Options.Limit = &limit
8890
}
8991

@@ -93,8 +95,8 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
9395
}
9496

9597
// ensure the result size is less than the configured limit
96-
if len(fes) > int(e.limit) {
97-
return restutil.Forbidden(fmt.Errorf("the number of filtered logs exceeds the maximum allowed value of %d, please use pagination", e.limit))
98+
if len(fes) > int(e.maxLimit) {
99+
return restutil.Forbidden(fmt.Errorf("the number of filtered logs exceeds the maximum allowed value of %d, please use pagination", e.maxLimit))
98100
}
99101

100102
return restutil.WriteJSON(w, fes)

‎api/events/events_test.go‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,18 @@ import (
2828
"github.com/vechain/thor/v2/tx"
2929
)
3030

31-
const defaultLogLimit uint64 = 1000
31+
const (
32+
defaultLogLimit uint64 = 1000
33+
defaultLogOffset uint64 = 100
34+
)
3235

3336
var (
3437
ts *httptest.Server
3538
tclient *thorclient.Client
3639
)
3740

3841
func TestEmptyEvents(t *testing.T) {
39-
initEventServer(t, defaultLogLimit)
42+
initEventServer(t, defaultLogLimit, defaultLogOffset)
4043
defer ts.Close()
4144

4245
tclient = thorclient.New(ts.URL)
@@ -49,7 +52,7 @@ func TestEmptyEvents(t *testing.T) {
4952
}
5053

5154
func TestEvents(t *testing.T) {
52-
thorChain := initEventServer(t, defaultLogLimit)
55+
thorChain := initEventServer(t, defaultLogLimit, defaultLogOffset)
5356
defer ts.Close()
5457

5558
blocksToInsert := 5
@@ -59,7 +62,7 @@ func TestEvents(t *testing.T) {
5962
}
6063

6164
func TestOptionalIndexes(t *testing.T) {
62-
thorChain := initEventServer(t, defaultLogLimit)
65+
thorChain := initEventServer(t, defaultLogLimit, defaultLogOffset)
6366
defer ts.Close()
6467
insertBlocks(t, thorChain, 5)
6568
tclient = thorclient.New(ts.URL)
@@ -109,7 +112,7 @@ func TestOptionalIndexes(t *testing.T) {
109112
}
110113

111114
func TestEvents_WithOptionsNoLimit(t *testing.T) {
112-
thorChain := initEventServer(t, defaultLogLimit)
115+
thorChain := initEventServer(t, defaultLogLimit, defaultLogOffset)
113116
defer ts.Close()
114117
insertBlocks(t, thorChain, 5)
115118

@@ -133,7 +136,7 @@ func TestEvents_WithOptionsNoLimit(t *testing.T) {
133136
}
134137

135138
func TestOption(t *testing.T) {
136-
thorChain := initEventServer(t, 5)
139+
thorChain := initEventServer(t, 5, defaultLogOffset)
137140
defer ts.Close()
138141
insertBlocks(t, thorChain, 5)
139142

@@ -197,10 +200,27 @@ func TestOption(t *testing.T) {
197200
assert.NoError(t, err)
198201
assert.Equal(t, http.StatusBadRequest, statusCode)
199202
assert.Equal(t, "number of criteria in criteriaSet: 11 cannot be greater than: 10\n", string(res))
203+
204+
filter = api.EventFilter{
205+
CriteriaSet: make([]*api.EventCriteria, 0),
206+
Range: nil,
207+
Options: &api.Options{Offset: defaultLogOffset, Limit: new(uint64(0))},
208+
Order: logdb.DESC,
209+
}
210+
211+
_, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter)
212+
require.NoError(t, err)
213+
assert.Equal(t, http.StatusOK, statusCode)
214+
215+
filter.Options.Offset = defaultLogOffset * 2
216+
res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter)
217+
require.NoError(t, err)
218+
assert.Equal(t, http.StatusForbidden, statusCode)
219+
assert.Equal(t, "options.offset exceeds the maximum allowed value of 100", strings.Trim(string(res), "\n"))
200220
}
201221

202222
func TestZeroFrom(t *testing.T) {
203-
thorChain := initEventServer(t, 100)
223+
thorChain := initEventServer(t, 100, defaultLogOffset)
204224
defer ts.Close()
205225
insertBlocks(t, thorChain, 5)
206226

@@ -234,7 +254,7 @@ func TestZeroFrom(t *testing.T) {
234254
}
235255

236256
func TestNullCriteriaSet(t *testing.T) {
237-
initEventServer(t, defaultLogLimit)
257+
initEventServer(t, defaultLogLimit, defaultLogOffset)
238258
defer ts.Close()
239259

240260
tclient = thorclient.New(ts.URL)
@@ -328,12 +348,12 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) {
328348
}
329349

330350
// Init functions
331-
func initEventServer(t *testing.T, limit uint64) *testchain.Chain {
351+
func initEventServer(t *testing.T, limit uint64, offset uint64) *testchain.Chain {
332352
thorChain, err := testchain.NewDefault()
333353
require.NoError(t, err)
334354

335355
router := mux.NewRouter()
336-
New(thorChain.Repo(), thorChain.LogDB(), limit, 10).Mount(router, "/logs/event")
356+
New(thorChain.Repo(), thorChain.LogDB(), limit, offset, 10).Mount(router, "/logs/event")
337357
ts = httptest.NewServer(router)
338358

339359
return thorChain

‎api/events_types.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ package api
77

88
import (
99
"fmt"
10-
"math"
1110

1211
"github.com/ethereum/go-ethereum/common/hexutil"
1312

@@ -73,15 +72,16 @@ type Options struct {
7372
IncludeIndexes bool `json:"includeIndexes,omitempty"`
7473
}
7574

76-
func (o *Options) Validate(limit uint64) error {
75+
func (o *Options) Validate(limit uint64, offset uint64) error {
7776
if o == nil {
7877
return nil
7978
}
8079
if o.Limit != nil && *o.Limit > limit {
8180
return fmt.Errorf("options.limit exceeds the maximum allowed value of %d", limit)
8281
}
83-
if o.Offset > math.MaxInt64 {
84-
return fmt.Errorf("options.offset exceeds the maximum allowed value of %d", math.MaxInt64)
82+
83+
if o.Offset > offset {
84+
return fmt.Errorf("options.offset exceeds the maximum allowed value of %d", offset)
8585
}
8686

8787
return nil

‎api/transfers/transfers.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,17 @@ import (
2222
type Transfers struct {
2323
repo *chain.Repository
2424
db *logdb.LogDB
25-
limit uint64
25+
maxLimit uint64
26+
maxOffset uint64
2627
maxCriteriaCount int
2728
}
2829

29-
func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64, maxCriteriaCount int) *Transfers {
30+
func New(repo *chain.Repository, db *logdb.LogDB, maxLimit uint64, maxOffset uint64, maxCriteriaCount int) *Transfers {
3031
return &Transfers{
3132
repo,
3233
db,
33-
logsLimit,
34+
maxLimit,
35+
maxOffset,
3436
maxCriteriaCount,
3537
}
3638
}
@@ -66,7 +68,7 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
6668
if err := restutil.ParseJSON(req.Body, &filter); err != nil {
6769
return restutil.BadRequest(errors.WithMessage(err, "body"))
6870
}
69-
if err := filter.Options.Validate(t.limit); err != nil {
71+
if err := filter.Options.Validate(t.maxLimit, t.maxOffset); err != nil {
7072
return restutil.Forbidden(err)
7173
}
7274
if err := filter.Range.Validate(); err != nil {
@@ -91,7 +93,7 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
9193
if filter.Options.Limit == nil {
9294
// if filter.Options.Limit is nil, set to the default limit +1
9395
// to detect whether there are more logs than the default limit
94-
limit := t.limit + 1
96+
limit := t.maxLimit + 1
9597
filter.Options.Limit = &limit
9698
}
9799

@@ -101,8 +103,8 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
101103
}
102104

103105
// ensure the result size is less than the configured limit
104-
if len(tLogs) > int(t.limit) {
105-
return restutil.Forbidden(fmt.Errorf("the number of filtered logs exceeds the maximum allowed value of %d, please use pagination", t.limit))
106+
if len(tLogs) > int(t.maxLimit) {
107+
return restutil.Forbidden(fmt.Errorf("the number of filtered logs exceeds the maximum allowed value of %d, please use pagination", t.maxLimit))
106108
}
107109

108110
return restutil.WriteJSON(w, tLogs)

‎api/transfers/transfers_test.go‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ import (
2828
"github.com/vechain/thor/v2/tx"
2929
)
3030

31-
const defaultLogLimit uint64 = 1000
31+
const (
32+
defaultLogLimit uint64 = 1000
33+
defaultLogOffset uint64 = 100
34+
)
3235

3336
var (
3437
ts *httptest.Server
@@ -37,7 +40,7 @@ var (
3740

3841
func TestEmptyTransfers(t *testing.T) {
3942
db := createDb(t)
40-
initTransferServer(t, db, defaultLogLimit)
43+
initTransferServer(t, db, defaultLogLimit, defaultLogOffset)
4144
defer ts.Close()
4245

4346
tclient = thorclient.New(ts.URL)
@@ -47,7 +50,7 @@ func TestEmptyTransfers(t *testing.T) {
4750

4851
func TestTransfers(t *testing.T) {
4952
db := createDb(t)
50-
initTransferServer(t, db, defaultLogLimit)
53+
initTransferServer(t, db, defaultLogLimit, defaultLogOffset)
5154
defer ts.Close()
5255

5356
tclient = thorclient.New(ts.URL)
@@ -59,7 +62,7 @@ func TestTransfers(t *testing.T) {
5962

6063
func TestOption(t *testing.T) {
6164
db := createDb(t)
62-
initTransferServer(t, db, 5)
65+
initTransferServer(t, db, 5, 100)
6366
defer ts.Close()
6467
insertBlocks(t, db, 5)
6568

@@ -100,11 +103,28 @@ func TestOption(t *testing.T) {
100103
require.NoError(t, err)
101104
assert.Equal(t, http.StatusForbidden, statusCode)
102105
assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n"))
106+
107+
filter = api.TransferFilter{
108+
CriteriaSet: make([]*logdb.TransferCriteria, 0),
109+
Range: nil,
110+
Options: &api.Options{Offset: defaultLogOffset, Limit: new(uint64(0))},
111+
Order: logdb.DESC,
112+
}
113+
114+
_, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/transfer", filter)
115+
require.NoError(t, err)
116+
assert.Equal(t, http.StatusOK, statusCode)
117+
118+
filter.Options.Offset = defaultLogOffset * 2
119+
res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/transfer", filter)
120+
require.NoError(t, err)
121+
assert.Equal(t, http.StatusForbidden, statusCode)
122+
assert.Equal(t, "options.offset exceeds the maximum allowed value of 100", strings.Trim(string(res), "\n"))
103123
}
104124

105125
func TestOptionalData(t *testing.T) {
106126
db := createDb(t)
107-
initTransferServer(t, db, defaultLogLimit)
127+
initTransferServer(t, db, defaultLogLimit, defaultLogOffset)
108128
defer ts.Close()
109129
insertBlocks(t, db, 5)
110130
tclient = thorclient.New(ts.URL)
@@ -155,7 +175,7 @@ func TestOptionalData(t *testing.T) {
155175

156176
func TestNullCriteriaSet(t *testing.T) {
157177
db := createDb(t)
158-
initTransferServer(t, db, defaultLogLimit)
178+
initTransferServer(t, db, defaultLogLimit, defaultLogOffset)
159179
defer ts.Close()
160180
tclient = thorclient.New(ts.URL)
161181

@@ -265,12 +285,12 @@ func insertBlocks(t *testing.T, db *logdb.LogDB, n int) {
265285
}
266286
}
267287

268-
func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64) {
288+
func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64, offset uint64) {
269289
thorChain, err := testchain.NewDefault()
270290
require.NoError(t, err)
271291

272292
router := mux.NewRouter()
273-
New(thorChain.Repo(), logDb, limit, 10).Mount(router, "/logs/transfer")
293+
New(thorChain.Repo(), logDb, limit, offset, 10).Mount(router, "/logs/transfer")
274294

275295
ts = httptest.NewServer(router)
276296
}

‎cmd/thor/flags.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,13 @@ var (
105105
Usage: "limit the number of logs returned by /logs API",
106106
Sources: envVar("API_LOGS_LIMIT"),
107107
}
108+
apiLogsMaxOffsetFlag = &cli.Uint64Flag{
109+
Name: "api-logs-max-offset",
110+
Local: true,
111+
Value: 100_000,
112+
Usage: "limit the maximum offset for /logs API",
113+
Sources: envVar("API_LOGS_MAX_OFFSET"),
114+
}
108115
apiEnableDeprecatedFlag = &cli.BoolFlag{
109116
Name: "api-enable-deprecated",
110117
Local: true,
@@ -130,6 +137,15 @@ var (
130137
Usage: "enable creation of additional indexes on startup",
131138
Sources: envVar("LOGDB_ADDITIONAL_INDEXES"),
132139
}
140+
logDbMaxReadConnsFlag = &cli.Uint64Flag{
141+
Name: "logdb-max-read-conns",
142+
Local: true,
143+
Hidden: true,
144+
Value: 64,
145+
Usage: "maximum number of read connections for the log database",
146+
Sources: envVar("LOGDB_MAX_READ_CONNS"),
147+
}
148+
133149
// priority fees API flags
134150
apiPriorityFeesPercentageFlag = &cli.Uint64Flag{
135151
Name: "api-priority-fees-percentage",

‎cmd/thor/httpserver/api_server.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ type APIConfig struct {
6565
Timeout int
6666
SlowQueriesThreshold int
6767
Log5XXErrors bool
68+
MaxLogsOffset uint64
6869
}
6970

7071
func StartAPIServer(
@@ -109,8 +110,8 @@ func StartAPIServer(
109110

110111
accounts.New(repo, stater, config.CallGasLimit, config.BatchDataMaxSize, forkConfig, bft, config.EnableDeprecated).Mount(router, "/accounts")
111112
if !config.SkipLogs {
112-
events.New(repo, logDB, config.LogsLimit, defaultMaxCriteriaCount).Mount(router, "/logs/event")
113-
transfers.New(repo, logDB, config.LogsLimit, defaultMaxCriteriaCount).Mount(router, "/logs/transfer")
113+
events.New(repo, logDB, config.LogsLimit, config.MaxLogsOffset, defaultMaxCriteriaCount).Mount(router, "/logs/event")
114+
transfers.New(repo, logDB, config.LogsLimit, config.MaxLogsOffset, defaultMaxCriteriaCount).Mount(router, "/logs/transfer")
114115
}
115116
blocks.New(repo, bft).Mount(router, "/blocks")
116117
transactions.New(repo, txPool).Mount(router, "/transactions")

0 commit comments

Comments
 (0)