Skip to content

Commit a8efe43

Browse files
urvisavlaurvisavla
authored
Merge branch 'main' into release/v23.0.1
2 parents 2f22576 + 17b1ac0 commit a8efe43

File tree

13 files changed

+81
-41
lines changed

13 files changed

+81
-41
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
## [v23.0.1](https://github.com/stellar/stellar-rpc/compare/v23.0.0...v23.0.1)
66
- Bump soroban-env lib to v23.0.1 ([#499](https://github.com/stellar/stellar-rpc/pull/499)).
77

8+
89
## [v23.0.0](https://github.com/stellar/stellar-rpc/compare/v22.1.5...v23.0.0): Protocol 23 Release
910

1011
### Breaking Changes

‎cmd/stellar-rpc/internal/db/db.go‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010
"strconv"
1111
"sync"
12+
"time"
1213

1314
sq "github.com/Masterminds/squirrel"
1415
_ "github.com/mattn/go-sqlite3"
@@ -41,7 +42,7 @@ type WriteTx interface {
4142
EventWriter() EventWriter
4243
LedgerWriter() LedgerWriter
4344

44-
Commit(ledgerCloseMeta xdr.LedgerCloseMeta) error
45+
Commit(ledgerCloseMeta xdr.LedgerCloseMeta, durationMetrics map[string]time.Duration) error
4546
Rollback() error
4647
}
4748

@@ -237,10 +238,17 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
237238
db := rw.db
238239
writer := writeTx{
239240
globalCache: db.cache,
240-
postCommit: func() error {
241+
postCommit: func(durationMetrics map[string]time.Duration) error {
241242
// TODO: this is sqlite-only, it shouldn't be here
243+
startTime := time.Now()
242244
_, err := db.ExecRaw(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
243-
return err
245+
if err != nil {
246+
return err
247+
}
248+
if durationMetrics != nil {
249+
durationMetrics["wal_checkpoint"] = time.Since(startTime)
250+
}
251+
return nil
244252
},
245253
tx: txSession,
246254
stmtCache: stmtCache,
@@ -269,7 +277,7 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
269277

270278
type writeTx struct {
271279
globalCache *dbCache
272-
postCommit func() error
280+
postCommit func(durationMetrics map[string]time.Duration) error
273281
tx db.SessionInterface
274282
stmtCache *sq.StmtCache
275283
ledgerWriter ledgerWriter
@@ -290,20 +298,33 @@ func (w writeTx) EventWriter() EventWriter {
290298
return &w.eventWriter
291299
}
292300

293-
func (w writeTx) Commit(ledgerCloseMeta xdr.LedgerCloseMeta) error {
301+
func (w writeTx) Commit(ledgerCloseMeta xdr.LedgerCloseMeta, durationMetrics map[string]time.Duration) error {
294302
ledgerSeq := ledgerCloseMeta.LedgerSequence()
295303
ledgerCloseTime := ledgerCloseMeta.LedgerCloseTime()
296304

305+
startTime := time.Now()
297306
if err := w.ledgerWriter.trimLedgers(ledgerSeq, w.historyRetentionWindow); err != nil {
298307
return err
299308
}
309+
if durationMetrics != nil {
310+
durationMetrics["trim_ledgers"] = time.Since(startTime)
311+
}
312+
313+
startTime = time.Now()
300314
if err := w.txWriter.trimTransactions(ledgerSeq, w.historyRetentionWindow); err != nil {
301315
return err
302316
}
317+
if durationMetrics != nil {
318+
durationMetrics["trim_transactions"] = time.Since(startTime)
319+
}
303320

321+
startTime = time.Now()
304322
if err := w.eventWriter.trimEvents(ledgerSeq, w.historyRetentionWindow); err != nil {
305323
return err
306324
}
325+
if durationMetrics != nil {
326+
durationMetrics["trim_events"] = time.Since(startTime)
327+
}
307328

308329
// We need to make the cache update atomic with the transaction commit.
309330
// Otherwise, the cache can be made inconsistent if a write transaction finishes
@@ -318,11 +339,15 @@ func (w writeTx) Commit(ledgerCloseMeta xdr.LedgerCloseMeta) error {
318339
w.globalCache.latestLedgerCloseTime = ledgerCloseTime
319340
return nil
320341
}
342+
startTime = time.Now()
321343
if err := commitAndUpdateCache(); err != nil {
322344
return err
323345
}
346+
if durationMetrics != nil {
347+
durationMetrics["commit"] = time.Since(startTime)
348+
}
324349

325-
return w.postCommit()
350+
return w.postCommit(durationMetrics)
326351
}
327352

328353
func (w writeTx) Rollback() error {

‎cmd/stellar-rpc/internal/db/ledger_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func TestLedgers(t *testing.T) {
8484

8585
ledgerCloseMeta := createLedger(ledgerSequence)
8686
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
87-
require.NoError(t, tx.Commit(ledgerCloseMeta))
87+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
8888
// rolling back after a commit is a no-op
8989
require.NoError(t, tx.Rollback())
9090
}
@@ -96,7 +96,7 @@ func TestLedgers(t *testing.T) {
9696
require.NoError(t, err)
9797
ledgerCloseMeta := createLedger(ledgerSequence)
9898
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
99-
require.NoError(t, tx.Commit(ledgerCloseMeta))
99+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
100100

101101
assertLedgerRange(t, reader, 1, 11)
102102

@@ -105,7 +105,7 @@ func TestLedgers(t *testing.T) {
105105
require.NoError(t, err)
106106
ledgerCloseMeta = createLedger(ledgerSequence)
107107
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
108-
require.NoError(t, tx.Commit(ledgerCloseMeta))
108+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
109109

110110
assertLedgerRange(t, reader, 8, 12)
111111
}
@@ -130,7 +130,7 @@ func TestGetLedgerRange_NonEmptyDB(t *testing.T) {
130130
require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1)
131131
require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1)
132132
}
133-
require.NoError(t, write.Commit(lcms[len(lcms)-1]))
133+
require.NoError(t, write.Commit(lcms[len(lcms)-1], nil))
134134

135135
reader := NewLedgerReader(db)
136136
ledgerRange, err := reader.GetLedgerRange(ctx)
@@ -158,7 +158,7 @@ func TestGetLedgerRange_SingleDBRow(t *testing.T) {
158158
require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1)
159159
require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1)
160160
}
161-
require.NoError(t, write.Commit(lcms[len(lcms)-1]))
161+
require.NoError(t, write.Commit(lcms[len(lcms)-1], nil))
162162

163163
reader := NewLedgerReader(db)
164164
ledgerRange, err := reader.GetLedgerRange(ctx)
@@ -242,6 +242,6 @@ func setupBenchmarkingDB(b *testing.B) (*DB, []xdr.LedgerCloseMeta) {
242242
require.NoError(b, ledgerW.InsertLedger(lcm))
243243
require.NoError(b, txW.InsertTransactions(lcm))
244244
}
245-
require.NoError(b, write.Commit(lcms[len(lcms)-1]))
245+
require.NoError(b, write.Commit(lcms[len(lcms)-1], nil))
246246
return testDB, lcms
247247
}

‎cmd/stellar-rpc/internal/db/transaction_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func TestTransactionEvent(t *testing.T) {
136136

137137
require.NoError(t, ledgerW.InsertLedger(lcm))
138138
require.NoError(t, txW.InsertTransactions(lcm))
139-
require.NoError(t, write.Commit(lcm))
139+
require.NoError(t, write.Commit(lcm, nil))
140140

141141
reader := NewTransactionReader(log, db, passphrase)
142142
tx, err := reader.GetTransaction(t.Context(), lcm.TransactionHash(0))
@@ -217,7 +217,7 @@ func TestTransactionFound(t *testing.T) {
217217
require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1)
218218
require.NoError(t, eventW.InsertEvents(lcm), "ingestion failed for ledger %+v", lcm.V1)
219219
}
220-
require.NoError(t, write.Commit(lcms[len(lcms)-1]))
220+
require.NoError(t, write.Commit(lcms[len(lcms)-1], nil))
221221

222222
// check 404 case
223223
reader := NewTransactionReader(log, db, passphrase)
@@ -265,7 +265,7 @@ func BenchmarkTransactionFetch(b *testing.B) {
265265
require.NoError(b, ledgerW.InsertLedger(lcm))
266266
require.NoError(b, txW.InsertTransactions(lcm))
267267
}
268-
require.NoError(b, write.Commit(lcms[len(lcms)-1]))
268+
require.NoError(b, write.Commit(lcms[len(lcms)-1], nil))
269269
reader := NewTransactionReader(log, db, passphrase)
270270

271271
randoms := make([]int, b.N)

‎cmd/stellar-rpc/internal/ingest/mock_db_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package ingest
22

33
import (
44
"context"
5+
"time"
56

67
"github.com/prometheus/client_golang/prometheus"
78
"github.com/stretchr/testify/mock"
@@ -54,8 +55,8 @@ func (m *MockTx) TransactionWriter() db.TransactionWriter {
5455
return args.Get(0).(db.TransactionWriter) //nolint:forcetypeassert
5556
}
5657

57-
func (m *MockTx) Commit(ledgerCloseMeta xdr.LedgerCloseMeta) error {
58-
args := m.Called(ledgerCloseMeta)
58+
func (m *MockTx) Commit(ledgerCloseMeta xdr.LedgerCloseMeta, durationMetrics map[string]time.Duration) error {
59+
args := m.Called(ledgerCloseMeta, durationMetrics)
5960
return args.Error(0)
6061
}
6162

‎cmd/stellar-rpc/internal/ingest/service.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88

99
"github.com/cenkalti/backoff/v4"
1010
"github.com/prometheus/client_golang/prometheus"
11-
1211
"github.com/stellar/go/historyarchive"
1312
backends "github.com/stellar/go/ingest/ledgerbackend"
1413
"github.com/stellar/go/support/log"
@@ -207,9 +206,16 @@ func (s *Service) ingest(ctx context.Context, sequence uint32) error {
207206
return err
208207
}
209208

210-
if err := tx.Commit(ledgerCloseMeta); err != nil {
209+
durationMetrics := map[string]time.Duration{}
210+
if err := tx.Commit(ledgerCloseMeta, durationMetrics); err != nil {
211211
return err
212212
}
213+
for key, duration := range durationMetrics {
214+
s.metrics.ingestionDurationMetric.
215+
With(prometheus.Labels{"type": key}).
216+
Observe(duration.Seconds())
217+
}
218+
213219
s.logger.
214220
WithField("duration", time.Since(startTime).Seconds()).
215221
Debugf("Ingested ledger %d", sequence)
@@ -238,13 +244,21 @@ func (s *Service) ingestLedgerCloseMeta(tx db.WriteTx, ledgerCloseMeta xdr.Ledge
238244
With(prometheus.Labels{"type": "transactions"}).
239245
Observe(time.Since(startTime).Seconds())
240246

247+
startTime = time.Now()
241248
if err := tx.EventWriter().InsertEvents(ledgerCloseMeta); err != nil {
242249
return err
243250
}
251+
s.metrics.ingestionDurationMetric.
252+
With(prometheus.Labels{"type": "events"}).
253+
Observe(time.Since(startTime).Seconds())
244254

255+
startTime = time.Now()
245256
if err := s.feeWindows.IngestFees(ledgerCloseMeta); err != nil {
246257
return err
247258
}
259+
s.metrics.ingestionDurationMetric.
260+
With(prometheus.Labels{"type": "fee-window"}).
261+
Observe(time.Since(startTime).Seconds())
248262

249263
return nil
250264
}

‎cmd/stellar-rpc/internal/ingest/service_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"time"
1010

1111
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/mock"
1213
"github.com/stretchr/testify/require"
1314

1415
"github.com/stellar/go/ingest/ledgerbackend"
@@ -257,7 +258,7 @@ func setupMockExpectations(ctx context.Context, t *testing.T, mockDB *MockDB,
257258
mockEventWriter := &MockEventWriter{}
258259

259260
mockDB.On("NewTx", ctx).Return(mockTx, nil).Once()
260-
mockTx.On("Commit", ledger).Return(nil).Once()
261+
mockTx.On("Commit", ledger, mock.AnythingOfType("map[string]time.Duration")).Return(nil).Once()
261262
mockTx.On("Rollback").Return(nil).Once()
262263
mockTx.On("LedgerWriter").Return(mockLedgerWriter).Once()
263264
mockTx.On("TransactionWriter").Return(mockTxWriter).Once()

‎cmd/stellar-rpc/internal/methods/get_events_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func TestGetEvents(t *testing.T) {
6464
ledgerCloseMeta := ledgerCloseMetaWithEvents(2, now.Unix(), txMeta...)
6565
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
6666
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta))
67-
require.NoError(t, write.Commit(ledgerCloseMeta))
67+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
6868

6969
handler := eventsRPCHandler{
7070
dbReader: store,
@@ -117,7 +117,7 @@ func TestGetEvents(t *testing.T) {
117117
ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...)
118118
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger")
119119
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta))
120-
require.NoError(t, write.Commit(ledgerCloseMeta))
120+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
121121

122122
handler := eventsRPCHandler{
123123
dbReader: store,
@@ -210,7 +210,7 @@ func TestGetEvents(t *testing.T) {
210210
ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...)
211211
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger")
212212
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events")
213-
require.NoError(t, write.Commit(ledgerCloseMeta))
213+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
214214

215215
handler := eventsRPCHandler{
216216
dbReader: store,
@@ -272,7 +272,7 @@ func TestGetEvents(t *testing.T) {
272272

273273
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
274274
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
275-
require.NoError(t, write.Commit(ledgerCloseMeta))
275+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
276276

277277
number := xdr.Uint64(4)
278278
handler := eventsRPCHandler{
@@ -415,7 +415,7 @@ func TestGetEvents(t *testing.T) {
415415

416416
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
417417
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
418-
require.NoError(t, write.Commit(ledgerCloseMeta))
418+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
419419

420420
number := xdr.Uint64(4)
421421
handler := eventsRPCHandler{
@@ -639,7 +639,7 @@ func TestGetEvents(t *testing.T) {
639639

640640
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
641641
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
642-
require.NoError(t, write.Commit(ledgerCloseMeta))
642+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
643643

644644
handler := eventsRPCHandler{
645645
dbReader: store,
@@ -741,7 +741,7 @@ func TestGetEvents(t *testing.T) {
741741
ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...)
742742
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
743743
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
744-
require.NoError(t, write.Commit(ledgerCloseMeta))
744+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
745745

746746
handler := eventsRPCHandler{
747747
dbReader: store,
@@ -819,7 +819,7 @@ func TestGetEvents(t *testing.T) {
819819
ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...)
820820
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
821821
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
822-
require.NoError(t, write.Commit(ledgerCloseMeta))
822+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
823823

824824
handler := eventsRPCHandler{
825825
dbReader: store,
@@ -931,7 +931,7 @@ func TestGetEvents(t *testing.T) {
931931
ledgerCloseMeta := ledgerCloseMetaWithEvents(5, now.Unix(), txMeta...)
932932
require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
933933
require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
934-
require.NoError(t, write.Commit(ledgerCloseMeta))
934+
require.NoError(t, write.Commit(ledgerCloseMeta, nil))
935935

936936
id := &protocol.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0}
937937
handler := eventsRPCHandler{
@@ -1039,7 +1039,7 @@ func BenchmarkGetEvents(b *testing.B) {
10391039
ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...)
10401040
require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ")
10411041
require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ")
1042-
require.NoError(b, write.Commit(ledgerCloseMeta))
1042+
require.NoError(b, write.Commit(ledgerCloseMeta, nil))
10431043
}
10441044

10451045
handler := eventsRPCHandler{

‎cmd/stellar-rpc/internal/methods/get_ledgers_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func setupTestDB(t *testing.T, numLedgers int) *db.DB {
3434
tx, err := db.NewReadWriter(log.DefaultLogger, testDB, daemon, 150, 100, passphrase).NewTx(context.Background())
3535
require.NoError(t, err)
3636
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
37-
require.NoError(t, tx.Commit(ledgerCloseMeta))
37+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
3838
}
3939
return testDB
4040
}
@@ -284,7 +284,7 @@ func setupBenchmarkingDB(b *testing.B) *db.DB {
284284
require.NoError(b, ledgerW.InsertLedger(lcm))
285285
require.NoError(b, txW.InsertTransactions(lcm))
286286
}
287-
require.NoError(b, write.Commit(lcms[len(lcms)-1]))
287+
require.NoError(b, write.Commit(lcms[len(lcms)-1], nil))
288288
return testDB
289289
}
290290

‎cmd/stellar-rpc/internal/methods/get_transactions_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ func setupDB(t *testing.T, numLedgers int, skipLedger int) *db.DB {
319319
tx, err := db.NewReadWriter(log.DefaultLogger, testDB, daemon, 150, 100, passphrase).NewTx(context.Background())
320320
require.NoError(t, err)
321321
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
322-
require.NoError(t, tx.Commit(ledgerCloseMeta))
322+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
323323
}
324324
return testDB
325325
}
@@ -333,7 +333,7 @@ func setupDBNoTxs(t *testing.T, numLedgers int) *db.DB {
333333
tx, err := db.NewReadWriter(log.DefaultLogger, testDB, daemon, 150, 100, passphrase).NewTx(context.Background())
334334
require.NoError(t, err)
335335
require.NoError(t, tx.LedgerWriter().InsertLedger(ledgerCloseMeta))
336-
require.NoError(t, tx.Commit(ledgerCloseMeta))
336+
require.NoError(t, tx.Commit(ledgerCloseMeta, nil))
337337
}
338338
return testDB
339339
}

0 commit comments

Comments
 (0)