@@ -10,8 +10,6 @@ import (
1010 "sync"
1111 "time"
1212
13- "github.com/prometheus/client_golang/prometheus"
14- "github.com/prometheus/client_golang/prometheus/promauto"
1513 "go.uber.org/multierr"
1614
1715 commonhex "github.com/smartcontractkit/chainlink-common/pkg/utils/hex"
@@ -33,48 +31,13 @@ const (
3331 processHeadTimeout = 10 * time .Minute
3432)
3533
36- var (
37- promNumGasBumps = promauto .NewCounterVec (prometheus.CounterOpts {
38- Name : "tx_manager_num_gas_bumps" ,
39- Help : "Number of gas bumps" ,
40- }, []string {"chainID" })
41-
42- promGasBumpExceedsLimit = promauto .NewCounterVec (prometheus.CounterOpts {
43- Name : "tx_manager_gas_bump_exceeds_limit" ,
44- Help : "Number of times gas bumping failed from exceeding the configured limit. Any counts of this type indicate a serious problem." ,
45- }, []string {"chainID" })
46- promNumConfirmedTxs = promauto .NewCounterVec (prometheus.CounterOpts {
47- Name : "tx_manager_num_confirmed_transactions" ,
48- Help : "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs" ,
49- }, []string {"chainID" })
50- promTimeUntilTxConfirmed = promauto .NewHistogramVec (prometheus.HistogramOpts {
51- Name : "tx_manager_time_until_tx_confirmed" ,
52- Help : "The amount of time elapsed from a transaction being broadcast to being included in a block." ,
53- Buckets : []float64 {
54- float64 (500 * time .Millisecond ),
55- float64 (time .Second ),
56- float64 (5 * time .Second ),
57- float64 (15 * time .Second ),
58- float64 (30 * time .Second ),
59- float64 (time .Minute ),
60- float64 (2 * time .Minute ),
61- float64 (5 * time .Minute ),
62- float64 (10 * time .Minute ),
63- },
64- }, []string {"chainID" })
65- promBlocksUntilTxConfirmed = promauto .NewHistogramVec (prometheus.HistogramOpts {
66- Name : "tx_manager_blocks_until_tx_confirmed" ,
67- Help : "The amount of blocks that have been mined from a transaction being broadcast to being included in a block." ,
68- Buckets : []float64 {
69- float64 (1 ),
70- float64 (5 ),
71- float64 (10 ),
72- float64 (20 ),
73- float64 (50 ),
74- float64 (100 ),
75- },
76- }, []string {"chainID" })
77- )
34+ type confimerMetrics interface {
35+ IncrementNumGasBumps (ctx context.Context )
36+ IncrementGasBumpExceedsLimit (ctx context.Context )
37+ IncrementNumConfirmedTxs (ctx context.Context , confirmedTransactions int )
38+ RecordTimeUntilTxConfirmed (ctx context.Context , duration float64 )
39+ RecordBlocksUntilTxConfirmed (ctx context.Context , blocksElapsed float64 )
40+ }
7841
7942// Confirmer is a broad service which performs four different tasks in sequence on every new longest chain
8043// Step 1: Mark that all currently pending transaction attempts were broadcast before this block
@@ -95,6 +58,7 @@ type Confirmer[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, THA
9558 txConfig types.ConfirmerTransactionsConfig
9659 dbConfig types.ConfirmerDatabaseConfig
9760 chainID CID
61+ metrics confimerMetrics
9862
9963 ks types.KeyStore [ADDR ]
10064 enabledAddresses []ADDR
@@ -127,6 +91,7 @@ func NewConfirmer[
12791 lggr logger.Logger ,
12892 isReceiptNil func (R ) bool ,
12993 stuckTxDetector types.StuckTxDetector [CHAIN_ID , ADDR , TX_HASH , BLOCK_HASH , SEQ , FEE ],
94+ metrics confimerMetrics ,
13095) * Confirmer [CHAIN_ID , HEAD , ADDR , TX_HASH , BLOCK_HASH , R , SEQ , FEE ] {
13196 lggr = logger .Named (lggr , "Confirmer" )
13297 return & Confirmer [CHAIN_ID , HEAD , ADDR , TX_HASH , BLOCK_HASH , R , SEQ , FEE ]{
@@ -143,6 +108,7 @@ func NewConfirmer[
143108 mb : mailbox .NewSingle [HEAD ](),
144109 isReceiptNil : isReceiptNil ,
145110 stuckTxDetector : stuckTxDetector ,
111+ metrics : metrics ,
146112 }
147113}
148114
@@ -368,7 +334,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded
368334 return nil
369335 }
370336 // Add newly confirmed transactions to the prom metric
371- promNumConfirmedTxs . WithLabelValues ( ec .chainID . String ()). Add ( float64 ( len (includedTxs ) ))
337+ ec .metrics . IncrementNumConfirmedTxs ( ctx , len (includedTxs ))
372338
373339 purgeTxIDs := make ([]int64 , 0 , len (includedTxs ))
374340 confirmedTxIDs := make ([]int64 , 0 , len (includedTxs ))
@@ -381,7 +347,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) ProcessIncluded
381347 continue
382348 }
383349 confirmedTxIDs = append (confirmedTxIDs , tx .ID )
384- observeUntilTxConfirmed (ec .chainID , tx . TxAttempts , head )
350+ observeUntilTxConfirmed (ctx , ec .metrics , tx , head )
385351 }
386352 // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message
387353 if err := ec .txStore .UpdateTxFatalError (ctx , purgeTxIDs , ec .stuckTxDetector .StuckTxFatalError ()); err != nil {
@@ -667,13 +633,13 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) bumpGas(ctx con
667633 // if no error, return attempt
668634 // if err, continue below
669635 if err == nil {
670- promNumGasBumps . WithLabelValues ( ec .chainID . String ()). Inc ( )
636+ ec .metrics . IncrementNumGasBumps ( ctx )
671637 ec .lggr .Debugw ("Rebroadcast bumping fee for tx" , append (logFields , "bumpedFee" , bumpedFee .String (), "bumpedFeeLimit" , bumpedFeeLimit )... )
672638 return bumpedAttempt , err
673639 }
674640
675641 if errors .Is (err , fees .ErrBumpFeeExceedsLimit ) {
676- promGasBumpExceedsLimit . WithLabelValues ( ec .chainID . String ()). Inc ( )
642+ ec .metrics . IncrementGasBumpExceedsLimit ( ctx )
677643 }
678644
679645 return bumpedAttempt , fmt .Errorf ("error bumping gas: %w" , err )
@@ -712,7 +678,7 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) handleInProgres
712678 if err != nil {
713679 return fmt .Errorf ("could not bump gas for terminally underpriced transaction: %w" , err )
714680 }
715- promNumGasBumps . WithLabelValues ( ec .chainID . String ()). Inc ( )
681+ ec .metrics . IncrementNumGasBumps ( ctx )
716682 lggr .With (
717683 "sendError" , sendError ,
718684 "maxGasPriceConfig" , ec .feeConfig .MaxFeePrice (),
@@ -853,38 +819,35 @@ func (ec *Confirmer[CID, HEAD, ADDR, THASH, BHASH, R, SEQ, FEE]) sendEmptyTransa
853819 return txhash , nil
854820}
855821
856- // observeUntilTxConfirmed observes the promBlocksUntilTxConfirmed metric for each confirmed
857- // transaction.
822+ // observeUntilTxConfirmed observes the timeUntilTxConfirmed and blocksUntilTxConfirmed metrics for each confirmed transaction.
858823func observeUntilTxConfirmed [
859824 CHAIN_ID chains.ID ,
860825 ADDR chains.Hashable ,
861826 TX_HASH , BLOCK_HASH chains.Hashable ,
862827 SEQ chains.Sequence ,
863828 FEE fees.Fee ,
864- ](chainID CHAIN_ID , attempts []types.TxAttempt [CHAIN_ID , ADDR , TX_HASH , BLOCK_HASH , SEQ , FEE ], head chains.Head [BLOCK_HASH ]) {
865- for _ , attempt := range attempts {
866- // We estimate the time until confirmation by subtracting from the time the tx (not the attempt)
867- // was created. We want to measure the amount of time taken from when a transaction is created
868- // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts
869- // were needed to achieve this.
870- duration := time .Since (attempt .Tx .CreatedAt )
871- promTimeUntilTxConfirmed .
872- WithLabelValues (chainID .String ()).
873- Observe (float64 (duration ))
874-
875- // Since a tx can have many attempts, we take the number of blocks to confirm as the block number
876- // of the receipt minus the block number of the first ever broadcast for this transaction.
877- var minBroadcastBefore int64
878- for _ , a := range attempt .Tx .TxAttempts {
879- if b := a .BroadcastBeforeBlockNum ; b != nil && * b < minBroadcastBefore {
880- minBroadcastBefore = * b
881- }
882- }
883- if minBroadcastBefore > 0 {
884- blocksElapsed := head .BlockNumber () - minBroadcastBefore
885- promBlocksUntilTxConfirmed .
886- WithLabelValues (chainID .String ()).
887- Observe (float64 (blocksElapsed ))
829+ ](ctx context.Context , metrics confimerMetrics , tx * types.Tx [CHAIN_ID , ADDR , TX_HASH , BLOCK_HASH , SEQ , FEE ], head chains.Head [BLOCK_HASH ]) {
830+ if tx == nil {
831+ return
832+ }
833+ // We estimate the time until confirmation by subtracting from the time the tx (not the attempt)
834+ // was created. We want to measure the amount of time taken from when a transaction is created
835+ // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts
836+ // were needed to achieve this.
837+ duration := time .Since (tx .CreatedAt )
838+ metrics .RecordTimeUntilTxConfirmed (ctx , float64 (duration ))
839+
840+ // Since a tx can have many attempts, we take the number of blocks to confirm as the current block number
841+ // minus the block number of the first ever broadcast for this transaction.
842+ var minBroadcastBefore int64
843+ for _ , a := range tx .TxAttempts {
844+ if b := a .BroadcastBeforeBlockNum ; b != nil && * b < minBroadcastBefore {
845+ minBroadcastBefore = * b
888846 }
889847 }
848+
849+ if minBroadcastBefore > 0 {
850+ blocksElapsed := head .BlockNumber () - minBroadcastBefore
851+ metrics .RecordBlocksUntilTxConfirmed (ctx , float64 (blocksElapsed ))
852+ }
890853}
0 commit comments