[SharovBot] fix(txpool): evict zombie queued txns exceeding MaxNonceGap (#19449)

**[SharovBot]**

## Split from #19393 per @yperbasis review

This PR contains **Bug #2 only** (zombie queued transaction eviction),
extracted from #19393 which was asked to be split into separate PRs.

## Problem

Queued transactions with an impossibly large nonce gap (e.g. on-chain
nonce=281, queued nonce=16,814 — gap of 16,533) sit in the pool forever.
They can never become pending without filling thousands of nonce
positions first, causing unbounded queued pool bloat (4,000+ txns
observed on Gnosis Chain, Erigon 3.3.8).

The existing blob-txn nonce-gap eviction only covered `BlobTxnType`.
Regular transactions had no gap limit.

## Fix

- Add `MaxNonceGap uint64` to `txpoolcfg.Config` (default: 64)
- Add `NonceTooDistant` (DiscardReason 37) for observability
- In `onSenderStateChange`, evict txns whose nonce exceeds `noGapsNonce`
by more than `MaxNonceGap`
- `noGapsNonce` accounts for consecutive txns already pooled, so
consecutive txns are never zombie-evicted
- Fix `toDelReasons` parallel slice to track correct discard reason per
evicted tx (was always logging `NonceTooLow`)

## Tests

- `TestZombieQueuedEviction` — 3 sub-tests:
1. Zombie tx (gap=65 > MaxNonceGap=64) is evicted with `NonceTooDistant`
  2. Tx at exactly MaxNonceGap boundary (gap=64) is kept
  3. Consecutive txns beyond MaxNonceGap are never zombie-evicted

## Testing

```
go build ./txnprovider/txpool/... ✅
go test ./txnprovider/txpool/... -run TestZombieQueuedEviction -count=1 ✅
```

## Related

- Bug #1 (stale pending / AuRa nonce) will be addressed separately per
@yperbasis feedback
- Backport to release/3.3 will follow once this is merged
- Original combined PR: #19393

Author: Giulio2002

txnprovider/txpool/pool.go

@@ -1863,12 +1863,22 @@ func (p *TxPool) onSenderStateChange(senderID uint64, senderNonce uint64, sender
 	cumulativeRequiredBalance := uint256.NewInt(0)
 	minFeeCap := uint256.NewInt(0).SetAllOne()
 	minTip := uint64(math.MaxUint64)
-	var toDel []*metaTxn // can't delete items while iterate them
+	var toDel []*metaTxn                       // can't delete items while iterate them
+	var toDelReasons []txpoolcfg.DiscardReason // parallel reasons slice for toDel
 
 	p.all.ascend(senderID, func(mt *metaTxn) bool {
 		deleteAndContinueReasonLog := ""
+		discardReason := txpoolcfg.NonceTooLow
 		if senderNonce > mt.TxnSlot.Nonce {
 			deleteAndContinueReasonLog = "low nonce"
+		} else if p.cfg.MaxNonceGap > 0 && mt.TxnSlot.Nonce > noGapsNonce && mt.TxnSlot.Nonce-noGapsNonce > p.cfg.MaxNonceGap {
+			// Evict "zombie" queued transactions whose nonce is so far ahead of the sender's
+			// on-chain nonce (accounting for any consecutive txns already in the pool) that they
+			// can practically never become pending. This prevents unbounded pool bloat from accounts
+			// that submitted transactions with impossibly large nonce gaps (e.g. nonce 144968 when
+			// on-chain nonce is 6398). The gap threshold is configurable via MaxNonceGap (default 64).
+			deleteAndContinueReasonLog = "nonce gap too large"
+			discardReason = txpoolcfg.NonceTooDistant
 		} else if mt.TxnSlot.Nonce != noGapsNonce && mt.TxnSlot.Type == BlobTxnType { // Discard nonce-gapped blob txns
 			deleteAndContinueReasonLog = "nonce-gapped blob txn"
 		}
@@ -1888,6 +1898,7 @@ func (p *TxPool) onSenderStateChange(senderID uint64, senderNonce uint64, sender
 				//already removed
 			}
 			toDel = append(toDel, mt)
+			toDelReasons = append(toDelReasons, discardReason)
 			return true
 		}
 
@@ -1956,8 +1967,8 @@ func (p *TxPool) onSenderStateChange(senderID uint64, senderNonce uint64, sender
 		return true
 	})
 
-	for _, mt := range toDel {
-		p.discardLocked(mt, txpoolcfg.NonceTooLow)
+	for i, mt := range toDel {
+		p.discardLocked(mt, toDelReasons[i])
 	}
 
 	logger.Trace("[txpool] onSenderStateChange", "sender", senderID, "count", p.all.count(senderID), "pending", p.pending.Len(), "baseFee", p.baseFee.Len(), "queued", p.queued.Len())

txnprovider/txpool/pool_test.go

@@ -1815,3 +1815,167 @@ func BenchmarkProcessRemoteTxns(b *testing.B) {
 	pending, baseFee, queued := pool.CountContent()
 	b.Logf("Final pool stats - pending: %d, baseFee: %d, queued: %d", pending, baseFee, queued)
 }
+
+// TestZombieQueuedEviction verifies that queued transactions whose nonce is so far ahead of
+// the sender's on-chain nonce that they can never become pending are evicted from the pool.
+// This covers Bug #2: "zombie" queued txns on Gnosis Chain (e.g. on-chain nonce=281 but
+// queued nonce=16814, a gap of 16,533 that can never be filled).
+func TestZombieQueuedEviction(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	ch := make(chan Announcements, 100)
+	coreDB := temporaltest.NewTestDB(t, datadir.New(t.TempDir()))
+	db := memdb.NewTestPoolDB(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	cfg := txpoolcfg.DefaultConfig
+	cfg.MaxNonceGap = 64 // explicit, same as default
+	sendersCache := kvcache.New(kvcache.DefaultCoherentConfig)
+	pool, err := New(ctx, ch, db, coreDB, cfg, sendersCache, chain.TestChainConfig, nil, nil, func() {}, nil, nil, log.New(), WithFeeCalculator(nil))
+	require.NoError(err)
+	require.NotNil(pool)
+
+	pendingBaseFee := uint64(200_000)
+	h1 := gointerfaces.ConvertHashToH256([32]byte{})
+	var senderAddr [20]byte
+	senderAddr[0] = 0x42
+
+	// Set sender's on-chain nonce = 5
+	acc := accounts3.Account{
+		Nonce:       5,
+		Balance:     *uint256.NewInt(1 * common.Ether),
+		CodeHash:    accounts.EmptyCodeHash,
+		Incarnation: 0,
+	}
+	v := accounts3.SerialiseV3(&acc)
+	change := &remoteproto.StateChangeBatch{
+		StateVersionId:      0,
+		PendingBlockBaseFee: pendingBaseFee,
+		BlockGasLimit:       1_000_000,
+		ChangeBatch: []*remoteproto.StateChange{
+			{
+				BlockHeight: 0,
+				BlockHash:   h1,
+				Changes: []*remoteproto.AccountChange{
+					{
+						Action:  remoteproto.Action_UPSERT,
+						Address: gointerfaces.ConvertAddressToH160(senderAddr),
+						Data:    v,
+					},
+				},
+			},
+		},
+	}
+	require.NoError(pool.OnNewBlock(ctx, change, TxnSlots{}, TxnSlots{}, TxnSlots{}))
+
+	t.Run("zombie tx with impossible nonce gap is evicted", func(t *testing.T) {
+		// nonce = 5 + 64 + 1 = 70, gap=65 > MaxNonceGap(64)
+		zombieNonce := uint64(5 + cfg.MaxNonceGap + 1)
+		var txnSlots TxnSlots
+		slot := &TxnSlot{
+			Tip:    *uint256.NewInt(300_000),
+			FeeCap: *uint256.NewInt(300_000),
+			Gas:    100_000,
+			Nonce:  zombieNonce,
+		}
+		slot.IDHash[0] = 0xAA
+		txnSlots.Append(slot, senderAddr[:], true)
+
+		reasons, err := pool.AddLocalTxns(ctx, txnSlots)
+		require.NoError(err)
+		require.Len(reasons, 1)
+		assert.Equal(txpoolcfg.NonceTooDistant, reasons[0],
+			"zombie tx (nonce gap %d > MaxNonceGap %d) should be evicted with NonceTooDistant",
+			zombieNonce-5, cfg.MaxNonceGap)
+
+		_, _, queued := pool.CountContent()
+		assert.Equal(0, queued, "queued pool should be empty after zombie eviction")
+	})
+
+	t.Run("tx at exactly MaxNonceGap boundary is kept", func(t *testing.T) {
+		// nonce = 5 + 64 = 69, gap=64 == MaxNonceGap (NOT evicted)
+		boundaryNonce := uint64(5 + cfg.MaxNonceGap)
+		var txnSlots TxnSlots
+		slot := &TxnSlot{
+			Tip:    *uint256.NewInt(300_000),
+			FeeCap: *uint256.NewInt(300_000),
+			Gas:    100_000,
+			Nonce:  boundaryNonce,
+		}
+		slot.IDHash[0] = 0xBB
+		txnSlots.Append(slot, senderAddr[:], true)
+
+		reasons, err := pool.AddLocalTxns(ctx, txnSlots)
+		require.NoError(err)
+		require.Len(reasons, 1)
+		// gap = boundaryNonce - noGapsNonce. noGapsNonce=5 (no consecutive txns).
+		// 64 is NOT > 64, so the tx should be kept (in queued due to nonce gap).
+		assert.Equal(txpoolcfg.Success, reasons[0],
+			"tx at exactly MaxNonceGap boundary (gap=%d) should be accepted", cfg.MaxNonceGap)
+	})
+
+	t.Run("consecutive txns beyond MaxNonceGap are kept", func(t *testing.T) {
+		// If there are consecutive txns 5, 6, 7, ..., 5+MaxNonceGap+5, they should all be kept
+		// because the gap from noGapsNonce is always 0 for consecutive txns.
+		var txnSlots TxnSlots
+		baseNonce := uint64(5)
+		// Clear the pool first by using a fresh pool
+		ch2 := make(chan Announcements, 100)
+		coreDB2 := temporaltest.NewTestDB(t, datadir.New(t.TempDir()))
+		db2 := memdb.NewTestPoolDB(t)
+		cfg2 := txpoolcfg.DefaultConfig
+		cfg2.MaxNonceGap = 10 // small gap for this test
+		pool2, err := New(ctx, ch2, db2, coreDB2, cfg2, kvcache.New(kvcache.DefaultCoherentConfig),
+			chain.TestChainConfig, nil, nil, func() {}, nil, nil, log.New(), WithFeeCalculator(nil))
+		require.NoError(err)
+
+		acc2 := accounts3.Account{
+			Nonce:    baseNonce,
+			Balance:  *uint256.NewInt(10 * common.Ether),
+			CodeHash: accounts.EmptyCodeHash,
+		}
+		v2 := accounts3.SerialiseV3(&acc2)
+		var addr2 [20]byte
+		addr2[0] = 0x99
+		change2 := &remoteproto.StateChangeBatch{
+			StateVersionId:      0,
+			PendingBlockBaseFee: pendingBaseFee,
+			BlockGasLimit:       1_000_000,
+			ChangeBatch: []*remoteproto.StateChange{
+				{
+					BlockHeight: 0,
+					BlockHash:   gointerfaces.ConvertHashToH256([32]byte{1}),
+					Changes: []*remoteproto.AccountChange{
+						{Action: remoteproto.Action_UPSERT, Address: gointerfaces.ConvertAddressToH160(addr2), Data: v2},
+					},
+				},
+			},
+		}
+		require.NoError(pool2.OnNewBlock(ctx, change2, TxnSlots{}, TxnSlots{}, TxnSlots{}))
+
+		// Add consecutive txns: nonces 5, 6, 7, ..., 5+MaxNonceGap+5 = 20
+		count := int(cfg2.MaxNonceGap + 5 + 1)
+		for i := 0; i < count; i++ {
+			txnSlots.Txns = nil
+			txnSlots.Senders = txnSlots.Senders[:0]
+			txnSlots.IsLocal = txnSlots.IsLocal[:0]
+			slot := &TxnSlot{
+				Tip:    *uint256.NewInt(300_000),
+				FeeCap: *uint256.NewInt(300_000),
+				Gas:    100_000,
+				Nonce:  baseNonce + uint64(i),
+			}
+			slot.IDHash[0] = uint8(0xC0 + i)
+			txnSlots.Append(slot, addr2[:], true)
+			reasons, err := pool2.AddLocalTxns(ctx, txnSlots)
+			require.NoError(err)
+			assert.Equal(txpoolcfg.Success, reasons[0],
+				"consecutive tx nonce=%d should not be evicted (gap from noGapsNonce is 0)", baseNonce+uint64(i))
+		}
+		// All consecutive txns (no nonce gaps) should be accepted and promoted to pending.
+		// The key check is that NONE were evicted with NonceTooDistant — verified above per-tx.
+		pending2, _, queued2 := pool2.CountContent()
+		assert.Equal(0, queued2, "no consecutive txns should be zombie-evicted (queued should be drained to pending)")
+		assert.Equal(count, pending2, "all consecutive txns should be pending (no gaps, sufficient balance)")
+	})
+}

txnprovider/txpool/txpoolcfg/txpoolcfg.go

@@ -41,6 +41,12 @@ type Config struct {
 	TotalBlobPoolLimit  uint64 // Total number of blobs (not txns) allowed within the txpool
 	PriceBump           uint64 // Price bump percentage to replace an already existing transaction
 	BlobPriceBump       uint64 //Price bump percentage to replace an existing 4844 blob txn (type-3)
+	// MaxNonceGap is the maximum allowed gap between a sender's on-chain nonce and the nonce of a
+	// queued transaction. Transactions whose nonce exceeds the on-chain nonce by more than this
+	// value are considered "zombie" transactions that can never become pending (they would require
+	// an impossibly large number of preceding transactions to fill the gap), and are evicted from
+	// the pool. A value of 0 disables this eviction. Default is 64.
+	MaxNonceGap uint64
 
 	// regular batch tasks processing
 	SyncToNewPeersEvery    time.Duration
@@ -77,6 +83,8 @@ var DefaultConfig = Config{
 	PriceBump:          10,   // Price bump percentage to replace an already existing transaction
 	BlobPriceBump:      100,
 
+	MaxNonceGap: 64, // Evict queued txns with a nonce gap > 64 from the on-chain nonce
+
 	NoGossip:     false,
 	MdbxWriteMap: false,
 }
@@ -121,6 +129,7 @@ const (
 	ErrAuthorityReserved DiscardReason = 34 // EIP-7702 transaction with authority already reserved
 	InvalidAA            DiscardReason = 35 // Invalid RIP-7560 transaction
 	ErrGetCode           DiscardReason = 36 // Error getting code during AA validation
+	NonceTooDistant      DiscardReason = 37 // Nonce gap between tx and on-chain nonce exceeds MaxNonceGap; tx can never become pending
 )
 
 func (r DiscardReason) String() string {
@@ -199,6 +208,8 @@ func (r DiscardReason) String() string {
 		return "RIP-7560 transaction failed validation"
 	case ErrGetCode:
 		return "error getting account code during RIP-7560 validation"
+	case NonceTooDistant:
+		return "nonce gap too large: transaction nonce is too far ahead of sender's on-chain nonce"
 	default:
 		panic(fmt.Sprintf("discard reason: %d", r))
 	}