Rework BatchTxPool functionality to a legitimate Ethereum tx mempool (#986)

* Rework BatchTxPool functionality to a legitimate Ethereum tx mempool

* Address review comments

* Log transaction submissions from BatchTxPool

* Close the window between batch detachment and the lastSubmittedAt update

* Deprecate --eoa-activity-cache-ttl CLI flag as it is no longer applicable

* Reject transactions with in-flight nonces

* Solidify BatchTxPool rework: fix log tag, race, sizing, and add unit tests

- flush success log tagged as flushReasonPrefix (metric already correct)
- flush success/failure paths merge lastSubmittedNonce/At with max() so a
  concurrent Add() fast-path is never regressed by a stale ack or rollback
- eoaEnqueueTxs preserves an existing same-nonce entry: last-write-wins
  keeps a client's fresher payload over a re-queued failed batch
- rename maxEOAPoolSize -> maxNonceLookahead (that's what it enforces and
  add a real per-EOA size cap at admission time via maxEOAQueueSize +
  ErrTxPoolFull
- staleEntry no longer marks a freshly-created queue (zero lastSubmittedAt)
  as stale
- selectSequentialNonces walks the nonce-keyed map directly (O(k)) instead
  of sorting the full queue every tick
- add batch_tx_pool_test.go: txQueue primitives + rollback preserve-fresh

* Revert logic change on staleEntry()

* Move state index nonce read after the duplication, in-flight checks

* Update comments on the new logic of BatchTxPool

* Remove unused from field from batchSubmission type

* Re-use eoaQueueEntry() in eoaEnqueueTxs()

* Move the cap check to just before the enqueue, so fast-path-eligible txs bypass it

* Prune transactions that exceed the queue TTL

* Cap retries per tx and drop with a WARN + TransactionsDropped after N attempts

* Add unit tests for ErrTxPoolFull and the flush failure/success merge branches

* Reset txQueue retries on submission success

* Update stalenessFactor so that it's about 10 seconds with the current tx-batch-interval

* Fix comment on eoaEnqueueTxs boolean value

Co-authored-by: Leo Zhang <zhangchiqing@gmail.com>

* Move rollback of nonce range reservation to eoaEnqueueTxs()

* Add comments on the fields of batchSubmission type

* Distinguish the 2 cases for fast-path submission for logging

* Guard elapsed spacing in processPooledTransactions() and add an E2E test

* test(requester): add BatchTxPool wedge -> recovery e2e test (#988)

Reproduces the wedge state from #983 (Cadence wrapper reverts before
advancing state, leaving the pool's in-flight marker ahead of the
on-chain frontier) and asserts BatchTxPool clears it within
TxBatchInterval*stalenessFactor. Auto-mine is disabled so the pool's
staleEntry timing is exposed - otherwise validateTransactionWithState
preempts once state catches up, masking the pool's recovery.

This is the counterpart to Test_BatchTxPool_InFlightNonceRejection,
which only verifies the wedge state is entered, not that it recovers.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: vishal <1117327+vishalchangrani@users.noreply.github.com>
Co-authored-by: Leo Zhang <zhangchiqing@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: 4 people

bootstrap/bootstrap.go

@@ -287,6 +287,12 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 			nonceProvider,
 		)
 	} else if b.config.TxBatchMode {
+		nonceProvider := requester.NewLocalNonceProvider(
+			b.config.FlowNetworkID,
+			b.storages.Registers,
+			b.storages.Blocks,
+			b.collector,
+		)
 		txPool, err = requester.NewBatchTxPool(
 			ctx,
 			b.client,
@@ -295,6 +301,7 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 			b.config,
 			b.collector,
 			b.keystore,
+			nonceProvider,
 		)
 	} else {
 		txPool, err = requester.NewSingleTxPool(

cmd/run/cmd.go

@@ -281,6 +281,7 @@ var (
 	txStateValidation string
 	initHeight,
 	forceStartHeight uint64
+	eoaActivityCacheTTL time.Duration
 )
 
 func init() {
@@ -321,7 +322,7 @@ func init() {
 	Cmd.Flags().DurationVar(&cfg.TxRequestLimitDuration, "tx-request-limit-duration", time.Second*3, "Time interval upon which to enforce transaction submission rate limiting.")
 	Cmd.Flags().BoolVar(&cfg.TxBatchMode, "tx-batch-mode", false, "Enable batch transaction submission, to avoid nonce mismatch issues for high-volume EOAs.")
 	Cmd.Flags().DurationVar(&cfg.TxBatchInterval, "tx-batch-interval", time.Millisecond*1200, "Time interval upon which to submit the transaction batches to the Flow network.")
-	Cmd.Flags().DurationVar(&cfg.EOAActivityCacheTTL, "eoa-activity-cache-ttl", time.Second*10, "Time interval used to track EOA activity. Tx send more frequently than this interval will be batched. Useful only when batch transaction submission is enabled.")
+	Cmd.Flags().DurationVar(&eoaActivityCacheTTL, "eoa-activity-cache-ttl", time.Second*10, "Time interval used to track EOA activity. Tx send more frequently than this interval will be batched. Useful only when batch transaction submission is enabled.")
 	Cmd.Flags().BoolVar(&cfg.ExperimentalSoftFinalityEnabled, "experimental-soft-finality-enabled", false, "Sets whether the gateway should use the experimental soft finality feature. This results in faster indexing time, because EVM state is fetched from finalized, instead of sealed Flow blocks.")
 	Cmd.Flags().BoolVar(&cfg.ExperimentalSealingVerificationEnabled, "experimental-sealing-verification-enabled", false, "Sets whether the gateway should use the experimental soft finality sealing verification feature. This is an extra safety check for --experimental-soft-finality-enabled=true, which verifies that all finalized Flow blocks that were indexed, have eventually been sealed. The ingestion will halt, even if a single Flow block was not found to be sealed.")
 	Cmd.Flags().BoolVar(&cfg.TxMemPoolMode, "tx-mempool-mode", false, "Enable the transaction mempool: expected-nonce transactions are submitted immediately, out-of-order transactions are held until their nonce gap fills. Mutually exclusive with --tx-batch-mode and requires --tx-state-validation=local-index.")
@@ -336,4 +337,9 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+
+	err = Cmd.Flags().MarkDeprecated("eoa-activity-cache-ttl", "This flag is no longer applicable and will be removed in future version. EOA activity is now preserved in a dedicated per-EOA queue, and not in a cache.")
+	if err != nil {
+		panic(err)
+	}
 }

config/config.go

@@ -120,10 +120,6 @@ type Config struct {
 	// TxBatchInterval is the time interval upon which to submit the transaction batches to the
 	// Flow network.
 	TxBatchInterval time.Duration
-	// EOAActivityCacheTTL is the time interval used to track EOA activity. Tx send more
-	// frequently than this interval will be batched.
-	// Useful only when batch transaction submission is enabled.
-	EOAActivityCacheTTL time.Duration
 	// ExperimentalSoftFinalityEnabled enables the experimental soft finality feature which syncs
 	// EVM block and transaction data from the upstream Access node before the block is sealed.
 	// CAUTION: This feature is experimental and may return incorrect data in certain circumstances.

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/ethereum/go-ethereum v1.17.4
 	github.com/goccy/go-json v0.10.4
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/holiman/uint256 v1.3.2
 	github.com/onflow/atree v0.16.1
 	github.com/onflow/cadence v1.10.5
@@ -98,6 +97,7 @@ require (
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/huandu/go-clone v1.6.0 // indirect

models/errors/errors.go

@@ -43,6 +43,9 @@ var (
 	// configured maximum gap ahead of the EOA's on-chain nonce. Such a tx cannot
 	// execute until the gap fills, so it is rejected up front for fast feedback.
 	ErrNonceTooHigh = fmt.Errorf("%w: %s", ErrInvalid, "nonce too high")
+	// ErrTxPoolFull is returned when the per-EOA pool has reached its size cap
+	// and cannot accept another transaction until existing ones drain.
+	ErrTxPoolFull = fmt.Errorf("%w: %s", ErrInvalid, "transaction pool is full for this account")
 
 	// Storage errors