Merge pull request #968 from onflow/vishal/batch_fix_with_soft_finality

fix(batch): always pool transactions to eliminate first-tx nonce race (soft-finality)

Author: vishalchangrani

cmd/run/cmd.go

@@ -298,7 +298,7 @@ func init() {
 	Cmd.Flags().DurationVar(&cfg.TxBatchInterval, "tx-batch-interval", time.Millisecond*1200, "Time interval upon which to submit the transaction batches to the Flow network.")
 	Cmd.Flags().BoolVar(&experimentalSoftFinalityEnabled, "experimental-soft-finality-enabled", false, "Sets whether the gateway should use the experimental soft finality feature. WARNING: This may result in incorrect results being returned in certain circumstances. Use only if you know what you are doing.")
 	Cmd.Flags().BoolVar(&experimentalSealingVerificationEnabled, "experimental-sealing-verification-enabled", true, "Sets whether the gateway should use the experimental soft finality sealing verification feature. WARNING: This may result in indexing halts if events do not match. Use only if you know what you are doing.")
-	Cmd.Flags().DurationVar(&cfg.EOAActivityCacheTTL, "eoa-activity-cache-ttl", time.Second*10, "Time interval used to track EOA activity. Tx send more frequently than this interval will be batched. Useful only when batch transaction submission is enabled.")
+	Cmd.Flags().DurationVar(&cfg.EOAActivityCacheTTL, "eoa-activity-cache-ttl", time.Second*10, "[DEPRECATED] No longer has any effect. BatchTxPool now always pools every transaction before submission.")
 	Cmd.Flags().DurationVar(&cfg.RpcRequestTimeout, "rpc-request-timeout", time.Second*120, "Sets the maximum duration at which JSON-RPC requests should generate a response, before they timeout. The default is 120 seconds.")
 
 	err := Cmd.Flags().MarkDeprecated("init-cadence-height", "This flag is no longer necessary and will be removed in future version. The initial Cadence height is known for testnet/mainnet and this was only required for fresh deployments of EVM Gateway. Once the DB has been initialized, the latest index Cadence height will be used upon start-up.")

config/config.go

@@ -129,9 +129,9 @@ type Config struct {
 	// of the events from the sealed block in the Flow network.
 	// CAUTION: This feature is experimental and will cause the node to halt if the events don't match.
 	ExperimentalSealingVerificationEnabled bool
-	// EOAActivityCacheTTL is the time interval used to track EOA activity. Tx send more
-	// frequently than this interval will be batched.
-	// Useful only when batch transaction submission is enabled.
+	// EOAActivityCacheTTL is no longer used. Kept for backwards-compatible config
+	// parsing so existing deployments with --eoa-activity-cache-ttl set do not break.
+	// Deprecated: has no effect since BatchTxPool now always pools every transaction.
 	EOAActivityCacheTTL time.Duration
 	// RpcRequestTimeout is the maximum duration at which JSON-RPC requests should generate
 	// a response, before they timeout.

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/ethereum/go-ethereum v1.16.8
 	github.com/goccy/go-json v0.10.4
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/holiman/uint256 v1.3.2
 	github.com/onflow/atree v0.16.0
 	github.com/onflow/cadence v1.10.3
@@ -99,6 +98,7 @@ require (
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/huandu/go-clone v1.6.0 // indirect

services/requester/batch_tx_pool.go

@@ -10,7 +10,6 @@ import (
 
 	gethCommon "github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-go-sdk"
 	"github.com/rs/zerolog"
@@ -22,34 +21,64 @@ import (
 	"github.com/onflow/flow-evm-gateway/services/requester/keystore"
 )
 
-const eoaActivityCacheSize = 10_000
+// BatchTxPool is a TxPool implementation that collects and groups transactions
+// by EOA signer, sorts them by nonce, and submits them as a batch via
+// EVM.batchRun on each flush interval.
+//
+// # Problem
+//
+// Flow does not have a traditional EVM mempool. On standard EVM chains, when a
+// wallet sends transactions out-of-nonce-sequence (e.g., nonces 5, 7, 6 in
+// parallel), the mempool holds future-nonce transactions until the gap is
+// filled. Flow EVM has no such holding mechanism — a transaction whose nonce
+// does not match the current account nonce is simply dropped.
+//
+// The original BatchTxPool implementation partially addressed this by batching
+// transactions that arrived from an EOA with "recent activity" (i.e., a prior
+// transaction within TxBatchInterval). However, it still submitted the FIRST
+// transaction from any burst immediately — before the rest of the burst had
+// a chance to arrive. If that first transaction happened to carry a future
+// nonce (due to parallel dispatch), it failed, and the gap it left caused all
+// subsequent nonces in the batch to fail as well.
+//
+// # Root cause (confirmed on testnet, 2026-06-03)
+//
+// A burst of 10 transactions was sent in parallel from a fresh wallet using
+// shuffled nonce order: [8 7 3 9 2 1 6 0 4 5].
+//
+//   - The gateway accepted all 10 (eth_sendRawTransaction returned success).
+//   - Nonce 6 arrived first → no prior EOA activity → submitted immediately
+//     as a standalone Cadence transaction → failed (on-chain nonce was 0).
+//   - The remaining 9 nonces [7 3 9 2 1 0 4 5 8] were pooled, sorted to
+//     [0 1 2 3 4 5 7 8 9], and submitted as EVM.batchRun.
+//   - EVM.batchRun executed 0→5 successfully, then encountered nonce 7.
+//     The expected nonce was 6 (which had already failed), so execution
+//     stopped. Nonces 7, 8, 9 were dropped.
+//   - Final result: 6/10 landed on-chain; 4 were silently dropped.
+//
+// # Fix
+//
+// All transactions are now always enqueued in the pool regardless of prior EOA
+// activity. The flush timer (TxBatchInterval) is the sole submission trigger.
+// This guarantees that parallel transactions from the same wallet accumulate
+// in the pool before being sorted by nonce and submitted atomically.
+//
+// Trade-off: every transaction now incurs up to TxBatchInterval of additional
+// latency before it is submitted to Flow. For the mainnet default of 2.5 s this
+// is acceptable; operators can lower the interval for latency-sensitive
+// deployments.
+type BatchTxPool struct {
+	*SingleTxPool
+	pooledTxs map[gethCommon.Address][]pooledEvmTx
+	txMux     sync.Mutex
+}
 
 type pooledEvmTx struct {
 	txPayload cadence.String
 	txHash    gethCommon.Hash
 	nonce     uint64
 }
 
-// BatchTxPool is a `TxPool` implementation that collects and groups
-// transactions based on their EOA signer, and submits them for execution
-// using a batch.
-//
-// The underlying Cadence EVM API used, is `EVM.batchRun`, instead of the
-// `EVM.run` used in `SingleTxPool`.
-//
-// The main advantage of this implementation over the `SingleTxPool`, is the
-// guarantee that transactions originated from the same EOA address, which
-// arrive in a short time interval (about the same as Flow's block production rate),
-// will be executed in the same order their arrived.
-// This helps to reduce the nonce mismatch errors which mainly occur from the
-// re-ordering of Cadence transactions that happens from Collection nodes.
-type BatchTxPool struct {
-	*SingleTxPool
-	pooledTxs   map[gethCommon.Address][]pooledEvmTx
-	txMux       sync.Mutex
-	eoaActivity *expirable.LRU[gethCommon.Address, time.Time]
-}
-
 var _ TxPool = &BatchTxPool{}
 
 func NewBatchTxPool(
@@ -77,27 +106,24 @@ func NewBatchTxPool(
 		return nil, err
 	}
 
-	eoaActivity := expirable.NewLRU[gethCommon.Address, time.Time](
-		eoaActivityCacheSize,
-		nil,
-		config.EOAActivityCacheTTL,
-	)
 	batchPool := &BatchTxPool{
 		SingleTxPool: singleTxPool,
 		pooledTxs:    make(map[gethCommon.Address][]pooledEvmTx),
 		txMux:        sync.Mutex{},
-		eoaActivity:  eoaActivity,
 	}
 
 	go batchPool.processPooledTransactions(ctx)
 
 	return batchPool, nil
 }
 
-// Add adds the EVM transaction to the tx pool, grouped with the rest of the
-// transactions from the same EOA signer.
-// After the configured `TxBatchInterval`, the collected transations
-// are batched and sent to the Flow network using `EVM.batchRun`, for execution.
+// Add enqueues the transaction in the per-EOA pool. It is never submitted
+// immediately; the flush goroutine (processPooledTransactions) handles
+// submission on every TxBatchInterval tick.
+//
+// Enqueueing everything — including the first transaction from a burst — is
+// the key invariant that prevents the "first tx escapes" race described in the
+// type-level comment above.
 func (t *BatchTxPool) Add(
 	ctx context.Context,
 	tx *gethTypes.Transaction,
@@ -123,44 +149,14 @@ func (t *BatchTxPool) Add(
 		return err
 	}
 
-	// Scenarios
-	// 1. EOA activity not found:
-	// => We send the transaction individually, without adding it
-	// to the batch pool.
-	//
-	// 2. EOA activity found AND it was more than [X] seconds ago:
-	// => We send the transaction individually, without adding it
-	// to the batch pool.
-	//
-	// 3. EOA activity found AND it was less than [X] seconds ago:
-	// => We add the transaction to the batch pool, so that it gets
-	// processed and submitted according to the configured
-	// `TxBatchInterval`.
-	//
-	// For all 3 cases, we record the activity time for the next
-	// transactions that might come from the same EOA.
-	// [X] is equal to the configured `TxBatchInterval` duration.
-	lastActivityTime, found := t.eoaActivity.Get(from)
-
-	if !found {
-		// Case 1. EOA activity not found:
-		err = t.submitSingleTransaction(ctx, hexEncodedTx)
-	} else if time.Since(lastActivityTime) > t.config.TxBatchInterval {
-		// Case 2. EOA activity found AND it was more than [X] seconds ago:
-		err = t.submitSingleTransaction(ctx, hexEncodedTx)
-	} else {
-		// Case 3. EOA activity found AND it was less than [X] seconds ago:
-		userTx := pooledEvmTx{txPayload: hexEncodedTx, txHash: tx.Hash(), nonce: tx.Nonce()}
-		// Prevent submission of duplicate transactions, based on their tx hash
-		if slices.Contains(t.pooledTxs[from], userTx) {
-			return errs.ErrDuplicateTransaction
-		}
-		t.pooledTxs[from] = append(t.pooledTxs[from], userTx)
+	userTx := pooledEvmTx{txPayload: hexEncodedTx, txHash: tx.Hash(), nonce: tx.Nonce()}
+	// Prevent submission of duplicate transactions, based on their tx hash
+	if slices.Contains(t.pooledTxs[from], userTx) {
+		return errs.ErrDuplicateTransaction
 	}
+	t.pooledTxs[from] = append(t.pooledTxs[from], userTx)
 
-	t.eoaActivity.Add(from, time.Now())
-
-	return err
+	return nil
 }
 
 func (t *BatchTxPool) processPooledTransactions(ctx context.Context) {
@@ -203,8 +199,8 @@ func (t *BatchTxPool) batchSubmitTransactionsForSameAddress(
 	referenceBlockHeader *flow.BlockHeader,
 	pooledTxs []pooledEvmTx,
 ) error {
-	// Sort the transactions based on their nonce, to make sure
-	// that no re-ordering has happened due to races etc.
+	// Sort by nonce to guarantee correct execution order regardless of the
+	// order in which transactions arrived at the gateway.
 	sort.Slice(pooledTxs, func(i, j int) bool {
 		return pooledTxs[i].nonce < pooledTxs[j].nonce
 	})
@@ -240,34 +236,3 @@ func (t *BatchTxPool) batchSubmitTransactionsForSameAddress(
 
 	return nil
 }
-
-func (t *BatchTxPool) submitSingleTransaction(
-	ctx context.Context,
-	hexEncodedTx cadence.String,
-) error {
-	coinbaseAddress, err := cadence.NewString(t.config.Coinbase.Hex())
-	if err != nil {
-		return err
-	}
-
-	script := replaceAddresses(runTxScript, t.config.FlowNetworkID)
-	flowTx, err := t.buildTransaction(
-		ctx,
-		t.getReferenceBlock(),
-		script,
-		cadence.NewArray([]cadence.Value{hexEncodedTx}),
-		coinbaseAddress,
-	)
-	if err != nil {
-		// If there was any error during the transaction build
-		// process, we record it as a dropped transaction.
-		t.collector.TransactionsDropped(1)
-		return err
-	}
-
-	if err := t.client.SendTransaction(ctx, *flowTx); err != nil {
-		return err
-	}
-
-	return nil
-}