Merge remote-tracking branch 'origin/main' into mpeter/soft-finality-sync-with-main

Author: m-Peter

bootstrap/bootstrap.go

@@ -267,7 +267,7 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 		}
 	}
 
-	b.keystore = keystore.New(accountKeys)
+	b.keystore = keystore.New(ctx, accountKeys, b.client, b.config, b.logger)
 
 	// create transaction pool
 	var txPool requester.TxPool

cmd/run/cmd.go

@@ -288,6 +288,7 @@ func init() {
 	Cmd.Flags().StringVar(&cloudKMSLocationID, "coa-cloud-kms-location-id", "", "The location ID where the key ring is grouped into, e.g. 'global'")
 	Cmd.Flags().StringVar(&cloudKMSKeyRingID, "coa-cloud-kms-key-ring-id", "", "The key ring ID where the KMS keys exist, e.g. 'tx-signing'")
 	Cmd.Flags().StringVar(&cloudKMSKey, "coa-cloud-kms-key", "", `Name of the KMS key and its version, e.g. "gw-key-6@1"`)
+	Cmd.Flags().BoolVar(&cfg.COATxLookupEnabled, "coa-tx-lookup-enabled", false, "Tracks cadence transactions to release COA signing keys more quickly. Use this on nodes with high tx volume that frequently run out of proposer keys.")
 	Cmd.Flags().StringVar(&walletKey, "wallet-api-key", "", "ECDSA private key used for wallet APIs. WARNING: This should only be used locally or for testing, never in production.")
 	Cmd.Flags().IntVar(&cfg.MetricsPort, "metrics-port", 9091, "Port for the metrics server")
 	Cmd.Flags().BoolVar(&cfg.IndexOnly, "index-only", false, "Run the gateway in index-only mode which only allows querying the state and indexing, but disallows sending transactions.")

config/config.go

@@ -56,6 +56,10 @@ type Config struct {
 	COAKey crypto.PrivateKey
 	// COACloudKMSKey is a Cloud KMS key that will be used for signing transactions.
 	COACloudKMSKey *flowGoKMS.Key
+	// COATxLookupEnabled enables tracking of Cadence transactions to release COA signing
+	// keys much faster. Increases capacity of the available COA signing keys for nodes
+	// with high tx volume.
+	COATxLookupEnabled bool
 	// GasPrice is a fixed gas price that will be used when submitting transactions.
 	GasPrice *big.Int
 	// EnforceGasPrice defines whether the minimum gas price should be enforced.

eth/types/types.go

@@ -359,10 +359,14 @@ func NewTransaction(
 		V:        (*hexutil.Big)(v),
 		R:        (*hexutil.Big)(r),
 		S:        (*hexutil.Big)(s),
-		ChainID:  (*hexutil.Big)(networkID),
 		size:     tx.Size(),
 	}
 
+	// if a legacy transaction has an EIP-155 chain id, include it explicitly
+	if id := tx.ChainId(); id.Sign() != 0 {
+		result.ChainID = (*hexutil.Big)(id)
+	}
+
 	// After the Pectra hard-fork, the full list of supported tx types is:
 	// LegacyTxType     = 0x00
 	// AccessListTxType = 0x01
@@ -391,6 +395,7 @@ func NewTransaction(
 		yparity := hexutil.Uint64(v.Sign())
 		result.Accesses = &al
 		result.YParity = &yparity
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
 	}
 
 	if tx.Type() > types.AccessListTxType {
@@ -399,17 +404,20 @@ func NewTransaction(
 		// Since BaseFee is `0`, this is the max gas price
 		// the sender is willing to pay.
 		result.GasPrice = (*hexutil.Big)(tx.GasFeeCap())
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
 	}
 
 	if tx.Type() > types.DynamicFeeTxType {
 		result.MaxFeePerBlobGas = (*hexutil.Big)(tx.BlobGasFeeCap())
 		result.BlobVersionedHashes = tx.BlobHashes()
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
 	}
 
 	// The `AuthorizationList` field became available with the introduction
 	// of https://eip7702.io/#specification, under the `SetCodeTxType`
 	if tx.Type() > types.BlobTxType {
 		result.AuthorizationList = tx.SetCodeAuthorizations()
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
 	}
 
 	return result, nil
@@ -528,7 +536,15 @@ func MarshalReceipt(
 		"effectiveGasPrice": (*hexutil.Big)(receipt.EffectiveGasPrice),
 	}
 
-	if _, ok := tx.(models.DirectCall); ok {
+	// Dynamically fallback to `BaseFeePerGas` when computing `EffectiveGasPrice`
+	// to fix historical gasPrice = 0 issues.
+	// This avoids the need to re-index the entire chain for previously stored
+	// transactions. For any transaction that had a `0` gas price, regardless
+	// whether they were COA interactions or regular EVM, we
+	// set the `effectiveGasPrice` to the value of `BaseFeePerGas`,
+	// which is the minimum amount of gas price required by any
+	// transaction, in order to comply with EIP-1559.
+	if receipt.EffectiveGasPrice.Sign() == 0 {
 		fields["effectiveGasPrice"] = (*hexutil.Big)(models.BaseFeePerGas)
 	}
 

models/receipt_test.go

@@ -13,6 +13,8 @@ func Test_DecodeReceipts(t *testing.T) {
 	_, receipt, _, err := decodeTransactionEvent(cdcEv)
 	require.NoError(t, err)
 
+	assert.Equal(t, BaseFeePerGas, receipt.EffectiveGasPrice)
+
 	for i, l := range rec.Logs {
 		assert.ObjectsAreEqualValues(l, receipt.Logs[i])
 		for j, tt := range l.Topics {

models/transaction.go

@@ -44,6 +44,7 @@ type Transaction interface {
 	GasFeeCap() *big.Int
 	GasTipCap() *big.Int
 	GasPrice() *big.Int
+	ChainId() *big.Int
 	BlobGas() uint64
 	BlobGasFeeCap() *big.Int
 	BlobHashes() []common.Hash
@@ -117,6 +118,10 @@ func (dc DirectCall) GasPrice() *big.Int {
 	return BaseFeePerGas
 }
 
+func (dc DirectCall) ChainId() *big.Int {
+	return big.NewInt(0)
+}
+
 func (dc DirectCall) BlobGas() uint64 {
 	return 0
 }
@@ -155,15 +160,64 @@ type TransactionCall struct {
 	*gethTypes.Transaction
 }
 
+func (tc TransactionCall) GasPrice() *big.Int {
+	// EIP-1559 introduced a new fee model in Ethereum that replaces the
+	// legacy `GasPrice` with `MaxFeePerGas` and `MaxPriorityFeePerGas`.
+	// However, many Ethereum tools and wallets (such as MetaMask, Hardhat, etc.)
+	// still rely on reading `GasPrice`, even if it’s not explicitly set.
+	//
+	// When a user submits an EIP-1559 transaction type, `GasPrice` is not
+	// specified, Ethereum nodes typically return the current `BaseFeePerGas`
+	// as the effective `GasPrice` for compatibility reasons.
+	//
+	// This behavior is mirrored here in Flow EVM Gateway: if `GasPrice` is zero,
+	// we return the configured `BaseFeePerGas` to satisfy the expectations of
+	// these tools.
+	//
+	// This does NOT affect Flow’s actual transaction fee calculation,
+	// this is purely for compatibility.
+	if tc.Transaction.GasPrice().Sign() == 0 {
+		return BaseFeePerGas
+	}
+	return tc.Transaction.GasPrice()
+}
+
+func (tc TransactionCall) GasFeeCap() *big.Int {
+	// `GasFeeCap` represents the `MaxFeePerGas` in EIP-1559, the maximum fee
+	// a user is willing to pay. Ethereum clients expect a non-zero value when
+	// calculating effective gas prices for compatibility.
+	//
+	// If the user does not provide a value (zero), this method returns the
+	// configured `BaseFeePerGas` to avoid confusion and comply with expected
+	// EVM behavior. This ensures Ethereum tooling can still function correctly
+	// when interacting with Flow EVM.
+	if tc.Transaction.GasFeeCap().Sign() == 0 {
+		return BaseFeePerGas
+	}
+	return tc.Transaction.GasFeeCap()
+}
+
+func (tc TransactionCall) GasTipCap() *big.Int {
+	// `GasTipCap` represents the `MaxPriorityFeePerGas` in EIP-1559, the optional
+	// "tip" to incentivize block inclusion. Ethereum expects this value to be
+	// explicitly defined or it defaults to something reasonable like the base fee.
+	//
+	// To satisfy Ethereum clients and maintain expected behavior, when this value
+	// is zero, Flow EVM returns the configured `BaseFeePerGas` as a safe default.
+	// This ensures Ethereum tools can continue to compute `EffectiveGasPrice`
+	// without errors.
+	if tc.Transaction.GasTipCap().Sign() == 0 {
+		return BaseFeePerGas
+	}
+	return tc.Transaction.GasTipCap()
+}
+
 func (tc TransactionCall) Hash() common.Hash {
 	return tc.Transaction.Hash()
 }
 
 func (tc TransactionCall) From() (common.Address, error) {
-	return gethTypes.Sender(
-		gethTypes.LatestSignerForChainID(tc.ChainId()),
-		tc.Transaction,
-	)
+	return DeriveTxSender(tc.Transaction)
 }
 
 func (tc TransactionCall) MarshalBinary() ([]byte, error) {
@@ -244,7 +298,11 @@ func decodeTransactionEvent(event cadence.Event) (
 				err,
 			)
 		}
-		receipt.EffectiveGasPrice = gethTx.EffectiveGasTipValue(nil)
+		if gethTx.GasPrice().Sign() == 0 {
+			receipt.EffectiveGasPrice = BaseFeePerGas
+		} else {
+			receipt.EffectiveGasPrice = gethTx.EffectiveGasTipValue(nil)
+		}
 		tx = TransactionCall{Transaction: gethTx}
 	}
 
@@ -305,3 +363,22 @@ func ValidateTransaction(
 
 	return nil
 }
+
+// DeriveTxSender returns the address derived from the signature (V, R, S)
+// using secp256k1 elliptic curve and an error if it failed deriving or
+// upon an incorrect signature.
+func DeriveTxSender(tx *gethTypes.Transaction) (common.Address, error) {
+	var signer gethTypes.Signer
+	if chainID := tx.ChainId(); chainID.Sign() != 0 {
+		signer = gethTypes.LatestSignerForChainID(chainID)
+	} else {
+		signer = gethTypes.HomesteadSigner{}
+	}
+
+	from, err := gethTypes.Sender(signer, tx)
+	if err != nil {
+		return common.Address{}, fmt.Errorf("failed to derive the sender: %w", err)
+	}
+
+	return from, nil
+}

models/transaction_test.go

@@ -123,7 +123,7 @@ func Test_DecodeEVMTransaction(t *testing.T) {
 	assert.Equal(t, big.NewInt(0), decTx.Value())
 	assert.Equal(t, uint8(0), decTx.Type())
 	assert.Equal(t, uint64(125_000), decTx.Gas())
-	assert.Equal(t, big.NewInt(0), decTx.GasPrice())
+	assert.Equal(t, BaseFeePerGas, decTx.GasPrice())
 	assert.Equal(t, uint64(0), decTx.BlobGas())
 	assert.Equal(t, uint64(347), decTx.Size())
 }
@@ -223,7 +223,7 @@ func Test_UnmarshalTransaction(t *testing.T) {
 		assert.Equal(t, big.NewInt(0), decTx.Value())
 		assert.Equal(t, uint8(0), decTx.Type())
 		assert.Equal(t, uint64(125_000), decTx.Gas())
-		assert.Equal(t, big.NewInt(0), decTx.GasPrice())
+		assert.Equal(t, BaseFeePerGas, decTx.GasPrice())
 		assert.Equal(t, uint64(0), decTx.BlobGas())
 		assert.Equal(t, uint64(347), decTx.Size())
 	})

services/ingestion/event_subscriber.go

@@ -176,7 +176,12 @@ func (r *RPCEventSubscriber) subscribe(ctx context.Context, height uint64) <-cha
 				for _, evt := range blockEvents.Events {
 					r.keyLock.NotifyTransaction(evt.TransactionID)
 				}
-				r.keyLock.NotifyBlock(blockEvents.Height)
+				r.keyLock.NotifyBlock(
+					flow.BlockHeader{
+						ID:     blockEvents.BlockID,
+						Height: blockEvents.Height,
+					},
+				)
 
 				eventsChan <- evmEvents
 

services/ingestion/event_subscriber_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/onflow/flow-go-sdk/access"
 	gethCommon "github.com/onflow/go-ethereum/common"
 
+	"github.com/onflow/flow-evm-gateway/config"
 	"github.com/onflow/flow-evm-gateway/models"
 	errs "github.com/onflow/flow-evm-gateway/models/errors"
 	"github.com/onflow/flow-evm-gateway/services/requester"
@@ -44,9 +45,14 @@ func Test_Subscribing(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	subscriber := NewRPCEventSubscriber(zerolog.Nop(), client, flowGo.Previewnet, keystore.New(nil), 1)
+	ctx := context.Background()
+	logger := zerolog.Nop()
+	cfg := config.Config{COATxLookupEnabled: true}
+	ks := keystore.New(ctx, nil, client, cfg, logger)
 
-	events := subscriber.Subscribe(context.Background())
+	subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
+
+	events := subscriber.Subscribe(ctx)
 
 	var prevHeight uint64
 
@@ -84,9 +90,14 @@ func Test_MissingBlockEvent(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	subscriber := NewRPCEventSubscriber(zerolog.Nop(), client, flowGo.Previewnet, keystore.New(nil), 1)
+	ctx := context.Background()
+	logger := zerolog.Nop()
+	cfg := config.Config{COATxLookupEnabled: true}
+	ks := keystore.New(ctx, nil, client, cfg, logger)
+
+	subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
 
-	events := subscriber.Subscribe(context.Background())
+	events := subscriber.Subscribe(ctx)
 
 	missingHashes := make([]gethCommon.Hash, 0)
 
@@ -186,9 +197,14 @@ func Test_SubscribingWithRetryOnError(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	subscriber := NewRPCEventSubscriber(zerolog.Nop(), client, flowGo.Previewnet, keystore.New(nil), 1)
+	ctx := context.Background()
+	logger := zerolog.Nop()
+	cfg := config.Config{COATxLookupEnabled: true}
+	ks := keystore.New(ctx, nil, client, cfg, logger)
+
+	subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
 
-	events := subscriber.Subscribe(context.Background())
+	events := subscriber.Subscribe(ctx)
 
 	var prevHeight uint64
 
@@ -249,9 +265,14 @@ func Test_SubscribingWithRetryOnErrorMultipleBlocks(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	subscriber := NewRPCEventSubscriber(zerolog.Nop(), client, flowGo.Previewnet, keystore.New(nil), 1)
+	ctx := context.Background()
+	logger := zerolog.Nop()
+	cfg := config.Config{COATxLookupEnabled: true}
+	ks := keystore.New(ctx, nil, client, cfg, logger)
 
-	events := subscriber.Subscribe(context.Background())
+	subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
+
+	events := subscriber.Subscribe(ctx)
 
 	var prevHeight uint64
 
@@ -311,9 +332,14 @@ func Test_SubscribingWithRetryOnErrorEmptyBlocks(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	subscriber := NewRPCEventSubscriber(zerolog.Nop(), client, flowGo.Previewnet, keystore.New(nil), 1)
+	ctx := context.Background()
+	logger := zerolog.Nop()
+	cfg := config.Config{COATxLookupEnabled: true}
+	ks := keystore.New(ctx, nil, client, cfg, logger)
+
+	subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
 
-	events := subscriber.Subscribe(context.Background())
+	events := subscriber.Subscribe(ctx)
 
 	var prevHeight uint64
 

services/requester/batch_tx_pool.go

@@ -120,10 +120,11 @@ func (t *BatchTxPool) Add(
 	t.txMux.Lock()
 	defer t.txMux.Unlock()
 
-	from, err := gethTypes.Sender(gethTypes.LatestSignerForChainID(tx.ChainId()), tx)
+	from, err := models.DeriveTxSender(tx)
 	if err != nil {
-		return fmt.Errorf("failed to derive the sender: %w", err)
+		return err
 	}
+
 	txData, err := tx.MarshalBinary()
 	if err != nil {
 		return err