refactor: added a total gas counter for a specific block and for each chain

Author: mschead

backfill/process/save_payloads.go

@@ -7,6 +7,7 @@ import (
 	"go-backfill/fetch"
 	"go-backfill/repository"
 	"log"
+	"strconv"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgxpool"
@@ -72,6 +73,7 @@ func savePayloads(network string, chainId int, processedPayloads []fetch.Process
 	}
 
 	var transactionIdsToSave [][]int64
+	var totalGasUsedInChain float64 = 0
 	for index, processedPayload := range processedPayloads {
 		var blockId = blockIds[index]
 		var currBlock = blocks[index]
@@ -90,6 +92,22 @@ func savePayloads(network string, chainId int, processedPayloads []fetch.Process
 			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transaction details for block %d -> %w", currBlock.Height, err)
 		}
 
+		var totalGasUsedInBlock float64 = 0
+		for _, txDetail := range txDetails {
+			gas, err := strconv.ParseFloat(txDetail.Gas, 64)
+			if err != nil {
+				return Counters{}, DataSizeTracker{}, fmt.Errorf("parsing gas for block %d: %w", currBlock.Height, err)
+			}
+			gasPrice, err := strconv.ParseFloat(txDetail.GasPrice, 64)
+			if err != nil {
+				return Counters{}, DataSizeTracker{}, fmt.Errorf("parsing gas price for block %d: %w", currBlock.Height, err)
+			}
+			totalGasUsedInBlock += gas * gasPrice
+		}
+
+		repository.SaveTotalGasUsed(tx, totalGasUsedInBlock, blockId)
+		totalGasUsedInChain += totalGasUsedInBlock
+
 		txsSize := approximateSize(txs)
 		dataSizeTracker.TransactionsKB += txsSize
 		transactionIdsToSave = append(transactionIdsToSave, transactionIds)
@@ -158,6 +176,7 @@ func savePayloads(network string, chainId int, processedPayloads []fetch.Process
 
 	repository.SaveCounters(tx, repository.CounterAttributes{
 		ChainId:              chainId,
+		TotalGasUsed:         totalGasUsedInChain,
 		CanonicalBlocksCount: len(blocks),
 		// ignore coinbase transactions on the canonical transaction counter
 		CanonicalTransactionsCount: counters.Transactions - len(blocks),

backfill/repository/counters_repository.go

@@ -2,6 +2,7 @@ package repository
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/jackc/pgx/v5"
 )
@@ -10,25 +11,35 @@ type CounterAttributes struct {
 	ChainId                    int
 	CanonicalBlocksCount       int
 	CanonicalTransactionsCount int
+	TotalGasUsed               float64
+}
+
+func SaveTotalGasUsed(tx pgx.Tx, totalGasUsed float64, blockId int64) error {
+	query := `
+		UPDATE "Blocks"
+		SET "totalGasUsed" = $1
+		WHERE "id" = $2
+	`
+
+	_, err := tx.Exec(context.Background(), query, totalGasUsed, blockId)
+	if err != nil {
+		return fmt.Errorf("saving total gas used: %w", err)
+	}
+
+	return nil
 }
 
 func SaveCounters(tx pgx.Tx, counters CounterAttributes) error {
 	query := `
 		UPDATE "Counters"
-		SET "canonicalBlocks" = "canonicalBlocks" + $1, "canonicalTransactions" = "canonicalTransactions" + $2
-		WHERE "chainId" = $3
+		SET "canonicalBlocks" = "canonicalBlocks" + $1, "canonicalTransactions" = "canonicalTransactions" + $2, "totalGasUsed" = "totalGasUsed" + $3
+		WHERE "chainId" = $4
 	`
 
-	batch := &pgx.Batch{}
-	batch.Queue(
-		query,
-		counters.CanonicalBlocksCount,
-		counters.CanonicalTransactionsCount,
-		counters.ChainId,
-	)
-
-	br := tx.SendBatch(context.Background(), batch)
-	defer br.Close()
+	_, err := tx.Exec(context.Background(), query, counters.CanonicalBlocksCount, counters.CanonicalTransactionsCount, counters.TotalGasUsed, counters.ChainId)
+	if err != nil {
+		return fmt.Errorf("saving counters: %w", err)
+	}
 
 	return nil
 }

indexer/migrations/20250722170139-create-counter-table.js

@@ -24,6 +24,10 @@ module.exports = {
         type: Sequelize.INTEGER,
         allowNull: false,
       },
+      totalGasUsed: {
+        type: Sequelize.DECIMAL(20, 10),
+        allowNull: false,
+      },
     });
 
     const chainIds = Array.from({ length: 20 }, (_, i) => ({
@@ -32,6 +36,7 @@ module.exports = {
       orphansBlocks: 0,
       canonicalTransactions: 0,
       orphanTransactions: 0,
+      totalGasUsed: 0,
     }));
 
     await queryInterface.bulkInsert('Counters', chainIds);

indexer/migrations/20250731194020-add-total-gas-used-in-block-table.js

@@ -0,0 +1,14 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addColumn('Blocks', 'totalGasUsed', {
+      type: Sequelize.DECIMAL(20, 10),
+    });
+  },
+
+  async down(queryInterface) {
+    await queryInterface.removeColumn('Blocks', 'totalGasUsed');
+  },
+};

indexer/src/models/block.ts

@@ -35,6 +35,7 @@ export interface BlockAttributes {
   coinbase: object;
   canonical?: boolean;
   transactionsCount: number;
+  totalGasUsed: number;
 }
 
 /**
@@ -102,6 +103,9 @@ class Block extends Model<BlockAttributes> implements BlockAttributes {
 
   /** The number of transactions in the block. */
   declare transactionsCount: number;
+
+  /** The total gas used in the block. */
+  declare totalGasUsed: number;
 }
 
 /**
@@ -202,6 +206,10 @@ Block.init(
       defaultValue: 0,
       comment: 'The number of transactions in the block.',
     },
+    totalGasUsed: {
+      type: DataTypes.DECIMAL(20, 10),
+      comment: 'The total gas used in the block.',
+    },
   },
   {
     sequelize,

indexer/src/models/counter.ts

@@ -8,6 +8,7 @@ class Counter extends Model {
   public orphansBlocks!: number;
   public canonicalTransactions!: number;
   public orphanTransactions!: number;
+  public totalGasUsed!: number;
 }
 
 Counter.init(
@@ -32,6 +33,10 @@ Counter.init(
       type: DataTypes.INTEGER,
       allowNull: false,
     },
+    totalGasUsed: {
+      type: DataTypes.DECIMAL(20, 10),
+      allowNull: false,
+    },
   },
   {
     sequelize,

indexer/src/services/counters.ts

@@ -7,13 +7,15 @@ export async function increaseCounters({
   canonicalTransactionsCount,
   orphanTransactionsCount,
   chainId,
+  totalGasUsed = 0,
   tx,
 }: {
   canonicalBlocksCount: number;
   orphansBlocksCount: number;
   canonicalTransactionsCount: number;
   orphanTransactionsCount: number;
   chainId: number;
+  totalGasUsed?: number;
   tx?: Transaction;
 }) {
   await Counter.increment('canonicalBlocks', {
@@ -36,4 +38,11 @@ export async function increaseCounters({
     transaction: tx,
     where: { chainId: { [Op.eq]: chainId } },
   });
+  if (totalGasUsed > 0) {
+    await Counter.increment('totalGasUsed', {
+      by: totalGasUsed,
+      transaction: tx,
+      where: { chainId: { [Op.eq]: chainId } },
+    });
+  }
 }

indexer/src/services/payload.ts

@@ -14,12 +14,12 @@
  * 6. Extracting guard information for security validation
  */
 
-import { BlockAttributes } from '@/models/block';
+import Block, { BlockAttributes } from '@/models/block';
 import TransactionModel, { TransactionAttributes } from '@/models/transaction';
 import Event, { EventAttributes } from '@/models/event';
 import Transfer, { TransferAttributes } from '@/models/transfer';
 import { getNftTransfers, getCoinTransfers } from './transfers';
-import { Transaction } from 'sequelize';
+import { Op, Transaction } from 'sequelize';
 import Signer from '@/models/signer';
 import Guard from '@/models/guard';
 import { handleSingleQuery } from '@/utils/raw-query';
@@ -74,11 +74,9 @@ export async function processPayloadKey(
   const transactions = payloadData.transactions || [];
 
   // Process all regular transactions in parallel
-  const transactionPromises: Array<
-    Promise<{ events: EventAttributes[]; transfers: TransferAttributes[] }>
-  > = transactions.map((transactionInfo: any) => {
-    return processTransaction(transactionInfo, block, tx);
-  });
+  const transactionPromises: Array<Promise<TransactionResult>> = transactions.map(
+    (transactionInfo: any) => processTransaction(transactionInfo, block, tx),
+  );
 
   const transactionResults = await Promise.all(transactionPromises);
   const normalEvents = transactionResults.map(t => t.events).flat();
@@ -87,15 +85,31 @@ export async function processPayloadKey(
   const coinbaseResult = await addCoinbaseTransactions([block], tx!);
   const coinbaseEvents = coinbaseResult.events;
 
+  const totalGasUsed = transactionResults.reduce((acc, t) => {
+    const gasUsed = new Decimal(t.gas).mul(t.gasprice).toNumber();
+    return acc.plus(gasUsed);
+  }, new Decimal(0));
+
   await increaseCounters({
     canonicalBlocksCount: 1,
     orphansBlocksCount: 0,
     canonicalTransactionsCount: transactionPromises.length,
     orphanTransactionsCount: 0,
     chainId: block.chainId,
+    totalGasUsed: totalGasUsed.toNumber(),
     tx,
   });
 
+  await Block.update(
+    {
+      totalGasUsed: sequelize.literal(`COALESCE("totalGasUsed", 0) + ${totalGasUsed}`),
+    },
+    {
+      transaction: tx,
+      where: { id: { [Op.eq]: block.id } },
+    },
+  );
+
   // Combine all events from both transaction types
   return [...normalEvents, ...coinbaseEvents];
 }
@@ -118,11 +132,19 @@ export async function processPayloadKey(
  * TODO: [OPTIMIZATION] The balance insert operation uses a raw SQL query which could potentially
  * be vulnerable to SQL injection. Consider using parameterized queries or ORM methods.
  */
+
+export interface TransactionResult {
+  events: EventAttributes[];
+  transfers: TransferAttributes[];
+  gas: string;
+  gasprice: string;
+}
+
 export async function processTransaction(
   transactionArray: any,
   block: BlockAttributes,
   tx?: Transaction,
-): Promise<{ events: EventAttributes[]; transfers: TransferAttributes[] }> {
+): Promise<TransactionResult> {
   const transactionInfo = transactionArray[TRANSACTION_INDEX];
   const receiptInfo = transactionArray[RECEIPT_INDEX];
 
@@ -312,6 +334,8 @@ export async function processTransaction(
       return {
         events: eventsWithTransactionId,
         transfers: transfersWithTransactionId,
+        gas: transactionDetailsAttributes.gas,
+        gasprice: transactionDetailsAttributes.gasprice,
       };
     }
 
@@ -338,6 +362,8 @@ export async function processTransaction(
     return {
       events: eventsWithTransactionId,
       transfers: transfersWithTransactionId,
+      gas: transactionDetailsAttributes.gas,
+      gasprice: transactionDetailsAttributes.gasprice,
     };
   } catch (error) {
     console.error(