Improve telemetry (#38)

* fix telemetry

* fix comment

* add doc

* fix comment

* add function name coment

* fix md

* fix metric logic

* change to sw case

* fix comment

---------

Co-authored-by: Tanut Lertwarachai <tanutlertwarachai@Tanuts-MacBook-Pro.local>

Author: tanut32039

docs/telemetry/README.md

@@ -0,0 +1,39 @@
+# Falcon Telemetry
+
+## Metrics Tracked
+
+The following telemetry metrics are captured and exposed via Prometheus:
+
+### Packet Relaying Metrics
+
+- `falcon_packets_relayed_success` (Counter): Total number of packets successfully relayed from BandChain
+
+- `falcon_unrelayed_packets` (Gauge): Number of unrelayed packets (the difference between total packets from BandChain and received packets from the target chain)
+
+### Task Execution Metrics
+
+- `falcon_tasks_count` (Counter): Total number of executed tasks
+- `falcon_finished_task_execution_time` (Summary): Execution time (ms) for finished tasks
+
+### Tunnel and Contract Metrics 
+- `falcon_tunnels_per_destination_chain` (Counter): Total number of tunnels per destination chain
+
+- `falcon_active_target_contracts_count` (Gauge): Number of active target chain contracts
+
+### Transaction Metrics
+- `falcon_txs_count` (Counter): Total number of transactions
+
+- `falcon_tx_process_time` (Summary): Processing time (ms) for transaction
+
+- `falcon_gas_used` (Summary): Amount of gas used per transaction
+
+## Grafana Dashboard
+Grafana provides a pre-built Falcon Dashboard to visualize relay metrics efficiently. You can download and import the dashboard from Grafana's official repository.
+
+
+- Dashboard Link: [Falcon Grafana Dashboard](https://grafana.com/grafana/dashboards/23038-falcon/)
+
+### Example Screenshot:
+![Packets Relayed Panel](example-packets-relayed.png)
+
+![Destination Chain Panel](example-destination-chain.png)

docs/telemetry/example-destination-chain.png

@@ -20,13 +20,20 @@ var metrics *PrometheusMetrics
 // It is set on initialization and does not change for the lifetime of the program.
 var globalTelemetryEnabled bool
 
+// Task statuses used as labels
+const (
+	FinishedTaskStatus = "finished"
+	ErrorTaskStatus    = "error"
+	SkippedTaskStatus  = "skipped"
+)
+
 type PrometheusMetrics struct {
 	PacketsRelayedSuccess      *prometheus.CounterVec
 	UnrelayedPackets           *prometheus.GaugeVec
 	TasksCount                 *prometheus.CounterVec
-	TaskExecutionTime          *prometheus.SummaryVec
+	FinishedTaskExecutionTime  *prometheus.SummaryVec
 	TunnelsPerDestinationChain *prometheus.CounterVec
-	ActiveTargetContractsCount prometheus.Gauge
+	ActiveTargetContractsCount *prometheus.GaugeVec
 	TxsCount                   *prometheus.CounterVec
 	TxProcessTime              *prometheus.SummaryVec
 	GasUsed                    *prometheus.SummaryVec
@@ -38,82 +45,93 @@ func updateMetrics(updateFn func()) {
 	}
 }
 
-// IncPacketsRelayedSuccess increments the count of successfully relayed packets for a specific tunnel.
+// IncPacketsRelayedSuccess increments the count of successfully relayed packets.
 func IncPacketsRelayedSuccess(tunnelID uint64) {
 	updateMetrics(func() {
 		metrics.PacketsRelayedSuccess.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Inc()
 	})
 }
 
-// SetUnrelayedPackets sets the number of unrelayed packets for a specific tunnel.
-func SetUnrelayedPackets(tunnelID uint64, unrelayedPackets float64) {
+// SetUnrelayedPackets sets the number of unrelayed packets.
+func SetUnrelayedPackets(tunnelID uint64, unrelayedPackets uint64) {
 	updateMetrics(func() {
-		metrics.UnrelayedPackets.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Set(unrelayedPackets)
+		metrics.UnrelayedPackets.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Set(float64(unrelayedPackets))
 	})
 }
 
-// IncTasksCount increments the total tasks count for a specific tunnel.
-func IncTasksCount(tunnelID uint64) {
+// IncTasksCount increments the total count of executed tasks.
+func IncTasksCount(tunnelID uint64, destinationChain string, taskStatus string) {
 	updateMetrics(func() {
-		metrics.TasksCount.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Inc()
+		metrics.TasksCount.WithLabelValues(fmt.Sprintf("%d", tunnelID), destinationChain, taskStatus).Inc()
 	})
 }
 
-// ObserveTaskExecutionTime records the execution time of a task for a specific tunnel.
-func ObserveTaskExecutionTime(tunnelID uint64, taskExecutionTime float64) {
+// ObserveFinishedTaskExecutionTime records the execution time (ms) of a finished task.
+func ObserveFinishedTaskExecutionTime(
+	tunnelID uint64,
+	destinationChain string,
+	finishedTaskExecutionTime int64,
+) {
 	updateMetrics(func() {
-		metrics.TaskExecutionTime.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Observe(taskExecutionTime)
+		metrics.FinishedTaskExecutionTime.WithLabelValues(fmt.Sprintf("%d", tunnelID), destinationChain).
+			Observe(float64(finishedTaskExecutionTime))
 	})
 }
 
-// IncTunnelsPerDestinationChain increments the total number of tunnels per specific destination chain.
+// IncTunnelsPerDestinationChain increments the count of tunnels per destination chain.
 func IncTunnelsPerDestinationChain(destinationChain string) {
 	updateMetrics(func() {
 		metrics.TunnelsPerDestinationChain.WithLabelValues(destinationChain).Inc()
 	})
 }
 
 // IncActiveTargetContractsCount increases the count of active target contracts.
-func IncActiveTargetContractsCount() {
+func IncActiveTargetContractsCount(destinationChain string) {
 	updateMetrics(func() {
-		metrics.ActiveTargetContractsCount.Inc()
+		metrics.ActiveTargetContractsCount.WithLabelValues(destinationChain).Inc()
 	})
 }
 
 // DecActiveTargetContractsCount decreases the count of active target contracts.
-func DecActiveTargetContractsCount() {
+func DecActiveTargetContractsCount(destinationChain string) {
 	updateMetrics(func() {
-		metrics.ActiveTargetContractsCount.Dec()
+		metrics.ActiveTargetContractsCount.WithLabelValues(destinationChain).Dec()
 	})
 }
 
-// IncTxsCount increments the transactions count metric for a specific tunnel.
-func IncTxsCount(tunnelID uint64) {
+// IncTxsCount increments the transactions count.
+func IncTxsCount(tunnelID uint64, destinationChain string, txStatus string) {
 	updateMetrics(func() {
-		metrics.TxsCount.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Inc()
+		metrics.TxsCount.WithLabelValues(fmt.Sprintf("%d", tunnelID), destinationChain, txStatus).Inc()
 	})
 }
 
-// ObserveTxProcessTime tracks transaction processing time in seconds with millisecond precision.
-func ObserveTxProcessTime(destinationChain string, txProcessTime float64) {
+// ObserveTxProcessTime records the processing time (ms) for each transaction.
+func ObserveTxProcessTime(tunnelID uint64, destinationChain string, txStatus string, txProcessTime int64) {
 	updateMetrics(func() {
-		metrics.TxProcessTime.WithLabelValues(destinationChain).Observe(txProcessTime)
+		metrics.TxProcessTime.WithLabelValues(fmt.Sprintf("%d", tunnelID), destinationChain, txStatus).
+			Observe(float64(txProcessTime))
 	})
 }
 
-// ObserveGasUsed tracks gas used for the each relayed transaction.
-func ObserveGasUsed(tunnelID uint64, gasUsed uint64) {
+// ObserveGasUsed tracks the amount of gas used for each transaction.
+func ObserveGasUsed(tunnelID uint64, destinationChain string, txStatus string, gasUsed float64) {
 	updateMetrics(func() {
-		metrics.GasUsed.WithLabelValues(fmt.Sprintf("%d", tunnelID)).Observe(float64(gasUsed))
+		metrics.GasUsed.WithLabelValues(fmt.Sprintf("%d", tunnelID), destinationChain, txStatus).
+			Observe(gasUsed)
 	})
 }
 
 func InitPrometheusMetrics() {
 	packetLabels := []string{"tunnel_id"}
-	taskLabels := []string{"tunnel_id"}
+	tasksCountLabels := []string{"tunnel_id", "destination_chain", "task_status"}
+	finishedTaskExecutionTimeLabels := []string{"tunnel_id", "destination_chain"}
 	tunnelPerDestinationChainLabels := []string{"destination_chain"}
-	txLabels := []string{"tunnel_id"}
-	gasUsedLabels := []string{"tunnel_id"}
+	activeTargetContractsLabels := []string{"destination_chain"}
+	txsCountLabels := []string{"tunnel_id", "destination_chain", "tx_status"}
+	txProcessTimeLabels := []string{"tunnel_id", "destination_chain", "tx_status"}
+
+	gasUsedLabels := []string{"tunnel_id", "destination_chain", "tx_status"}
 
 	metrics = &PrometheusMetrics{
 		PacketsRelayedSuccess: promauto.NewCounterVec(prometheus.CounterOpts{
@@ -126,38 +144,38 @@ func InitPrometheusMetrics() {
 		}, packetLabels),
 		TasksCount: promauto.NewCounterVec(prometheus.CounterOpts{
 			Name: "falcon_tasks_count",
-			Help: "Total number of successfully executed tasks",
-		}, taskLabels),
-		TaskExecutionTime: promauto.NewSummaryVec(prometheus.SummaryOpts{
-			Name: "falcon_task_execution_time",
-			Help: "Task execution time in milliseconds",
+			Help: "Total number of executed tasks",
+		}, tasksCountLabels),
+		FinishedTaskExecutionTime: promauto.NewSummaryVec(prometheus.SummaryOpts{
+			Name: "falcon_finished_task_execution_time",
+			Help: "Execution time (ms) for finished tasks",
 			Objectives: map[float64]float64{
 				0.5:  0.05,
 				0.9:  0.01,
 				0.99: 0.001,
 			},
-		}, taskLabels),
+		}, finishedTaskExecutionTimeLabels),
 		TunnelsPerDestinationChain: promauto.NewCounterVec(prometheus.CounterOpts{
 			Name: "falcon_tunnels_per_destination_chain",
-			Help: "Total number of destination chains",
+			Help: "Total number of tunnels per destination chain",
 		}, tunnelPerDestinationChainLabels),
-		ActiveTargetContractsCount: promauto.NewGauge(prometheus.GaugeOpts{
+		ActiveTargetContractsCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "falcon_active_target_contracts_count",
 			Help: "Number of active target chain contracts",
-		}),
+		}, activeTargetContractsLabels),
 		TxsCount: promauto.NewCounterVec(prometheus.CounterOpts{
 			Name: "falcon_txs_count",
-			Help: "Total number of transactions per tunnel",
-		}, txLabels),
+			Help: "Total number of transactions",
+		}, txsCountLabels),
 		TxProcessTime: promauto.NewSummaryVec(prometheus.SummaryOpts{
 			Name: "falcon_tx_process_time",
-			Help: "Transaction processing time in milliseconds",
+			Help: "Processing time (ms) for transaction",
 			Objectives: map[float64]float64{
 				0.5:  0.05,
 				0.9:  0.01,
 				0.99: 0.001,
 			},
-		}, txLabels),
+		}, txProcessTimeLabels),
 		GasUsed: promauto.NewSummaryVec(prometheus.SummaryOpts{
 			Name: "falcon_gas_used",
 			Help: "Amount of gas used per transaction",
@@ -178,7 +196,7 @@ func StartMetricsServer(ctx context.Context, log *zap.Logger, metricsListenAddr
 	ln, err := net.Listen("tcp", metricsListenAddr)
 	if err != nil {
 		log.Error(
-			"Failed to start metrics server you can change the address and port using metrics-listen-addr config setting or --metrics-listen-flag",
+			"Failed to start metrics server you can change the address and port using metrics-listen-addr config setting or --metrics-listen-addr flag",
 		)
 
 		return fmt.Errorf("failed to listen on metrics address %q: %w", metricsListenAddr, err)

docs/telemetry/example-packets-relayed.png

@@ -386,7 +386,6 @@ func (a *App) Relay(ctx context.Context, tunnelID uint64) error {
 	tr := NewTunnelRelayer(
 		a.Log,
 		tunnel.ID,
-		tunnel.TargetAddress,
 		a.Config.Global.CheckingPacketInterval,
 		a.BandClient,
 		chainProvider,

internal/relayermetrics/metrics.go

@@ -182,9 +182,6 @@ func (cp *EVMChainProvider) RelayPacket(ctx context.Context, packet *bandtypes.P
 
 		createdAt := time.Now()
 
-		// increment the transactions count metric for the current tunnel
-		relayermetrics.IncTxsCount(packet.TunnelID)
-
 		log.Info(
 			"Submitted a message; checking transaction status",
 			zap.String("tx_hash", txHash),
@@ -193,6 +190,7 @@ func (cp *EVMChainProvider) RelayPacket(ctx context.Context, packet *bandtypes.P
 
 		var checkTxErr error
 		var txStatus TxStatus
+		var gasUsed decimal.NullDecimal
 	checkTxLogic:
 		for time.Since(createdAt) < cp.Config.WaitingTxDuration {
 			result, err := cp.CheckConfirmedTx(ctx, txHash)
@@ -212,13 +210,18 @@ func (cp *EVMChainProvider) RelayPacket(ctx context.Context, packet *bandtypes.P
 
 			checkTxErr = nil
 			txStatus = result.Status
+			gasUsed = result.GasUsed
+
 			switch result.Status {
 			case TX_STATUS_SUCCESS:
-				// track transaction processing time in seconds with millisecond precision
-				relayermetrics.ObserveTxProcessTime(cp.ChainName, float64(time.Since(createdAt).Milliseconds()))
+				// increment the transactions count metric
+				relayermetrics.IncTxsCount(packet.TunnelID, cp.ChainName, TX_STATUS_SUCCESS.String())
+
+				// track transaction processing time (ms)
+				relayermetrics.ObserveTxProcessTime(packet.TunnelID, cp.ChainName, TX_STATUS_SUCCESS.String(), time.Since(createdAt).Milliseconds())
 
 				// track gas used for the relayed transaction
-				relayermetrics.ObserveGasUsed(packet.TunnelID, result.GasUsed.Decimal.BigInt().Uint64())
+				relayermetrics.ObserveGasUsed(packet.TunnelID, cp.ChainName, TX_STATUS_SUCCESS.String(), gasUsed.Decimal.InexactFloat64())
 
 				log.Info(
 					"Packet is successfully relayed",
@@ -227,6 +230,16 @@ func (cp *EVMChainProvider) RelayPacket(ctx context.Context, packet *bandtypes.P
 				)
 				return nil
 			case TX_STATUS_FAILED:
+				// track transaction processing time (ms)
+				relayermetrics.ObserveTxProcessTime(
+					packet.TunnelID,
+					cp.ChainName,
+					TX_STATUS_FAILED.String(),
+					time.Since(createdAt).Milliseconds(),
+				)
+
+				// track gas used for the relayed transaction
+				relayermetrics.ObserveGasUsed(packet.TunnelID, cp.ChainName, TX_STATUS_FAILED.String(), gasUsed.Decimal.InexactFloat64())
 				log.Debug(
 					"Transaction failed during relay attempt",
 					zap.Error(err),
@@ -246,6 +259,9 @@ func (cp *EVMChainProvider) RelayPacket(ctx context.Context, packet *bandtypes.P
 			}
 		}
 
+		// increment the transactions count metric
+		relayermetrics.IncTxsCount(packet.TunnelID, cp.ChainName, txStatus.String())
+
 		log.Error(
 			"Failed to relaying a packet with status and error",
 			zap.Error(checkTxErr),
@@ -296,19 +312,16 @@ func (cp *EVMChainProvider) CheckConfirmedTx(
 	ctx context.Context,
 	txHash string,
 ) (*ConfirmTxResult, error) {
-	failResult := NewConfirmTxResult(
-		txHash,
-		TX_STATUS_UNMINED,
-		decimal.NullDecimal{},
-	)
-
 	receipt, err := cp.Client.GetTxReceipt(ctx, txHash)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get tx receipt: %w", err)
 	}
 
+	// calculate gas used and effective gas price
+	gasUsed := decimal.NewNullDecimal(decimal.New(int64(receipt.GasUsed), 0))
+
 	if receipt.Status == gethtypes.ReceiptStatusFailed {
-		return failResult.WithStatus(TX_STATUS_FAILED), nil
+		return NewConfirmTxResult(txHash, TX_STATUS_FAILED, gasUsed), nil
 	}
 
 	latestBlock, err := cp.Client.GetBlockHeight(ctx)
@@ -318,11 +331,9 @@ func (cp *EVMChainProvider) CheckConfirmedTx(
 
 	// if tx block is not confirmed and waiting too long return status with timeout
 	if receipt.BlockNumber.Uint64() > latestBlock-cp.Config.BlockConfirmation {
-		return failResult.WithStatus(TX_STATUS_UNMINED), nil
+		return NewConfirmTxResult(txHash, TX_STATUS_UNMINED, decimal.NullDecimal{}), nil
 	}
 
-	// calculate gas used and effective gas price
-	gasUsed := decimal.NewNullDecimal(decimal.New(int64(receipt.GasUsed), 0))
 	return NewConfirmTxResult(txHash, TX_STATUS_SUCCESS, gasUsed), nil
 }
 
@@ -574,6 +585,11 @@ func (cp *EVMChainProvider) QueryBalance(
 	return cp.Client.GetBalance(ctx, address)
 }
 
+// GetChainName retrieves the chain name from the chain provider.
+func (cp *EVMChainProvider) GetChainName() string {
+	return cp.ChainName
+}
+
 // queryRelayerGasFee queries the relayer gas fee being set on tunnel router.
 func (cp *EVMChainProvider) queryRelayerGasFee(ctx context.Context) (*big.Int, error) {
 	calldata, err := cp.TunnelRouterABI.Pack("gasFee")

internal/relayertest/mocks/chain_provider.go

@@ -259,7 +259,7 @@ func (s *ProviderTestSuite) TestCheckConfirmedTx() {
 			out: evm.NewConfirmTxResult(
 				txHash,
 				evm.TX_STATUS_FAILED,
-				decimal.NullDecimal{},
+				decimal.NewNullDecimal(decimal.New(21000, 0)),
 			),
 		},
 		{