fix: creationtime value of coinbase transactions

Author: mschead

backfill/Dockerfile.creationtime

@@ -0,0 +1,11 @@
+FROM golang:1.23.7 AS builder
+WORKDIR /app
+COPY . .
+RUN go mod download
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o fix-creationtime ./fix-creationtime/fix-creationtime.go
+
+FROM scratch
+WORKDIR /app
+COPY ./global-bundle.pem ./global-bundle.pem
+COPY --from=builder /app/fix-creationtime .
+CMD ["./fix-creationtime"]

backfill/fix-creationtime/fix-creationtime.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"go-backfill/config"
+	"log"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+const (
+	batchSize = 1000
+	lastId    = 105126770
+)
+
+func fixBatchCreationTime(conn *pgx.Conn, lastId int64) (bool, int64, error) {
+	startTime := time.Now()
+
+	// Start transaction for writes
+	tx, err := conn.Begin(context.Background())
+	if err != nil {
+		return false, lastId, fmt.Errorf("failed to begin transaction: %v", err)
+	}
+	defer tx.Rollback(context.Background())
+
+	// Update creation time for coinbase transactions in batch
+	updateQuery := `
+		WITH OrderedUpdates AS (
+			SELECT id
+			FROM "Transactions"
+			WHERE id > $1 AND sender = 'coinbase'
+			ORDER BY id ASC
+			LIMIT $2
+		)
+		UPDATE "Transactions"
+		SET creationtime = (CAST(creationtime AS BIGINT) / 1000000)::TEXT
+		FROM OrderedUpdates
+		WHERE "Transactions".id = OrderedUpdates.id
+		RETURNING "Transactions".id, "Transactions".creationtime
+	`
+
+	rows, err := tx.Query(context.Background(), updateQuery, lastId, batchSize)
+	if err != nil {
+		return false, lastId, fmt.Errorf("failed to execute update: %v", err)
+	}
+	defer rows.Close()
+
+	var updatedCount int
+	var lastProcessedId int64
+	var lastCreationTime string
+	for rows.Next() {
+		if err := rows.Scan(&lastProcessedId, &lastCreationTime); err != nil {
+			return false, lastId, fmt.Errorf("failed to scan row: %v", err)
+		}
+		updatedCount++
+
+	}
+
+	if err := tx.Commit(context.Background()); err != nil {
+		return false, lastId, fmt.Errorf("failed to commit transaction: %v", err)
+	}
+
+	elapsed := time.Since(startTime)
+	log.Printf("Fixed creation time for %d coinbase transactions. Batch time: %.2fs", updatedCount, elapsed.Seconds())
+
+	return updatedCount == batchSize, lastProcessedId, nil
+}
+
+func main() {
+	envFile := flag.String("env", ".env", "Path to the .env file")
+	flag.Parse()
+
+	config.InitEnv(*envFile)
+	env := config.GetConfig()
+
+	// Database connection
+	connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
+		env.DbHost, env.DbPort, env.DbUser, env.DbPassword, env.DbName)
+
+	conn, err := pgx.Connect(context.Background(), connStr)
+	if err != nil {
+		log.Fatalf("Failed to connect to database: %v", err)
+	}
+	defer conn.Close(context.Background())
+
+	log.Println("Connected to database")
+
+	lastId := int64(0)
+	hasMore := true
+	processedTransactions := int64(0)
+
+	for hasMore {
+		var err error
+		hasMore, lastId, err = fixBatchCreationTime(conn, lastId)
+		if err != nil {
+			log.Fatalf("Error during batch processing: %v", err)
+		}
+		processedTransactions += batchSize
+		progress := float64(processedTransactions) / float64(105126770) * 100
+
+		if hasMore {
+			log.Printf("Progress: %.2f%% (%d/%d transactions processed)", progress, processedTransactions, lastId)
+		} else {
+			log.Printf("Progress: 100.00%%")
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	log.Println("Creation time fix completed successfully")
+}

backfill/process/coinbase.go

@@ -40,7 +40,7 @@ func ProcessCoinbaseTransaction(coinbase string, blockId int64, creationTime int
 	txAttribute := repository.TransactionAttributes{
 		BlockId:      blockId,
 		ChainId:      int(chainId),
-		CreationTime: strconv.FormatInt(creationTime, 10),
+		CreationTime: strconv.FormatInt(creationTime/1000000, 10),
 		Hash:         coinbaseDecoded.ReqKey,
 		Result:       coinbaseDecoded.Result,
 		Logs:         coinbaseDecoded.Logs,

indexer/src/kadena-server/repository/infra/repository/transaction-db-repository.ts

@@ -203,7 +203,7 @@ export default class TransactionDbRepository implements TransactionRepository {
         JOIN "Transactions" t ON b.id = t."blockId"
         LEFT JOIN "TransactionDetails" td ON t.id = td."transactionId"
         ${transactionsConditions}
-        ORDER BY t.id ${order}
+        ORDER BY t.creationtime ${order}
         LIMIT $1
       `;
     } else {
@@ -212,7 +212,7 @@ export default class TransactionDbRepository implements TransactionRepository {
           SELECT t.id, t."blockId", t.hash, t.num_events, t.txid, t.logs, t.result, t.requestkey, t."chainId"
           FROM "Transactions" t
           ${transactionsConditions}
-          ORDER BY t.id ${order}
+          ORDER BY t.creationtime ${order}
           LIMIT $1
         )
         SELECT
@@ -425,7 +425,7 @@ export default class TransactionDbRepository implements TransactionRepository {
       ) filtered_signers ON t.id = filtered_signers."transactionId"
       LEFT JOIN "TransactionDetails" td on t.id = td."transactionId"
       ${cursorCondition}
-      ORDER BY t.id ${order}
+      ORDER BY t.creationtime ${order}
       LIMIT $1;
     `;
 

indexer/src/kadena-server/repository/infra/schema-validator/transaction-meta-schema-validator.ts

@@ -14,7 +14,7 @@ function validate(row: any): TransactionMetaOutput {
   const res = schema.parse(row);
   return {
     chainId: res.chainId,
-    creationTime: new Date(Number(res.creationTime) / 1000),
+    creationTime: new Date(Number(res.creationTime) * 1000),
     gasLimit: res.gasLimit ?? 0,
     gasPrice: Number(res.gasPrice) ?? 0,
     sender: res.sender,

indexer/src/services/sync/coinbase.ts

@@ -122,7 +122,7 @@ export async function processCoinbaseTransaction(
   const transactionAttributes = {
     blockId: block.id,
     chainId: block.chainId,
-    creationtime: block.creationTime.toString(),
+    creationtime: Math.trunc(Number(block.creationTime) / 1000000).toString(),
     hash: coinbase.reqKey,
     result: coinbase.result,
     logs: coinbase.logs,