Merge pull request #500 from hack-a-chain-software/develop

Merge 'develop' into 'main'

Author: mschead

.gitignore

@@ -3,6 +3,9 @@ dist/
 data/
 draft/
 .env
+.env.testing
+.env.mainnet01
+.env.testnet04
 .terraform
 .terraform.lock.hcl
 .terraform.tfstate.lock.info
@@ -12,3 +15,4 @@ draft/
 indexer/indexer
 .cursor
 indexer/tests/logs/*
+indexer/tests/integration/api-parity/logs/*

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## Aug 13 - Sep 11, 2025
+
+- fix: remove unnecessary error throwing in dataloaders
+- refactor: removed CORS origin handler
+- fix: remove token price cache, causing inaccurate usd prices
+- feat: added errors monitoring for Kadena report
+- fix: liquidity can be object or number
+- fix: reverted orphan blocks algorithm behavior
+- refactor: changed indexes approach and graphql resolvers to make transfers query faster for all use cases
+- fix: tvl history query to better capture daily tvl values
+- refactor: improved transactions count when using minHeight and maxHeight
+- fix: added hasTokenId to all transfers fields and their counters
+- feat: add optional code field to TransactionSummary and related repository updates
+- refactor: improved blocks-from-height query
+- feat: optional port for migration
+- fix: created a missing migration to add an index on the module and chainId columns in the Events table
+- feat: added module as a new param in events query and improved the indexes to make the query performance better
+- fix: dex metrics tvl history query
+- refactor: transactions query improvement
+- refactor: separated pairCreation process from indexing
+- feat: created a query to get the token price based on its latest swap
+- feat: allowing to query holders of certain fungible module
+- fix: manually upsert poolstats to not overwrite db transaction
+- fix: added badResult field to the transactions-by-pact-code query and changed the creationTime field to DateTime type
+- fix: created a migration to add an index on the code column in the TransactionDetails table
+- refactor: added token id field in the transfers query for the case where isNft = true
+- refactor: started to track RECONCILE events and transform them in NFT transfers
+- refactor: changed the totalGasUsed field name to totalGasUsedInKda
+- refactor: created the transactionsByPactCode query to avoid performance issues
+- feat: balances queries
+- feat: added counters to the network info query
+- feat: added total count in blocksFromHeight query
+- fix: adjusted orphaned transactions field to use the canonical flag
+- feat: added total gas used in block type of graphql
+- tests: improved api parity tests to use random block hashes in queries like block, transactions, transfers and events
+
 ## Jul 28 - Aug 13, 2025
 
 - refactor: removed unnecessary environment variables and made some previously required variables optional

backfill/db-migrator/creationtime-events.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"go-backfill/config"
+	"log"
+
+	_ "github.com/lib/pq" // PostgreSQL driver
+)
+
+const (
+	creationTimeBatchSizeEvents = 500
+	startTransactionIdEvents    = 1
+	endTransactionIdEvents      = 1000
+)
+
+// This script was created to duplicate the creation time of transaction to the events table.
+// The main motivation was to improve the performance of the events query.
+
+func updateCreationTimesForEvents() error {
+	envFile := flag.String("env", ".env", "Path to the .env file")
+	flag.Parse()
+	config.InitEnv(*envFile)
+	env := config.GetConfig()
+	connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
+		env.DbHost, env.DbPort, env.DbUser, env.DbPassword, env.DbName)
+
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return fmt.Errorf("failed to connect to database: %v", err)
+	}
+	defer db.Close()
+
+	log.Println("Connected to database")
+
+	// Test database connection
+	if err := db.Ping(); err != nil {
+		return fmt.Errorf("failed to ping database: %v", err)
+	}
+
+	// Process transactions in batches
+	if err := processTransactionsBatchForEvents(db); err != nil {
+		return fmt.Errorf("failed to process transactions: %v", err)
+	}
+
+	log.Println("Successfully updated all events creation times")
+	return nil
+}
+
+func processTransactionsBatchForEvents(db *sql.DB) error {
+	currentId := startTransactionIdEvents
+	totalProcessed := 0
+	totalTransactions := endTransactionIdEvents - startTransactionIdEvents + 1
+	lastProgressPrinted := -1.0
+
+	log.Printf("Starting to process transactions from ID %d to %d",
+		startTransactionIdEvents, endTransactionIdEvents)
+	log.Printf("Total transactions to process: %d", totalTransactions)
+
+	for currentId <= endTransactionIdEvents {
+		// Calculate batch end
+		batchEnd := currentId + creationTimeBatchSizeEvents - 1
+		if batchEnd > endTransactionIdEvents {
+			batchEnd = endTransactionIdEvents
+		}
+
+		// Process this batch
+		processed, err := processBatchForEvents(db, currentId, batchEnd)
+		if err != nil {
+			return fmt.Errorf("failed to process batch %d-%d: %v", currentId, batchEnd, err)
+		}
+
+		totalProcessed += processed
+
+		// Calculate progress percentage
+		transactionsProcessed := batchEnd - startTransactionIdEvents + 1
+		progressPercent := (float64(transactionsProcessed) / float64(totalTransactions)) * 100.0
+
+		// Only print progress if it has increased by at least 0.1%
+		if progressPercent-lastProgressPrinted >= 0.1 {
+			log.Printf("Progress: %.1f%%", progressPercent)
+			lastProgressPrinted = progressPercent
+		}
+
+		// Move to next batch
+		currentId = batchEnd + 1
+	}
+
+	log.Printf("Completed processing. Total events updated: %d (100.0%%)", totalProcessed)
+	return nil
+}
+
+func processBatchForEvents(db *sql.DB, startId, endId int) (int, error) {
+	// Begin transaction for atomic operation
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, fmt.Errorf("failed to begin transaction: %v", err)
+	}
+	defer tx.Rollback() // Will be ignored if tx.Commit() succeeds
+
+	// Update events with creation time from transactions in a single query
+	updateQuery := `
+		UPDATE "Events" 
+		SET creationtime = t.creationtime, "updatedAt" = CURRENT_TIMESTAMP
+		FROM "Transactions" t
+		WHERE "Events"."transactionId" = t.id 
+		AND t.id >= $1 AND t.id <= $2
+	`
+
+	result, err := tx.Exec(updateQuery, startId, endId)
+	if err != nil {
+		return 0, fmt.Errorf("failed to update events: %v", err)
+	}
+
+	rowsAffected, err := result.RowsAffected()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get rows affected: %v", err)
+	}
+
+	// Commit the transaction
+	if err := tx.Commit(); err != nil {
+		return 0, fmt.Errorf("failed to commit transaction: %v", err)
+	}
+
+	return int(rowsAffected), nil
+}
+
+func creationTimesEvents() {
+	if err := updateCreationTimesForEvents(); err != nil {
+		log.Fatalf("Error: %v", err)
+	}
+}

backfill/db-migrator/creationtime.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"go-backfill/config"
+	"log"
+
+	_ "github.com/lib/pq" // PostgreSQL driver
+)
+
+const (
+	creationTimeBatchSize = 500
+	startTransactionId    = 1
+	endTransactionId      = 110239835
+)
+
+// This script was created to duplicate the creation time of transaction to the events and transfers tables.
+// The main motivation was to improve the performance of the events and transfers queries.
+
+func updateCreationTimes() error {
+	envFile := flag.String("env", ".env", "Path to the .env file")
+	flag.Parse()
+	config.InitEnv(*envFile)
+	env := config.GetConfig()
+	connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
+		env.DbHost, env.DbPort, env.DbUser, env.DbPassword, env.DbName)
+
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return fmt.Errorf("failed to connect to database: %v", err)
+	}
+	defer db.Close()
+
+	log.Println("Connected to database")
+
+	// Test database connection
+	if err := db.Ping(); err != nil {
+		return fmt.Errorf("failed to ping database: %v", err)
+	}
+
+	// Process transactions in batches
+	if err := processTransactionsBatch(db); err != nil {
+		return fmt.Errorf("failed to process transactions: %v", err)
+	}
+
+	log.Println("Successfully updated all events and transfers creation times")
+	return nil
+}
+
+func processTransactionsBatch(db *sql.DB) error {
+	currentId := startTransactionId
+	totalProcessed := 0
+	totalTransactions := endTransactionId - startTransactionId + 1
+	lastProgressPrinted := -1.0
+
+	log.Printf("Starting to process transactions from ID %d to %d",
+		startTransactionId, endTransactionId)
+	log.Printf("Total transactions to process: %d", totalTransactions)
+
+	for currentId <= endTransactionId {
+		// Calculate batch end
+		batchEnd := currentId + creationTimeBatchSize - 1
+		if batchEnd > endTransactionId {
+			batchEnd = endTransactionId
+		}
+
+		// Process this batch
+		processed, err := processBatch(db, currentId, batchEnd)
+		if err != nil {
+			return fmt.Errorf("failed to process batch %d-%d: %v", currentId, batchEnd, err)
+		}
+
+		totalProcessed += processed
+
+		// Calculate progress percentage
+		transactionsProcessed := batchEnd - startTransactionId + 1
+		progressPercent := (float64(transactionsProcessed) / float64(totalTransactions)) * 100.0
+
+		// Only print progress if it has increased by at least 0.1%
+		if progressPercent-lastProgressPrinted >= 0.1 {
+			log.Printf("Progress: %.1f%%", progressPercent)
+			lastProgressPrinted = progressPercent
+		}
+
+		// Move to next batch
+		currentId = batchEnd + 1
+	}
+
+	log.Printf("Completed processing. Total records updated: %d (100.0%%)", totalProcessed)
+	return nil
+}
+
+func processBatch(db *sql.DB, startId, endId int) (int, error) {
+	// Begin transaction for atomic operation
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, fmt.Errorf("failed to begin transaction: %v", err)
+	}
+	defer tx.Rollback() // Will be ignored if tx.Commit() succeeds
+
+	// Update events with creation time from transactions
+	eventsUpdateQuery := `
+		UPDATE "Events" 
+		SET creationtime = t.creationtime, "updatedAt" = CURRENT_TIMESTAMP
+		FROM "Transactions" t
+		WHERE "Events"."transactionId" = t.id 
+		AND t.id >= $1 AND t.id <= $2
+	`
+
+	eventsResult, err := tx.Exec(eventsUpdateQuery, startId, endId)
+	if err != nil {
+		return 0, fmt.Errorf("failed to update events: %v", err)
+	}
+
+	eventsRowsAffected, err := eventsResult.RowsAffected()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get events rows affected: %v", err)
+	}
+
+	// Update transfers with creation time from transactions
+	transfersUpdateQuery := `
+		UPDATE "Transfers" 
+		SET creationtime = t.creationtime, "updatedAt" = CURRENT_TIMESTAMP
+		FROM "Transactions" t
+		WHERE "Transfers"."transactionId" = t.id 
+		AND t.id >= $1 AND t.id <= $2
+	`
+
+	transfersResult, err := tx.Exec(transfersUpdateQuery, startId, endId)
+	if err != nil {
+		return 0, fmt.Errorf("failed to update transfers: %v", err)
+	}
+
+	transfersRowsAffected, err := transfersResult.RowsAffected()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get transfers rows affected: %v", err)
+	}
+
+	// Commit the transaction
+	if err := tx.Commit(); err != nil {
+		return 0, fmt.Errorf("failed to commit transaction: %v", err)
+	}
+
+	totalRowsAffected := int(eventsRowsAffected + transfersRowsAffected)
+	return totalRowsAffected, nil
+}
+
+func creationTimes() {
+	if err := updateCreationTimes(); err != nil {
+		log.Fatalf("Error: %v", err)
+	}
+}