Merge pull request #336 from hack-a-chain-software/bare-server

Merge 'bare-metal-server' into 'develop'

Author: mschead

backfill/main.go

@@ -19,7 +19,14 @@ func main() {
 	go config.StartMemoryMonitoring()
 	cut := fetch.FetchCut()
 	ChainId := env.ChainId
-	minHeights := config.GetMinHeights(env.Network)
-	SyncMinHeight := minHeights[ChainId]
-	process.StartBackfill(cut.Height, cut.Hash, ChainId, SyncMinHeight, pool)
+
+	var effectiveSyncMinHeight int
+	if env.SyncMinHeight > 0 {
+		effectiveSyncMinHeight = env.SyncMinHeight
+	} else {
+		chainGenesisHeights := config.GetMinHeights(env.Network)
+		effectiveSyncMinHeight = chainGenesisHeights[ChainId]
+	}
+
+	process.StartBackfill(cut.Height, cut.Hash, ChainId, effectiveSyncMinHeight, pool)
 }

backfill/process/process.go

@@ -20,7 +20,14 @@ func StartBackfill(LastHeight int, Hash string, ChainId int, SyncMinHeight int,
 
 	for CurrentHeight >= SyncMinHeight {
 		startTime := time.Now()
-		nextHeight := max(CurrentHeight-env.SyncFetchIntervalInBlocks+1, SyncMinHeight)
+
+		var nextHeight int
+		if CurrentHeight == SyncMinHeight {
+			nextHeight = SyncMinHeight
+		} else {
+			nextHeight = Max(CurrentHeight-env.SyncFetchIntervalInBlocks+1, SyncMinHeight)
+		}
+
 		log.Printf("Processing height %d to %d...\n", CurrentHeight, nextHeight)
 
 		blocks, err := fetch.FetchPayloadsWithHeaders(network, ChainId, Hash, nextHeight, CurrentHeight)

backfill/process/save_payloads.go

@@ -72,17 +72,17 @@ func savePayloads(network string, chainId int, processedPayloads []fetch.Process
 		var currBlock = blocks[index]
 		txs, txDetails, txCoinbase, err := PrepareTransactions(network, blockId, processedPayload, currBlock)
 		if err != nil {
-			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transactions -> %w", err)
+			return Counters{}, DataSizeTracker{}, fmt.Errorf("preparing transactions for block %d -> %w", currBlock.Height, err)
 		}
 
 		transactionIds, err := repository.SaveTransactions(tx, txs, txCoinbase)
 		if err != nil {
-			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transactions -> %w", err)
+			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transactions for block %d -> %w", currBlock.Height, err)
 		}
 
 		err = repository.SaveTransactionDetails(tx, txDetails, transactionIds)
 		if err != nil {
-			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transactions -> %w", err)
+			return Counters{}, DataSizeTracker{}, fmt.Errorf("saving transaction details for block %d -> %w", currBlock.Height, err)
 		}
 
 		txsSize := approximateSize(txs)
@@ -144,15 +144,18 @@ func savePayloads(network string, chainId int, processedPayloads []fetch.Process
 	// 	counters.Guards += len(guards)
 	// }
 
+	log.Printf("Saved payloads in %fs\n", time.Since(startTime).Seconds())
+
+	commitStartTime := time.Now()
 	if err := tx.Commit(context.Background()); err != nil {
 		return Counters{}, DataSizeTracker{}, fmt.Errorf("committing transaction: %w", err)
 	}
+	log.Printf("DB commit took %fs\n", time.Since(commitStartTime).Seconds())
 
 	dataSizeTracker.TransactionsKB /= 1024
 	dataSizeTracker.EventsKB /= 1024
 	dataSizeTracker.TransfersKB /= 1024
 	dataSizeTracker.SignersKB /= 1024
 
-	log.Printf("Saved payloads in %fs\n", time.Since(startTime).Seconds())
 	return counters, dataSizeTracker, nil
 }

backfill/process/utils.go

@@ -161,7 +161,7 @@ func buildModuleName(namespace *string, name string) string {
 	return name
 }
 
-func max(a, b int) int {
+func Max(a, b int) int {
 	if a > b {
 		return a
 	}

backfill/repository/transaction_repository.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -119,8 +120,7 @@ func SaveTransactionDetails(db pgx.Tx, details []TransactionDetailsAttributes, t
 		return nil
 	}
 
-	query := `
-		INSERT INTO "TransactionDetails" (
+	query := `INSERT INTO "TransactionDetails" (
 			"transactionId", code, continuation, data, gas, gaslimit, gasprice,
 			nonce, pactid, proof, rollback, sigs, step, ttl, "createdAt", "updatedAt"
 		)
@@ -131,6 +131,10 @@ func SaveTransactionDetails(db pgx.Tx, details []TransactionDetailsAttributes, t
 	batch := &pgx.Batch{}
 
 	for index := 0; index < len(details); index++ {
+		// Sanitize data before marshalling to prevent "unsupported Unicode escape sequence" errors.
+		// PostgreSQL's JSONB type does not support null characters (\u0000), so we remove them.
+		details[index].Data = bytes.ReplaceAll(details[index].Data, []byte(`\u0000`), []byte{})
+
 		detail := details[index]
 		code, err := json.Marshal(detail.Code)
 		if err != nil {
@@ -179,9 +183,11 @@ func SaveTransactionDetails(db pgx.Tx, details []TransactionDetailsAttributes, t
 	// Execute all queued queries
 	for i := 0; i < len(details); i++ {
 		if _, err := br.Exec(); err != nil {
-			return fmt.Errorf("failed to execute batch for transaction details %d: %v", i, err)
+			failedDetail := details[i]
+			failedData, _ := json.MarshalIndent(failedDetail, "", "  ")
+			return fmt.Errorf("failed to execute batch for transaction details %d: %v\nFailing data:\n%s", i, err, string(failedData))
 		}
 	}
 
 	return nil
-}
+}

indexer/docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.8'
 services:
   indexer-db:
     image: postgres
-    container_name: kad-indexer-postgres
+    container_name: postgres-indexer
     environment:
       POSTGRES_USER: ${DB_USERNAME}
       POSTGRES_PASSWORD: ${DB_PASSWORD}
@@ -12,6 +12,24 @@ services:
       - '5432:5432'
     volumes:
       - ${PWD}/indexer/postgres:/var/lib/postgresql/data
+    command:
+      - 'postgres'
+      - '-c'
+      - 'shared_buffers=8GB'
+      - '-c'
+      - 'effective_cache_size=24GB'
+      - '-c'
+      - 'maintenance_work_mem=2GB'
+      - '-c'
+      - 'work_mem=64MB'
+      - '-c'
+      - 'max_wal_size=16GB'
+      - '-c'
+      - 'random_page_cost=1.1'
+      - '-c'
+      - 'effective_io_concurrency=200'
+      - '-c'
+      - 'synchronous_commit=off'
     healthcheck:
       test: ['CMD-SHELL', 'pg_isready -U ${DB_USERNAME}']
       interval: 10s

indexer/migrations/20250610124420-update-pool-stats-timestamp.js

@@ -29,27 +29,13 @@ module.exports = {
       type: Sequelize.DATEONLY,
       allowNull: false,
     });
-
-    // Create unique index
-    await queryInterface.addIndex('PoolStats', ['pairId', 'timestamp'], {
-      unique: true,
-      name: 'pool_stats_pairid_timestamp_idx',
-    });
   },
 
   async down(queryInterface, Sequelize) {
-    // Remove unique index
-    await queryInterface.removeIndex('PoolStats', 'pool_stats_pairid_timestamp_idx');
-
     // Convert back to TIMESTAMP WITH TIME ZONE
     await queryInterface.changeColumn('PoolStats', 'timestamp', {
       type: Sequelize.DATE,
       allowNull: false,
     });
-
-    // Recreate original non-unique index
-    await queryInterface.addIndex('PoolStats', ['pairId', 'timestamp'], {
-      name: 'pool_stats_pairid_timestamp_idx',
-    });
   },
 };

indexer/src/services/payload.ts

@@ -198,13 +198,14 @@ export async function processTransaction(
     );
 
     const events = await Promise.all(eventsAttributes);
-    const swapEvents = events.filter(event => event.name === 'SWAP');
-    console.log('swapEvents', JSON.stringify(swapEvents, null, 2));
-    const addLiquidityEvents = events.filter(event => event.name === 'ADD_LIQUIDITY');
-    console.log('addLiquidityEvents', JSON.stringify(addLiquidityEvents, null, 2));
-    const mintEvents = events.filter(event => event.name === 'MINT_EVENT');
-    console.log('mintEvents', JSON.stringify(mintEvents, null, 2));
-    console.log('------------------------------------- end -------------------------------');
+    // Note: Should not summit debug to the 'main' branch
+    // const swapEvents = events.filter(event => event.name === 'SWAP');
+    // console.log('swapEvents', JSON.stringify(swapEvents, null, 2));
+    // const addLiquidityEvents = events.filter(event => event.name === 'ADD_LIQUIDITY');
+    // console.log('addLiquidityEvents', JSON.stringify(addLiquidityEvents, null, 2));
+    // const mintEvents = events.filter(event => event.name === 'MINT_EVENT');
+    // console.log('mintEvents', JSON.stringify(mintEvents, null, 2));
+    // console.log('------------------------------------- end -------------------------------');
     const eventsWithTransactionId = events.map(event => ({
       ...event,
       transactionId,