From ffeab2df32dcad4d646154ccfd08dc15761e6860 Mon Sep 17 00:00:00 2001 From: nman98 Date: Fri, 3 Oct 2025 13:33:49 +0200 Subject: [PATCH 1/6] Changed the transaction general to receive the block height, split the queries in the database for indexer and the api --- indexer/address_cache/address.go | 16 +-- indexer/address_cache/types.go | 2 +- indexer/data_processor/operator.go | 51 ++++----- indexer/data_processor/types.go | 9 ++ indexer/orchestrator/operator.go | 44 ++++++-- indexer/orchestrator/operator_test.go | 5 +- indexer/orchestrator/types.go | 5 +- pkgs/database/insert.go | 1 + pkgs/database/queries_api.go | 100 ++++++++++++++++++ .../{queries.go => queries_indexer.go} | 38 +++---- pkgs/database/types.go | 10 ++ pkgs/sql_data_types/table.go | 9 +- 12 files changed, 217 insertions(+), 73 deletions(-) create mode 100644 pkgs/database/queries_api.go rename pkgs/database/{queries.go => queries_indexer.go} (86%) diff --git a/indexer/address_cache/address.go b/indexer/address_cache/address.go index 160011c..b0a4b9e 100644 --- a/indexer/address_cache/address.go +++ b/indexer/address_cache/address.go @@ -22,25 +22,25 @@ import ( func NewAddressCache(chainName string, db DatabaseForAddresses, loadVal bool) *AddressCache { if loadVal { // if true load the validator addresses - addresses, err := loadAddresses(chainName, loadVal, db) + addresses, maxIndex, err := loadAddresses(chainName, loadVal, db) if err != nil { log.Fatalf("failed to load addresses: %v", err) } return &AddressCache{ address: addresses, db: db, - highestIndex: 0, + highestIndex: maxIndex, } } else { // if false load the regular addresses - addresses, err := loadAddresses(chainName, loadVal, db) + addresses, maxIndex, err := loadAddresses(chainName, loadVal, db) if err != nil { log.Fatalf("failed to load addresses: %v", err) } return &AddressCache{ address: addresses, db: db, - highestIndex: 0, + highestIndex: maxIndex, } } } @@ -194,10 +194,10 @@ func (a *AddressCache) GetAddress(address string) int32 { // Returns: // - map[string]int32: the map of addresses and their ids // - error: if the query fails -func loadAddresses(chainName string, loadVal bool, db DatabaseForAddresses) (map[string]int32, error) { - addresses, err := db.GetAllAddresses(chainName, loadVal) +func loadAddresses(chainName string, loadVal bool, db DatabaseForAddresses) (map[string]int32, int32, error) { + addresses, maxIndex, err := db.GetAllAddresses(chainName, loadVal, nil) if err != nil { - return nil, err + return nil, 0, err } - return addresses, nil + return addresses, maxIndex, nil } diff --git a/indexer/address_cache/types.go b/indexer/address_cache/types.go index 6842c10..d5407eb 100644 --- a/indexer/address_cache/types.go +++ b/indexer/address_cache/types.go @@ -4,7 +4,7 @@ package addresscache type DatabaseForAddresses interface { FindExistingAccounts(addresses []string, chainName string, searchValidators bool) (map[string]int32, error) InsertAddresses(addresses []string, chainName string, insertValidators bool) error - GetAllAddresses(chainName string, searchValidators bool) (map[string]int32, error) + GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) } // AddressCache is a map of addresses tied to their int32 index in the database diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index d8e99ae..c8dcd14 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -1,13 +1,13 @@ package dataprocessor import ( + "crypto/sha256" "encoding/base64" "fmt" "log" "strconv" "strings" "sync" - "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/decoder" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" @@ -138,11 +138,14 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei if block.Result.Block.Data.Txs != nil { for _, tx := range *block.Result.Block.Data.Txs { txHash, err := base64.StdEncoding.DecodeString(tx) + // turn to sha256 and then turn it to raw bytes to + // match the transaction hash + txHashSha256 := sha256.Sum256(txHash) if err != nil { log.Printf("Failed to decode tx hash %s: %v", tx, err) continue } - txs = append(txs, txHash) + txs = append(txs, txHashSha256[:]) } } @@ -187,7 +190,7 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei // // The method will not throw an error if the transactions are not found, it will just return nil func (d *DataProcessor) ProcessTransactions( - transactions map[*rpcClient.TxResponse]time.Time, + transactions []TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) { @@ -196,21 +199,20 @@ func (d *DataProcessor) ProcessTransactions( wg := sync.WaitGroup{} wg.Add(len(transactions)) - for transaction, timestamp := range transactions { + for _, transaction := range transactions { go func( - transaction *rpcClient.TxResponse, - timestamp time.Time, + transaction TrasnactionsData, compressEvents bool) { defer wg.Done() - txResult := transaction.Result.TxResult + txResult := transaction.Response.Result.TxResult - decodedMsg := decoder.NewDecodedMsg(transaction.Result.Tx) + decodedMsg := decoder.NewDecodedMsg(transaction.Response.Result.Tx) fee := decodedMsg.GetFee() msgTypes := decodedMsg.GetMsgTypes() // convert the tx hash from base64 to sha256 - txHash, err := base64.StdEncoding.DecodeString(transaction.GetHash()) + txHash, err := base64.StdEncoding.DecodeString(transaction.Response.GetHash()) if err != nil { return } @@ -226,7 +228,7 @@ func (d *DataProcessor) ProcessTransactions( } // solve the events - events, err := EventSolver(transaction, compressEvents) + events, err := EventSolver(transaction.Response, compressEvents) if err != nil { return } @@ -235,7 +237,8 @@ func (d *DataProcessor) ProcessTransactions( transactionChan <- &sqlDataTypes.TransactionGeneral{ TxHash: txHash, ChainName: d.chainName, - Timestamp: timestamp, + Timestamp: transaction.Timestamp, + BlockHeight: transaction.BlockHeight, MsgTypes: msgTypes, TxEvents: events.GetNativeEvents(), TxEventsCompressed: events.GetCompressedData(), @@ -245,7 +248,7 @@ func (d *DataProcessor) ProcessTransactions( Fee: fee, } - }(transaction, timestamp, compressEvents) + }(transaction, compressEvents) } go func() { @@ -278,7 +281,7 @@ func (d *DataProcessor) ProcessTransactions( // Returns: // - error: if processing fails func (d *DataProcessor) ProcessMessages( - transactions map[*rpcClient.TxResponse]time.Time, + transactions []TrasnactionsData, fromHeight uint64, toHeight uint64) error { @@ -289,10 +292,10 @@ func (d *DataProcessor) ProcessMessages( wg1.Add(len(transactions)) // Launch goroutines to collect addresses concurrently - for transaction := range transactions { - go func(transaction *rpcClient.TxResponse) { + for _, transaction := range transactions { + go func(transaction TrasnactionsData) { defer wg1.Done() - decodedMsg := decoder.NewDecodedMsg(transaction.Result.Tx) + decodedMsg := decoder.NewDecodedMsg(transaction.Response.Result.Tx) if decodedMsg == nil { addressCollectionChan <- []*decoder.DecodedMsg{nil} return @@ -344,41 +347,41 @@ func (d *DataProcessor) ProcessMessages( // Launch goroutines to process messages concurrently txIndex := 0 - for transaction, timestamp := range transactions { + for _, transaction := range transactions { if txIndex >= len(allDecodedMsgs) { wg2.Done() continue } - go func(transaction *rpcClient.TxResponse, timestamp time.Time, decodedMsg *decoder.DecodedMsg) { + go func(transaction TrasnactionsData, decodedMsg *decoder.DecodedMsg) { defer wg2.Done() if decodedMsg == nil { // There might be an error here // but any kind of retry mechanism will probably not help // log it for. - log.Printf("The transaction couldn't be decoded, tx hash: %s", transaction.GetHash()) + log.Printf("The transaction couldn't be decoded, tx hash: %s", transaction.Response.GetHash()) resultChan <- processedResult{nil, nil} return } // Convert directly to database-ready messages with address IDs - txHash, err := base64.StdEncoding.DecodeString(transaction.GetHash()) + txHash, err := base64.StdEncoding.DecodeString(transaction.Response.GetHash()) if err != nil { - log.Printf("Failed to decode tx hash %s: %v", transaction.GetHash(), err) + log.Printf("Failed to decode tx hash %s: %v", transaction.Response.GetHash(), err) resultChan <- processedResult{nil, err} return } - dbMessageGroups, err := decodedMsg.ConvertToDbMessages(d.addressCache, txHash, d.chainName, timestamp, decodedMsg.GetSigners()) + dbMessageGroups, err := decodedMsg.ConvertToDbMessages(d.addressCache, txHash, d.chainName, transaction.Timestamp, decodedMsg.GetSigners()) if err != nil { - log.Printf("Failed to convert messages for tx %s: %v", transaction.GetHash(), err) + log.Printf("Failed to convert messages for tx %s: %v", transaction.Response.GetHash(), err) resultChan <- processedResult{nil, err} return } resultChan <- processedResult{dbMessageGroups, nil} - }(transaction, timestamp, allDecodedMsgs[txIndex]) + }(transaction, allDecodedMsgs[txIndex]) txIndex++ } diff --git a/indexer/data_processor/types.go b/indexer/data_processor/types.go index affcb0d..906f04c 100644 --- a/indexer/data_processor/types.go +++ b/indexer/data_processor/types.go @@ -1,6 +1,9 @@ package dataprocessor import ( + "time" + + rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" sqlDataTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types" ) @@ -28,3 +31,9 @@ type DataProcessor struct { validatorCache AddressCache chainName string } + +type TrasnactionsData struct { + Response *rpcClient.TxResponse + Timestamp time.Time + BlockHeight uint64 +} diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go index 95176af..dd7cdc9 100644 --- a/indexer/orchestrator/operator.go +++ b/indexer/orchestrator/operator.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config" + dataprocessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" ) @@ -241,10 +242,14 @@ func (or *Orchestrator) updateProgressMetrics(chunkStart, chunkEnd uint64, block // collectTransactionsFromBlocks extracts all transactions from blocks and queries them concurrently // This mimics the Python _process_historical_block_chunk behavior -func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockResponse) map[*rpcClient.TxResponse]time.Time { +func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockResponse) []dataprocessor.TrasnactionsData { // Collect all transaction hashes from all blocks var allTxHashes []string - blockTimestamps := make(map[string]time.Time) // txHash -> block timestamp + blockTxData := make([]struct { + txHash string + blockHeight uint64 + timestamp time.Time + }, 0) for _, block := range blocks { if block == nil { @@ -265,13 +270,26 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR } txHashSha256 := sha256.Sum256(txHashBytes) txHashFinal := base64.StdEncoding.EncodeToString(txHashSha256[:]) + blockHeight, err := block.GetHeight() + if err != nil { + log.Printf("Failed to get block height: %v", err) + continue + } allTxHashes = append(allTxHashes, txHashFinal) - blockTimestamps[txHashFinal] = block.GetTimestamp() + blockTxData = append(blockTxData, struct { + txHash string + blockHeight uint64 + timestamp time.Time + }{ + txHash: txHashFinal, + blockHeight: blockHeight, + timestamp: block.GetTimestamp(), + }) } } if len(allTxHashes) == 0 { - return make(map[*rpcClient.TxResponse]time.Time) + return make([]dataprocessor.TrasnactionsData, 0) } log.Printf("Fetching %d transactions concurrently", len(allTxHashes)) @@ -280,17 +298,23 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR transactions := or.queryOperator.GetTransactions(allTxHashes) // Build map of transactions with their timestamps - txMap := make(map[*rpcClient.TxResponse]time.Time) + txData := make([]dataprocessor.TrasnactionsData, 0) for _, tx := range transactions { if tx != nil { - if timestamp, exists := blockTimestamps[tx.GetHash()]; exists { - txMap[tx] = timestamp + for _, blockTx := range blockTxData { + if blockTx.txHash == tx.GetHash() { + txData = append(txData, dataprocessor.TrasnactionsData{ + Response: tx, + Timestamp: blockTx.timestamp, + BlockHeight: blockTx.blockHeight, + }) + } } } } - log.Printf("Successfully collected %d valid transactions", len(txMap)) - return txMap + log.Printf("Successfully collected %d valid transactions", len(txData)) + return txData } // This function processes all data using optimized concurrent execution @@ -308,7 +332,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // The method will not throw an error if the data is not found, it will just return nil func (or *Orchestrator) processAllConcurrently( blocks []*rpcClient.BlockResponse, - transactions map[*rpcClient.TxResponse]time.Time, + transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) error { diff --git a/indexer/orchestrator/operator_test.go b/indexer/orchestrator/operator_test.go index 938fcf8..6e96d90 100644 --- a/indexer/orchestrator/operator_test.go +++ b/indexer/orchestrator/operator_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config" + dataprocessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/orchestrator" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" ) @@ -35,12 +36,12 @@ func (m *MockDataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fro } // Mock method for ProcessTransactions -func (m *MockDataProcessor) ProcessTransactions(transactions map[*rpcClient.TxResponse]time.Time, compressEvents bool, fromHeight uint64, toHeight uint64) { +func (m *MockDataProcessor) ProcessTransactions(transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) { m.ProcessTransactionsCalled = true } // Mock method for ProcessMessages -func (m *MockDataProcessor) ProcessMessages(transactions map[*rpcClient.TxResponse]time.Time, fromHeight uint64, toHeight uint64) error { +func (m *MockDataProcessor) ProcessMessages(transactions []dataprocessor.TrasnactionsData, fromHeight uint64, toHeight uint64) error { m.ProcessMessagesCalled = true return m.ProcessMessagesError } diff --git a/indexer/orchestrator/types.go b/indexer/orchestrator/types.go index 4a0732a..0264708 100644 --- a/indexer/orchestrator/types.go +++ b/indexer/orchestrator/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config" + dataprocessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" ) @@ -11,8 +12,8 @@ import ( type DataProcessor interface { ProcessValidatorAddresses(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) - ProcessTransactions(transactions map[*rpcClient.TxResponse]time.Time, compressEvents bool, fromHeight uint64, toHeight uint64) - ProcessMessages(transactions map[*rpcClient.TxResponse]time.Time, fromHeight uint64, toHeight uint64) error + ProcessTransactions(transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) + ProcessMessages(transactions []dataprocessor.TrasnactionsData, fromHeight uint64, toHeight uint64) error ProcessValidatorSignings(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) } diff --git a/pkgs/database/insert.go b/pkgs/database/insert.go index 3b0ba99..de696d3 100644 --- a/pkgs/database/insert.go +++ b/pkgs/database/insert.go @@ -139,6 +139,7 @@ func (t *TimescaleDb) InsertTransactionsGeneral(transactionsGeneral []sql_data_t transactionsGeneral[i].TxHash, transactionsGeneral[i].ChainName, transactionsGeneral[i].Timestamp, + transactionsGeneral[i].BlockHeight, makePgxArray(transactionsGeneral[i].MsgTypes), makePgxArray(transactionsGeneral[i].TxEvents), transactionsGeneral[i].TxEventsCompressed, diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go new file mode 100644 index 0000000..be55555 --- /dev/null +++ b/pkgs/database/queries_api.go @@ -0,0 +1,100 @@ +package database + +import "context" + +// GetBlock gets a block from the database for a given height and chain name +// +// Usage: +// +// # Used to get a block from the database for a given height and chain name +// +// Args: +// - height: the height of the block +// - chainName: the name of the chain +// +// Returns: +// - *BlockData: the block data +// - error: if the query fails +func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, error) { + query := ` + SELECT encode(hash, 'base64'), + height, + timestamp, + chain_id, + (SELECT array_agg(upper(encode(tx, 'base64'))) + FROM unnest(blocks.txs) AS tx + ) AS txs + FROM blocks + WHERE height = $1 + AND chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, height, chainName) + var block BlockData + err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) + if err != nil { + return nil, err + } + return &block, nil +} + +// GetAllIntAddresses gets all the addresses from the database for a given chain name. +// Unlike the address cache for the indexer, the difference here is that map here is of int32 showing +// the string address. It should be the same as GetAllAddresses but with the key being the int32 address. +// +// Usage: +// +// # Used to get all the addresses from the database for a given chain name +// +// Args: +// - chainName: the name of the chain +// - searchValidators: whether to search for validators or accounts +// - highestIndex: the highest index of the addresses already recorded or it could be a 0 +// +// Returns: +// - map[int32]string: the map of all addresses and their ids +// - error: if the query fails +func (t *TimescaleDb) GetAllIntAddresses( + chainName string, + searchValidators bool, + highestIndex *int32) (map[int32]string, int32, error) { + addressesMap := make(map[int32]string) + var maxIndex int32 = 0 + if highestIndex != nil { + maxIndex = *highestIndex + } + query := "" + if searchValidators { + query += ` + SELECT address, id + FROM gno_validators + WHERE chain_name = $1 + AND id > $2 + ` + } else { + query += ` + SELECT address, id + FROM gno_addresses + WHERE chain_name = $1 + AND id > $2 + ` + } + rows, err := t.pool.Query(context.Background(), query, chainName, highestIndex) + if err != nil { + return nil, 0, err + } + defer rows.Close() + + for rows.Next() { + var address string + var id int32 + err := rows.Scan(&address, &id) + if err != nil { + return nil, 0, err + } + addressesMap[id] = address + if id > maxIndex { + maxIndex = id + } + } + return addressesMap, maxIndex, nil +} diff --git a/pkgs/database/queries.go b/pkgs/database/queries_indexer.go similarity index 86% rename from pkgs/database/queries.go rename to pkgs/database/queries_indexer.go index e4fe3f2..ec486f6 100644 --- a/pkgs/database/queries.go +++ b/pkgs/database/queries_indexer.go @@ -2,8 +2,6 @@ package database import ( "context" - - "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types" ) // FindExistingAccounts finds the existing accounts in the database @@ -70,13 +68,19 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, // Args: // // - chainName: the name of the chain +// - searchValidators: whether to search for validators or accounts +// - highestIndex: the highest index of the addresses already recorded or it could be a 0 // // Returns: // // - map[string]int32: the map of all accounts and their ids // - error: if the query fails -func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool) (map[string]int32, error) { +func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) { addressesMap := make(map[string]int32) + var maxIndex int32 = 0 + if highestIndex != nil { + maxIndex = *highestIndex + } // we need to check if we are searching for validators or accounts query := "" if searchValidators { @@ -84,17 +88,19 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool) ( SELECT address, id FROM gno_validators WHERE chain_name = $1 + AND id > $2 ` } else { query += ` SELECT address, id FROM gno_addresses WHERE chain_name = $1 + AND id > $2 ` } - rows, err := t.pool.Query(context.Background(), query, chainName) + rows, err := t.pool.Query(context.Background(), query, chainName, highestIndex) if err != nil { - return nil, err + return nil, 0, err } defer rows.Close() @@ -103,11 +109,14 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool) ( var id int32 err := rows.Scan(&address, &id) if err != nil { - return nil, err + return nil, 0, err } addressesMap[address] = id + if id > maxIndex { + maxIndex = id + } } - return addressesMap, nil + return addressesMap, maxIndex, nil } // CheckCurrentDatabaseName checks the current database name @@ -159,18 +168,3 @@ func (t *TimescaleDb) GetLastBlockHeight(chainName string) (uint64, error) { } return lastBlockHeight, nil } - -func (t *TimescaleDb) GetBlock(height uint64) (*sql_data_types.Blocks, error) { - query := ` - SELECT encode(hash, 'base64'), height, timestamp, chain_id, txs, chain_name - FROM blocks - WHERE height = $1 - ` - row := t.pool.QueryRow(context.Background(), query, height) - var block sql_data_types.Blocks - err := row.Scan(&block) - if err != nil { - return nil, err - } - return &block, nil -} diff --git a/pkgs/database/types.go b/pkgs/database/types.go index 7efaf97..b53b5dd 100644 --- a/pkgs/database/types.go +++ b/pkgs/database/types.go @@ -29,3 +29,13 @@ type DatabasePoolConfig struct { PoolHealthCheckPeriod time.Duration PoolMaxConnLifetimeJitter time.Duration } + +// BlockData represents the actual block data returned in the response body +type BlockData struct { + Hash string `json:"hash" doc:"Block hash (hex-encoded)"` + Height uint64 `json:"height" doc:"Block height"` + Timestamp time.Time `json:"timestamp" doc:"Block timestamp"` + ChainID string `json:"chain_id" doc:"Chain identifier"` + Txs []string `json:"txs" doc:"Transactions (base64 encoded)"` + TxCount int `json:"tx_count" doc:"Number of transactions in the block"` +} diff --git a/pkgs/sql_data_types/table.go b/pkgs/sql_data_types/table.go index 45cf37a..9413ae3 100644 --- a/pkgs/sql_data_types/table.go +++ b/pkgs/sql_data_types/table.go @@ -227,10 +227,11 @@ func (at AddressTx) TableColumns() []string { // But what kind of data will be stored should be managed by the config. // It is not recommended to use both modes at the same time. type TransactionGeneral struct { - TxHash []byte `db:"tx_hash" dbtype:"bytea" nullable:"false" primary:"true"` - ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` - Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"` - MsgTypes []string `db:"msg_types" dbtype:"TEXT[]" nullable:"false" primary:"false"` + TxHash []byte `db:"tx_hash" dbtype:"bytea" nullable:"false" primary:"true"` + ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` + Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"` + BlockHeight uint64 `db:"block_height" dbtype:"bigint" nullable:"false" primary:"false"` + MsgTypes []string `db:"msg_types" dbtype:"TEXT[]" nullable:"false" primary:"false"` // tx events in the future there should be an option to have this compressed // for now only store the native format but keep the option to have it compressed TxEvents []Event `db:"tx_events" dbtype:"event[]" nullable:"true" primary:"false"` From d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e Mon Sep 17 00:00:00 2001 From: nman98 Date: Sat, 4 Oct 2025 10:38:03 +0200 Subject: [PATCH 2/6] Fixed a bug where the msg_types would insert a empty string, added more queries for the api and added handlers for some api queries --- api/handlers/blocks.go | 67 ++++++++ api/handlers/transactions.go | 95 ++++++++++++ api/huma-types/block.go | 24 +++ api/huma-types/transaction.go | 15 ++ experiments/experiment11/main.go | 19 +++ indexer/decoder/product.go | 2 +- pkgs/database/queries_api.go | 253 +++++++++++++++++++++++++------ pkgs/database/types.go | 10 -- pkgs/database/types_api.go | 85 +++++++++++ 9 files changed, 514 insertions(+), 56 deletions(-) create mode 100644 api/handlers/blocks.go create mode 100644 api/handlers/transactions.go create mode 100644 api/huma-types/block.go create mode 100644 api/huma-types/transaction.go create mode 100644 experiments/experiment11/main.go create mode 100644 pkgs/database/types_api.go diff --git a/api/handlers/blocks.go b/api/handlers/blocks.go new file mode 100644 index 0000000..00f1fb0 --- /dev/null +++ b/api/handlers/blocks.go @@ -0,0 +1,67 @@ +package handlers + +import ( + "context" + "fmt" + + humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types" + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" + "github.com/danielgtaylor/huma/v2" +) + +// BlocksHandler handles block-related API requests +type BlocksHandler struct { + db *database.TimescaleDb + chainName string +} + +// NewBlocksHandler creates a new blocks handler +func NewBlocksHandler(db *database.TimescaleDb, chainName string) *BlocksHandler { + return &BlocksHandler{db: db, chainName: chainName} +} + +// GetBlock retrieves a block by height +func (h *BlocksHandler) GetBlock(ctx context.Context, input *humatypes.BlockGetInput) (*humatypes.BlockGetOutput, error) { + // Fetch from database + block, err := h.db.GetBlock(input.Height, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Block at height %d not found", input.Height), err) + } + + response := &humatypes.BlockGetOutput{ + Body: database.BlockData{ + Hash: block.Hash, + Height: block.Height, + Timestamp: block.Timestamp, + ChainID: block.ChainID, + Txs: block.Txs, + TxCount: len(block.Txs), + }, + } + return response, nil +} + +func (h *BlocksHandler) GetFromToBlocks( + ctx context.Context, + input *humatypes.FromToBlocksGetInput, +) (*humatypes.FromToBlocksGetOutput, error) { + // Fetch from database + blocks, err := h.db.GetFromToBlocks(input.FromHeight, input.ToHeight, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Blocks from height %d to height %d not found", input.FromHeight, input.ToHeight), err) + } + response := &humatypes.FromToBlocksGetOutput{ + Body: make([]database.BlockData, 0, len(blocks)), + } + for _, block := range blocks { + response.Body = append(response.Body, database.BlockData{ + Hash: block.Hash, + Height: block.Height, + Timestamp: block.Timestamp, + ChainID: block.ChainID, + Txs: block.Txs, + TxCount: len(block.Txs), + }) + } + return response, nil +} diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go new file mode 100644 index 0000000..e2b7bca --- /dev/null +++ b/api/handlers/transactions.go @@ -0,0 +1,95 @@ +package handlers + +import ( + "context" + "encoding/base64" + "fmt" + "log" + "strings" + + humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types" + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" + "github.com/danielgtaylor/huma/v2" +) + +type TransactionsHandler struct { + db *database.TimescaleDb + chainName string +} + +func NewTransactionsHandler(db *database.TimescaleDb, chainName string) *TransactionsHandler { + return &TransactionsHandler{db: db, chainName: chainName} +} + +func (h *TransactionsHandler) GetTransactionBasic( + ctx context.Context, + input *humatypes.TransactionGetInput, +) (*humatypes.TransactionBasicGetOutput, error) { + input.TxHash = strings.Trim(input.TxHash, " ") + txHash, err := base64.URLEncoding.DecodeString(input.TxHash) + log.Println("txHash base64url", input.TxHash) + txHashBase64 := base64.StdEncoding.EncodeToString(txHash) + log.Println("txHash base64", txHashBase64) + if err != nil { + return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) + } + transaction, err := h.db.GetTransaction(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + return &humatypes.TransactionBasicGetOutput{ + Body: *transaction, + }, nil +} + +func (h *TransactionsHandler) GetTransactionMessage( + ctx context.Context, + input *humatypes.TransactionGetInput, +) (*humatypes.TransactionMessageGetOutput, error) { + input.TxHash = strings.Trim(input.TxHash, " ") + txHash, err := base64.URLEncoding.DecodeString(input.TxHash) + txHashBase64 := base64.StdEncoding.EncodeToString(txHash) + if err != nil { + return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) + } + msgType, err := h.db.GetMsgType(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + log.Println("msgType", msgType) + switch msgType { + case "bank_msg_send": + data, err := h.db.GetBankSend(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + return &humatypes.TransactionMessageGetOutput{ + Body: *data, + }, nil + case "vm_msg_call": + data, err := h.db.GetMsgCall(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + return &humatypes.TransactionMessageGetOutput{ + Body: *data, + }, nil + case "vm_msg_add_package": + data, err := h.db.GetMsgAddPackage(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + return &humatypes.TransactionMessageGetOutput{ + Body: *data, + }, nil + case "vm_msg_run": + data, err := h.db.GetMsgRun(txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + return &humatypes.TransactionMessageGetOutput{ + Body: *data, + }, nil + } + return nil, huma.Error400BadRequest("Transaction message type not found", nil) +} diff --git a/api/huma-types/block.go b/api/huma-types/block.go new file mode 100644 index 0000000..9f50fe2 --- /dev/null +++ b/api/huma-types/block.go @@ -0,0 +1,24 @@ +package humatypes + +import ( + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" +) + +// BlockGetInput represents the input for getting a block by height +type BlockGetInput struct { + Height uint64 `path:"height" minimum:"1" example:"12345" doc:"Block height to retrieve" required:"true"` +} + +type FromToBlocksGetInput struct { + FromHeight uint64 `path:"from_height" minimum:"1" example:"12345" doc:"From block height" required:"true"` + ToHeight uint64 `path:"to_height" minimum:"1" example:"12345" doc:"To block height" required:"true"` +} + +// BlockGetOutput represents the response structure for a single block +type BlockGetOutput struct { + Body database.BlockData +} + +type FromToBlocksGetOutput struct { + Body []database.BlockData +} diff --git a/api/huma-types/transaction.go b/api/huma-types/transaction.go new file mode 100644 index 0000000..63bd00b --- /dev/null +++ b/api/huma-types/transaction.go @@ -0,0 +1,15 @@ +package humatypes + +import "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" + +type TransactionGetInput struct { + TxHash string `path:"tx_hash" doc:"Transaction hash (base64url encoded)" required:"true"` +} + +type TransactionBasicGetOutput struct { + Body database.Transaction +} + +type TransactionMessageGetOutput struct { + Body any // database.MsgRun | database.MsgCall | database.MsgAddPackage | database.BankSend +} diff --git a/experiments/experiment11/main.go b/experiments/experiment11/main.go new file mode 100644 index 0000000..0fffdeb --- /dev/null +++ b/experiments/experiment11/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "encoding/base64" + "fmt" + "log" +) + +func main() { + fmt.Println("Enter base64 to encode to base64url: ") + var input string + fmt.Scanln(&input) + base64Decoded, err := base64.StdEncoding.DecodeString(input) + if err != nil { + log.Fatal(err) + } + base64url := base64.URLEncoding.EncodeToString(base64Decoded) + fmt.Println("Base64url: ", base64url) +} diff --git a/indexer/decoder/product.go b/indexer/decoder/product.go index 5bc3601..12cbd54 100644 --- a/indexer/decoder/product.go +++ b/indexer/decoder/product.go @@ -58,7 +58,7 @@ func (dm *DecodedMsg) GetMessages() []map[string]any { // // The method will not throw an error if the message types are not found, it will just return nil func (dm *DecodedMsg) GetMsgTypes() []string { - msgTypes := make([]string, len(dm.Messages)) + msgTypes := make([]string, 0, len(dm.Messages)) for _, message := range dm.Messages { msgTypes = append(msgTypes, message["msg_type"].(string)) } diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go index be55555..1c1588c 100644 --- a/pkgs/database/queries_api.go +++ b/pkgs/database/queries_api.go @@ -1,6 +1,9 @@ package database -import "context" +import ( + "context" + "log" +) // GetBlock gets a block from the database for a given height and chain name // @@ -37,64 +40,224 @@ func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, err return &block, nil } -// GetAllIntAddresses gets all the addresses from the database for a given chain name. -// Unlike the address cache for the indexer, the difference here is that map here is of int32 showing -// the string address. It should be the same as GetAllAddresses but with the key being the int32 address. +// GetFromToBlocks gets a range of blocks from the database for a given height range and chain name // // Usage: // -// # Used to get all the addresses from the database for a given chain name +// # Used to get a range of blocks from the database for a given height range and chain name // // Args: +// - fromHeight: the starting height of the block +// - toHeight: the ending height of the block (inclusive) // - chainName: the name of the chain -// - searchValidators: whether to search for validators or accounts -// - highestIndex: the highest index of the addresses already recorded or it could be a 0 // // Returns: -// - map[int32]string: the map of all addresses and their ids +// - []*BlockData: the range of block data // - error: if the query fails -func (t *TimescaleDb) GetAllIntAddresses( - chainName string, - searchValidators bool, - highestIndex *int32) (map[int32]string, int32, error) { - addressesMap := make(map[int32]string) - var maxIndex int32 = 0 - if highestIndex != nil { - maxIndex = *highestIndex - } - query := "" - if searchValidators { - query += ` - SELECT address, id - FROM gno_validators - WHERE chain_name = $1 - AND id > $2 - ` - } else { - query += ` - SELECT address, id - FROM gno_addresses - WHERE chain_name = $1 - AND id > $2 - ` - } - rows, err := t.pool.Query(context.Background(), query, chainName, highestIndex) +func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) { + query := ` + SELECT encode(hash, 'base64'), + height, + timestamp, + chain_id, + (SELECT array_agg(upper(encode(tx, 'base64'))) + FROM unnest(blocks.txs) AS tx + ) AS txs + FROM blocks + WHERE height >= $1 AND height <= $2 + AND chain_name = $3 + ` + rows, err := t.pool.Query(context.Background(), query, fromHeight, toHeight, chainName) if err != nil { - return nil, 0, err + return nil, err } defer rows.Close() - + blocks := make([]*BlockData, 0) for rows.Next() { - var address string - var id int32 - err := rows.Scan(&address, &id) + var block BlockData + err := rows.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) if err != nil { - return nil, 0, err - } - addressesMap[id] = address - if id > maxIndex { - maxIndex = id + return nil, err } + blocks = append(blocks, &block) + } + return blocks, nil +} + +func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, error) { + query := ` + SELECT + encode(bms.tx_hash, 'base64') AS tx_hash, + bms.timestamp, + gn_from.address AS from_address, + gn_to.address AS to_address, + bms.amount, + array( + SELECT gn.address + FROM unnest(bms.signers) AS signer_id + JOIN gno_addresses gn ON gn.id = signer_id + ) AS signers + FROM bank_msg_send bms + LEFT JOIN gno_addresses gn_from ON bms.from_address = gn_from.id + LEFT JOIN gno_addresses gn_to ON bms.to_address = gn_to.id + WHERE bms.tx_hash = decode($1, 'base64') + AND bms.chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var bankSend BankSend + err := row.Scan(&bankSend.TxHash, &bankSend.Timestamp, &bankSend.FromAddress, &bankSend.ToAddress, &bankSend.Amount, &bankSend.Signers) + if err != nil { + return nil, err + } + return &bankSend, nil +} + +func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, error) { + query := ` + SELECT + encode(vmc.tx_hash, 'base64') AS tx_hash, + vmc.timestamp, + gn.address AS caller, + vmc.pkg_path, + vmc.func_name, + vmc.args, + vmc.send, + vmc.max_deposit, + array( + SELECT gn.address + FROM unnest(vmc.signers) AS signer_id + JOIN gno_addresses gn ON gn.id = signer_id + ) AS signers + FROM vm_msg_call vmc + LEFT JOIN gno_addresses gn ON vmc.caller = gn.id + WHERE vmc.tx_hash = decode($1, 'base64') + AND vmc.chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var msgCall MsgCall + err := row.Scan(&msgCall.TxHash, &msgCall.Timestamp, &msgCall.Caller, &msgCall.PkgPath, &msgCall.FuncName, &msgCall.Args, &msgCall.Send, &msgCall.MaxDeposit, &msgCall.Signers) + if err != nil { + return nil, err + } + return &msgCall, nil +} + +func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAddPackage, error) { + query := ` + SELECT + encode(vmap.tx_hash, 'base64') AS tx_hash, + vmap.timestamp, + gn.address AS creator, + vmap.pkg_path, + vmap.pkg_name, + vmap.pkg_file_names, + vmap.send, + vmap.max_deposit, + array( + SELECT gn.address + FROM unnest(vmap.signers) AS signer_id + JOIN gno_addresses gn ON gn.id = signer_id + ) AS signers + FROM vm_msg_add_package vmap + LEFT JOIN gno_addresses gn ON vmap.creator = gn.id + WHERE vmap.tx_hash = decode($1, 'base64') + AND vmap.chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var msgAddPackage MsgAddPackage + err := row.Scan(&msgAddPackage.TxHash, &msgAddPackage.Timestamp, &msgAddPackage.Creator, &msgAddPackage.PkgPath, &msgAddPackage.PkgName, &msgAddPackage.PkgFileNames, &msgAddPackage.Send, &msgAddPackage.MaxDeposit, &msgAddPackage.Signers) + if err != nil { + return nil, err + } + return &msgAddPackage, nil +} + +func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error) { + query := ` + SELECT + encode(vmr.tx_hash, 'base64') AS tx_hash, + vmr.timestamp, + gn.address AS caller, + vmr.pkg_path, + vmr.pkg_name, + vmr.pkg_file_names, + vmr.send, + vmr.max_deposit, + array( + SELECT gn.address + FROM unnest(vmr.signers) AS signer_id + JOIN gno_addresses gn ON gn.id = signer_id + ) AS signers + FROM vm_msg_run vmr + LEFT JOIN gno_addresses gn ON vmr.caller = gn.id + WHERE vmr.tx_hash = decode($1, 'base64') + AND vmr.chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var msgRun MsgRun + err := row.Scan(&msgRun.TxHash, &msgRun.Timestamp, &msgRun.Caller, &msgRun.PkgPath, &msgRun.PkgName, &msgRun.PkgFileNames, &msgRun.Send, &msgRun.MaxDeposit, &msgRun.Signers) + if err != nil { + log.Println("error getting msg run", err) + return nil, err + } + return &msgRun, nil +} + +func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transaction, error) { + var messageType []string + query := ` + SELECT + encode(tx.tx_hash, 'base64') AS tx_hash, + tx.timestamp, + tx.block_height, + tx.tx_events, + tx.gas_used, + tx.gas_wanted, + tx.fee, + tx.msg_types + FROM transaction_general tx + WHERE tx.tx_hash = decode($1, 'base64') + AND tx.chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var transaction Transaction + err := row.Scan( + &transaction.TxHash, + &transaction.Timestamp, + &transaction.BlockHeight, + &transaction.TxEvents, + &transaction.GasUsed, + &transaction.GasWanted, + &transaction.Fee, + &messageType, + ) + if err != nil { + log.Println("error getting transaction", err) + return nil, err + } + return &transaction, nil +} + +func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error) { + query := ` + SELECT msg_types + FROM transaction_general + WHERE tx_hash = decode($1, 'base64') + AND chain_name = $2 + ` + row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + var msgType []string + // to future me + // in the events the transactions can harbor more transaction types this + // will not work, for now I only have seen with one message type per transaction + // if this happens at least throw some log warning + if len(msgType) >= 2 { + log.Println("warning: transaction has more than one message type", msgType) + return msgType[0], nil + } + err := row.Scan(&msgType) + if err != nil { + return "", err } - return addressesMap, maxIndex, nil + return msgType[0], nil } diff --git a/pkgs/database/types.go b/pkgs/database/types.go index b53b5dd..7efaf97 100644 --- a/pkgs/database/types.go +++ b/pkgs/database/types.go @@ -29,13 +29,3 @@ type DatabasePoolConfig struct { PoolHealthCheckPeriod time.Duration PoolMaxConnLifetimeJitter time.Duration } - -// BlockData represents the actual block data returned in the response body -type BlockData struct { - Hash string `json:"hash" doc:"Block hash (hex-encoded)"` - Height uint64 `json:"height" doc:"Block height"` - Timestamp time.Time `json:"timestamp" doc:"Block timestamp"` - ChainID string `json:"chain_id" doc:"Chain identifier"` - Txs []string `json:"txs" doc:"Transactions (base64 encoded)"` - TxCount int `json:"tx_count" doc:"Number of transactions in the block"` -} diff --git a/pkgs/database/types_api.go b/pkgs/database/types_api.go new file mode 100644 index 0000000..27a02b1 --- /dev/null +++ b/pkgs/database/types_api.go @@ -0,0 +1,85 @@ +package database + +import "time" + +// BlockData represents the actual block data returned in the response body +type BlockData struct { + Hash string `json:"hash" doc:"Block hash (base64 encoded)"` + Height uint64 `json:"height" doc:"Block height"` + Timestamp time.Time `json:"timestamp" doc:"Block timestamp"` + ChainID string `json:"chain_id" doc:"Chain identifier"` + Txs []string `json:"txs" doc:"Transactions (base64 encoded)"` + TxCount int `json:"tx_count" doc:"Number of transactions in the block"` +} + +type Event struct { + AtType string `json:"at_type" doc:"Event type"` + Type string `json:"type" doc:"Event type"` + Attributes []Attribute `json:"attributes" doc:"Event attributes"` + PkgPath string `json:"pkg_path" doc:"Package path"` +} + +type Attribute struct { + Key string `json:"key" doc:"Attribute key"` + Value string `json:"value" doc:"Attribute value"` +} + +type Amount struct { + Amount string `json:"amount" doc:"Amount"` + Denom string `json:"denom" doc:"Denom"` +} + +type BankSend struct { + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + FromAddress string `json:"from_address" doc:"From address (addresses)"` + ToAddress string `json:"to_address" doc:"To address (addresses)"` + Amount []Amount `json:"amount" doc:"Amount"` + Signers []string `json:"signers" doc:"Signers (addresses)"` +} + +type MsgCall struct { + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + Send []Amount `json:"send" doc:"Send amount"` + PkgPath string `json:"pkg_path" doc:"Package path"` + FuncName string `json:"func_name" doc:"Function name"` + Args string `json:"args" doc:"Arguments"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` +} + +type MsgAddPackage struct { + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Creator string `json:"creator" doc:"Creator address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` +} + +type MsgRun struct { + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` +} + +type Transaction struct { + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + BlockHeight uint64 `json:"block_height" doc:"Block height"` + TxEvents []Event `json:"tx_events" doc:"Transaction events"` + GasUsed uint64 `json:"gas_used" doc:"Gas used"` + GasWanted uint64 `json:"gas_wanted" doc:"Gas wanted"` + Fee Amount `json:"fee" doc:"Fee"` +} From f2a39d65c5622e0a5953d1f6113ab5eea1996cad Mon Sep 17 00:00:00 2001 From: nman98 Date: Sat, 4 Oct 2025 13:25:41 +0200 Subject: [PATCH 3/6] Added main.go for the api, added config example for api and added more data for the env example, fixed versioning --- .env.example | 21 +++- .github/workflows/release.yml | 5 +- .gitignore | 5 + Dockerfile | 7 +- Makefile | 19 +++- api/config/loader.go | 70 +++++++++++++ api/config/types.go | 29 ++++++ api/handlers/transactions.go | 4 - api/main.go | 169 +++++++++++++++++++++++++++++++ config-api.yml.example | 11 ++ experiments/experiment11/main.go | 5 +- go.mod | 1 + go.sum | 2 + indexer/main.go | 6 +- 14 files changed, 335 insertions(+), 19 deletions(-) create mode 100644 api/config/loader.go create mode 100644 api/config/types.go create mode 100644 api/main.go create mode 100644 config-api.yml.example diff --git a/.env.example b/.env.example index 16dbf7e..3a00667 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,24 @@ # Environment example use your own values +# just do not use these values unless you are testing or developing DB_HOST=127.0.0.1 DB_PORT=5432 -DB_USER=postgres +DB_USER=writer DB_SSLMODE=disable DB_PASSWORD=12345678 -DB_NAME=gnoland \ No newline at end of file +DB_NAME=gnoland + +# Envs for the API +# If you plan to use the API than use this values +API_DB_HOST=127.0.0.1 +API_DB_PORT=5432 +API_DB_USER=reader +API_DB_SSLMODE=disable +API_DB_PASSWORD=12345678 +API_DB_NAME=gnoland + +API_DB_POOL_MAX_CONNS=50 +API_DB_POOL_MIN_CONNS=10 +API_DB_POOL_MAX_CONN_LIFETIME=10s +API_DB_POOL_MAX_CONN_IDLE_TIME=5m +API_DB_POOL_HEALTH_CHECK_PERIOD=1m +API_DB_POOL_MAX_CONN_LIFETIME_JITTER=1m \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index beee1a1..660e6fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,8 +40,11 @@ jobs: run: | mkdir -p build + # Get git commit hash + GIT_COMMIT=$(git rev-parse --short HEAD) + # Linux AMD64 - GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Version=${{ steps.version.outputs.version }}" -o build/indexer indexer/main.go + GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Commit=${GIT_COMMIT} -X main.Version=${{ steps.version.outputs.version }}" -o build/indexer-linux-amd64 indexer/main.go - name: Create GitHub Release uses: softprops/action-gh-release@v1 diff --git a/.gitignore b/.gitignore index bc68b22..f2b4099 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,12 @@ go.work.sum config.yml test_config.yml .env +config-api.yml # State dumps and diagnostics directories state_dumps/ diagnostics/ + +# Cert files +server.crt +server.key diff --git a/Dockerfile b/Dockerfile index dfa8968..e2bb99f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,10 @@ WORKDIR /app COPY . . -RUN go build -o indexer indexer/main.go - -RUN chmod +x indexer +ARG VERSION=unknown +RUN GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \ + go build -ldflags="-X main.Commit=${GIT_COMMIT} -X main.Version=${VERSION}" -o indexer indexer/main.go && \ + chmod +x indexer FROM debian:trixie-slim diff --git a/Makefile b/Makefile index b003f1d..b69a6b4 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,20 @@ -.PHONY: build install clean build-experimental install-experimental +.PHONY: build install clean build-experimental install-experimental build-api + +# Get git information +GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +GIT_TAG := $(shell git describe --tags --exact-match 2>/dev/null || echo "") +VERSION := $(if $(GIT_TAG),$(GIT_TAG),dev-$(GIT_COMMIT)) build: mkdir -p build - go build -o build/indexer indexer/main.go + go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer indexer/main.go install: - cd indexer && go install ./... + cd indexer && go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" + +build-api: + mkdir -p build + go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/api api/main.go clean: rm -rf build @@ -13,9 +22,9 @@ clean: # experimental build with greentea garbage collection # use at your own risk build-experimental: - GOEXPERIMENT=greenteagc go build -o build/indexer-tea indexer/main.go + GOEXPERIMENT=greenteagc go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer-tea indexer/main.go # experimental install with greentea garbage collection # use at your own risk install-experimental: - cd indexer && GOEXPERIMENT=greenteagc go install ./... \ No newline at end of file + cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" \ No newline at end of file diff --git a/api/config/loader.go b/api/config/loader.go new file mode 100644 index 0000000..a9b3f28 --- /dev/null +++ b/api/config/loader.go @@ -0,0 +1,70 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/caarlos0/env/v6" + "github.com/joho/godotenv" + "go.yaml.in/yaml/v4" +) + +func LoadConfig(path string) (*ApiConfig, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config ApiConfig + err = yaml.Unmarshal(yamlFile, &config) + if err != nil { + return nil, err + } + // check if any of the fields are empty + if config.Host == "" { + config.Host = "localhost" + } + if config.Port == 0 { + config.Port = 8080 + } + // any cors method should be auto filled by the cors middleware + // TODO: maybe add more options later + return &config, nil +} + +func LoadEnvironment(path string) (*ApiEnv, error) { + possibleEnvFiles := []string{ + ".env", // accept only .env for now + } + // check if any of the files exist within the current path + existingFiles := []string{} + for _, envFile := range possibleEnvFiles { + formPath := filepath.Join(path, envFile) + if _, err := os.Stat(formPath); err == nil { + existingFiles = append(existingFiles, formPath) + } + } + // if there are multiple files, decide which has highest priority + // 1. production , 2. development, 3. local, 4. default + // only use the regular .env for now return to this laster + if len(existingFiles) == 0 { + fmt.Println("No environment file found. Searching for os environment variables.") + } else if len(existingFiles) == 1 { + absPath, err := filepath.Abs(existingFiles[0]) + if err != nil { + return nil, err + } + err = godotenv.Load(absPath) + if err != nil { + return nil, fmt.Errorf("error loading .env file: %w", err) + } + fmt.Printf("Loaded environment variables from %s\n", absPath) + } + + environment := ApiEnv{} + if err := env.Parse(&environment); err != nil { + return nil, fmt.Errorf("failed to parse environment variables: %w", err) + } + + return &environment, nil +} diff --git a/api/config/types.go b/api/config/types.go new file mode 100644 index 0000000..5fb6466 --- /dev/null +++ b/api/config/types.go @@ -0,0 +1,29 @@ +package config + +import "time" + +type ApiConfig struct { + // Basic connection info + Host string `yaml:"host"` + Port int `yaml:"port"` + // CORS config + CorsAllowedOrigins []string `yaml:"cors_allowed_origins"` + CorsAllowedMethods []string `yaml:"cors_allowed_methods"` + CorsAllowedHeaders []string `yaml:"cors_allowed_headers"` + CorsMaxAge int `yaml:"cors_max_age"` +} + +type ApiEnv struct { + ApiDbHost string `env:"API_DB_HOST" envDefault:"localhost"` + ApiDbPort int `env:"API_DB_PORT" envDefault:"5432"` + ApiDbUser string `env:"API_DB_USER" envDefault:"postgres"` + ApiDbPassword string `env:"API_DB_PASSWORD" envDefault:"12345678"` + ApiDbName string `env:"API_DB_NAME" envDefault:"gnoland"` + ApiDbSslmode string `env:"API_DB_SSLMODE" envDefault:"disable"` + ApiDbPoolMaxConns int `env:"API_DB_POOL_MAX_CONNS" envDefault:"50"` + ApiDbPoolMinConns int `env:"API_DB_POOL_MIN_CONNS" envDefault:"10"` + ApiDbPoolMaxConnLifetime time.Duration `env:"API_DB_POOL_MAX_CONN_LIFETIME" envDefault:"10m"` + ApiDbPoolMaxConnIdleTime time.Duration `env:"API_DB_POOL_MAX_CONN_IDLE_TIME" envDefault:"5m"` + ApiDbPoolHealthCheckPeriod time.Duration `env:"API_DB_POOL_HEALTH_CHECK_PERIOD" envDefault:"1m"` + ApiDbPoolMaxConnLifetimeJitter time.Duration `env:"API_DB_POOL_MAX_CONN_LIFETIME_JITTER" envDefault:"1m"` +} diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go index e2b7bca..abf8cf3 100644 --- a/api/handlers/transactions.go +++ b/api/handlers/transactions.go @@ -4,7 +4,6 @@ import ( "context" "encoding/base64" "fmt" - "log" "strings" humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types" @@ -27,9 +26,7 @@ func (h *TransactionsHandler) GetTransactionBasic( ) (*humatypes.TransactionBasicGetOutput, error) { input.TxHash = strings.Trim(input.TxHash, " ") txHash, err := base64.URLEncoding.DecodeString(input.TxHash) - log.Println("txHash base64url", input.TxHash) txHashBase64 := base64.StdEncoding.EncodeToString(txHash) - log.Println("txHash base64", txHashBase64) if err != nil { return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) } @@ -56,7 +53,6 @@ func (h *TransactionsHandler) GetTransactionMessage( if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - log.Println("msgType", msgType) switch msgType { case "bank_msg_send": data, err := h.db.GetBankSend(txHashBase64, h.chainName) diff --git a/api/main.go b/api/main.go new file mode 100644 index 0000000..cf34bdb --- /dev/null +++ b/api/main.go @@ -0,0 +1,169 @@ +package main + +import ( + "fmt" + "log" + "net/http" + + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/config" + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/handlers" + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" + "github.com/danielgtaylor/huma/v2" + "github.com/danielgtaylor/huma/v2/adapters/humachi" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/cors" + "github.com/spf13/cobra" + // "github.com/go-chi/httprate" +) + +var ( + Commit = "unknown" // Set via ldflags at build time + Version = "unknown" // Set via ldflags at build time +) + +var rootCmd = &cobra.Command{ + Use: "api", + Short: "Spectra Gnoland Indexer API", + Long: "API for the Spectra Gnoland Indexer", + Version: Version + " (commit: " + Commit + ")", + Run: func(cmd *cobra.Command, args []string) { + var configPath string + var err error + var certFilePath string + var keyFilePath string + + // check all of the flags + + // check config file + configPath, err = cmd.Flags().GetString("config") + if err != nil { + log.Fatalf("failed to get config path: %v", err) + } + conf, err := config.LoadConfig(configPath) + if err != nil { + log.Fatalf("failed to load config: %v", err) + } + + env, err := config.LoadEnvironment(".") + if err != nil { + log.Fatalf("failed to load environment: %v", err) + } + + // check cert file and key file + certFilePath, err = cmd.Flags().GetString("cert-file") + if err != nil { + log.Fatalf("failed to get cert file path: %v", err) + } + keyFilePath, err = cmd.Flags().GetString("key-file") + if err != nil { + log.Fatalf("failed to get key file path: %v", err) + } + + // Setup router and middleware + router := chi.NewMux() + router.Use(middleware.Logger) + router.Use(middleware.Recoverer) + router.Use(middleware.CleanPath) + router.Use(middleware.Compress(5, "application/json")) + // heartbeat route + router.Use(middleware.Heartbeat("/")) + + // Configure CORS from config file + corsOptions := cors.Options{ + AllowedOrigins: conf.CorsAllowedOrigins, + AllowedMethods: conf.CorsAllowedMethods, + AllowedHeaders: conf.CorsAllowedHeaders, + MaxAge: conf.CorsMaxAge, + } + // Set defaults if not provided + if len(corsOptions.AllowedOrigins) == 0 { + corsOptions.AllowedOrigins = []string{"*"} + } + if len(corsOptions.AllowedMethods) == 0 { + corsOptions.AllowedMethods = []string{"GET"} + } + if len(corsOptions.AllowedHeaders) == 0 { + corsOptions.AllowedHeaders = []string{"Origin", "Content-Type", "Accept", "Origin"} + } + if corsOptions.MaxAge == 0 { + corsOptions.MaxAge = 600 + } + router.Use(cors.Handler(corsOptions)) + + /* rate limiting middleware + I think it would be easier to implement the rate limit to some reverse proxy like nginx, apache, etc. + but I will leave it here for now + if you do decide you want more control over the rate limiting, to use the chi rate limiting middleware + just remove the comment lines and build it with the chi rate limiting middleware + and adjust the rate limit and time window as you see fit + TODO: add proper documentation for this and maybe add it as a option in the config file? + router.Use(httprate.LimitByIP(100, 1*time.Minute)) + */ + + api := humachi.New(router, huma.DefaultConfig("Spectra Gnoland Indexer API", Version)) + + api.OpenAPI().Info.Title = "Spectra Gnoland Indexer API" + api.OpenAPI().Info.Version = Version + api.OpenAPI().Info.Description = "API for the Spectra Gnoland Indexer" + + // Initialize database connection from environment variables + db := database.NewTimescaleDb(database.DatabasePoolConfig{ + Host: env.ApiDbHost, + Port: env.ApiDbPort, + User: env.ApiDbUser, + Password: env.ApiDbPassword, + Dbname: env.ApiDbName, + Sslmode: env.ApiDbSslmode, + PoolMaxConns: env.ApiDbPoolMaxConns, + PoolMinConns: env.ApiDbPoolMinConns, + PoolMaxConnLifetime: env.ApiDbPoolMaxConnLifetime, + PoolMaxConnIdleTime: env.ApiDbPoolMaxConnIdleTime, + PoolHealthCheckPeriod: env.ApiDbPoolHealthCheckPeriod, + PoolMaxConnLifetimeJitter: env.ApiDbPoolMaxConnLifetimeJitter, + }) + + // Initialize handlers with dependencies + blocksHandler := handlers.NewBlocksHandler(db, env.ApiDbName) + transactionsHandler := handlers.NewTransactionsHandler(db, env.ApiDbName) + + // Register Block API routes + huma.Get(api, "/block/{height}", blocksHandler.GetBlock) + huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks) + + // Register Transaction API routes + huma.Get(api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic) + huma.Get(api, "/transaction/{tx_hash}/message", transactionsHandler.GetTransactionMessage) + + // Start server using config values + addr := fmt.Sprintf("%s:%d", conf.Host, conf.Port) + log.Printf("Starting server on %s", addr) + + // if cert file and key file are provided, use https + if certFilePath != "" && keyFilePath != "" { + err = http.ListenAndServeTLS(addr, certFilePath, keyFilePath, router) + if err != nil { + log.Fatalf("failed to start server: %v", err) + } + } else { + err = http.ListenAndServe(addr, router) + if err != nil { + log.Fatalf("failed to start server: %v", err) + } + } + + }, +} + +func init() { + rootCmd.PersistentFlags().StringP("config", "c", "config-api.yml", "config file path") + rootCmd.PersistentFlags().StringP("cert-file", "t", "", "certificate file path") + rootCmd.PersistentFlags().StringP("key-file", "k", "", "key file path") + +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatalf("failed to execute command: %v", err) + } +} diff --git a/config-api.yml.example b/config-api.yml.example new file mode 100644 index 0000000..f484b80 --- /dev/null +++ b/config-api.yml.example @@ -0,0 +1,11 @@ +host: 127.0.0.1 +port: 8080 +cors_allowed_origins: + - "*" +cors_allowed_methods: + - "GET" +cors_allowed_headers: + - "Origin" + - "Content-Type" + - "Accept" +cors_max_age: 600 \ No newline at end of file diff --git a/experiments/experiment11/main.go b/experiments/experiment11/main.go index 0fffdeb..fa423fc 100644 --- a/experiments/experiment11/main.go +++ b/experiments/experiment11/main.go @@ -9,7 +9,10 @@ import ( func main() { fmt.Println("Enter base64 to encode to base64url: ") var input string - fmt.Scanln(&input) + _, err := fmt.Scanln(&input) + if err != nil { + log.Fatal(err) + } base64Decoded, err := base64.StdEncoding.DecodeString(input) if err != nil { log.Fatal(err) diff --git a/go.mod b/go.mod index 6c0abb1..2d71f44 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/danielgtaylor/huma/v2 v2.34.1 github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd github.com/go-chi/chi/v5 v5.2.3 + github.com/go-chi/cors v1.2.2 github.com/jackc/pgx/v5 v5.7.6 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.18.0 diff --git a/go.sum b/go.sum index 5c22eff..9f7acd6 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd h1:r/uiCyhbsQU+cs51/nf github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd/go.mod h1:j9wKq29meqwktEj2ReqPbSkeYUwoisfxHHVeV20lhtw= github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= +github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/indexer/main.go b/indexer/main.go index 68d80cc..81dfa24 100644 --- a/indexer/main.go +++ b/indexer/main.go @@ -10,15 +10,15 @@ import ( ) var ( - Commit = "unknown" // try to get from git else leave unknown - Version = "unknown; commit: " + Commit // try to get from git else leave unknown + Commit = "unknown" // Set via ldflags at build time + Version = "unknown" // Set via ldflags at build time ) var rootCmd = &cobra.Command{ Use: "indexer", Short: "Spectra Gnoland Indexer", Long: "A blockchain indexer for Gnoland that processes blocks and transactions.", - Version: Version, + Version: Version + " (commit: " + Commit + ")", } var liveCmd = &cobra.Command{ From 90bef0ec5a0bff468d4a1d6771b82706029f4ea9 Mon Sep 17 00:00:00 2001 From: nman98 Date: Sat, 4 Oct 2025 15:50:58 +0200 Subject: [PATCH 4/6] Small name change for the sql data tables for the validator signing, added address txs query and get all validator signers --- api/config/types.go | 1 + api/handlers/address.go | 31 +++++++ api/handlers/blocks.go | 24 +++++ api/handlers/transactions.go | 67 +++++++++++--- api/huma-types/address.go | 17 ++++ api/huma-types/block.go | 8 ++ api/huma-types/transaction.go | 44 ++++++++- api/main.go | 13 ++- config-api.yml.example | 3 +- pkgs/database/queries_api.go | 167 ++++++++++++++++++++++++++++++++++ pkgs/database/types_api.go | 12 +++ pkgs/sql_data_types/table.go | 2 +- 12 files changed, 368 insertions(+), 21 deletions(-) create mode 100644 api/handlers/address.go create mode 100644 api/huma-types/address.go diff --git a/api/config/types.go b/api/config/types.go index 5fb6466..ca9fd41 100644 --- a/api/config/types.go +++ b/api/config/types.go @@ -11,6 +11,7 @@ type ApiConfig struct { CorsAllowedMethods []string `yaml:"cors_allowed_methods"` CorsAllowedHeaders []string `yaml:"cors_allowed_headers"` CorsMaxAge int `yaml:"cors_max_age"` + ChainName string `yaml:"chain_name"` } type ApiEnv struct { diff --git a/api/handlers/address.go b/api/handlers/address.go new file mode 100644 index 0000000..3e75a3d --- /dev/null +++ b/api/handlers/address.go @@ -0,0 +1,31 @@ +package handlers + +import ( + "context" + + humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types" + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" + "github.com/danielgtaylor/huma/v2" +) + +type AddressHandler struct { + db *database.TimescaleDb + chainName string +} + +func NewAddressHandler(db *database.TimescaleDb, chainName string) *AddressHandler { + return &AddressHandler{db: db, chainName: chainName} +} + +func (h *AddressHandler) GetAddressTxs( + ctx context.Context, + input *humatypes.AddressGetInput, +) (*humatypes.AddressGetOutput, error) { + address, err := h.db.GetAddressTxs(input.Address, h.chainName, input.FromTimestamp, input.ToTimestamp) + if err != nil { + return nil, huma.Error404NotFound("Address not found", err) + } + return &humatypes.AddressGetOutput{ + Body: *address, + }, nil +} diff --git a/api/handlers/blocks.go b/api/handlers/blocks.go index 00f1fb0..c57f816 100644 --- a/api/handlers/blocks.go +++ b/api/handlers/blocks.go @@ -41,11 +41,16 @@ func (h *BlocksHandler) GetBlock(ctx context.Context, input *humatypes.BlockGetI return response, nil } +// Get from block height a to block height b func (h *BlocksHandler) GetFromToBlocks( ctx context.Context, input *humatypes.FromToBlocksGetInput, ) (*humatypes.FromToBlocksGetOutput, error) { // Fetch from database + // validate input + if input.FromHeight > input.ToHeight { + return nil, huma.Error400BadRequest("From height must be less than to height", nil) + } blocks, err := h.db.GetFromToBlocks(input.FromHeight, input.ToHeight, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Blocks from height %d to height %d not found", input.FromHeight, input.ToHeight), err) @@ -65,3 +70,22 @@ func (h *BlocksHandler) GetFromToBlocks( } return response, nil } + +func (h *BlocksHandler) GetAllBlockSigners( + ctx context.Context, + input *humatypes.AllBlockSignersGetInput, +) (*humatypes.AllBlockSignersGetOutput, error) { + // Fetch from database + blockSigners, err := h.db.GetAllBlockSigners(h.chainName, input.BlockHeight) + if err != nil { + return nil, huma.Error404NotFound("Block signers not found", err) + } + response := &humatypes.AllBlockSignersGetOutput{ + Body: database.BlockSigners{ + BlockHeight: blockSigners.BlockHeight, + Proposer: blockSigners.Proposer, + SignedVals: blockSigners.SignedVals, + }, + } + return response, nil +} diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go index abf8cf3..345180d 100644 --- a/api/handlers/transactions.go +++ b/api/handlers/transactions.go @@ -53,39 +53,80 @@ func (h *TransactionsHandler) GetTransactionMessage( if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } + + var message humatypes.TransactionMessage + switch msgType { case "bank_msg_send": data, err := h.db.GetBankSend(txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - return &humatypes.TransactionMessageGetOutput{ - Body: *data, - }, nil + message = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + FromAddress: data.FromAddress, + ToAddress: data.ToAddress, + Amount: data.Amount, + } case "vm_msg_call": data, err := h.db.GetMsgCall(txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - return &humatypes.TransactionMessageGetOutput{ - Body: *data, - }, nil + message = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + Send: data.Send, + PkgPath: data.PkgPath, + FuncName: data.FuncName, + Args: data.Args, + MaxDeposit: data.MaxDeposit, + } case "vm_msg_add_package": data, err := h.db.GetMsgAddPackage(txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - return &humatypes.TransactionMessageGetOutput{ - Body: *data, - }, nil + message = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Creator: data.Creator, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } case "vm_msg_run": data, err := h.db.GetMsgRun(txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - return &humatypes.TransactionMessageGetOutput{ - Body: *data, - }, nil + message = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } + default: + return nil, huma.Error400BadRequest("Transaction message type not found", nil) } - return nil, huma.Error400BadRequest("Transaction message type not found", nil) + + return &humatypes.TransactionMessageGetOutput{ + Body: message, + }, nil } diff --git a/api/huma-types/address.go b/api/huma-types/address.go new file mode 100644 index 0000000..ad5ec42 --- /dev/null +++ b/api/huma-types/address.go @@ -0,0 +1,17 @@ +package humatypes + +import ( + "time" + + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" +) + +type AddressGetInput struct { + Address string `path:"address" doc:"Address" required:"true"` + FromTimestamp time.Time `query:"from_timestamp" doc:"From timestamp" format:"2025-01-01T00:00:00+00:00 or 2025-01-01T00:00:00Z" required:"true"` + ToTimestamp time.Time `query:"to_timestamp" doc:"To timestamp" format:"2025-01-01T00:00:00+00:00 or 2025-01-01T00:00:00Z" required:"true"` +} + +type AddressGetOutput struct { + Body []database.AddressTx +} diff --git a/api/huma-types/block.go b/api/huma-types/block.go index 9f50fe2..07fa577 100644 --- a/api/huma-types/block.go +++ b/api/huma-types/block.go @@ -22,3 +22,11 @@ type BlockGetOutput struct { type FromToBlocksGetOutput struct { Body []database.BlockData } + +type AllBlockSignersGetInput struct { + BlockHeight uint64 `path:"block_height" minimum:"1" example:"12345" doc:"Block height" required:"true"` +} + +type AllBlockSignersGetOutput struct { + Body database.BlockSigners +} diff --git a/api/huma-types/transaction.go b/api/huma-types/transaction.go index 63bd00b..1e90743 100644 --- a/api/huma-types/transaction.go +++ b/api/huma-types/transaction.go @@ -1,15 +1,53 @@ package humatypes -import "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" +import ( + "time" + + "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" +) type TransactionGetInput struct { - TxHash string `path:"tx_hash" doc:"Transaction hash (base64url encoded)" required:"true"` + // tx hash needs to be exactly 44 characters long + TxHash string `path:"tx_hash" minLength:"44" maxLength:"44" doc:"Transaction hash (base64url encoded)" required:"true"` } type TransactionBasicGetOutput struct { Body database.Transaction } +// TransactionMessage represents a unified transaction message type that can be one of: +// bank_msg_send, vm_msg_call, vm_msg_add_package, or vm_msg_run +// not maybe the best implementation, but this one works for now +// to future me, if you figure out a better way to do this, please do so +// for now this is good enough +type TransactionMessage struct { + // Common fields (always present) + MessageType string `json:"message_type" doc:"Type of message: bank_msg_send, vm_msg_call, vm_msg_add_package, or vm_msg_run" enum:"bank_msg_send,vm_msg_call,vm_msg_add_package,vm_msg_run"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Signers []string `json:"signers" doc:"Signers (addresses)"` + + // BankSend specific fields + FromAddress string `json:"from_address,omitempty" doc:"From address (only for bank_msg_send)"` + ToAddress string `json:"to_address,omitempty" doc:"To address (only for bank_msg_send)"` + Amount []database.Amount `json:"amount,omitempty" doc:"Amount (only for bank_msg_send)"` + + // MsgCall specific fields + Caller string `json:"caller,omitempty" doc:"Caller address (for vm_msg_call and vm_msg_run)"` + FuncName string `json:"func_name,omitempty" doc:"Function name (only for vm_msg_call)"` + Args string `json:"args,omitempty" doc:"Arguments (only for vm_msg_call)"` + + // MsgAddPackage and MsgRun specific fields + Creator string `json:"creator,omitempty" doc:"Creator address (only for vm_msg_add_package)"` + PkgName string `json:"pkg_name,omitempty" doc:"Package name (for vm_msg_add_package and vm_msg_run)"` + PkgFileNames []string `json:"pkg_file_names,omitempty" doc:"Package file names (for vm_msg_add_package and vm_msg_run)"` + + // Shared fields for vm_* messages + PkgPath string `json:"pkg_path,omitempty" doc:"Package path (for vm_msg_call, vm_msg_add_package, and vm_msg_run)"` + Send []database.Amount `json:"send,omitempty" doc:"Send amount (for vm_msg_call, vm_msg_add_package, and vm_msg_run)"` + MaxDeposit []database.Amount `json:"max_deposit,omitempty" doc:"Max deposit (for vm_msg_call, vm_msg_add_package, and vm_msg_run)"` +} + type TransactionMessageGetOutput struct { - Body any // database.MsgRun | database.MsgCall | database.MsgAddPackage | database.BankSend + Body TransactionMessage } diff --git a/api/main.go b/api/main.go index cf34bdb..a0326ab 100644 --- a/api/main.go +++ b/api/main.go @@ -84,7 +84,7 @@ var rootCmd = &cobra.Command{ corsOptions.AllowedMethods = []string{"GET"} } if len(corsOptions.AllowedHeaders) == 0 { - corsOptions.AllowedHeaders = []string{"Origin", "Content-Type", "Accept", "Origin"} + corsOptions.AllowedHeaders = []string{"Origin", "Content-Type", "Accept"} } if corsOptions.MaxAge == 0 { corsOptions.MaxAge = 600 @@ -124,17 +124,22 @@ var rootCmd = &cobra.Command{ }) // Initialize handlers with dependencies - blocksHandler := handlers.NewBlocksHandler(db, env.ApiDbName) - transactionsHandler := handlers.NewTransactionsHandler(db, env.ApiDbName) + blocksHandler := handlers.NewBlocksHandler(db, conf.ChainName) + transactionsHandler := handlers.NewTransactionsHandler(db, conf.ChainName) + addressHandler := handlers.NewAddressHandler(db, conf.ChainName) // Register Block API routes huma.Get(api, "/block/{height}", blocksHandler.GetBlock) huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks) + huma.Get(api, "/blocks/{block_height}/signers", blocksHandler.GetAllBlockSigners) // Register Transaction API routes huma.Get(api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic) huma.Get(api, "/transaction/{tx_hash}/message", transactionsHandler.GetTransactionMessage) + // Register Address API routes + huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs) + // Start server using config values addr := fmt.Sprintf("%s:%d", conf.Host, conf.Port) log.Printf("Starting server on %s", addr) @@ -142,11 +147,13 @@ var rootCmd = &cobra.Command{ // if cert file and key file are provided, use https if certFilePath != "" && keyFilePath != "" { err = http.ListenAndServeTLS(addr, certFilePath, keyFilePath, router) + log.Printf("Starting server on %s with HTTPS", addr) if err != nil { log.Fatalf("failed to start server: %v", err) } } else { err = http.ListenAndServe(addr, router) + log.Printf("Starting server on %s with HTTP", addr) if err != nil { log.Fatalf("failed to start server: %v", err) } diff --git a/config-api.yml.example b/config-api.yml.example index f484b80..b432523 100644 --- a/config-api.yml.example +++ b/config-api.yml.example @@ -8,4 +8,5 @@ cors_allowed_headers: - "Origin" - "Content-Type" - "Accept" -cors_max_age: 600 \ No newline at end of file +cors_max_age: 600 +chain_name: gnoland \ No newline at end of file diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go index 1c1588c..c4a5f17 100644 --- a/pkgs/database/queries_api.go +++ b/pkgs/database/queries_api.go @@ -3,6 +3,7 @@ package database import ( "context" "log" + "time" ) // GetBlock gets a block from the database for a given height and chain name @@ -84,6 +85,56 @@ func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainN return blocks, nil } +// GetAllBlockSigners gets all of the validators that signed that block + the proposer +// +// Usage: +// +// # Used to get all of the validators that signed that block + the proposer +// +// Args: +// - chainName: the name of the chain +// - blockHeight: the height of the block +// +// Returns: +// - *BlockSigners: the block signers +// - error: if the query fails +func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) (*BlockSigners, error) { + query := ` + SELECT + vb.block_height, + gv.address AS proposer, + array( + SELECT gv.address + FROM unnest(vb.signed_vals) AS signed_val_id + JOIN gno_validators gv ON gv.id = signed_val_id + ) AS signed_vals + FROM validator_block_signing vb + LEFT JOIN gno_validators gv ON vb.proposer = gv.id + WHERE vb.chain_name = $1 + AND vb.block_height = $2 + ` + row := t.pool.QueryRow(context.Background(), query, chainName, blockHeight) + var blockSigners BlockSigners + err := row.Scan(&blockSigners.BlockHeight, &blockSigners.Proposer, &blockSigners.SignedVals) + if err != nil { + return nil, err + } + return &blockSigners, nil +} + +// GetBankSend gets the bank send message for a given transaction hash +// +// Usage: +// +// # Used to get the bank send message for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - *BankSend: the bank send message +// - error: if the query fails func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, error) { query := ` SELECT @@ -112,6 +163,19 @@ func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, e return &bankSend, nil } +// GetMsgCall gets the msg call message for a given transaction hash +// +// Usage: +// +// # Used to get the msg call message for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - *MsgCall: the msg call message +// - error: if the query fails func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, error) { query := ` SELECT @@ -142,6 +206,19 @@ func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, err return &msgCall, nil } +// GetMsgAddPackage gets the msg add package message for a given transaction hash +// +// Usage: +// +// # Used to get the msg add package message for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - *MsgAddPackage: the msg add package message +// - error: if the query fails func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAddPackage, error) { query := ` SELECT @@ -172,6 +249,19 @@ func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAdd return &msgAddPackage, nil } +// GetMsgRun gets the msg run message for a given transaction hash +// +// Usage: +// +// # Used to get the msg run message for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - *MsgRun: the msg run message +// - error: if the query fails func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error) { query := ` SELECT @@ -203,6 +293,19 @@ func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error return &msgRun, nil } +// GetTransaction gets the transaction for a given transaction hash +// +// Usage: +// +// # Used to get the transaction for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - *Transaction: the transaction +// - error: if the query fails func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transaction, error) { var messageType []string query := ` @@ -238,6 +341,19 @@ func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transact return &transaction, nil } +// GetMsgType gets the message type for a given transaction hash +// +// Usage: +// +// # Used to get the message type for a given transaction hash +// +// Args: +// - txHash: the hash of the transaction +// - chainName: the name of the chain +// +// Returns: +// - string: the message type +// - error: if the query fails func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error) { query := ` SELECT msg_types @@ -261,3 +377,54 @@ func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error } return msgType[0], nil } + +// GetAddressTxs gets the transactions for a given address for a certain time period +// +// Usage: +// +// # Used to get the transactions for a given address for a certain time period +// +// Args: +// - address: the address +// - chainName: the name of the chain +// - fromTimestamp: the starting timestamp +// - toTimestamp: the ending timestamp +// +// Returns: +// - []*AddressTx: the transactions +// - error: if the query fails +func (t *TimescaleDb) GetAddressTxs( + address string, + chainName string, + fromTimestamp time.Time, + toTimestamp time.Time, +) (*[]AddressTx, error) { + query := ` + SELECT + encode(tx.tx_hash, 'base64') AS tx_hash, + tx.timestamp, + tx.msg_types + FROM address_tx tx + WHERE tx.address = (SELECT id FROM gno_addresses WHERE address = $1 AND chain_name = $2) + AND tx.chain_name = $2 + AND tx.timestamp >= $3 + AND tx.timestamp <= $4 + ` + + addressTxs := make([]AddressTx, 0) + + rows, err := t.pool.Query(context.Background(), query, address, chainName, fromTimestamp, toTimestamp) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var addressTx AddressTx + err := rows.Scan(&addressTx.Hash, &addressTx.Timestamp, &addressTx.MsgTypes) + if err != nil { + return nil, err + } + addressTxs = append(addressTxs, addressTx) + } + return &addressTxs, nil +} diff --git a/pkgs/database/types_api.go b/pkgs/database/types_api.go index 27a02b1..15166cc 100644 --- a/pkgs/database/types_api.go +++ b/pkgs/database/types_api.go @@ -83,3 +83,15 @@ type Transaction struct { GasWanted uint64 `json:"gas_wanted" doc:"Gas wanted"` Fee Amount `json:"fee" doc:"Fee"` } + +type BlockSigners struct { + BlockHeight uint64 `json:"block_height" doc:"Block height"` + Proposer string `json:"proposer" doc:"Proposer (addresses)"` + SignedVals []string `json:"signed_vals" doc:"Signed validators (addresses)"` +} + +type AddressTx struct { + Hash string `json:"hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + MsgTypes []string `json:"msg_types" doc:"Message types"` +} diff --git a/pkgs/sql_data_types/table.go b/pkgs/sql_data_types/table.go index 9413ae3..9057f92 100644 --- a/pkgs/sql_data_types/table.go +++ b/pkgs/sql_data_types/table.go @@ -140,7 +140,7 @@ type ValidatorBlockSigning struct { BlockHeight uint64 `db:"block_height" dbtype:"bigint" nullable:"false" primary:"true"` Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"` Proposer int32 `db:"proposer" dbtype:"integer" nullable:"false" primary:"false"` - SignedVals []int32 `db:"address" dbtype:"integer[]" nullable:"false" primary:"false"` + SignedVals []int32 `db:"signed_vals" dbtype:"integer[]" nullable:"false" primary:"false"` ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` // use type enum chain_name from postgres // MissedVals []int32 `db:"missed_vals" dbtype:"integer" nullable:"false" primary:"false"` // can't confirm who is in the active set without making a smart contract query to the DAO smart contract From b2c20d222e4f52b74b333974a7930df3cddad29e Mon Sep 17 00:00:00 2001 From: nman98 Date: Sat, 4 Oct 2025 18:30:50 +0200 Subject: [PATCH 5/6] Added more docs and polished the existing ones --- README.md | 47 ++++++++++------------- docs/README.md | 4 +- docs/api.md | 91 ++++++++++++++++++++++++++++++++++++++++++++ docs/benchmarks.md | 23 +++++++++++ docs/requirements.md | 72 +++++++++++++++++++++++++++++++++++ docs/requriements.md | 43 --------------------- 6 files changed, 210 insertions(+), 70 deletions(-) create mode 100644 docs/api.md create mode 100644 docs/benchmarks.md create mode 100644 docs/requirements.md delete mode 100644 docs/requriements.md diff --git a/README.md b/README.md index 7b69b08..623ae80 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@
Spectra Gnoland Indexer
+

This project is still in development. Future releases might have some breaking changes.

The Spectra Gnoland Indexer(SGI) is a tool that records the data from the Gnoland blockchain and stores the data in the timeseries database, Timescale DB. This indexer will be a part of the Spectra explorer and will be used to @@ -9,19 +10,19 @@ store the data for the explorer. This program can be used for any kind of data/a The biggest problem when dealing with the blockchain data is how to store the data in a way that is easy to query and analyze. Some projects rely on the direct access to the data from the blockchain nodes and the problem in this -case is that the nodes are meant for multiple purposes. They usually provide at least one, sometimes even more, -endpoints for querying the data, then this node need to be able to work with other nodes and to acquire the block -data from other nodes via P2P network. Most nodes can be used as a miners/validators that need to sign the blocks, -and maintain the blockchain state. While they are the first point of contact for data and contact with the chain -they are not most efficient when it comes to making some queries easy and fast. Plus the nodes are made to be fast -and efficient when it comes to preformance of the nodes, so they are not most efficient when it comes to storing -the data. +case is that the nodes are meant for multiple purposes. + +They usually provide at least one, sometimes even more, endpoints for querying the data, then this node need to be +able to work with other nodes and to acquire the block data from other nodes via P2P network. Most nodes can be +used as a miners/validators that need to sign the blocks. Even if these nodes are only to be used as a RPC nodes +they can still provide the data but depending on the underlying SDK, tech stack and other factors it might be +not the best choice. And finally the nodes are not the best when it comes to storing the data and scalability. ## Table of content - [Table of content](#table-of-content) - [The solution](#the-solution) - [Why TimescaleDB? And can it work on other SQL databases?](#why-timescaledb-and-can-it-work-on-other-sql-databases) -- [The indexer and how it works](#the-indexer-and-how-it-works) +- [How does the indexer work?](#how-does-the-indexer-work) - [What is indexed and what is not](#what-is-indexed-and-what-is-not) - [Blocks:](#blocks) - [Validator signings:](#validator-signings) @@ -53,6 +54,12 @@ TimescaleDB (Tiger Data is their commercial version) is a extension of the Postg store the time series data in a way that is easy to query and analyze. It also has a lot of features that can extend the capabilities of the PostgreSQL and make it more powerful. +The TimescaleDB has also feature of it's own that is not present in the Postgres. The user could extend the indexer +by adding the data aggregation features, automatic jobs scheduling, hyperfunctions and more. + +This database extension also has a data compression feature that can be used to reduce the storage space and +segment the data into smaller chunks by using time based intervals making the queries faster. + Technically speaking the indexer can work on Postgres database, however you would need to create the tables and types manually. This might be added later in the future if there is a demand for it but for now the focus is on the TimescaleDB. @@ -65,21 +72,7 @@ There are some other SQL databases that could work in theory. For them to work t If all of the above are met then the indexer can work on them. It might work on CockroachDB for example but this is out of the scope of this project. Maybe in the future it might be an interesting idea to support it. -There were some other contenders for the database choice. They were: InfluxDB and QuestDB. - -At the time when the cosmos indexer was made the InfluxDB had a opensource version 2 but at the time it was lacking -some of the features that are now present in the version 3. Maybe it could have been used for the indexer but at -the time it seemed to be not the best choice. - -QuestDB was a interesting choice and it is probably even better choice than TimescaleDB. But it does have some -limitations but nothing that would make it impossible to use it. However with the TimescaleDB it is pretty much an -extention of Postgres and it is more mature and has more features and it can be extended with any other extensions -that are available for the Postgres. - -The TimescaleDB has also feature of it's own that is not present in the Postgres. The user could extend the indexer -by adding the data aggregation features, automatic jobs scheduling, hyperfunctions and more. - -## The indexer and how it works +## How does the indexer work? The indexer has 2 main modes of operation: - Live mode @@ -136,6 +129,7 @@ Stored data: - Validator block signing height - Validator block signing timestamp - Validator block signing signed validators +- Proposer address Not stored data: - Missed validators @@ -149,11 +143,12 @@ VM message Add Package and Call where in theory one could extract even the body ### 🦾 Pros: -- The indexer process the data using goroutines and channels, which can provide a faster processing of the data. Some early test on the real data showed about 2vCPU can index 10K blocks in about 2m30s while the 4vCPU can index 10K blocks in about 1m30s. +- The indexer process the data using goroutines and channels, which can provide a faster processing of the data. +- Fast data processing. [see benchmarks](./docs/benchmarks.md) - The program has 2 modes that can be used for the indexing of the data. This can be useful for the testing, partial indexing of the chain or gradual indexing of the chain. - The data is stored ready to be used for any kind of analytics and visualization with any programming language. - No need to deal with Amino encoding and decoding for the messagess as the indexer decodes the messages and stores them in the database. -- It comes with a REST API to get you started quickly.(To be added in the future) +- It comes with a REST API to get you started quickly. - It relies on a SQL database, which can provide a easier experience for any user that is familiar with the SQL. - It uses TimescaleDB, a PostgresSQL extenstion that can be extended with any other extensions, plus the TimescaleDB has a lot of features that are not present in the Postgres. @@ -165,4 +160,4 @@ VM message Add Package and Call where in theory one could extract even the body ## In depth documentation -For more detailed documentation, please refer to the [docs](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/blob/main/docs/README.md) directory. \ No newline at end of file +For more detailed documentation, please refer to the [docs](./docs/README.md) directory. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index c727a03..b51f794 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,4 +7,6 @@ This documentation is a work in progress and will be updated as the indexer is u - [Requirements](./requirements.md) - [Setup](./setup.md) - [Integration Testing](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/blob/main/integration) -- [Scaling](./scaling.md) \ No newline at end of file +- [Scaling](./scaling.md) +- [Benchmarks](./benchmarks.md) +- [API](./api.md) \ No newline at end of file diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..7f1f826 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,91 @@ +# REST API + +With this indexer you can use the REST API from the api directory. It is built with chi framework and huma. + +Out of the box you get a [Spotlight UI](https://stoplight.io/) to interact with the API on the /docs route. +You can also use some other API UIs but you will need to make the changes yourself. See docs about changing the UI [here](https://huma.rocks/features/api-docs/). + +The huma is framework agnostic and you could modify the API to use some other framework, use another middleware +or maybe use the stdlib http package. This API provides the most basic and necessary features for querying the database. + +## API routes + +There are total of 5 routes available. This are the basic routes that are needed for the most part. This might be expanded in the future. + +### Blocks + +- /block/{height} - Get a specific block data by height +- /blocks/{from_height}/{to_height} - Get a range of blocks data by height range +- /blocks/{block_height}/signers - Get all of the validators that signed that block + the proposer + +### Transactions + +- /transaction/{tx_hash} - Get a specific basic transaction data by hash, this gives the basic data about the transaction like hash, timestamp, block height, gas used, gas wanted, fee and more. +- /transaction/{tx_hash}/message - Get a specific transaction message data by hash, this gives more detailed data about type of transaction, specific data for that message type and more. + +### Addresses + +- /address/{address}/txs?from_timestamp={from_timestamp}&to_timestamp={to_timestamp} - Get all of the transactions for a given address for a certain time period + +## Setup API + +To setup the API you can use the config file. The example config file is in the root under config-api.yml.example. +```yaml +# Example config file for the API +host: 127.0.0.1 +port: 8080 +cors_allowed_origins: + - "*" +cors_allowed_methods: + - "GET" +cors_allowed_headers: + - "Origin" + - "Content-Type" + - "Accept" +cors_max_age: 600 +chain_name: gnoland +``` + +Some of the environment variables are located under the .env file. The example .env file is in the root under .env.example. + +```env +# Example .env file for the API +# do not use password default unless for development or testing!!! +API_DB_HOST=127.0.0.1 +API_DB_PORT=5432 +API_DB_USER=reader +API_DB_SSLMODE=disable +API_DB_PASSWORD=12345678 +API_DB_NAME=gnoland + +# these are the default values for the database connection pool +# if they are not filled the API will load the default values +API_DB_POOL_MAX_CONNS=50 +API_DB_POOL_MIN_CONNS=10 +API_DB_POOL_MAX_CONN_LIFETIME=10s +API_DB_POOL_MAX_CONN_IDLE_TIME=5m +API_DB_POOL_HEALTH_CHECK_PERIOD=1m +API_DB_POOL_MAX_CONN_LIFETIME_JITTER=1m +``` + +You can make the API by running the following command from the project root: + +```bash +make build-api +``` +This command will build the API and it will be located in the build directory. + +To run the API you can use the following command: + +```bash +./build/api -c config-api.yml +``` + +You can also use the following command to run the API with HTTPS if you have the cert and key files: + +```bash +./build/api -c config-api.yml -t cert.pem -k key.pem +``` + +The docker image doesn't exist for now and it will be added in the future. + diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..c679ee2 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,23 @@ +# Benchmarks + +## Date 04/10/2025 + +The benchmarks were ran on different machines with different max block chunk size in the settings. It was done on +the old Gnoland 7.2 testnet from block height 950866 to 960866. The chunk size declares how much blocks can be +processed at the same time. The max transaction chunk was left at 100 as it is the default value. + +| Cpu Count | 50 chunks | 100 chunks | 200 chunks | 500 chunks | +| 2 vCPUs AMD Epyc | 2m25s | 1m23s | 50s | 28s | +| 4 vCPUs AMD Epyc | 1m30s | 1m05s | 37s | 18s | +| 16 core/32 threads AMD Ryzen 9 7900x3d | 24s | 16s | 10s | 10-5.4s* | + +* The 16 core/32 threads benchmark was ran 2/4 times the resault was at 6.5s however at this point the RPC node might be a bottleneck. One time it got 10 seconds and another time it was 5.4 seconds. + +This blocks are mostly empty of transactions so it might not be the best benchmark. But it is a good indication of the performance and can be used as a reference. + +However for the safety of the indexer I would recommend to use at the 50 - 100 chunks for now. Having 200 is a bit risky at this point untill more testing is done. + + + + + diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 0000000..1bd8620 --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,72 @@ +# Requirements + +The indexer could can run on one machine while the database is located on another machine. You can also use the +Tiger Data (Timescaledb cloud edition) which is a managed service by Tiger Data which is a company that builds the +Timescaledb. + +However for the ease of understanding through the documentation will refer that the indexer and the database are located on the same machine. + +## Hardware requirements + +To run the indexer you need to have the following system requirements: + +Minimum system requirements: + +- 2vCPUs +- 8GB RAM + +Recommended system requirements: + +- 4vCPUs +- 16GB RAM + +The indexer could probably run on ARM64 architecture but it is not tested yet. So stick with the x86_64 +architecture. There shouldn't be any major difference but if you really want to run it on ARM64 you might need to compile +the indexer from the source code. + +For storage HDD should be enough. Durring the whole time of the development it was tested on old smal laptop HDD +with 5400 RPM i think. So it should work for the most part. But for any bigger projects where write and read +speeds are important you might needs SSD depenging on what you are doing. The SATA SSD is a good choice for the +most part but you can also use the NVMe SSD for better performance. + +As for the size it depends on the amount of the data that you are indexing. +For example the integration test did about 400K blocks and 1 million +transactions. So it took around 2.2 GB of disk space. There are a lot of things that can affect the size of the +database but at least use this as some reference point to how much space you might need. + +For the RAM and CPU it kinda depends but for now this is a good starting point. As the database size grows, you +might need to increase the RAM and CPU. + +Aditional info for RAM. It is not that simple for more info look [here](https://docs.tigerdata.com/use-timescale/latest/hypertables/improve-query-performance/). + +To make queries and compression efficient you need to have the necessary amount of RAM. This is kinda hard to calculate but according to the Tiger Data documentation: + +```text +The default chunk interval is 7 days. However, best practice is to set chunk_interval so that prior to processing, +the indexes for chunks currently being ingested into fit within 25% of main memory. For example, on a system with +64 GB of memory, if index growth is approximately 2 GB per day, a 1-week chunk interval is appropriate. If index +growth is around 10 GB per day, use a 1-day interval. +``` + +So this is a bit hard to calculate. For example the Spectra cosmos indexer works in a similar way. At the testing +phase on Osmosis the 100K of block data took about 400MB-700MB of storage space. Now not all of the data is indexed +but for the sake of example let's say it is 700MB. The Osmosis has around 1s block production rate so this is 2 +days and maybe around 2/3 of one day. Let's assume that for the week it will take arround 2GB. If Gnoland ever +reached these amount of data you might need to increase the RAM size or modify the chunk interval. But this is if +the indexed data was 2GB. Realistically this is not the case and the indexer data would need a lot of indexed +data. + +Good rule of thumb is if the queries are slow increase the RAM size or modify the chunk interval or look at the +queries and see if they are efficient. + +8 GB of RAM is a good starting point. 16 GB should be enough for most cases. Depending on how popular the chain is +and how much data is indexed you might need to increase the RAM size. + +## Software and OS requirements + +The following software and OS requirements are required to run the indexer: + +- Go 1.25.1 +- TimescaleDB 2.18 or higher but with PostgresSQL 16 or higher +- OS: Linux, anything based on Debian(Ubuntu, Mint, etc.) or RHEL(CentOS stream, Rocky Linux, etc.) should work, openSUSE also ok +- Docker (optional) \ No newline at end of file diff --git a/docs/requriements.md b/docs/requriements.md deleted file mode 100644 index 96d7d25..0000000 --- a/docs/requriements.md +++ /dev/null @@ -1,43 +0,0 @@ -# Requirements - -The indexer could can run on one machine while the database is located on another machine. You can also use the -Tiger Data (Timescaledb cloud edition) which is a managed service by Tiger Data which is a company that builds the -Timescaledb. - -However for the ease of understanding through the documentation will refer that the indexer and the database are located on the same machine. - -## Hardware requirements - -To run the indexer you need to have the following system requirements: - -Minimum system requirements: - -- 2vCPUs -- 8GB RAM - -Recommended system requirements: - -- 4vCPUs -- 16GB RAM - -The indexer could probably run on ARM64 architecture but it is not tested yet. So stick with the x86_64 architecture. - -For the storage it can work on the HDD, but if the Gnoland chain grows in popularity and number of users increases, -the HDD might not be a good choice in the long term. So maybe a regular SATA SSD could be a better choice. -But if you are just running some testing or just a small deployment, the HDD will do the job most of the time. -As for the size it depends on the amount of the data that you are indexing. -For example the integration test did about 400K blocks and 1 million -transactions. So it took around 2.2 GB of disk space. There are a lot of things that can affect the size of the -database but at least use this as some reference point to how much space you might need. - -For the RAM and CPU it kinda depends but for now this is a good starting point. As the database size grows, you -might need to increase the RAM and CPU. - -## Software and OS requirements - -The following software and OS requirements are required to run the indexer: - -- Go 1.25.1 -- TimescaleDB 2.18 or higher but with PostgresSQL 16 or higher -- OS: Linux, anything based on Debian(Ubuntu, Mint, etc.) or RHEL(CentOS stream, Rocky Linux, etc.) should work, openSUSE also ok -- Docker (optional) \ No newline at end of file From 17255997b4bab03c30d03a9074554fa6a438bf32 Mon Sep 17 00:00:00 2001 From: nman98 Date: Sat, 4 Oct 2025 18:56:58 +0200 Subject: [PATCH 6/6] 0.2.0 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2c929..96e0560 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.0] - 2025-10-04 + +Added the REST API with some basic routes that will come in handy. Small bug fixes and changes. + +### Added + +- Rest API with 5 basic routes that will come in handy. [90bef0e](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/90bef0ec5a0bff468d4a1d6771b82706029f4ea9),[f2a39d6](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/f2a39d65c5622e0a5953d1f6113ab5eea1996cad),[d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e) +- Some basic documentation for the API and modified the existing docs. [b2c20d2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/b2c20d222e4f52b74b333974a7930df3cddad29e) +- Database queries for the API ( althoguh they can be used for any other app or service if needed ) [d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e), [f2a39d6](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/f2a39d65c5622e0a5953d1f6113ab5eea1996cad) + +### Fixed + +- Some table fixes, the msg_types would insert the empty string because when the make slice function was called by the accident it added the empty string instead of just allocating the size of the slice. [d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e) +- The validator signing has the column name addresses for the signed validators. While this is not a bug it wasn't intended. [90bef0e](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/90bef0ec5a0bff468d4a1d6771b82706029f4ea9) + ## [0.1.1] - 2025-10-02 Mostly some fixes. The live should work now there was a bug with the RPC client when making a request to the last block height recorded. The retry worker could sometimes send the data to the closed channel. Now the query operator calls the wg.Done functions directly. It should work now but there might be some other bugs.