diff --git a/.github/workflows/golangci.yml b/.github/workflows/golangci.yml index f0704b6..32614d1 100644 --- a/.github/workflows/golangci.yml +++ b/.github/workflows/golangci.yml @@ -1,4 +1,8 @@ name: Golangci-lint +permissions: + contents: read + packages: read + on: push: branches: [dev, release/**] diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 5bdaab1..a56871d 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -1,4 +1,8 @@ name: Go Test +permissions: + contents: read + packages: read + on: push: branches: [dev, release/**] diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d284a02..a298fe5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,5 +1,7 @@ name: Integration Tests - +permissions: + contents: read + packages: read on: pull_request: branches: [ main, dev ] @@ -55,7 +57,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db --db-user postgres --db-name postgres + go run indexer/indexer.go setup create-db --db-user postgres --db-name postgres env: DB_HOST: localhost DB_PORT: 5432 @@ -129,7 +131,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db + go run indexer/indexer.go setup create-db --db-user postgres --db-name postgres env: DB_HOST: localhost DB_PORT: 5432 @@ -203,7 +205,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db + go run indexer/indexer.go setup create-db env: DB_HOST: localhost DB_PORT: 5432 diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d3805..e953e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2025-11-26 + +Mostly it has some bug fixes. + +### Fixed + +- The indexer would go into the blocks data and store the signers from the last commit, which is actually all of the block signers from the previous block. So it would insert it like it was meant for that block height. From now on the indexer will fetch data from the /commit method and insert it properly. [de40740](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/de407407e67ae588141b361307fae18215f53a18) +- Gnoland can indeed execute multiple message types in the same transaction. The indexer wasn't able to hold this data properly and would cause an error because in the postgres primary key was attached making each message type unique. Now each message type has a message_counter which is a smallint(int16) which is used as a index for that transaction. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) + +### Changes + +- The REST API now returns a map of int16 and message data for the `/transaction/{tx_hash}/message` route. [74a35b30](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/74a35b301070bb1dab5cbadec6fe64b16ad7eb3b) +- The indexer should stop using sync.Map and use sync.Mutex with regular map to store the addresses before handing the operation to the AddressSolver function. [dde82a4](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/dde82a4ecc9bce50963fbff3f4e13a3a20b47f9d) +- The Orchestrator needs to make a fetch to the commit RPC method. This operation is done side by side with fetching the blocks method. A side effect of this is that indexer at that moment will fetch 2 times more request to the RPC, and if the Gnoland node has a limit on the amount of RPC clients that can use it, it might cause the indexer to slow down and throw errors on this requests. In future releases this should be improved. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) +- Updated the go version to 1.25.4 + ## [0.3.0] - 2025-11-10 In this release there are some fixes and improvements. The live process should work properly now and the REST API has some new routes. CLI commands are now combined with the ones from the setup cli. Some processes have been improved to use less memory. diff --git a/Makefile b/Makefile index b69a6b4..e77b38f 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,22 @@ -.PHONY: build install clean build-experimental install-experimental build-api +.PHONY: build install clean build-experimental install-experimental build-api test-race integration-test test + +######################################################## +# Build and install the indexer +######################################################## + # Get git information GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") GIT_TAG := $(shell git describe --tags --exact-match 2>/dev/null || echo "") -VERSION := $(if $(GIT_TAG),$(GIT_TAG),dev-$(GIT_COMMIT)) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "") +VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_BRANCH)-$(GIT_COMMIT)) build: mkdir -p build - go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer indexer/main.go + go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer indexer/indexer.go install: - cd indexer && go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" + go install -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" indexer/indexer.go build-api: mkdir -p build @@ -22,9 +28,17 @@ clean: # experimental build with greentea garbage collection # use at your own risk build-experimental: - GOEXPERIMENT=greenteagc go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer-tea indexer/main.go + GOEXPERIMENT=greenteagc go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer-tea indexer/indexer.go -# experimental install with greentea garbage collection -# use at your own risk -install-experimental: - cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" \ No newline at end of file +######################################################## +# Test the indexer +######################################################## + +test: + go test -v ./... + +test-race: + go test -v -race ./... + +integration-test: + cd integration && go test -v -tags=integration -timeout=20m ./... \ No newline at end of file diff --git a/api/handlers/address.go b/api/handlers/address.go index 0983cf8..6d12638 100644 --- a/api/handlers/address.go +++ b/api/handlers/address.go @@ -20,7 +20,13 @@ func (h *AddressHandler) GetAddressTxs( ctx context.Context, input *humatypes.AddressGetInput, ) (*humatypes.AddressGetOutput, error) { - address, err := h.db.GetAddressTxs(input.Address, h.chainName, input.FromTimestamp, input.ToTimestamp) + address, err := h.db.GetAddressTxs( + ctx, + input.Address, + h.chainName, + input.FromTimestamp, + input.ToTimestamp, + ) if err != nil { return nil, huma.Error404NotFound("Address not found", err) } diff --git a/api/handlers/blocks.go b/api/handlers/blocks.go index f92dced..43e60eb 100644 --- a/api/handlers/blocks.go +++ b/api/handlers/blocks.go @@ -23,7 +23,7 @@ func NewBlocksHandler(db DatabaseHandler, chainName string) *BlocksHandler { // GetBlock retrieves a block by height func (h *BlocksHandler) GetBlock(ctx context.Context, input *humatypes.BlockGetInput) (*humatypes.BlockGetOutput, error) { // Fetch from database - block, err := h.db.GetBlock(input.Height, h.chainName) + block, err := h.db.GetBlock(ctx, input.Height, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Block at height %d not found", input.Height), err) } @@ -51,7 +51,7 @@ func (h *BlocksHandler) GetFromToBlocks( if input.FromHeight > input.ToHeight { return nil, huma.Error400BadRequest("From height must be less than to height", nil) } - blocks, err := h.db.GetFromToBlocks(input.FromHeight, input.ToHeight, h.chainName) + blocks, err := h.db.GetFromToBlocks(ctx, input.FromHeight, input.ToHeight, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Blocks from height %d to height %d not found", input.FromHeight, input.ToHeight), err) } @@ -76,7 +76,7 @@ func (h *BlocksHandler) GetAllBlockSigners( input *humatypes.AllBlockSignersGetInput, ) (*humatypes.AllBlockSignersGetOutput, error) { // Fetch from database - blockSigners, err := h.db.GetAllBlockSigners(h.chainName, input.BlockHeight) + blockSigners, err := h.db.GetAllBlockSigners(ctx, h.chainName, input.BlockHeight) if err != nil { return nil, huma.Error404NotFound("Block signers not found", err) } @@ -92,7 +92,7 @@ func (h *BlocksHandler) GetAllBlockSigners( // Get latest block height func (h *BlocksHandler) GetLatestBlock(ctx context.Context, _ *humatypes.LatestBlockHeightGetInput) (*humatypes.LatestBlockHeightGetOutput, error) { - block, err := h.db.GetLatestBlock(h.chainName) + block, err := h.db.GetLatestBlock(ctx, h.chainName) if err != nil { return nil, huma.Error404NotFound("Latest block height not found", err) } @@ -112,7 +112,7 @@ func (h *BlocksHandler) GetLatestBlock(ctx context.Context, _ *humatypes.LatestB // Get last x blocks func (h *BlocksHandler) GetLastXBlocks(ctx context.Context, input *humatypes.LastXBlocksGetInput) (*humatypes.LastXBlocksGetOutput, error) { // Fetch from database - blocks, err := h.db.GetLastXBlocks(h.chainName, input.Amount) + blocks, err := h.db.GetLastXBlocks(ctx, h.chainName, input.Amount) if err != nil { return nil, huma.Error404NotFound("Last x blocks not found", err) } diff --git a/api/handlers/database_test.go b/api/handlers/database_test.go index 2aeb6b8..c8488ae 100644 --- a/api/handlers/database_test.go +++ b/api/handlers/database_test.go @@ -1,6 +1,7 @@ package handlers_test import ( + "context" "fmt" "time" @@ -18,13 +19,13 @@ type MockDatabase struct { msgCall map[string]*database.MsgCall msgAddPackage map[string]*database.MsgAddPackage msgRun map[string]*database.MsgRun - msgTypes map[string]string + msgTypes map[string][]string shouldError bool errorMsg string } -func (m *MockDatabase) GetBlock(height uint64, chainName string) (*database.BlockData, error) { +func (m *MockDatabase) GetBlock(ctx context.Context, height uint64, chainName string) (*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -35,7 +36,7 @@ func (m *MockDatabase) GetBlock(height uint64, chainName string) (*database.Bloc return block, nil } -func (m *MockDatabase) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) { +func (m *MockDatabase) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -50,7 +51,7 @@ func (m *MockDatabase) GetFromToBlocks(fromHeight uint64, toHeight uint64, chain return result, nil } -func (m *MockDatabase) GetAllBlockSigners(chainName string, blockHeight uint64) (*database.BlockSigners, error) { +func (m *MockDatabase) GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*database.BlockSigners, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -61,7 +62,7 @@ func (m *MockDatabase) GetAllBlockSigners(chainName string, blockHeight uint64) return blockSigners, nil } -func (m *MockDatabase) GetTransaction(txHash string, chainName string) (*database.Transaction, error) { +func (m *MockDatabase) GetTransaction(ctx context.Context, txHash string, chainName string) (*database.Transaction, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -72,7 +73,7 @@ func (m *MockDatabase) GetTransaction(txHash string, chainName string) (*databas return transaction, nil } -func (m *MockDatabase) GetAddressTxs(address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) { +func (m *MockDatabase) GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -83,7 +84,7 @@ func (m *MockDatabase) GetAddressTxs(address string, chainName string, fromTimes return addressTxs, nil } -func (m *MockDatabase) GetLatestBlock(chainName string) (*database.BlockData, error) { +func (m *MockDatabase) GetLatestBlock(ctx context.Context, chainName string) (*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -93,7 +94,7 @@ func (m *MockDatabase) GetLatestBlock(chainName string) (*database.BlockData, er return m.latestBlock, nil } -func (m *MockDatabase) GetLastXBlocks(chainName string, x uint64) ([]*database.BlockData, error) { +func (m *MockDatabase) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -104,7 +105,7 @@ func (m *MockDatabase) GetLastXBlocks(chainName string, x uint64) ([]*database.B return blocks, nil } -func (m *MockDatabase) GetLastXTransactions(chainName string, x uint64) ([]*database.Transaction, error) { +func (m *MockDatabase) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*database.Transaction, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -115,18 +116,18 @@ func (m *MockDatabase) GetLastXTransactions(chainName string, x uint64) ([]*data return transactions, nil } -func (m *MockDatabase) GetMsgType(txHash string, chainName string) (string, error) { +func (m *MockDatabase) GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) { if m.shouldError { - return "", fmt.Errorf("%s", m.errorMsg) + return nil, fmt.Errorf("%s", m.errorMsg) } - msgType, ok := m.msgTypes[txHash] + msgTypes, ok := m.msgTypes[txHash] if !ok { - return "", fmt.Errorf("message type not found") + return nil, fmt.Errorf("message type not found") } - return msgType, nil + return msgTypes, nil } -func (m *MockDatabase) GetBankSend(txHash string, chainName string) (*database.BankSend, error) { +func (m *MockDatabase) GetBankSend(ctx context.Context, txHash string, chainName string) ([]*database.BankSend, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -134,10 +135,10 @@ func (m *MockDatabase) GetBankSend(txHash string, chainName string) (*database.B if !ok { return nil, fmt.Errorf("bank send not found") } - return bankSend, nil + return []*database.BankSend{bankSend}, nil } -func (m *MockDatabase) GetMsgCall(txHash string, chainName string) (*database.MsgCall, error) { +func (m *MockDatabase) GetMsgCall(ctx context.Context, txHash string, chainName string) ([]*database.MsgCall, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -145,10 +146,10 @@ func (m *MockDatabase) GetMsgCall(txHash string, chainName string) (*database.Ms if !ok { return nil, fmt.Errorf("message call not found") } - return msgCall, nil + return []*database.MsgCall{msgCall}, nil } -func (m *MockDatabase) GetMsgAddPackage(txHash string, chainName string) (*database.MsgAddPackage, error) { +func (m *MockDatabase) GetMsgAddPackage(ctx context.Context, txHash string, chainName string) ([]*database.MsgAddPackage, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -156,10 +157,10 @@ func (m *MockDatabase) GetMsgAddPackage(txHash string, chainName string) (*datab if !ok { return nil, fmt.Errorf("message add package not found") } - return msgAddPackage, nil + return []*database.MsgAddPackage{msgAddPackage}, nil } -func (m *MockDatabase) GetMsgRun(txHash string, chainName string) (*database.MsgRun, error) { +func (m *MockDatabase) GetMsgRun(ctx context.Context, txHash string, chainName string) ([]*database.MsgRun, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -167,5 +168,5 @@ func (m *MockDatabase) GetMsgRun(txHash string, chainName string) (*database.Msg if !ok { return nil, fmt.Errorf("message run not found") } - return msgRun, nil + return []*database.MsgRun{msgRun}, nil } diff --git a/api/handlers/interface.go b/api/handlers/interface.go index 739fc7c..962bca1 100644 --- a/api/handlers/interface.go +++ b/api/handlers/interface.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" @@ -9,21 +10,21 @@ import ( // DatabaseHandler interface for the database type DatabaseHandler interface { // Addresses - GetAddressTxs(address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) + GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) // Blocks - GetBlock(height uint64, chainName string) (*database.BlockData, error) - GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) - GetAllBlockSigners(chainName string, blockHeight uint64) (*database.BlockSigners, error) - GetLatestBlock(chainName string) (*database.BlockData, error) - GetLastXBlocks(chainName string, x uint64) ([]*database.BlockData, error) + GetBlock(ctx context.Context, height uint64, chainName string) (*database.BlockData, error) + GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) + GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*database.BlockSigners, error) + GetLatestBlock(ctx context.Context, chainName string) (*database.BlockData, error) + GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*database.BlockData, error) // Transactions - GetTransaction(txHash string, chainName string) (*database.Transaction, error) - GetLastXTransactions(chainName string, x uint64) ([]*database.Transaction, error) - GetMsgType(txHash string, chainName string) (string, error) - GetBankSend(txHash string, chainName string) (*database.BankSend, error) - GetMsgCall(txHash string, chainName string) (*database.MsgCall, error) - GetMsgAddPackage(txHash string, chainName string) (*database.MsgAddPackage, error) - GetMsgRun(txHash string, chainName string) (*database.MsgRun, error) + GetTransaction(ctx context.Context, txHash string, chainName string) (*database.Transaction, error) + GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*database.Transaction, error) + GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) + GetBankSend(ctx context.Context, txHash string, chainName string) ([]*database.BankSend, error) + GetMsgCall(ctx context.Context, txHash string, chainName string) ([]*database.MsgCall, error) + GetMsgAddPackage(ctx context.Context, txHash string, chainName string) ([]*database.MsgAddPackage, error) + GetMsgRun(ctx context.Context, txHash string, chainName string) ([]*database.MsgRun, error) } diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go index cb72d92..1083e3c 100644 --- a/api/handlers/transactions.go +++ b/api/handlers/transactions.go @@ -20,6 +20,7 @@ func NewTransactionsHandler(db DatabaseHandler, chainName string) *TransactionsH return &TransactionsHandler{db: db, chainName: chainName} } +// GetTransactionBasic retrieves basic transaction details by tx hash func (h *TransactionsHandler) GetTransactionBasic( ctx context.Context, input *humatypes.TransactionGetInput, @@ -30,7 +31,7 @@ func (h *TransactionsHandler) GetTransactionBasic( if err != nil { return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) } - transaction, err := h.db.GetTransaction(txHashBase64, h.chainName) + transaction, err := h.db.GetTransaction(ctx, txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } @@ -39,6 +40,7 @@ func (h *TransactionsHandler) GetTransactionBasic( }, nil } +// GetTransactionMessage retrieves all messages within a transaction by tx hash func (h *TransactionsHandler) GetTransactionMessage( ctx context.Context, input *humatypes.TransactionGetInput, @@ -46,93 +48,106 @@ func (h *TransactionsHandler) GetTransactionMessage( input.TxHash = strings.Trim(input.TxHash, " ") txHash, err := base64.URLEncoding.DecodeString(input.TxHash) txHashBase64 := base64.StdEncoding.EncodeToString(txHash) + response := make(map[int16]humatypes.TransactionMessage) if err != nil { return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) } - msgType, err := h.db.GetMsgType(txHashBase64, h.chainName) + msgTypes, err := h.db.GetMsgTypes(ctx, txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - var message humatypes.TransactionMessage - - switch msgType { - case "bank_msg_send": - data, err := h.db.GetBankSend(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - FromAddress: data.FromAddress, - ToAddress: data.ToAddress, - Amount: data.Amount, - } - case "vm_msg_call": - data, err := h.db.GetMsgCall(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Caller: data.Caller, - Send: data.Send, - PkgPath: data.PkgPath, - FuncName: data.FuncName, - Args: data.Args, - MaxDeposit: data.MaxDeposit, - } - case "vm_msg_add_package": - data, err := h.db.GetMsgAddPackage(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + for _, msgType := range msgTypes { + switch msgType { + case "bank_msg_send": + data, err := h.db.GetBankSend(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + FromAddress: data.FromAddress, + ToAddress: data.ToAddress, + Amount: data.Amount, + } + } + case "vm_msg_call": + data, err := h.db.GetMsgCall(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + Send: data.Send, + PkgPath: data.PkgPath, + FuncName: data.FuncName, + Args: data.Args, + MaxDeposit: data.MaxDeposit, + } + } + case "vm_msg_add_package": + data, err := h.db.GetMsgAddPackage(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Creator: data.Creator, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } + } + case "vm_msg_run": + data, err := h.db.GetMsgRun(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } + } + default: + return nil, huma.Error400BadRequest("Transaction message type not found", nil) } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Creator: data.Creator, - PkgPath: data.PkgPath, - PkgName: data.PkgName, - PkgFileNames: data.PkgFileNames, - Send: data.Send, - MaxDeposit: data.MaxDeposit, - } - case "vm_msg_run": - data, err := h.db.GetMsgRun(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Caller: data.Caller, - PkgPath: data.PkgPath, - PkgName: data.PkgName, - PkgFileNames: data.PkgFileNames, - Send: data.Send, - MaxDeposit: data.MaxDeposit, - } - default: - return nil, huma.Error400BadRequest("Transaction message type not found", nil) } - return &humatypes.TransactionMessageGetOutput{ - Body: message, + Body: response, }, nil } +// GetLastXTransactions retrieves the last X transactions func (h *TransactionsHandler) GetLastXTransactions(ctx context.Context, input *humatypes.TransactionGeneralListGetInput) (*humatypes.TransactionGeneralListGetOutput, error) { - transactions, err := h.db.GetLastXTransactions(h.chainName, input.Amount) + transactions, err := h.db.GetLastXTransactions(ctx, h.chainName, input.Amount) if err != nil { return nil, huma.Error404NotFound("Last x transactions not found", err) } diff --git a/api/huma-types/transaction.go b/api/huma-types/transaction.go index 68996fa..137e1d9 100644 --- a/api/huma-types/transaction.go +++ b/api/huma-types/transaction.go @@ -11,6 +11,7 @@ type TransactionGetInput struct { TxHash string `path:"tx_hash" minLength:"44" maxLength:"44" doc:"Transaction hash (base64url encoded)" required:"true"` } +// TransactionBasicGetOutput represents the response for basic transaction details type TransactionBasicGetOutput struct { Body database.Transaction } @@ -48,8 +49,9 @@ type TransactionMessage struct { MaxDeposit []database.Amount `json:"max_deposit,omitempty" doc:"Max deposit (for vm_msg_call, vm_msg_add_package, and vm_msg_run)"` } +// TransactionMessageGetOutput represents the response containing all messages within a transaction type TransactionMessageGetOutput struct { - Body TransactionMessage + Body map[int16]TransactionMessage } type TransactionGeneralListGetInput struct { diff --git a/api/main.go b/api/main.go index b1fc333..10728ce 100644 --- a/api/main.go +++ b/api/main.go @@ -1,9 +1,11 @@ package main import ( + _ "embed" "fmt" "log" "net/http" + "strings" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/config" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/handlers" @@ -17,6 +19,9 @@ import ( // "github.com/go-chi/httprate" ) +//go:embed public/favicon.ico +var favicon []byte + var ( Commit = "unknown" // Set via ldflags at build time Version = "unknown" // Set via ldflags at build time @@ -103,9 +108,24 @@ var rootCmd = &cobra.Command{ api := humachi.New(router, huma.DefaultConfig("Spectra Gnoland Indexer API", Version)) - api.OpenAPI().Info.Title = "Spectra Gnoland Indexer API" - api.OpenAPI().Info.Version = Version - api.OpenAPI().Info.Description = "API for the Spectra Gnoland Indexer" + openApi := api.OpenAPI() + openApi.Info = &huma.Info{ + Title: "Spectra Gnoland Indexer API", + Version: strings.TrimPrefix(Version, "v"), + Description: "API for the Spectra Gnoland Indexer", + Contact: &huma.Contact{ + Name: "Cogwheel Validator", + URL: "https://cogwheel.zone", + Email: "info@cogwheel.zone", + }, + License: &huma.License{ + Name: "Apache 2.0", + URL: "https://github.com/Cogwheel-Validator/spectra-gnoland-indexer?tab=Apache-2.0-1-ov-file#readme", + }, + } + openApi.ExternalDocs = &huma.ExternalDocs{ + URL: "https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/tree/main/docs", + } // Initialize database connection from environment variables db := database.NewTimescaleDb(database.DatabasePoolConfig{ @@ -128,20 +148,71 @@ var rootCmd = &cobra.Command{ transactionsHandler := handlers.NewTransactionsHandler(db, conf.ChainName) addressHandler := handlers.NewAddressHandler(db, conf.ChainName) + // favicon route + router.Get("/favicon.ico", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/x-icon") + _, err := w.Write(favicon) + if err != nil { + log.Printf("failed to write favicon: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + }) + // Register Block API routes - huma.Get(api, "/block/{height}", blocksHandler.GetBlock) - huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks) - huma.Get(api, "/blocks/{block_height}/signers", blocksHandler.GetAllBlockSigners) - huma.Get(api, "/blocks/latest", blocksHandler.GetLatestBlock) - huma.Get(api, "/blocks", blocksHandler.GetLastXBlocks) + huma.Get(api, "/block/{height}", blocksHandler.GetBlock, + func(op *huma.Operation) { + op.Summary = "Get Block Height" + op.Description = "Retrieve block data by its height" + }) + huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks, + func(op *huma.Operation) { + op.Summary = "Get From To Blocks" + op.Description = "Retrieve blocks data by its height range" + }) + huma.Get(api, "/blocks/{block_height}/signers", blocksHandler.GetAllBlockSigners, + func(op *huma.Operation) { + op.Summary = "Get All Block Signers" + op.Description = "Retrieve all validators that signed a block by its height" + }) + huma.Get(api, "/blocks/latest", blocksHandler.GetLatestBlock, + func(op *huma.Operation) { + op.Summary = "Get Latest Block" + op.Description = "Retrieve the latest block data" + }) + huma.Get(api, "/blocks", blocksHandler.GetLastXBlocks, + func(op *huma.Operation) { + op.Summary = "Get Last X Blocks" + op.Description = "Retrieve the last X blocks data" + }) // Register Transaction API routes - huma.Get(api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic) - huma.Get(api, "/transaction/{tx_hash}/message", transactionsHandler.GetTransactionMessage) - huma.Get(api, "/transactions", transactionsHandler.GetLastXTransactions) + huma.Get( + api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic, + func(op *huma.Operation) { + op.Summary = "Get Transaction Basic" + op.Description = "Retrieve basic transaction data by its hash" + }) + huma.Get( + api, + "/transaction/{tx_hash}/message", + transactionsHandler.GetTransactionMessage, + func(op *huma.Operation) { + op.Summary = "Get All Transaction Messages" + op.Description = "Retrieve all messages contained within a transaction by its hash" + }) + huma.Get(api, "/transactions", transactionsHandler.GetLastXTransactions, + func(op *huma.Operation) { + op.Summary = "Get Last X Transactions" + op.Description = "Retrieve the last X transactions data" + }) // Register Address API routes - huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs) + huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs, + func(op *huma.Operation) { + op.Summary = "Get Address Transactions" + op.Description = "Retrieve all transactions for a given address for a certain time period" + }) // Start server using config values addr := fmt.Sprintf("%s:%d", conf.Host, conf.Port) @@ -169,7 +240,6 @@ func init() { rootCmd.PersistentFlags().StringP("config", "c", "config-api.yml", "config file path") rootCmd.PersistentFlags().StringP("cert-file", "t", "", "certificate file path") rootCmd.PersistentFlags().StringP("key-file", "k", "", "key file path") - } func main() { diff --git a/api/public/favicon.ico b/api/public/favicon.ico new file mode 100644 index 0000000..f0e5dff Binary files /dev/null and b/api/public/favicon.ico differ diff --git a/experiments/experiment1/enc_dec_test.go b/experiments/experiment1/enc_dec_test.go index 7ddf26e..680abf6 100644 --- a/experiments/experiment1/enc_dec_test.go +++ b/experiments/experiment1/enc_dec_test.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "log" - "testing" ) // In Gnoland txs are marked with base64 not in hex like in other Cosmos SDK chains @@ -15,8 +14,8 @@ import ( // To get something that resembles txhash we need then to encode the data to sha256 and then to base64 // This can be later used to query the tx from the rpc client via the tx method // Same data is pressent within the tx query method so this should be useful for the indexer -func TestEncodeDecode(t *testing.T) { - raw_data := "CnQKDS9iYW5rLk1zZ1NlbmQSYwooZzE0ODU4M3Q1eDY2enM2cDkwZWhhZDZsNHFlZmV5YWY1NHM2OXdxbBIoZzFrM2NjcDA5NzdhajMyeXJqeHBtdjA5d2hxZ2w3bjgzcjZjc2pyYxoNMTAwMDAwMDB1Z25vdBIRCIDaxAkSCjEwMDAwdWdub3Qafgo6ChMvdG0uUHViS2V5U2VjcDI1NmsxEiMKIQMcQg5ueuQfcNtMh5w7PjWaTKO72epQaK9kax/g3+C9NRJAYMsAIGMPOzc964cBKZ49xg+KBO/5T3RJLhjDzO9x5a5Kr4sX+78p3QLvOV6a30htoJc11XYGV5tymCLgewf8lg==" +func main() { + raw_data := "CpABCgovdm0ubV9jYWxsEoEBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIhZnbm8ubGFuZC9yL2dub3N3YXAvZ25zKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpwBCgovdm0ubV9jYWxsEo0BCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiJnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi91c2RjKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9iYXIqCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL2JheioIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMAqbAQoKL3ZtLm1fY2FsbBKMAQooZzE3MjkwY3d2bXJhcHZwODY5eGZuaGhhd2E4c205ZWRwdWZ6YXQ3ZCIhZ25vLmxhbmQvci9nbm9zd2FwL3Rlc3RfdG9rZW4vb2JsKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9mb28qCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL3F1eCoIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMBITCICEr18SDDEwMDAwMDB1Z25vdBp+CjoKEy90bS5QdWJLZXlTZWNwMjU2azESIwohAtGyA6l3UIrUup5z7yXo90bXDcXUOmLiK34YPffgQ6pAEkCg8pAtBv6Fhw98bKYdqrEX2UrcjYvIUpbGdxMyc5Zpfl1cMNo8G6vzpnFaQESX+7eZIFTfO5BCFjVJLfC5Ur5C" // decode the raw data from base64 to string base64Decoded, err := base64.StdEncoding.DecodeString(raw_data) diff --git a/experiments/experiment4/encode_proto.go b/experiments/experiment4/encode_proto.go index 7ee570e..082f1ae 100644 --- a/experiments/experiment4/encode_proto.go +++ b/experiments/experiment4/encode_proto.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/hex" "fmt" "log" @@ -82,18 +83,52 @@ func main() { fmt.Printf("Length: %d bytes\n", len(encoded)) // try different compressions - zstdOptions := zstandard.WithEncoderLevel(zstandard.SpeedBetterCompression) - zstdEncoder, err := zstandard.NewWriter(nil, zstdOptions) + // Test with different compression levels + for _, level := range []int{1, 2, 3, 10, 15, 18, 20} { + encoderLevel := zstandard.EncoderLevelFromZstd(level) + encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel)) + if err != nil { + log.Fatal(err) + } + + var buf bytes.Buffer + encoder.Reset(&buf) + _, err = encoder.Write(encoded) + if err != nil { + log.Fatal(err) + } + err = encoder.Close() + if err != nil { + log.Fatal(err) + } + + compressed := buf.Bytes() + fmt.Printf("Compressed (level %d): %d bytes (%.1f%% of original)\n", level, len(compressed), float64(len(compressed))/float64(len(encoded))*100) + } + // Final compression with level 22 + encoderLevel := zstandard.EncoderLevelFromZstd(22) + encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel)) if err != nil { log.Fatal(err) } - compressed := zstdEncoder.EncodeAll(encoded, nil) + + var buf bytes.Buffer + encoder.Reset(&buf) + _, err = encoder.Write(encoded) if err != nil { log.Fatal(err) } - fmt.Println("Compressed bytes:", string(compressed)) - fmt.Printf("Length: %d bytes\n", len(compressed)) - + err = encoder.Close() + if err != nil { + log.Fatal(err) + } + err = encoder.Close() + if err != nil { + log.Fatal(err) + } + compressed := buf.Bytes() + fmt.Println("Final compressed bytes:", hex.EncodeToString(compressed)) + fmt.Printf("Final length: %d bytes\n", len(compressed)) decoded, err := DecodeProto(encoded) if err != nil { log.Fatal(err) diff --git a/go.mod b/go.mod index 898a65b..26eb2d0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Cogwheel-Validator/spectra-gnoland-indexer -go 1.25.1 +go 1.25.4 require ( github.com/caarlos0/env/v6 v6.10.1 diff --git a/indexer/address_cache/address.go b/indexer/address_cache/address.go index b0a4b9e..b8ca1ed 100644 --- a/indexer/address_cache/address.go +++ b/indexer/address_cache/address.go @@ -1,8 +1,10 @@ package addresscache import ( + "context" "log" "maps" + "time" ) // NewAddressCache is a constructor for the AddressCache struct @@ -10,7 +12,7 @@ import ( // it will also load the validator addresses if the loadVal is true // it will load the regular addresses if the loadVal is false // -// Args: +// Parameters: // - chainName: the name of the chain // - db: the database connection interface // - loadVal: whether to load the validator addresses @@ -50,7 +52,7 @@ func NewAddressCache(chainName string, db DatabaseForAddresses, loadVal bool) *A // This method is used to add addresses to the cache // It will add the addresses to the cache and update the highest index // -// Args: +// Parameters: // - newAddresses: the new addresses to add to the cache // // Returns: @@ -80,7 +82,7 @@ func (a *AddressCache) addAddresses(newAddresses map[string]int32) { // - add more documentation // - add more logging // -// Args: +// Parameters: // - address: the addresses to solve // - chainName: the chain name // - insertValidators: whether to insert validators @@ -110,7 +112,9 @@ func (a *AddressCache) AddressSolver( // chech if there is already recorded addresses in the db // technically this should be handled by LoadAddresses but let's make one more check // probably not needed but just in case - existingAddresses, err := a.db.FindExistingAccounts(address, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + existingAddresses, err := a.db.FindExistingAccounts(ctx, address, chainName, insertValidators) + cancel() if err != nil { return } @@ -128,13 +132,18 @@ func (a *AddressCache) AddressSolver( if len(addressToAdd) > 1 { for i := range retryAttempts { - loopErr := a.db.InsertAddresses(addressToAdd, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, addressToAdd, chainName, insertValidators) + cancel() + if loopErr != nil { // in the events the oneByOne is true the program will try to insert the addresses one by one // as a final resort with this some might be inserted but some might not if oneByOne != nil && *oneByOne && i == retryAttempts-1 { for _, addr := range addressToAdd { - loopErr := a.db.InsertAddresses([]string{addr}, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, []string{addr}, chainName, insertValidators) + cancel() if loopErr != nil { // this is a final resort, so we can log the error for debugging purposes log.Println("Error inserting address:", addr, "error:", loopErr) @@ -147,14 +156,18 @@ func (a *AddressCache) AddressSolver( } } else if len(addressToAdd) == 1 { // if there is only one address to insert, we can do it directly - loopErr := a.db.InsertAddresses([]string{addressToAdd[0]}, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, []string{addressToAdd[0]}, chainName, insertValidators) + cancel() if loopErr != nil { return } } // at the end of the function we should add the addresses to the cache // we need to make a query of the added addresses to get the ids along with the addresses - newAddrMap, err := a.db.FindExistingAccounts(addressToAdd, chainName, insertValidators) + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + newAddrMap, err := a.db.FindExistingAccounts(ctx, addressToAdd, chainName, insertValidators) + cancel() if err != nil { log.Println("Error finding existing accounts:", err) return @@ -167,7 +180,7 @@ func (a *AddressCache) AddressSolver( // This method is used to get the address from the cache // If the address is not in the cache, it will return 0 // -// Args: +// Parameters: // - address: the address to get // // Returns: @@ -186,7 +199,7 @@ func (a *AddressCache) GetAddress(address string) int32 { // This method is called when the program starts and when the cache is empty // Should only be called once per program start // -// Args: +// Parameters: // - chainName: the name of the chain // - loadVal: whether to load the validator addresses // - db: the database connection interface @@ -195,7 +208,9 @@ func (a *AddressCache) GetAddress(address string) int32 { // - map[string]int32: the map of addresses and their ids // - error: if the query fails func loadAddresses(chainName string, loadVal bool, db DatabaseForAddresses) (map[string]int32, int32, error) { - addresses, maxIndex, err := db.GetAllAddresses(chainName, loadVal, nil) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + addresses, maxIndex, err := db.GetAllAddresses(ctx, chainName, loadVal, nil) if err != nil { return nil, 0, err } diff --git a/indexer/address_cache/types.go b/indexer/address_cache/types.go index d5407eb..33bee23 100644 --- a/indexer/address_cache/types.go +++ b/indexer/address_cache/types.go @@ -1,10 +1,12 @@ package addresscache +import "context" + // A database interface for what AddressCache needs from database type DatabaseForAddresses interface { - FindExistingAccounts(addresses []string, chainName string, searchValidators bool) (map[string]int32, error) - InsertAddresses(addresses []string, chainName string, insertValidators bool) error - GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) + FindExistingAccounts(ctx context.Context, addresses []string, chainName string, searchValidators bool) (map[string]int32, error) + InsertAddresses(ctx context.Context, addresses []string, chainName string, insertValidators bool) error + GetAllAddresses(ctx context.Context, chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) } // AddressCache is a map of addresses tied to their int32 index in the database diff --git a/indexer/cmd/setup.go b/indexer/cmd/setup.go index a192255..b48c856 100644 --- a/indexer/cmd/setup.go +++ b/indexer/cmd/setup.go @@ -1,7 +1,9 @@ package cmd import ( + "context" "log" + "time" dbinit "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/db_init" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" @@ -95,7 +97,9 @@ var createDbCmd = &cobra.Command{ // create a new database named "gnoland" // but check if the current database is "gnoland" - currentDb, err := db.CheckCurrentDatabaseName() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + currentDb, err := db.CheckCurrentDatabaseName(ctx) if err != nil { log.Fatalf("failed to check current database name: %v", err) } diff --git a/indexer/context_hook/hook.go b/indexer/context_hook/hook.go index 2fad928..981a91d 100644 --- a/indexer/context_hook/hook.go +++ b/indexer/context_hook/hook.go @@ -12,7 +12,7 @@ import ( // NewSignalHandler is a constructor function that creates a new signal handler with // cleanup and state dump functions // -// Args: +// Parameters: // - cleanup: the cleanup function // - stateDump: the state dump function // @@ -32,7 +32,7 @@ func NewSignalHandler(cleanup func() error, stateDump func() error) *SignalHandl // Context returns the context that will be cancelled on shutdown signals // -// Args: +// Parameters: // - sh: the signal handler // // Returns: diff --git a/indexer/data_processor/event.go b/indexer/data_processor/event.go index c4dfa0c..4bfabf6 100644 --- a/indexer/data_processor/event.go +++ b/indexer/data_processor/event.go @@ -62,7 +62,7 @@ func (er *EventResult) GetCompressedData() []byte { // 1. Native postgres format ([]sqlDataTypes.Event) // 2. Compressed protobuf format ([]byte) // -// Args: +// Parameters: // - txResponse: a transaction response // - useCompressed: if true, returns compressed format; otherwise native format // diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index abca23b..b5b38bc 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -1,6 +1,7 @@ package dataprocessor import ( + "context" "crypto/sha256" "encoding/base64" "fmt" @@ -8,6 +9,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/decoder" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" @@ -16,7 +18,7 @@ import ( // Constructor function for the DataProcessor struct // -// Args: +// Parameters: // - db: the database connection interface // - addressCache: the address cache interface // - validatorCache: the validator cache interface @@ -40,10 +42,10 @@ func NewDataProcessor( } // ProcessValidatorAddresses is a method to process the validator addresses from a slice of blocks -// it will process the validator addresses from the blocks and store them in a sync.Map -// it will then extract the addresses from the sync.Map and insert them into the address cache +// it will process the validator addresses from the blocks and store them in a map[string]struct{} +// it will then extract the addresses from the map[string]struct{} and insert them into the address cache // -// Args: +// Parameters: // - blocks: a slice of blocks // - fromHeight: the start height // - toHeight: the end height @@ -57,9 +59,10 @@ func (d *DataProcessor) ProcessValidatorAddresses( fromHeight uint64, toHeight uint64, ) { - // sync.Map for thread-safe concurrent access and inser it as address/bool + // map[string]struct{} for thread-safe concurrent access and insert it as address/struct{} // the program should be able to avoid duplicates since it is a map - var addressesMap sync.Map + var mu sync.Mutex + addressesMap := make(map[string]struct{}) wg := sync.WaitGroup{} wg.Add(len(blocks)) @@ -72,24 +75,27 @@ func (d *DataProcessor) ProcessValidatorAddresses( precommits := block.Result.Block.LastCommit.Precommits for _, precommit := range precommits { if precommit != nil { - addressesMap.Store(precommit.ValidatorAddress, true) + mu.Lock() + addressesMap[precommit.ValidatorAddress] = struct{}{} + mu.Unlock() } } // Process proposer proposer := block.Result.Block.Header.ProposerAddress - addressesMap.Store(proposer, true) + mu.Lock() + addressesMap[proposer] = struct{}{} + mu.Unlock() }(block) } wg.Wait() - // Extract unique addresses from sync.Map + // Extract unique addresses from map[string]struct{} addresses := make([]string, 0) - addressesMap.Range(func(key, value interface{}) bool { - addresses = append(addresses, key.(string)) - return true - }) + for address := range addressesMap { + addresses = append(addresses, address) + } // retry 3 times just for the sake of it d.validatorCache.AddressSolver(addresses, d.chainName, true, 3, nil) @@ -100,7 +106,7 @@ func (d *DataProcessor) ProcessValidatorAddresses( // it will process the blocks using async workers and store them directly into a result slice // it will then insert the blocks into the database // -// Args: +// Parameters: // - blocks: a slice of blocks // - fromHeight: the start height // - toHeight: the end height @@ -111,10 +117,11 @@ func (d *DataProcessor) ProcessValidatorAddresses( // The method will not throw an error if the blocks are not found, it will just return nil func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) { // Preallocate slice to avoid growing allocations - blocksData := make([]sqlDataTypes.Blocks, len(blocks)) + blockAmount := len(blocks) + blocksData := make([]sqlDataTypes.Blocks, blockAmount) var mu sync.Mutex wg := sync.WaitGroup{} - wg.Add(len(blocks)) + wg.Add(blockAmount) for idx, block := range blocks { go func(idx int, block *rpcClient.BlockResponse) { @@ -168,7 +175,11 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei wg.Wait() - err := d.dbPool.InsertBlocks(blocksData) + // add multiplier for the timeout depending on the block amount + timeout := 10*time.Second + (time.Duration(blockAmount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := d.dbPool.InsertBlocks(ctx, blocksData) if err != nil { log.Printf("Failed to insert blocks: %v", err) } @@ -179,7 +190,7 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei // it will process the transactions using async workers and collect them in a preallocated slice // it will then insert the transactions into the database // -// Args: +// Parameters: // - transactions: a map of transactions and timestamps // - compressEvents: if true, compress the events // @@ -194,11 +205,12 @@ func (d *DataProcessor) ProcessTransactions( toHeight uint64) { // Preallocate slice to avoid growing allocations - transactionsData := make([]sqlDataTypes.TransactionGeneral, len(transactions)) + transactionAmount := len(transactions) + transactionsData := make([]sqlDataTypes.TransactionGeneral, transactionAmount) var mu sync.Mutex var validCount int wg := sync.WaitGroup{} - wg.Add(len(transactions)) + wg.Add(transactionAmount) for idx, transaction := range transactions { go func(idx int, transaction TrasnactionsData) { @@ -258,19 +270,24 @@ func (d *DataProcessor) ProcessTransactions( // It is more of a safety feature than nececity transactionsData = transactionsData[:validCount] - err := d.dbPool.InsertTransactionsGeneral(transactionsData) + // add multiplier for the timeout depending on the transaction amount + timeout := 10*time.Second + (time.Duration(transactionAmount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := d.dbPool.InsertTransactionsGeneral(ctx, transactionsData) if err != nil { log.Printf("Failed to insert transactions: %v", err) + return } log.Printf("Transactions processed from %d to %d", fromHeight, toHeight) } // ProcessMessages processes all messages from transactions using concurrent "swarm method" // This method uses a two-phase concurrent approach: -// 1. Collect and resolve all addresses to IDs using concurrent workers and sync.Map +// 1. Collect and resolve all addresses to IDs using concurrent workers and map[string]struct{} // 2. Convert messages to database-ready format with address IDs using concurrent processing // -// Args: +// Parameters: // - transactions: a map of transactions and timestamps // - fromHeight: the start height // - toHeight: the end height @@ -282,11 +299,13 @@ func (d *DataProcessor) ProcessMessages( fromHeight uint64, toHeight uint64) error { - // Phase 1: Concurrent address collection using sync.Map - var addressesMap sync.Map - addressCollectionChan := make(chan []*decoder.DecodedMsg, len(transactions)) + // Phase 1: Concurrent address collection using map[strinct]struct{} + var mu sync.Mutex + transactionAmount := len(transactions) + addressesMap := make(map[string]struct{}) + addressCollectionChan := make(chan []*decoder.DecodedMsg, transactionAmount) wg1 := sync.WaitGroup{} - wg1.Add(len(transactions)) + wg1.Add(transactionAmount) // Launch goroutines to collect addresses concurrently for _, transaction := range transactions { @@ -298,11 +317,13 @@ func (d *DataProcessor) ProcessMessages( return } - // Collect addresses from this transaction and store in sync.Map + // Collect addresses from this transaction and store in map[string]struct{} addresses := decodedMsg.CollectAllAddresses() + mu.Lock() for _, address := range addresses { - addressesMap.Store(address, true) // Thread-safe deduplication + addressesMap[address] = struct{}{} } + mu.Unlock() addressCollectionChan <- []*decoder.DecodedMsg{decodedMsg} }(transaction) @@ -320,12 +341,11 @@ func (d *DataProcessor) ProcessMessages( allDecodedMsgs = append(allDecodedMsgs, decodedMsgs[0]) } - // Extract addresses from sync.Map and resolve to IDs + // Extract addresses from map[string]struct{} and resolve to IDs allAddresses := make([]string, 0) - addressesMap.Range(func(key, value interface{}) bool { - allAddresses = append(allAddresses, key.(string)) - return true - }) + for address := range addressesMap { + allAddresses = append(allAddresses, address) + } if len(allAddresses) > 0 { d.addressCache.AddressSolver(allAddresses, d.chainName, false, 3, nil) @@ -343,7 +363,7 @@ func (d *DataProcessor) ProcessMessages( var aggregationMutex sync.Mutex wg2 := sync.WaitGroup{} - wg2.Add(len(transactions)) + wg2.Add(transactionAmount) // Launch goroutines to process messages concurrently for idx, transaction := range transactions { @@ -394,7 +414,11 @@ func (d *DataProcessor) ProcessMessages( // Create a slice of sqlDataType.AddressTx // we need to get all of the addresses from the aggregatedDbGroups addresses := createAddressTx(aggregatedDbGroups) - err := d.dbPool.InsertAddressTx(addresses) + // add multiplier for the timeout depending on the address amount + timeout := 10*time.Second + (time.Duration(len(addresses)) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertAddressTx(ctx, addresses) + cancel() if err != nil { return fmt.Errorf("failed to insert address tx: %w", err) } @@ -418,31 +442,68 @@ func (d *DataProcessor) ProcessMessages( func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) error { var insertErrors []error + msgSendCount := len(groups.MsgSend) + msgCallCount := len(groups.MsgCall) + msgAddPkgCount := len(groups.MsgAddPkg) + msgRunCount := len(groups.MsgRun) + // Insert DbMsgSend messages with address IDs - if len(groups.MsgSend) > 0 { - if err := d.dbPool.InsertMsgSend(groups.MsgSend); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgSend: %w", err)) + if msgSendCount > 0 { + timeout := 10*time.Second + (time.Duration(msgSendCount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgSend(ctx, groups.MsgSend) + cancel() + if err != nil { + hashes := make([]string, 0, len(groups.MsgSend)) + for _, msg := range groups.MsgSend { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgSend: %w, hashes: %v", err, hashes)) } } // Insert DbMsgCall messages with address IDs - if len(groups.MsgCall) > 0 { - if err := d.dbPool.InsertMsgCall(groups.MsgCall); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgCall: %w", err)) + if msgCallCount > 0 { + timeout := 10*time.Second + (time.Duration(msgCallCount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgCall(ctx, groups.MsgCall) + cancel() + if err != nil { + hashes := make([]string, 0, len(groups.MsgCall)) + for _, msg := range groups.MsgCall { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgCall: %w, hashes: %v", err, hashes)) } } // Insert DbMsgAddPackage messages with address IDs - if len(groups.MsgAddPkg) > 0 { - if err := d.dbPool.InsertMsgAddPackage(groups.MsgAddPkg); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgAddPackage: %w", err)) + if msgAddPkgCount > 0 { + timeout := 10*time.Second + (time.Duration(msgAddPkgCount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgAddPackage(ctx, groups.MsgAddPkg) + cancel() + if err != nil { + hashes := make([]string, 0, len(groups.MsgAddPkg)) + for _, msg := range groups.MsgAddPkg { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgAddPackage: %w, hashes: %v", err, hashes)) } } // Insert DbMsgRun messages with address IDs - if len(groups.MsgRun) > 0 { - if err := d.dbPool.InsertMsgRun(groups.MsgRun); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgRun: %w", err)) + if msgRunCount > 0 { + timeout := 10*time.Second + (time.Duration(msgRunCount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgRun(ctx, groups.MsgRun) + cancel() + if err != nil { + hashes := make([]string, 0, len(groups.MsgRun)) + for _, msg := range groups.MsgRun { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgRun: %w, hashes: %v", err, hashes)) } } @@ -459,27 +520,28 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e } func (d *DataProcessor) ProcessValidatorSignings( - blocks []*rpcClient.BlockResponse, + commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) { - validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, len(blocks)) + commitAmount := len(commits) + validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, commitAmount) wg := sync.WaitGroup{} - wg.Add(len(blocks)) + wg.Add(commitAmount) // Process blocks concurrently - for _, block := range blocks { - go func(block *rpcClient.BlockResponse) { + for _, commit := range commits { + go func(commit *rpcClient.CommitResponse) { defer wg.Done() signedVals := struct { Proposer int32 SignedVals []int32 }{ - Proposer: d.validatorCache.GetAddress(block.Result.Block.Header.ProposerAddress), + Proposer: d.validatorCache.GetAddress(commit.GetProposerAddress()), SignedVals: make([]int32, 0), } - precommits := block.Result.Block.LastCommit.Precommits + precommits := commit.GetSigners() for _, precommit := range precommits { if precommit != nil { signedVals.SignedVals = append( @@ -488,20 +550,20 @@ func (d *DataProcessor) ProcessValidatorSignings( } } - height, err := strconv.ParseUint(block.Result.Block.Header.Height, 10, 64) + height, err := commit.GetHeight() if err != nil { - log.Printf("Failed to parse block height %s: %v", block.Result.Block.Header.Height, err) + log.Printf("Failed to get commit height: %v", err) return } validatorChan <- &sqlDataTypes.ValidatorBlockSigning{ BlockHeight: height, - Timestamp: block.Result.Block.Header.Time, + Timestamp: commit.GetTimestamp(), Proposer: signedVals.Proposer, SignedVals: signedVals.SignedVals, ChainName: d.chainName, } - }(block) + }(commit) } // Close channel when all goroutines finish @@ -510,23 +572,30 @@ func (d *DataProcessor) ProcessValidatorSignings( close(validatorChan) }() - validatorData := make([]sqlDataTypes.ValidatorBlockSigning, 0, len(blocks)) + validatorData := make([]sqlDataTypes.ValidatorBlockSigning, 0, len(commits)) for validator := range validatorChan { validatorData = append(validatorData, *validator) } - err := d.dbPool.InsertValidatorBlockSignings(validatorData) + timeout := 10*time.Second + (time.Duration(commitAmount) * time.Second / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertValidatorBlockSignings(ctx, validatorData) + cancel() if err != nil { - log.Printf("Failed to insert validator block signings: %v", err) + log.Printf("Failed to insert validator commit signings: %v", err) } - log.Printf("Validator block signings processed from %d to %d", fromHeight, toHeight) + log.Printf("Validator commit signings processed from %d to %d", fromHeight, toHeight) } // createAddressTx is a private func that creates a slice of sqlDataTypes.AddressTx from a // decoder.DbMessageGroups using concurrent workers with mutex-based aggregation // it should be used to create the data for the address_tx table func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { - txAmount := len(msg.MsgSend) + len(msg.MsgCall) + len(msg.MsgAddPkg) + len(msg.MsgRun) + msgSendCount := len(msg.MsgSend) + msgCallCount := len(msg.MsgCall) + msgAddPkgCount := len(msg.MsgAddPkg) + msgRunCount := len(msg.MsgRun) + txAmount := msgSendCount + msgCallCount + msgAddPkgCount + msgRunCount if txAmount == 0 { return []sqlDataTypes.AddressTx{} } @@ -536,7 +605,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { wg := sync.WaitGroup{} // Process MsgSend messages - wg.Add(len(msg.MsgSend)) + wg.Add(msgSendCount) for _, msgItem := range msg.MsgSend { go func(msgItem sqlDataTypes.MsgSend) { defer wg.Done() @@ -564,7 +633,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgCall messages - wg.Add(len(msg.MsgCall)) + wg.Add(msgCallCount) for _, msgItem := range msg.MsgCall { go func(msgItem sqlDataTypes.MsgCall) { defer wg.Done() @@ -590,7 +659,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgAddPkg messages - wg.Add(len(msg.MsgAddPkg)) + wg.Add(msgAddPkgCount) for _, msgItem := range msg.MsgAddPkg { go func(msgItem sqlDataTypes.MsgAddPackage) { defer wg.Done() @@ -616,7 +685,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgRun messages - wg.Add(len(msg.MsgRun)) + wg.Add(msgRunCount) for _, msgItem := range msg.MsgRun { go func(msgItem sqlDataTypes.MsgRun) { defer wg.Done() diff --git a/indexer/data_processor/operator_test.go b/indexer/data_processor/operator_test.go index c362891..0bfa462 100644 --- a/indexer/data_processor/operator_test.go +++ b/indexer/data_processor/operator_test.go @@ -1,6 +1,7 @@ package dataprocessor_test import ( + "context" "testing" dataProcessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor" @@ -14,37 +15,37 @@ type MockDatabase struct { LastInsertError error } -func (m *MockDatabase) InsertBlocks(blocks []sqlDataTypes.Blocks) error { +func (m *MockDatabase) InsertBlocks(ctx context.Context, blocks []sqlDataTypes.Blocks) error { m.InsertBlocksCalled = true return m.LastInsertError } -func (m *MockDatabase) InsertValidatorBlockSignings(signings []sqlDataTypes.ValidatorBlockSigning) error { +func (m *MockDatabase) InsertValidatorBlockSignings(ctx context.Context, signings []sqlDataTypes.ValidatorBlockSigning) error { return m.LastInsertError } -func (m *MockDatabase) InsertTransactionsGeneral(transactions []sqlDataTypes.TransactionGeneral) error { +func (m *MockDatabase) InsertTransactionsGeneral(ctx context.Context, transactions []sqlDataTypes.TransactionGeneral) error { m.InsertTransactionsCalled = true return m.LastInsertError } -func (m *MockDatabase) InsertMsgSend(messages []sqlDataTypes.MsgSend) error { +func (m *MockDatabase) InsertMsgSend(ctx context.Context, messages []sqlDataTypes.MsgSend) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgCall(messages []sqlDataTypes.MsgCall) error { +func (m *MockDatabase) InsertMsgCall(ctx context.Context, messages []sqlDataTypes.MsgCall) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgAddPackage(messages []sqlDataTypes.MsgAddPackage) error { +func (m *MockDatabase) InsertMsgAddPackage(ctx context.Context, messages []sqlDataTypes.MsgAddPackage) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgRun(messages []sqlDataTypes.MsgRun) error { +func (m *MockDatabase) InsertMsgRun(ctx context.Context, messages []sqlDataTypes.MsgRun) error { return m.LastInsertError } -func (m *MockDatabase) InsertAddressTx(addresses []sqlDataTypes.AddressTx) error { +func (m *MockDatabase) InsertAddressTx(ctx context.Context, addresses []sqlDataTypes.AddressTx) error { return m.LastInsertError } @@ -111,17 +112,17 @@ func TestDataProcessor_DatabaseInterface(t *testing.T) { var db dataProcessor.Database = &MockDatabase{} // Test interface methods - err := db.InsertBlocks([]sqlDataTypes.Blocks{}) + err := db.InsertBlocks(context.Background(), []sqlDataTypes.Blocks{}) if err != nil { t.Errorf("InsertBlocks should not return error with nil input, got: %v", err) } - err = db.InsertTransactionsGeneral([]sqlDataTypes.TransactionGeneral{}) + err = db.InsertTransactionsGeneral(context.Background(), []sqlDataTypes.TransactionGeneral{}) if err != nil { t.Errorf("InsertTransactionsGeneral should not return error with nil input, got: %v", err) } - err = db.InsertAddressTx([]sqlDataTypes.AddressTx{}) + err = db.InsertAddressTx(context.Background(), []sqlDataTypes.AddressTx{}) if err != nil { t.Errorf("InsertAddressTx should not return error with nil input, got: %v", err) } diff --git a/indexer/data_processor/types.go b/indexer/data_processor/types.go index 906f04c..7dc183d 100644 --- a/indexer/data_processor/types.go +++ b/indexer/data_processor/types.go @@ -1,6 +1,7 @@ package dataprocessor import ( + "context" "time" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" @@ -9,14 +10,14 @@ import ( // Define interface for what DataProcessor needs from database type Database interface { - InsertBlocks(blocks []sqlDataTypes.Blocks) error - InsertValidatorBlockSignings(validatorBlockSignings []sqlDataTypes.ValidatorBlockSigning) error - InsertTransactionsGeneral(transactionsGeneral []sqlDataTypes.TransactionGeneral) error - InsertMsgSend(messages []sqlDataTypes.MsgSend) error - InsertMsgCall(messages []sqlDataTypes.MsgCall) error - InsertMsgAddPackage(messages []sqlDataTypes.MsgAddPackage) error - InsertMsgRun(messages []sqlDataTypes.MsgRun) error - InsertAddressTx(addresses []sqlDataTypes.AddressTx) error + InsertBlocks(ctx context.Context, blocks []sqlDataTypes.Blocks) error + InsertValidatorBlockSignings(ctx context.Context, validatorBlockSignings []sqlDataTypes.ValidatorBlockSigning) error + InsertTransactionsGeneral(ctx context.Context, transactionsGeneral []sqlDataTypes.TransactionGeneral) error + InsertMsgSend(ctx context.Context, messages []sqlDataTypes.MsgSend) error + InsertMsgCall(ctx context.Context, messages []sqlDataTypes.MsgCall) error + InsertMsgAddPackage(ctx context.Context, messages []sqlDataTypes.MsgAddPackage) error + InsertMsgRun(ctx context.Context, messages []sqlDataTypes.MsgRun) error + InsertAddressTx(ctx context.Context, addresses []sqlDataTypes.AddressTx) error } // Define interface for what DataProcessor needs from AddressCache diff --git a/indexer/db_init/hypertable.go b/indexer/db_init/hypertable.go index 3d42939..aeeacca 100644 --- a/indexer/db_init/hypertable.go +++ b/indexer/db_init/hypertable.go @@ -19,20 +19,12 @@ import ( // - Community Edition 2.19.3+ (TimescaleDB) // - Cloud edition (Tiger Data) // -// Not Recommended: -// - Apache TimescaleDB 2.19.3+ (Apache TimescaleDB) -// To be clear it is not recommended to use the Apache TimescaleDB because it is not tested on and it won't -// be officially supported. The reason is most of the features that are available in the Community Edition are not -// available in the Apache TimescaleDB. However it might be possible to use it since we only need to use hypertables -// which should exist in the Apache TimescaleDB. -// -// For maximum compatibility, this implementation supports both approaches. // ConvertToHypertables is a method that converts the given table names to hypertables // // This function will only start this process however the whole process will run through the 3 steps // This is first step in the process -// Args: +// Parameters: // - tableNames: a slice of table names to convert to hypertables // // Returns: @@ -59,7 +51,7 @@ func (init *DBInitializer) ConvertToHypertables(tableNames []string) { // // This function will only start this process however the whole process will run through the 3 steps // This is second step in the process -// Args: +// Parameters: // - tables: a map of table names to their columns // // Returns: @@ -89,7 +81,7 @@ func (init *DBInitializer) AlterCompressionSegments(tables map[string][]string) // // This function will only start this process however the whole process will run through the 3 steps // This is third step in the process -// Args: +// Parameters: // - tableNames: a slice of table names to add the compression policy to // // Returns: diff --git a/indexer/db_init/tables.go b/indexer/db_init/tables.go index a50a16e..57d542d 100644 --- a/indexer/db_init/tables.go +++ b/indexer/db_init/tables.go @@ -41,7 +41,7 @@ type SpecialType struct { // GetTableInfo extracts database table information from a struct using reflection // This function reads the struct tags and converts them to database metadata // -// Args: +// Parameters: // - structType: the struct type to get the table info from // - tableName: the name of the table to get the table info from // @@ -113,7 +113,7 @@ func GetTableInfo(structType interface{}, tableName string) (*TableInfo, error) // GenerateCreateTableSQL generates a PostgreSQL CREATE TABLE statement from struct metadata // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // // Returns: @@ -174,7 +174,7 @@ func GenerateCreateTableSQL(tableInfo *TableInfo) string { // GenerateCreateHypertableSQL generates a PostgreSQL CREATE TABLE statement with modern TimescaleDB hypertable syntax // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // - chunkInterval: the interval to chunk the table by // - partitionColumn: the column to partition the table by @@ -234,7 +234,7 @@ func GenerateCreateHypertableSQL(tableInfo *TableInfo, chunkInterval string, par // GenerateSpecialTypeSQL generates a PostgreSQL CREATE TYPE statement from struct metadata // -// Args: +// Parameters: // - specialType: the special type to generate the SQL for // // Returns: @@ -260,7 +260,7 @@ func GenerateSpecialTypeSQL(specialType *SpecialType) string { // CreateTableSQL creates a table in the database based on struct metadata // -// Args: +// Parameters: // - t: the table info for the table to create // // Returns: @@ -350,7 +350,7 @@ func (db *DBInitializer) GetTimescaleDBVersion() (*TimescaleDBVersion, error) { // CreateTableFromStruct creates a database table based on struct tags // -// Args: +// Parameters: // - structType: the struct type to create the table from // - tableName: the name of the table to create // @@ -419,7 +419,7 @@ func GetSpecialTypeInfo(structType interface{}, typeName string) (*SpecialType, // CreateSpecialTypeFromStruct creates a database type based on struct tags // -// Args: +// Parameters: // - structType: the struct type to create the special type from // - typeName: the name of the special type to create // @@ -444,7 +444,7 @@ func (db *DBInitializer) CreateSpecialTypeFromStruct(structType interface{}, typ // CreateHypertableFromStruct creates a hypertable using the appropriate method based on TimescaleDB version // -// Args: +// Parameters: // - structType: the struct type to create the hypertable from // - tableName: the name of the table to create the hypertable from // - partitionColumn: the column to partition the table by @@ -493,7 +493,7 @@ func (db *DBInitializer) createHypertableModern(tableInfo *TableInfo, partitionC // createHypertableLegacy creates a hypertable using the legacy 3-step process // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // - partitionColumn: the column to partition the table by // - chunkInterval: the interval to chunk the table by @@ -542,7 +542,7 @@ func (db *DBInitializer) CreateChainTypeEnum(enumValues []string) error { // CreateUser creates a user in the database // // The function will -// Args: +// Parameters: // - name: the name of the user to create // // Returns: diff --git a/indexer/decoder/decoder.go b/indexer/decoder/decoder.go index ed86815..b2f80c6 100644 --- a/indexer/decoder/decoder.go +++ b/indexer/decoder/decoder.go @@ -23,7 +23,7 @@ type Decoder struct { // NewDecoder creates a new Decoder struct // -// Args: +// Parameters: // - encodedTx: base64 encoded stdTx // // Returns: @@ -39,7 +39,7 @@ func NewDecoder(encodedTx string) *Decoder { // The function decodes the data and unmarshals it into a std.Tx struct // The struct contains the transaction data and the messages // -// Args: +// Parameters: // - s: base64 encoded stdTx // // Returns: @@ -64,7 +64,7 @@ func (d *Decoder) DecodeStdTxFromBase64() (*std.Tx, error) { // to be reflected here, mostly... // Some will need to be updated and added manually in the future but for now this is good enough // -// Args: +// Parameters: // - none // // Returns: @@ -108,7 +108,11 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { var messages []map[string]any // Process each message in the transaction - for _, msg := range tx.GetMsgs() { + for i, msg := range tx.GetMsgs() { + if i > 32767 { + return BasicTxData{}, nil, fmt.Errorf("transaction message count exceeds maximum: %d", i) + } + messageCounter := int16(i) switch m := msg.(type) { case bank.MsgSend: // amount should have something like 1000000 ugnot we just need to split it and convert it to uint64 @@ -117,10 +121,11 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { amount = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "bank_msg_send", - "from_address": m.FromAddress.String(), - "to_address": m.ToAddress.String(), - "amount": amount, + "msg_type": "bank_msg_send", + "from_address": m.FromAddress.String(), + "to_address": m.ToAddress.String(), + "amount": amount, + "message_counter": messageCounter, }) case vm.MsgCall: caller := m.Caller.String() @@ -139,13 +144,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { // combine the args into a string args := strings.Join(m.Args, ",") messages = append(messages, map[string]any{ - "msg_type": "vm_msg_call", - "caller": caller, - "pkg_path": pkgPath, - "func_name": funcName, - "args": args, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_call", + "caller": caller, + "pkg_path": pkgPath, + "func_name": funcName, + "args": args, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) case vm.MsgAddPackage: pkgPath := m.Package.Path @@ -161,13 +167,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { maxDeposit = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "vm_msg_add_package", - "pkg_path": pkgPath, - "pkg_name": pkgName, - "pkg_file_names": pkgFileNames, - "creator": creator, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_add_package", + "pkg_path": pkgPath, + "pkg_name": pkgName, + "pkg_file_names": pkgFileNames, + "creator": creator, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) case vm.MsgRun: @@ -186,13 +193,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { maxDeposit = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "vm_msg_run", - "caller": caller, - "pkg_path": pkgPath, - "pkg_name": pkgName, - "pkg_file_names": pkgFileNames, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_run", + "caller": caller, + "pkg_path": pkgPath, + "pkg_name": pkgName, + "pkg_file_names": pkgFileNames, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) // case for AnyNewMessage add here: default: diff --git a/indexer/decoder/product.go b/indexer/decoder/product.go index 12cbd54..60e25bf 100644 --- a/indexer/decoder/product.go +++ b/indexer/decoder/product.go @@ -11,7 +11,7 @@ import ( // NewDecodedMsg creates a new DecodedMsg struct // -// Args: +// Parameters: // - encodedTx: the encoded transaction // // Returns: @@ -229,6 +229,11 @@ func (dm *DecodedMsg) convertToDbMsgSend( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgSend, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + fromAddress, ok := msgMap["from_address"].(string) if !ok { return nil, fmt.Errorf("missing from_address") @@ -255,13 +260,14 @@ func (dm *DecodedMsg) convertToDbMsgSend( } return &dataTypes.MsgSend{ - TxHash: txHash, - ChainName: chainName, - FromAddress: addressResolver.GetAddress(fromAddress), - ToAddress: addressResolver.GetAddress(toAddress), - Amount: amount, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + ChainName: chainName, + FromAddress: addressResolver.GetAddress(fromAddress), + ToAddress: addressResolver.GetAddress(toAddress), + Amount: amount, + Signers: signerIds, + Timestamp: timestamp, + MessageCounter: messageCounter, }, nil } @@ -274,6 +280,11 @@ func (dm *DecodedMsg) convertToDbMsgCall( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgCall, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + caller, ok := msgMap["caller"].(string) if !ok { return nil, fmt.Errorf("missing caller") @@ -325,16 +336,17 @@ func (dm *DecodedMsg) convertToDbMsgCall( } return &dataTypes.MsgCall{ - TxHash: txHash, - ChainName: chainName, - Caller: addressResolver.GetAddress(caller), - Send: send, - PkgPath: pkgPath, - FuncName: funcName, - Args: argsStr, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Caller: addressResolver.GetAddress(caller), + Send: send, + PkgPath: pkgPath, + FuncName: funcName, + Args: argsStr, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } @@ -347,6 +359,11 @@ func (dm *DecodedMsg) convertToDbMsgAddPackage( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgAddPackage, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + creator, ok := msgMap["creator"].(string) if !ok { return nil, fmt.Errorf("missing creator") @@ -393,15 +410,16 @@ func (dm *DecodedMsg) convertToDbMsgAddPackage( } return &dataTypes.MsgAddPackage{ - TxHash: txHash, - ChainName: chainName, - Creator: addressResolver.GetAddress(creator), - PkgPath: pkgPath, - PkgName: pkgName, - Send: send, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Creator: addressResolver.GetAddress(creator), + PkgPath: pkgPath, + PkgName: pkgName, + Send: send, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } @@ -414,6 +432,11 @@ func (dm *DecodedMsg) convertToDbMsgRun( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgRun, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + caller, ok := msgMap["caller"].(string) if !ok { return nil, fmt.Errorf("missing caller") @@ -460,14 +483,15 @@ func (dm *DecodedMsg) convertToDbMsgRun( } return &dataTypes.MsgRun{ - TxHash: txHash, - ChainName: chainName, - Caller: addressResolver.GetAddress(caller), - PkgPath: pkgPath, - PkgName: pkgName, - Send: send, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Caller: addressResolver.GetAddress(caller), + PkgPath: pkgPath, + PkgName: pkgName, + Send: send, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } diff --git a/indexer/main.go b/indexer/indexer.go similarity index 100% rename from indexer/main.go rename to indexer/indexer.go diff --git a/indexer/main_operator/operator.go b/indexer/main_operator/operator.go index 72d82ff..c737a45 100644 --- a/indexer/main_operator/operator.go +++ b/indexer/main_operator/operator.go @@ -111,7 +111,7 @@ func InitMainOperator( // initializeDatabase is a private function to initialize the database // it is used to initialize the database for the main operator // -// Args: +// Parameters: // - conf: the config // - env: the environment // @@ -174,7 +174,7 @@ func initializeDatabase(conf *config.Config, env *config.Environment) *database. // initializeMajorConstructors is a private function to initialize the major constructors // it is used to initialize the major constructors for the main operator // -// Args: +// Parameters: // - conf: the config // - env: the environment // - chainName: the chain name diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go index 3239294..a895759 100644 --- a/indexer/orchestrator/operator.go +++ b/indexer/orchestrator/operator.go @@ -65,33 +65,16 @@ func (or *Orchestrator) HistoricProcess( for startHeight := fromHeight; startHeight <= toHeight; { chunkEndHeight := min(startHeight+or.config.MaxBlockChunkSize-1, toHeight) - chunkStartTime := time.Now() log.Printf("Processing chunk from %d to %d", startHeight, chunkEndHeight) // Update current processing height or.currentProcessingHeight = startHeight - // Step 1: Get blocks concurrently - blocks := or.queryOperator.GetFromToBlocks(startHeight, chunkEndHeight) - - if len(blocks) == 0 { - log.Printf("No valid blocks in chunk %d-%d", startHeight, chunkEndHeight) - } else { - // Step 2: Collect all transactions from all blocks in this chunk - allTransactions := or.collectTransactionsFromBlocks(blocks) - - log.Printf("Collected %d transactions from %d blocks in chunk", len(allTransactions), len(blocks)) - - // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, allTransactions, false, startHeight, chunkEndHeight); err != nil { - log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err) - } else { - // Progress logging - chunkDuration := time.Since(chunkStartTime) - totalDuration := time.Since(startTime) - log.Printf("Chunk %d-%d completed in %v, total time: %v", - startHeight, chunkEndHeight, chunkDuration, totalDuration) - } + // Process the chunk + err := or.processChunk(startHeight, chunkEndHeight) + if err != nil { + log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err) + } // Always advance to next chunk, regardless of whether blocks were found @@ -120,13 +103,15 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool // Initial setup - get starting height if !skipInitialDbCheck { - lastProcessedHeight, err = or.db.GetLastBlockHeight(or.chainName) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + lastProcessedHeight, err = or.db.GetLastBlockHeight(ctx, or.chainName) if err != nil { log.Printf("Failed to get last block height from database: %v", err) log.Printf("Either there are no blocks in the database or the database is not properly configured.") log.Printf("Use skipInitialDbCheck=true if this is expected to run from the latest chain height without previous data.") log.Printf("Starting from height 1") - lastProcessedHeight = 1 + lastProcessedHeight = 0 } log.Printf("Retrieved last processed height from database: %d", lastProcessedHeight) } else { @@ -161,7 +146,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool continue } - blocksBehind := int64(latestHeight) - int64(lastProcessedHeight) + blocksBehind := latestHeight - lastProcessedHeight // If caught up, wait and continue if blocksBehind <= 0 { @@ -171,7 +156,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool } // Adjust chunk size based on how far behind we are - currentChunkSize := min(uint64(blocksBehind), or.config.MaxBlockChunkSize) + currentChunkSize := min(blocksBehind, or.config.MaxBlockChunkSize) chunkStart := lastProcessedHeight + 1 chunkEnd := min(chunkStart+currentChunkSize-1, latestHeight) @@ -182,7 +167,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool or.currentProcessingHeight = chunkStart // Process this chunk - err = or.processLiveChunk(chunkStart, chunkEnd) + err = or.processChunk(chunkStart, chunkEnd) if err != nil { log.Printf("Error processing live chunk %d-%d: %v", chunkStart, chunkEnd, err) time.Sleep(or.config.LivePooling) @@ -199,14 +184,29 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool } } -// processLiveChunk processes a single chunk of blocks for live processing -func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { +// processChunk processes a single chunk of blocks for live processing +func (or *Orchestrator) processChunk(chunkStart, chunkEnd uint64) error { chunkStartTime := time.Now() // Step 1: Get blocks concurrently - blocks := or.queryOperator.GetFromToBlocks(chunkStart, chunkEnd) + var wg sync.WaitGroup + wg.Add(2) + + var blocks []*rpcClient.BlockResponse + var commits []*rpcClient.CommitResponse - if len(blocks) == 0 { + go func() { + defer wg.Done() + blocks = or.queryOperator.GetFromToBlocks(chunkStart, chunkEnd) + }() + go func() { + defer wg.Done() + commits = or.queryOperator.GetFromToCommits(chunkStart, chunkEnd) + }() + + wg.Wait() + + if len(blocks) == 0 && len(commits) == 0 { log.Printf("No valid blocks in live chunk %d-%d", chunkStart, chunkEnd) return nil } @@ -217,18 +217,21 @@ func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { log.Printf("Collected %d transactions from %d blocks in live chunk", len(allTransactions), len(blocks)) // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, allTransactions, false, chunkStart, chunkEnd); err != nil { + if err := or.processAll(blocks, commits, allTransactions, false, chunkStart, chunkEnd); err != nil { return fmt.Errorf("failed to process live chunk %d-%d: %w", chunkStart, chunkEnd, err) } chunkDuration := time.Since(chunkStartTime) - log.Printf("Live chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration) + log.Printf("Chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration) return nil } // updateProgressMetrics updates and logs progress metrics for live processing -func (or *Orchestrator) updateProgressMetrics(chunkStart, chunkEnd uint64, blocksBehind int64, lastProgressTime *time.Time) { +func (or *Orchestrator) updateProgressMetrics( + chunkStart, chunkEnd, blocksBehind uint64, + lastProgressTime *time.Time, +) { now := time.Now() timeSinceLastProgress := now.Sub(*lastProgressTime) @@ -241,8 +244,17 @@ func (or *Orchestrator) updateProgressMetrics(chunkStart, chunkEnd uint64, block } } -// collectTransactionsFromBlocks extracts all transactions from blocks and queries them concurrently -// This mimics the Python _process_historical_block_chunk behavior +/* + collectTransactionsFromBlocks extracts all transactions from blocks and queries them concurrently + +Parameters: + - blocks: a slice of blocks + +Returns: + - a slice of transactions + +The method will not throw an error if the transactions are not found, it will just return an empty slice. +*/ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockResponse) []dataprocessor.TrasnactionsData { // Collect all transaction hashes from all blocks var allTxHashes []string @@ -320,7 +332,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // This function processes all data using optimized concurrent execution // -// Args: +// Parameters: // - blocks: a slice of blocks // - transactions: a map of transactions and timestamps // - compressEvents: if true, compress the events @@ -331,8 +343,9 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // - error: if processing fails // // The method will not throw an error if the data is not found, it will just return nil -func (or *Orchestrator) processAllConcurrently( +func (or *Orchestrator) processAll( blocks []*rpcClient.BlockResponse, + commits []*rpcClient.CommitResponse, transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, @@ -409,7 +422,7 @@ func (or *Orchestrator) processAllConcurrently( go func() { defer wg2.Done() log.Printf("Phase 2: Starting ProcessValidatorSignings") - or.dataProcessor.ProcessValidatorSignings(blocks, fromHeight, toHeight) + or.dataProcessor.ProcessValidatorSignings(commits, fromHeight, toHeight) log.Printf("Phase 2: ProcessValidatorSignings completed") }() @@ -423,7 +436,7 @@ func (or *Orchestrator) processAllConcurrently( // saveProcessingState is a private method that saves // the current processing state to a file // -// Args: +// Parameters: // - height: the height of the processing state // - reason: the reason for the processing state // diff --git a/indexer/orchestrator/operator_test.go b/indexer/orchestrator/operator_test.go index 6e96d90..37cc34f 100644 --- a/indexer/orchestrator/operator_test.go +++ b/indexer/orchestrator/operator_test.go @@ -47,7 +47,7 @@ func (m *MockDataProcessor) ProcessMessages(transactions []dataprocessor.Trasnac } // Mock method for ProcessValidatorSignings -func (m *MockDataProcessor) ProcessValidatorSignings(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) { +func (m *MockDataProcessor) ProcessValidatorSignings(commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) { m.ProcessValidatorSigningsCalled = true } @@ -55,8 +55,9 @@ func (m *MockDataProcessor) ProcessValidatorSignings(blocks []*rpcClient.BlockRe // The idea should be to count the number of calls to the method // and check if the method is called with the correct parameters type MockQueryOperator struct { - ShouldReturnBlocks bool - CallCount int + ShouldReturnBlocks bool + CallCount int + ShouldReturnCommits bool } // Mock method for GetFromToBlocks @@ -70,6 +71,17 @@ func (m *MockQueryOperator) GetFromToBlocks(fromHeight uint64, toHeight uint64) return []*rpcClient.BlockResponse{{}} } +// Mock method for GetFromToCommits +func (m *MockQueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse { + m.CallCount++ + if !m.ShouldReturnCommits { + return []*rpcClient.CommitResponse{} // Return empty slice + } + + // Return a single empty commit + return []*rpcClient.CommitResponse{{}} +} + // Mock method for GetTransactions func (m *MockQueryOperator) GetTransactions(txs []string) []*rpcClient.TxResponse { return []*rpcClient.TxResponse{} // Empty transactions @@ -88,7 +100,7 @@ type MockDatabaseHeight struct { } // Mock method for GetLastBlockHeight -func (m *MockDatabaseHeight) GetLastBlockHeight(chainName string) (uint64, error) { +func (m *MockDatabaseHeight) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { if m.ShouldError { return 0, &DatabaseError{"database error"} } @@ -132,7 +144,8 @@ func TestOrchestrator_HistoricProcess_CallsAllProcessors(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} mockQueryOperator := &MockQueryOperator{ - ShouldReturnBlocks: true, // Return at least one block + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} @@ -179,7 +192,8 @@ func TestOrchestrator_HistoricProcess_SkipsProcessingWhenNoBlocks(t *testing.T) // Setup mocks - no blocks returned mockDataProcessor := &MockDataProcessor{} mockQueryOperator := &MockQueryOperator{ - ShouldReturnBlocks: false, // Return empty blocks + ShouldReturnBlocks: false, // Return empty blocks + ShouldReturnCommits: false, // Return empty commits } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} @@ -213,7 +227,10 @@ func TestOrchestrator_HistoricProcess_SkipsProcessingWhenNoBlocks(t *testing.T) func TestOrchestrator_LiveProcess_RespectsContext(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{HeightToReturn: 50} mockRPC := &MockGnolandRpcClient{HeightToReturn: 100} @@ -251,7 +268,10 @@ func TestOrchestrator_LiveProcess_RespectsContext(t *testing.T) { func TestOrchestrator_LiveProcess_SkipDbCheck(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{ShouldError: true} // Database will error mockRPC := &MockGnolandRpcClient{HeightToReturn: 100} @@ -284,7 +304,10 @@ func TestOrchestrator_Constructor_ValidatesMode(t *testing.T) { }() mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} diff --git a/indexer/orchestrator/types.go b/indexer/orchestrator/types.go index 0264708..8421fc7 100644 --- a/indexer/orchestrator/types.go +++ b/indexer/orchestrator/types.go @@ -1,6 +1,7 @@ package orchestrator import ( + "context" "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config" @@ -14,19 +15,20 @@ type DataProcessor interface { ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) ProcessTransactions(transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) ProcessMessages(transactions []dataprocessor.TrasnactionsData, fromHeight uint64, toHeight uint64) error - ProcessValidatorSignings(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) + ProcessValidatorSignings(commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) } type QueryOperator interface { GetFromToBlocks(fromHeight uint64, toHeight uint64) []*rpcClient.BlockResponse GetTransactions(txs []string) []*rpcClient.TxResponse GetLatestBlockHeight() (uint64, error) + GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse } // Only needed for one opetaion // Part of the timescaledb interface type DatabaseHeight interface { - GetLastBlockHeight(chainName string) (uint64, error) + GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) } // Only needed for one opetaion diff --git a/indexer/query/operator.go b/indexer/query/operator.go index 958b0ba..41ee6ce 100644 --- a/indexer/query/operator.go +++ b/indexer/query/operator.go @@ -51,7 +51,7 @@ func NewQueryOperator( // the indexer should store them all together as one huge slice of blocks, so the order is not important // the speed is what matters here. // -// Args: +// Parameters: // - fromHeight: the start height // - toHeight: the end height // @@ -140,11 +140,71 @@ func (q *QueryOperator) GetFromToBlocks(fromHeight uint64, toHeight uint64) []*r return blocks } +func (q *QueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rc.CommitResponse { + diff := toHeight - fromHeight + 1 + if diff < 1 { + return nil + } + + commits := make([]*rc.CommitResponse, diff) + var mu sync.Mutex + wg := sync.WaitGroup{} + wg.Add(int(diff)) + + // Launch goroutines to get the commits + for i := range diff { + height := fromHeight + i + idx := i // Capture index + go func(height uint64, idx int) { + commit, err := q.rpcClient.GetCommit(height) + if err != nil { + // Use retry mechanism with callback pattern + retry.RetryWithContext( + q.retryAmount, + q.pause, + q.pauseTime, + q.exponentialBackoff, + func(args ...any) (*rc.CommitResponse, error) { + h := args[0].(uint64) + result, rpcErr := q.rpcClient.GetCommit(h) + if rpcErr != nil { + return nil, rpcErr + } + return result, nil + }, + func(result *rc.CommitResponse) { + mu.Lock() + commits[idx] = result + mu.Unlock() + wg.Done() + }, + func(retryErr error) { + log.Printf("failed to get commit %d after retries: %v", height, retryErr) + mu.Lock() + commits[idx] = nil + mu.Unlock() + wg.Done() + }, + height, + ) + return + } + mu.Lock() + commits[idx] = commit + mu.Unlock() + wg.Done() + }(height, int(idx)) + } + + wg.Wait() + return commits +} + // A swarm method to get transactions from a slice of tx hashes // This is a fan out method that lauches async workers for each tx and wait to get the resaults // the indexer should store them all together as one huge slice of transactions, // -// Args: +// Parameters: // - txs: a slice of tx hashes // // Returns: diff --git a/indexer/query/operator_test.go b/indexer/query/operator_test.go index 6a16b38..df68d91 100644 --- a/indexer/query/operator_test.go +++ b/indexer/query/operator_test.go @@ -1,41 +1,61 @@ package query_test import ( + "sync" "testing" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/query" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" + "github.com/stretchr/testify/assert" ) // MockRpcClient - focuses on tracking what was called type MockRpcClient struct { + mu sync.Mutex GetBlockCalled bool GetLatestBlockHeightCalled bool GetTxCalled bool GetBlockCallCount int GetTxCallCount int + GetCommitCalled bool + GetCommitCallCount int } // Mock method for GetBlock func (m *MockRpcClient) GetBlock(height uint64) (*rpcClient.BlockResponse, *rpcClient.RpcHeightError) { + m.mu.Lock() m.GetBlockCalled = true m.GetBlockCallCount++ + m.mu.Unlock() return &rpcClient.BlockResponse{}, nil } // Mock method for GetLatestBlockHeight func (m *MockRpcClient) GetLatestBlockHeight() (uint64, *rpcClient.RpcHeightError) { + m.mu.Lock() m.GetLatestBlockHeightCalled = true + m.mu.Unlock() return 1, nil } // Mock method for GetTx func (m *MockRpcClient) GetTx(txHash string) (*rpcClient.TxResponse, *rpcClient.RpcStringError) { + m.mu.Lock() m.GetTxCalled = true m.GetTxCallCount++ + m.mu.Unlock() return &rpcClient.TxResponse{}, nil } +// Mock method for GetCommit +func (m *MockRpcClient) GetCommit(height uint64) (*rpcClient.CommitResponse, *rpcClient.RpcCommitError) { + m.mu.Lock() + m.GetCommitCalled = true + m.GetCommitCallCount++ + m.mu.Unlock() + return &rpcClient.CommitResponse{}, nil +} + // TestQueryOperator - tests the query operator func TestQueryOperator(t *testing.T) { mockRpcClient := &MockRpcClient{} @@ -43,32 +63,26 @@ func TestQueryOperator(t *testing.T) { // Test GetFromToBlocks - should call GetBlock multiple times (1 to 10 = 10 calls) queryOperator.GetFromToBlocks(1, 10) - if !mockRpcClient.GetBlockCalled { - t.Errorf("GetFromToBlocks should call GetBlock") - } - if mockRpcClient.GetBlockCallCount != 10 { - t.Errorf("GetFromToBlocks(1, 10) should call GetBlock 10 times, got %d", mockRpcClient.GetBlockCallCount) - } + assert.True(t, mockRpcClient.GetBlockCalled) + assert.Equal(t, 10, mockRpcClient.GetBlockCallCount) // Reset mock for next test mockRpcClient.GetLatestBlockHeightCalled = false // Test GetLatestBlockHeight _, err := queryOperator.GetLatestBlockHeight() - if err != nil { - t.Errorf("GetLatestBlockHeight should not return an error") - } - if !mockRpcClient.GetLatestBlockHeightCalled { - t.Errorf("GetLatestBlockHeight should be called") - } + assert.NoError(t, err) + assert.True(t, mockRpcClient.GetLatestBlockHeightCalled) // Test GetTransactions - should call GetTx for each transaction hash txHashes := []string{"txHash1", "txHash2", "txHash3"} queryOperator.GetTransactions(txHashes) - if !mockRpcClient.GetTxCalled { - t.Errorf("GetTransactions should call GetTx") - } - if mockRpcClient.GetTxCallCount != len(txHashes) { - t.Errorf("GetTransactions should call GetTx %d times, got %d", len(txHashes), mockRpcClient.GetTxCallCount) - } + assert.True(t, mockRpcClient.GetTxCalled) + assert.Equal(t, len(txHashes), mockRpcClient.GetTxCallCount) + + // Test GetFromToCommits - should call GetCommit multiple times (1 to 10 = 10 calls) + queryOperator.GetFromToCommits(1, 10) + assert.True(t, mockRpcClient.GetCommitCalled) + + assert.Equal(t, 10, mockRpcClient.GetCommitCallCount) } diff --git a/indexer/query/types.go b/indexer/query/types.go index 4aeca03..b19df5c 100644 --- a/indexer/query/types.go +++ b/indexer/query/types.go @@ -33,4 +33,5 @@ type RpcClient interface { GetBlock(height uint64) (*rpcClient.BlockResponse, *rpcClient.RpcHeightError) GetLatestBlockHeight() (uint64, *rpcClient.RpcHeightError) GetTx(txHash string) (*rpcClient.TxResponse, *rpcClient.RpcStringError) + GetCommit(height uint64) (*rpcClient.CommitResponse, *rpcClient.RpcCommitError) } diff --git a/indexer/retry/worker.go b/indexer/retry/worker.go index d4f8dee..d5fa5e2 100644 --- a/indexer/retry/worker.go +++ b/indexer/retry/worker.go @@ -18,7 +18,7 @@ type RetryResult[T any] struct { // GenericRetryQuery is a generic retry wrapper that can work with any query function // It handles the retry logic and sends the result to a channel // -// Args: +// Parameters: // - retryAmount: the number of retry attempts, pulled from the query operator retry options // - pause: the number of attempts to pause after failing, pulled from the query operator retry options // - pauseTime: the time to pause after failing, pulled from the query operator retry options diff --git a/indexer/rpc_client/client.go b/indexer/rpc_client/client.go index 999beda..33b8e04 100644 --- a/indexer/rpc_client/client.go +++ b/indexer/rpc_client/client.go @@ -27,7 +27,7 @@ import ( // - GetTx: call to get a tx from the rpc client // - GetAbciQuery: call to get a abci query from the rpc client // -// Args: +// Parameters: // // - rpcURL: the url of the rpc client // - timeout: the timeout for the rpc client(optional) @@ -68,8 +68,9 @@ const ( Block = "block" AbciQuery = "abci_query" // might be useful for health check - Health = "health" - Tx = "tx" + Health = "health" + Tx = "tx" + RequestCommit = "commit" ) func (r *RpcGnoland) performRequest(method string, params map[string]any, result interface{}) error { @@ -118,7 +119,7 @@ func (r *RpcGnoland) Health() error { // GetValidators method to get validators from the rpc client. // -// Args: +// Parameters: // - height: the height of the block to get the validators for // // Returns: @@ -149,7 +150,7 @@ func (r *RpcGnoland) GetValidators(height uint64) (*ValidatorsResponse, *RpcHeig // GetBlock method to get a block from the rpc client. // -// Args: +// Parameters: // - height: the height of the block to get // // Returns: @@ -210,7 +211,7 @@ func (r *RpcGnoland) GetLatestBlockHeight() (uint64, *RpcHeightError) { // GetTx method to get a tx from the rpc client. // -// Args: +// Parameters: // - txHash: the base64 encoded string of the tx to get // // Returns: @@ -243,7 +244,7 @@ func (r *RpcGnoland) GetTx(txHash string) (*TxResponse, *RpcStringError) { // This method is used to get any kind of data from the rpc client. // This might not be used in the indexer but let's keep it here for now. // -// Args: +// Parameters: // - path: the path of the abci query // - data: the data of the abci query // - height: the height of the block to get the abci query for(optional, if not specified it will get the latest block) @@ -274,3 +275,25 @@ func (r *RpcGnoland) GetAbciQuery(path string, data string, height *uint64, prov return response["result"], nil } + +func (r *RpcGnoland) GetCommit(height uint64) (*CommitResponse, *RpcCommitError) { + response := &CommitResponse{} + params := map[string]any{ + "height": strconv.FormatUint(height, 10), + } + if err := r.performRequest(RequestCommit, params, response); err != nil { + return nil, &RpcCommitError{ + Height: height, + HasHeight: true, + Err: err, + } + } + if response.Error != nil { + return nil, &RpcCommitError{ + Height: height, + HasHeight: true, + Err: fmt.Errorf("rpc error: %v, %s", response.Error.Code, response.Error.Message), + } + } + return response, nil +} diff --git a/indexer/rpc_client/commit.go b/indexer/rpc_client/commit.go new file mode 100644 index 0000000..5df52d9 --- /dev/null +++ b/indexer/rpc_client/commit.go @@ -0,0 +1,68 @@ +package rpcclient + +import ( + "fmt" + "strconv" + "time" +) + +// CommitResponse is the response from the commit endpoint +type CommitResponse struct { + Jsonrpc string `json:"jsonrpc"` + ID int `json:"id"` + Error *JsonRpcError `json:"error,omitempty"` + Result CommitResult `json:"result"` +} + +// CommitResult is the result from the commit endpoint +type CommitResult struct { + SignedHeader SignedHeader `json:"signed_header"` +} + +// SignedHeader is the struct for the signed header, which is a part of commit result +type SignedHeader struct { + Header BlockHeader `json:"header"` + Commit Commit `json:"commit"` +} + +// Commit is the struct for the commit, which is a part of signed header +type Commit struct { + BlockID BlockID `json:"block_id"` + Precommits []*Precommit `json:"precommits"` +} + +// GetHeight returns the height of the commit +func (cr *CommitResponse) GetHeight() (uint64, error) { + if cr == nil { + return 0, fmt.Errorf("CommitResponse Header Height is nil") + } + height, err := strconv.ParseUint(cr.Result.SignedHeader.Header.Height, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse height: %v", err) + } + return height, nil +} + +// GetTimestamp returns the timestamp of the commit +func (cr *CommitResponse) GetTimestamp() time.Time { + if cr == nil { + return time.Time{} + } + return cr.Result.SignedHeader.Header.Time +} + +// GetSigners returns the signers of the commit +func (cr *CommitResponse) GetSigners() []*Precommit { + if cr == nil { + return nil + } + return cr.Result.SignedHeader.Commit.Precommits +} + +// GetProposerAddress returns the proposer address of the commit +func (cr *CommitResponse) GetProposerAddress() string { + if cr == nil { + return "" + } + return cr.Result.SignedHeader.Header.ProposerAddress +} diff --git a/indexer/rpc_client/errors.go b/indexer/rpc_client/errors.go index dfb1520..71efaf7 100644 --- a/indexer/rpc_client/errors.go +++ b/indexer/rpc_client/errors.go @@ -31,3 +31,18 @@ func (e *RpcStringError) Error() string { } return fmt.Sprintf("rpc error: %v", e.Err) } + +// RpcCommitError represents an RPC error that includes the commit context for retry purposes +type RpcCommitError struct { + Height uint64 + HasHeight bool // indicates if Height field is valid + Err error +} + +// Error implements the error interface +func (e *RpcCommitError) Error() string { + if e.HasHeight { + return fmt.Sprintf("rpc error at height %d: %v", e.Height, e.Err) + } + return fmt.Sprintf("rpc error: %v", e.Err) +} diff --git a/indexer/rpc_client/rate_limit/rate_limit.go b/indexer/rpc_client/rate_limit/rate_limit.go index 09f464b..9029e7d 100644 --- a/indexer/rpc_client/rate_limit/rate_limit.go +++ b/indexer/rpc_client/rate_limit/rate_limit.go @@ -14,7 +14,7 @@ type ChannelRateLimiter struct { // NewChannelRateLimiter creates a new ChannelRateLimiter // -// Args: +// Parameters: // - requestsAllowed: the number of requests allowed per time window // - timeWindow: the time window for rate limiting // diff --git a/indexer/rpc_client/rate_limited_client.go b/indexer/rpc_client/rate_limited_client.go index 1f9e1ba..8fddd34 100644 --- a/indexer/rpc_client/rate_limited_client.go +++ b/indexer/rpc_client/rate_limited_client.go @@ -8,7 +8,7 @@ import ( // NewRateLimitedRpcClient creates a new rate-limited RPC client wrapper // -// Args: +// Parameters: // - rpcURL: the url of the rpc client // - timeout: the timeout for the rpc client (optional) // - requestsPerWindow: number of requests allowed per time window @@ -79,6 +79,12 @@ func (r *RateLimitedRpcClient) GetAbciQuery(path string, data string, height *ui return r.client.GetAbciQuery(path, data, height, prove) } +// GetCommit method with rate limiting +func (r *RateLimitedRpcClient) GetCommit(height uint64) (*CommitResponse, *RpcCommitError) { + r.rateLimiter.Wait() + return r.client.GetCommit(height) +} + // TryHealth - non-blocking version that returns false if rate limited func (r *RateLimitedRpcClient) TryHealth() (error, bool) { if !r.rateLimiter.Allow() { @@ -132,6 +138,15 @@ func (r *RateLimitedRpcClient) TryGetAbciQuery(path string, data string, height return response, err, true } +// TryGetCommit - non-blocking version that returns false if rate limited +func (r *RateLimitedRpcClient) TryGetCommit(height uint64) (*CommitResponse, *RpcCommitError, bool) { + if !r.rateLimiter.Allow() { + return nil, nil, false // rate limited + } + response, err := r.client.GetCommit(height) + return response, err, true +} + // GetRateLimiterStatus returns information about the current rate limiter status func (r *RateLimitedRpcClient) GetRateLimiterStatus() rate_limit.ChannelRateLimiterStatus { return r.rateLimiter.GetStatus() diff --git a/indexer/rpc_client/types.go b/indexer/rpc_client/types.go index 765f6e0..40b1c94 100644 --- a/indexer/rpc_client/types.go +++ b/indexer/rpc_client/types.go @@ -34,6 +34,7 @@ type Client interface { GetLatestBlockHeight() (uint64, *RpcHeightError) GetTx(txHash string) (*TxResponse, *RpcStringError) GetAbciQuery(path string, data string, height *uint64, prove *bool) (any, error) + GetCommit(height uint64) (*CommitResponse, *RpcCommitError) } type RateLimiter interface { diff --git a/integration/synthetic/init.go b/integration/synthetic/init.go index 09cf467..cf642dc 100644 --- a/integration/synthetic/init.go +++ b/integration/synthetic/init.go @@ -1,6 +1,7 @@ package synthetic import ( + "context" "log" addressCache "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/address_cache" @@ -81,7 +82,7 @@ type MockDatabaseHeight struct { lastHeight uint64 } -func (m *MockDatabaseHeight) GetLastBlockHeight(chainName string) (uint64, error) { +func (m *MockDatabaseHeight) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { return m.lastHeight, nil } diff --git a/integration/synthetic/query_operator.go b/integration/synthetic/query_operator.go index 569217e..c083fb7 100644 --- a/integration/synthetic/query_operator.go +++ b/integration/synthetic/query_operator.go @@ -30,6 +30,9 @@ type SyntheticQueryOperator struct { // Pre-generated data for consistent testing blocks map[uint64]*rpcClient.BlockResponse transactions map[string]*rpcClient.TxResponse + commits map[uint64]*rpcClient.CommitResponse + // response maker + responseMaker *ResponseMaker } // NewSyntheticQueryOperator creates a new synthetic query operator @@ -52,6 +55,8 @@ func NewSyntheticQueryOperator(chainID string, fromHeight uint64, maxHeight uint signedValidators: validators, blocks: make(map[uint64]*rpcClient.BlockResponse), transactions: make(map[string]*rpcClient.TxResponse), + commits: make(map[uint64]*rpcClient.CommitResponse), + responseMaker: NewResponseMaker(gen), } // Pre-generate some blocks and transactions for consistency @@ -92,17 +97,33 @@ func (sq *SyntheticQueryOperator) GetTransactions(txHashes []string) []*rpcClien return transactions } +// GetFromToCommits implements the QueryOperator interface by returning synthetic commits +func (sq *SyntheticQueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse { + diff := toHeight - fromHeight + 1 + if diff < 1 { + return nil + } + + commits := make([]*rpcClient.CommitResponse, 0, diff) + + for height := fromHeight; height <= toHeight; height++ { + commits = append(commits, sq.getCommit(height)) + } + return commits +} + // GetLatestBlockHeight implements the QueryOperator interface func (sq *SyntheticQueryOperator) GetLatestBlockHeight() (uint64, error) { return sq.currentHeight, nil } -// preGenerateData creates a consistent dataset of blocks and transactions +// preGenerateData creates a consistent dataset of blocks, transactions and commits func (sq *SyntheticQueryOperator) preGenerateData(fromHeight uint64, maxHeight uint64) { startTime := time.Now() // Generate blocks from height start to maxHeight for height := fromHeight; height <= maxHeight; height++ { sq.createSynthBlock(height) + sq.createCommit(height) // Log progress every 100 blocks if height%100 == 0 { @@ -111,6 +132,7 @@ func (sq *SyntheticQueryOperator) preGenerateData(fromHeight uint64, maxHeight u } log.Printf("Pre-generated data for blocks from %d to %d in %v", fromHeight, maxHeight, time.Since(startTime)) log.Printf("Total transactions generated: %d", len(sq.transactions)) + log.Printf("Total commits generated: %d", len(sq.commits)) } // getBlock returns existing block or creates a new one @@ -173,15 +195,13 @@ func (sq *SyntheticQueryOperator) createSynthBlock(height uint64) (*rpcClient.Bl // Create the block using existing synthetic response maker blockInput := GenBlockInput{ - Height: height, - ChainID: sq.chainID, - Timestamp: blockTimestamp, - ProposerAddress: sq.signedValidators[height%uint64(len(sq.signedValidators))], // random validator - SignedValidators: sq.signedValidators, - TxsRaw: txRaws, + Height: height, + ChainID: sq.chainID, + Timestamp: blockTimestamp, + TxsRaw: txRaws, } - block := GenerateBlockResponse(blockInput) + block := sq.responseMaker.GenerateBlockResponse(blockInput) sq.blocks[height] = block return block, txResponses @@ -211,12 +231,34 @@ func (sq *SyntheticQueryOperator) createTransaction( Events: txEvents, } - tx := GenerateTransactionResponse(txInput) + tx := sq.responseMaker.GenerateTransactionResponse(txInput) sq.transactions[txHash] = tx return tx } +// createCommit generates a synthetic commit for the given height +func (sq *SyntheticQueryOperator) createCommit(height uint64) *rpcClient.CommitResponse { + commitInput := GenCommitInput{ + Height: height, + ChainID: sq.chainID, + Timestamp: baseTimestamp.Add(time.Duration(height-1) * blockProductionRate), + ProposerAddress: sq.signedValidators[height%uint64(len(sq.signedValidators))], // random validator + SignedValidators: sq.signedValidators, + } + commit := sq.responseMaker.GenerateCommitResponse(commitInput) + sq.commits[height] = commit + return commit +} + +func (sq *SyntheticQueryOperator) getCommit(height uint64) *rpcClient.CommitResponse { + if commit, ok := sq.commits[height]; ok { + return commit + } + log.Fatal("commit not found") + return nil +} + // SetCurrentHeight allows updating the current height for live testing scenarios func (sq *SyntheticQueryOperator) SetCurrentHeight(height uint64) { sq.currentHeight = height diff --git a/integration/synthetic/respone_maker.go b/integration/synthetic/respone_maker.go index ff746dd..01e7284 100644 --- a/integration/synthetic/respone_maker.go +++ b/integration/synthetic/respone_maker.go @@ -8,53 +8,39 @@ import ( "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/generator" ) +type ResponseMaker struct { + generator *generator.DataGenerator +} + +func NewResponseMaker(generator *generator.DataGenerator) *ResponseMaker { + return &ResponseMaker{ + generator: generator, + } +} + type GenBlockInput struct { - Height uint64 - ChainID string - Timestamp time.Time - ProposerAddress string - SignedValidators []string - TxsRaw []string + Height uint64 + ChainID string + Timestamp time.Time + TxsRaw []string } -func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { - gen := generator.NewDataGenerator(500) - - // here is the deal the progrma will just make some hash here - // usually there are 2 hashes, new hash and hash from the previous block - // however for this instance we will just ignore it because - // this level of precision is not needed since the purpose is to - // test the indexer - blockHash := gen.GenerateBlockHash() - partsHash := gen.GenerateBlockHash() - - // the program will generate some hash for the previous block - // so it has some similaraties to the real response - lastBlockHash := gen.GenerateBlockHash() - lastPartsHash := gen.GenerateBlockHash() - - // just make some random hash here - lastCommitHash := gen.GenerateBlockHash() - dataHash := gen.GenerateBlockHash() - validatorsHash := gen.GenerateBlockHash() - nextValidatorsHash := gen.GenerateBlockHash() - consensusHash := gen.GenerateBlockHash() - appHash := gen.GenerateBlockHash() - lastResultsHash := gen.GenerateBlockHash() +func (rm *ResponseMaker) GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { + // just generate the hash for block and last block + // for others just use empty strings since this is all just to mock the similar expereience + blockHash := rm.generator.GenerateBlockHash() + lastBlockHash := rm.generator.GenerateBlockHash() + partsHash := "" + lastPartsHash := "" + lastCommitHash := "" + dataHash := "" + validatorsHash := "" + nextValidatorsHash := "" + consensusHash := "" + appHash := "" + lastResultsHash := "" - precommits := make([]*rpcClient.Precommit, 0, len(input.SignedValidators)) - for i, validator := range input.SignedValidators { - precommits = append(precommits, &rpcClient.Precommit{ - ValidatorAddress: validator, - ValidatorIndex: strconv.Itoa(i), - Signature: gen.GenerateBlockHash(), - Type: 2, // not sure why 2 but that is what I saw in the real data - Height: strconv.FormatUint(input.Height, 10), - Round: "0", - BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, - Timestamp: input.Timestamp, - }) - } + precommits := make([]*rpcClient.Precommit, 0) // just mock it since we do not need it for the block block := rpcClient.BlockResponse{ Jsonrpc: "2.0", @@ -74,7 +60,7 @@ func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { NumTxs: "0", // it is not really important for the indexer TotalTxs: "0", // it is not really important for the indexer AppVersion: "1.0.0", - ProposerAddress: input.ProposerAddress, + ProposerAddress: "", // just mock it since we do not need it for the block LastBlockID: rpcClient.BlockID{Hash: lastBlockHash, Parts: rpcClient.Parts{Total: "1", Hash: lastPartsHash}}, // just make some random hash here LastCommitHash: lastCommitHash, @@ -95,7 +81,7 @@ func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { NumTxs: "0", // it is not really important for the indexer TotalTxs: "0", // it is not really important for the indexer AppVersion: "1.0.0", - ProposerAddress: input.ProposerAddress, + ProposerAddress: "", // just mock it since we do not need it for the block LastBlockID: rpcClient.BlockID{Hash: lastBlockHash, Parts: rpcClient.Parts{Total: "1", Hash: lastPartsHash}}, LastCommitHash: lastCommitHash, DataHash: dataHash, @@ -129,7 +115,7 @@ type GenTransactionInput struct { Events *generator.TxEvents } -func GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxResponse { +func (rm *ResponseMaker) GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxResponse { // because I pretty much complicated this part of the code the // program will now need to pull the data from proto events // and convert them to rpc client events @@ -179,3 +165,67 @@ func GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxRespons } return &txResponse } + +type GenCommitInput struct { + Height uint64 + ChainID string + Timestamp time.Time + ProposerAddress string + SignedValidators []string +} + +func (rm *ResponseMaker) GenerateCommitResponse(input GenCommitInput) *rpcClient.CommitResponse { + // just like in the gen block we need the data for the header but not really + // just generate the hash for block and last block + // for others just use empty strings since this is all just to mock the similar experience + blockHash := rm.generator.GenerateBlockHash() + partsHash := "" + lastCommitHash := "" + dataHash := "" + validatorsHash := "" + nextValidatorsHash := "" + consensusHash := "" + appHash := "" + lastResultsHash := "" + + precommits := make([]*rpcClient.Precommit, 0, len(input.SignedValidators)) + for i, validator := range input.SignedValidators { + precommits = append(precommits, &rpcClient.Precommit{ + ValidatorAddress: validator, + ValidatorIndex: strconv.Itoa(i), + Signature: rm.generator.GenerateBlockHash(), + Type: 2, // not sure why 2 but that is what I saw in the real data + Height: strconv.FormatUint(input.Height, 10), + Round: "0", + BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, + Timestamp: input.Timestamp, + }) + } + + commitResponse := rpcClient.CommitResponse{ + Jsonrpc: "2.0", + ID: 1, + Result: rpcClient.CommitResult{ + SignedHeader: rpcClient.SignedHeader{ + Header: rpcClient.BlockHeader{ + Height: strconv.FormatUint(input.Height, 10), + Time: input.Timestamp, + ChainID: input.ChainID, + ProposerAddress: input.ProposerAddress, + LastCommitHash: lastCommitHash, + DataHash: dataHash, + ValidatorsHash: validatorsHash, + NextValidatorsHash: nextValidatorsHash, + ConsensusHash: consensusHash, + AppHash: appHash, + LastResultsHash: lastResultsHash, + }, + Commit: rpcClient.Commit{ + BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, + Precommits: precommits, + }, + }, + }, + } + return &commitResponse +} diff --git a/integration/test_config.example.yml b/integration/test_config.example.yml index 506ca0c..27861f7 100644 --- a/integration/test_config.example.yml +++ b/integration/test_config.example.yml @@ -18,6 +18,6 @@ pool_max_conn_lifetime_jitter: 30s # synthetic test config chain_id: gnoland -max_height: 100000 +max_height: 5000 from_height: 1 -to_height: 100000 \ No newline at end of file +to_height: 5000 \ No newline at end of file diff --git a/pkgs/database/insert.go b/pkgs/database/insert.go index de696d3..ae1a74f 100644 --- a/pkgs/database/insert.go +++ b/pkgs/database/insert.go @@ -19,13 +19,19 @@ import ( // # Used inside of the address cache package to insert the addresses to the database // // Parameters: +// - ctx: the context to use for the insert // - addresses: a slice of addresses to insert // - chainName: the name of the chain to insert the addresses to // - insertValidators: a boolean to indicate if the addresses are validators or accounts // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, insertValidators bool) error { +func (t *TimescaleDb) InsertAddresses( + ctx context.Context, + addresses []string, + chainName string, + insertValidators bool, +) error { column_names := []string{"address", "chain_name"} var table_name string if insertValidators { @@ -38,7 +44,7 @@ func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, inse return []any{addresses[i], chainName}, nil }) // copy the addresses to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{table_name}, column_names, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{table_name}, column_names, pgxSlice) return err } @@ -49,12 +55,13 @@ func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, inse // // # Used for inserting a large number of blocks to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - blocks: a slice of blocks to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { +func (t *TimescaleDb) InsertBlocks(ctx context.Context, blocks []sql_data_types.Blocks) error { // Return early if no blocks to insert if len(blocks) == 0 { return nil @@ -75,7 +82,7 @@ func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { columns := blocks[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"blocks"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"blocks"}, columns, pgxSlice) return err } @@ -86,12 +93,16 @@ func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { // // # Used for inserting a large number of validator block signings to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - validatorBlockSigning: a slice of validator block signings to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_data_types.ValidatorBlockSigning) error { +func (t *TimescaleDb) InsertValidatorBlockSignings( + ctx context.Context, + validatorBlockSigning []sql_data_types.ValidatorBlockSigning, +) error { // Return early if no validator block signings to insert if len(validatorBlockSigning) == 0 { return nil @@ -111,7 +122,7 @@ func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_d columns := validatorBlockSigning[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"validator_block_signing"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"validator_block_signing"}, columns, pgxSlice) return err } @@ -122,12 +133,16 @@ func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_d // // # Used for inserting a large number of transaction general data to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - transactionsGeneral: a slice of transaction general data to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertTransactionsGeneral(transactionsGeneral []sql_data_types.TransactionGeneral) error { +func (t *TimescaleDb) InsertTransactionsGeneral( + ctx context.Context, + transactionsGeneral []sql_data_types.TransactionGeneral, +) error { // Return early if no transactions to insert if len(transactionsGeneral) == 0 { return nil @@ -154,18 +169,19 @@ func (t *TimescaleDb) InsertTransactionsGeneral(transactionsGeneral []sql_data_t columns := transactionsGeneral[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"transaction_general"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"transaction_general"}, columns, pgxSlice) return err } // InsertAddressTx inserts a slice of AddressTx into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - addresses: a slice of AddressTx to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertAddressTx(addresses []sql_data_types.AddressTx) error { +func (t *TimescaleDb) InsertAddressTx(ctx context.Context, addresses []sql_data_types.AddressTx) error { // Return early if no addresses to insert if len(addresses) == 0 { return nil @@ -182,18 +198,19 @@ func (t *TimescaleDb) InsertAddressTx(addresses []sql_data_types.AddressTx) erro }) columns := addresses[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"address_tx"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"address_tx"}, columns, pgxSlice) return err } // InsertMsgSend inserts a slice of MsgSend messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgSend messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgSend(messages []sql_data_types.MsgSend) error { +func (t *TimescaleDb) InsertMsgSend(ctx context.Context, messages []sql_data_types.MsgSend) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -208,22 +225,24 @@ func (t *TimescaleDb) InsertMsgSend(messages []sql_data_types.MsgSend) error { messages[i].ToAddress, makePgxArray(messages[i].Amount), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"bank_msg_send"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"bank_msg_send"}, columns, pgxSlice) return err } // InsertMsgCall inserts a slice of MsgCall messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgCall messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgCall(messages []sql_data_types.MsgCall) error { +func (t *TimescaleDb) InsertMsgCall(ctx context.Context, messages []sql_data_types.MsgCall) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -241,22 +260,27 @@ func (t *TimescaleDb) InsertMsgCall(messages []sql_data_types.MsgCall) error { makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_call"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_call"}, columns, pgxSlice) return err } // InsertMsgAddPackage inserts a slice of MsgAddPackage messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgAddPackage messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgAddPackage(messages []sql_data_types.MsgAddPackage) error { +func (t *TimescaleDb) InsertMsgAddPackage( + ctx context.Context, + messages []sql_data_types.MsgAddPackage, +) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -274,22 +298,27 @@ func (t *TimescaleDb) InsertMsgAddPackage(messages []sql_data_types.MsgAddPackag makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_add_package"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_add_package"}, columns, pgxSlice) return err } // InsertMsgRun inserts a slice of MsgRun messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgRun messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { +func (t *TimescaleDb) InsertMsgRun( + ctx context.Context, + messages []sql_data_types.MsgRun, +) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -307,11 +336,12 @@ func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_run"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_run"}, columns, pgxSlice) return err } @@ -321,7 +351,7 @@ func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { // bytearrays but to be sure it should be usable on any type that is supposed to be inserted into the database as // an array // -// Args: +// Parameters: // - v: a slice of any type // // Returns: diff --git a/pkgs/database/pool.go b/pkgs/database/pool.go index 9b6f9a6..3d31856 100644 --- a/pkgs/database/pool.go +++ b/pkgs/database/pool.go @@ -13,7 +13,7 @@ import ( // NewTimescaleDb is a constructor function that creates a new TimescaleDb instance // -// Args: +// Parameters: // - config: the database config, a struct that contains the necessary data from the config file // // Returns: @@ -44,7 +44,7 @@ func NewTimescaleDbSetup(config DatabasePoolConfig) *TimescaleDb { // connectToDb is a internal function that connects to the database for the indexer // -// Args: +// Parameters: // - config: the database config, a struct that contains the necessary data from the config file // // Returns: @@ -135,7 +135,7 @@ func setupConnection(config DatabasePoolConfig) (*pgxpool.Pool, error) { // create a new database with the given name // -// Args: +// Parameters: // - db: the database connection pool // - dbname: the name of the database to create // @@ -156,7 +156,7 @@ func CreateDatabase(db *TimescaleDb, dbname string) error { // most of the time when the postgres server is running, it will be in the "postgres" database // only to be used initiating command // -// Args: +// Parameters: // - db: the database connection pool // - config: the database config, a struct that contains the necessary data from the config file // @@ -186,7 +186,7 @@ func SwitchDatabase(db *TimescaleDb, config DatabasePoolConfig, dbname string) e // GetPool returns the database connection pool // -// Args: +// Parameters: // - db: the database connection pool // // Returns: diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go index 41203be..216907a 100644 --- a/pkgs/database/queries_api.go +++ b/pkgs/database/queries_api.go @@ -12,14 +12,14 @@ import ( // // # Used to get a block from the database for a given height and chain name // -// Args: +// Parameters: // - height: the height of the block // - chainName: the name of the chain // // Returns: // - *BlockData: the block data // - error: if the query fails -func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, error) { +func (t *TimescaleDb) GetBlock(ctx context.Context, height uint64, chainName string) (*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -32,7 +32,7 @@ func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, err WHERE height = $1 AND chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, height, chainName) + row := t.pool.QueryRow(ctx, query, height, chainName) var block BlockData err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) if err != nil { @@ -41,7 +41,7 @@ func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, err return &block, nil } -func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { +func (t *TimescaleDb) GetLatestBlock(ctx context.Context, chainName string) (*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -55,7 +55,7 @@ func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { ORDER BY height DESC LIMIT 1 ` - row := t.pool.QueryRow(context.Background(), query, chainName) + row := t.pool.QueryRow(ctx, query, chainName) var block BlockData err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) if err != nil { @@ -70,14 +70,14 @@ func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { // // # Used to get the last x blocks from the database for a given chain name // -// Args: +// Parameters: // - chainName: the name of the chain // - x: the number of blocks to get // // Returns: // - []*BlockData: the last x blocks // - error: if the query fails -func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, error) { +func (t *TimescaleDb) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -91,7 +91,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, ORDER BY height DESC LIMIT $2 ` - rows, err := t.pool.Query(context.Background(), query, chainName, x) + rows, err := t.pool.Query(ctx, query, chainName, x) if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, // // # Used to get a range of blocks from the database for a given height range and chain name // -// Args: +// Parameters: // - fromHeight: the starting height of the block // - toHeight: the ending height of the block (inclusive) // - chainName: the name of the chain @@ -125,7 +125,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, // Returns: // - []*BlockData: the range of block data // - error: if the query fails -func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) { +func (t *TimescaleDb) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -138,7 +138,7 @@ func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainN WHERE height >= $1 AND height <= $2 AND chain_name = $3 ` - rows, err := t.pool.Query(context.Background(), query, fromHeight, toHeight, chainName) + rows, err := t.pool.Query(ctx, query, fromHeight, toHeight, chainName) if err != nil { return nil, err } @@ -161,14 +161,18 @@ func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainN // // # Used to get all of the validators that signed that block + the proposer // -// Args: +// Parameters: // - chainName: the name of the chain // - blockHeight: the height of the block // // Returns: // - *BlockSigners: the block signers // - error: if the query fails -func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) (*BlockSigners, error) { +func (t *TimescaleDb) GetAllBlockSigners( + ctx context.Context, + chainName string, + blockHeight uint64, +) (*BlockSigners, error) { query := ` SELECT vb.block_height, @@ -183,7 +187,7 @@ func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) ( WHERE vb.chain_name = $1 AND vb.block_height = $2 ` - row := t.pool.QueryRow(context.Background(), query, chainName, blockHeight) + row := t.pool.QueryRow(ctx, query, chainName, blockHeight) var blockSigners BlockSigners err := row.Scan(&blockSigners.BlockHeight, &blockSigners.Proposer, &blockSigners.SignedVals) if err != nil { @@ -198,14 +202,18 @@ func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) ( // // # Used to get the bank send message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: -// - *BankSend: the bank send message +// - []*BankSend: the bank send messages // - error: if the query fails -func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, error) { +func (t *TimescaleDb) GetBankSend( + ctx context.Context, + txHash string, + chainName string, +) ([]*BankSend, error) { query := ` SELECT encode(bms.tx_hash, 'base64') AS tx_hash, @@ -224,13 +232,31 @@ func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, e WHERE bms.tx_hash = decode($1, 'base64') AND bms.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) - var bankSend BankSend - err := row.Scan(&bankSend.TxHash, &bankSend.Timestamp, &bankSend.FromAddress, &bankSend.ToAddress, &bankSend.Amount, &bankSend.Signers) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &bankSend, nil + defer rows.Close() + bankSends := make([]*BankSend, 0) + for rows.Next() { + bankSend := &BankSend{} + err := rows.Scan( + &bankSend.TxHash, + &bankSend.Timestamp, + &bankSend.FromAddress, + &bankSend.ToAddress, + &bankSend.Amount, + &bankSend.Signers, + ) + if err != nil { + return nil, err + } + bankSends = append(bankSends, bankSend) + } + if err := rows.Err(); err != nil { + return nil, err + } + return bankSends, nil } // GetMsgCall gets the msg call message for a given transaction hash @@ -239,17 +265,22 @@ func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, e // // # Used to get the msg call message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: -// - *MsgCall: the msg call message +// - []*MsgCall: the msg call messages // - error: if the query fails -func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, error) { +func (t *TimescaleDb) GetMsgCall( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgCall, error) { query := ` SELECT encode(vmc.tx_hash, 'base64') AS tx_hash, + vmc.message_counter, vmc.timestamp, gn.address AS caller, vmc.pkg_path, @@ -267,13 +298,35 @@ func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, err WHERE vmc.tx_hash = decode($1, 'base64') AND vmc.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) - var msgCall MsgCall - err := row.Scan(&msgCall.TxHash, &msgCall.Timestamp, &msgCall.Caller, &msgCall.PkgPath, &msgCall.FuncName, &msgCall.Args, &msgCall.Send, &msgCall.MaxDeposit, &msgCall.Signers) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &msgCall, nil + defer rows.Close() + msgCalls := make([]*MsgCall, 0) + for rows.Next() { + msgCall := &MsgCall{} + err := rows.Scan( + &msgCall.TxHash, + &msgCall.MessageCounter, + &msgCall.Timestamp, + &msgCall.Caller, + &msgCall.PkgPath, + &msgCall.FuncName, + &msgCall.Args, + &msgCall.Send, + &msgCall.MaxDeposit, + &msgCall.Signers, + ) + if err != nil { + return nil, err + } + msgCalls = append(msgCalls, msgCall) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgCalls, nil } // GetMsgAddPackage gets the msg add package message for a given transaction hash @@ -282,17 +335,22 @@ func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, err // // # Used to get the msg add package message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: -// - *MsgAddPackage: the msg add package message +// - []*MsgAddPackage: the msg add package messages // - error: if the query fails -func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAddPackage, error) { +func (t *TimescaleDb) GetMsgAddPackage( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgAddPackage, error) { query := ` SELECT encode(vmap.tx_hash, 'base64') AS tx_hash, + vmap.message_counter, vmap.timestamp, gn.address AS creator, vmap.pkg_path, @@ -310,13 +368,35 @@ func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAdd WHERE vmap.tx_hash = decode($1, 'base64') AND vmap.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) - var msgAddPackage MsgAddPackage - err := row.Scan(&msgAddPackage.TxHash, &msgAddPackage.Timestamp, &msgAddPackage.Creator, &msgAddPackage.PkgPath, &msgAddPackage.PkgName, &msgAddPackage.PkgFileNames, &msgAddPackage.Send, &msgAddPackage.MaxDeposit, &msgAddPackage.Signers) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &msgAddPackage, nil + defer rows.Close() + msgAddPackages := make([]*MsgAddPackage, 0) + for rows.Next() { + msgAddPackage := &MsgAddPackage{} + err := rows.Scan( + &msgAddPackage.TxHash, + &msgAddPackage.MessageCounter, + &msgAddPackage.Timestamp, + &msgAddPackage.Creator, + &msgAddPackage.PkgPath, + &msgAddPackage.PkgName, + &msgAddPackage.PkgFileNames, + &msgAddPackage.Send, + &msgAddPackage.MaxDeposit, + &msgAddPackage.Signers, + ) + if err != nil { + return nil, err + } + msgAddPackages = append(msgAddPackages, msgAddPackage) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgAddPackages, nil } // GetMsgRun gets the msg run message for a given transaction hash @@ -325,17 +405,22 @@ func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAdd // // # Used to get the msg run message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: -// - *MsgRun: the msg run message +// - []*MsgRun: the msg run messages // - error: if the query fails -func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error) { +func (t *TimescaleDb) GetMsgRun( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgRun, error) { query := ` SELECT encode(vmr.tx_hash, 'base64') AS tx_hash, + vmr.message_counter, vmr.timestamp, gn.address AS caller, vmr.pkg_path, @@ -353,14 +438,35 @@ func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error WHERE vmr.tx_hash = decode($1, 'base64') AND vmr.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) - var msgRun MsgRun - err := row.Scan(&msgRun.TxHash, &msgRun.Timestamp, &msgRun.Caller, &msgRun.PkgPath, &msgRun.PkgName, &msgRun.PkgFileNames, &msgRun.Send, &msgRun.MaxDeposit, &msgRun.Signers) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { - log.Println("error getting msg run", err) return nil, err } - return &msgRun, nil + defer rows.Close() + msgRuns := make([]*MsgRun, 0) + for rows.Next() { + msgRun := &MsgRun{} + err := rows.Scan( + &msgRun.TxHash, + &msgRun.MessageCounter, + &msgRun.Timestamp, + &msgRun.Caller, + &msgRun.PkgPath, + &msgRun.PkgName, + &msgRun.PkgFileNames, + &msgRun.Send, + &msgRun.MaxDeposit, + &msgRun.Signers, + ) + if err != nil { + return nil, err + } + msgRuns = append(msgRuns, msgRun) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgRuns, nil } // GetTransaction gets the transaction for a given transaction hash @@ -369,14 +475,14 @@ func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error // // # Used to get the transaction for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *Transaction: the transaction // - error: if the query fails -func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transaction, error) { +func (t *TimescaleDb) GetTransaction(ctx context.Context, txHash string, chainName string) (*Transaction, error) { query := ` SELECT encode(tx.tx_hash, 'base64') AS tx_hash, @@ -391,7 +497,7 @@ func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transact WHERE tx.tx_hash = decode($1, 'base64') AND tx.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var transaction Transaction err := row.Scan( &transaction.TxHash, @@ -416,10 +522,10 @@ func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transact // // # Used to get the last x transactions from the database for a given chain name // -// Args: +// Parameters: // - chainName: the name of the chain // - x: the number of transactions to get -func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Transaction, error) { +func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*Transaction, error) { query := ` SELECT encode(tx.tx_hash, 'base64') AS tx_hash, @@ -435,7 +541,7 @@ func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Trans ORDER BY tx.timestamp DESC LIMIT $2 ` - rows, err := t.pool.Query(context.Background(), query, chainName, x) + rows, err := t.pool.Query(ctx, query, chainName, x) if err != nil { return nil, err } @@ -455,41 +561,33 @@ func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Trans return transactions, nil } -// GetMsgType gets the message type for a given transaction hash +// GetMsgTypes gets the message type for a given transaction hash // // Usage: // // # Used to get the message type for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: -// - string: the message type +// - []string: the message types // - error: if the query fails -func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error) { +func (t *TimescaleDb) GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) { query := ` SELECT msg_types FROM transaction_general WHERE tx_hash = decode($1, 'base64') AND chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) - var msgType []string - // to future me - // in the events the transactions can harbor more transaction types this - // will not work, for now I only have seen with one message type per transaction - // if this happens at least throw some log warning - if len(msgType) >= 2 { - log.Println("warning: transaction has more than one message type", msgType) - return msgType[0], nil - } - err := row.Scan(&msgType) + row := t.pool.QueryRow(ctx, query, txHash, chainName) + var msgTypes []string + err := row.Scan(&msgTypes) if err != nil { - return "", err + return nil, err } - return msgType[0], nil + return msgTypes, nil } // GetAddressTxs gets the transactions for a given address for a certain time period @@ -498,7 +596,7 @@ func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error // // # Used to get the transactions for a given address for a certain time period // -// Args: +// Parameters: // - address: the address // - chainName: the name of the chain // - fromTimestamp: the starting timestamp @@ -508,6 +606,7 @@ func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error // - []*AddressTx: the transactions // - error: if the query fails func (t *TimescaleDb) GetAddressTxs( + ctx context.Context, address string, chainName string, fromTimestamp time.Time, @@ -527,7 +626,7 @@ func (t *TimescaleDb) GetAddressTxs( addressTxs := make([]AddressTx, 0) - rows, err := t.pool.Query(context.Background(), query, address, chainName, fromTimestamp, toTimestamp) + rows, err := t.pool.Query(ctx, query, address, chainName, fromTimestamp, toTimestamp) if err != nil { return nil, err } diff --git a/pkgs/database/queries_indexer.go b/pkgs/database/queries_indexer.go index ec486f6..18870cb 100644 --- a/pkgs/database/queries_indexer.go +++ b/pkgs/database/queries_indexer.go @@ -11,8 +11,9 @@ import ( // Used within the account cache package to get query about the existing accounts // and then we can know which ones to insert // -// Args: +// Parameters: // +// - ctx: the context to use for the query // - addresses: the addresses to check // - chainName: the name of the chain // @@ -20,7 +21,12 @@ import ( // // - map[string]int32: the map of existing addresses and their ids // - error: if the query fails -func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, searchValidators bool) (map[string]int32, error) { +func (t *TimescaleDb) FindExistingAccounts( + ctx context.Context, + addresses []string, + chainName string, + searchValidators bool, +) (map[string]int32, error) { addressesMap := make(map[string]int32) // we need to check if the addresses are already in the map // so we make this query to the db to get the addresses that are already in the map @@ -40,7 +46,7 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, AND address = ANY($2) ` } - rows, err := t.pool.Query(context.Background(), query, chainName, addresses) + rows, err := t.pool.Query(ctx, query, chainName, addresses) if err != nil { return nil, err } @@ -65,8 +71,9 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, // // # Only used when the program is initializing to get all the accounts with their ids // -// Args: +// Parameters: // +// - ctx: the context to use for the query // - chainName: the name of the chain // - searchValidators: whether to search for validators or accounts // - highestIndex: the highest index of the addresses already recorded or it could be a 0 @@ -75,7 +82,12 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, // // - map[string]int32: the map of all accounts and their ids // - error: if the query fails -func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) { +func (t *TimescaleDb) GetAllAddresses( + ctx context.Context, + chainName string, + searchValidators bool, + highestIndex *int32, +) (map[string]int32, int32, error) { addressesMap := make(map[string]int32) var maxIndex int32 = 0 if highestIndex != nil { @@ -98,7 +110,7 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, h AND id > $2 ` } - rows, err := t.pool.Query(context.Background(), query, chainName, highestIndex) + rows, err := t.pool.Query(ctx, query, chainName, maxIndex) if err != nil { return nil, 0, err } @@ -125,15 +137,18 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, h // // Used to check if the current database is "gnoland" // +// Parameters: +// - ctx: the context to use for the query +// // Returns: // // - string: the name of the current database // - error: if the query fails -func (t *TimescaleDb) CheckCurrentDatabaseName() (string, error) { +func (t *TimescaleDb) CheckCurrentDatabaseName(ctx context.Context) (string, error) { query := ` SELECT current_database() ` - row := t.pool.QueryRow(context.Background(), query) + row := t.pool.QueryRow(ctx, query) var currentDb string err := row.Scan(¤tDb) if err != nil { @@ -148,19 +163,20 @@ func (t *TimescaleDb) CheckCurrentDatabaseName() (string, error) { // // # Used to get the last block height from the database for a given chain // -// Args: +// Parameters: +// - ctx: the context to use for the query // - chainName: the name of the chain // // Returns: // - uint64: the last block height // - error: if the query fails -func (t *TimescaleDb) GetLastBlockHeight(chainName string) (uint64, error) { +func (t *TimescaleDb) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { query := ` SELECT MAX(height) FROM blocks WHERE chain_name = $1 ` - row := t.pool.QueryRow(context.Background(), query, chainName) + row := t.pool.QueryRow(ctx, query, chainName) var lastBlockHeight uint64 err := row.Scan(&lastBlockHeight) if err != nil { diff --git a/pkgs/database/types_api.go b/pkgs/database/types_api.go index 34782dc..7824e54 100644 --- a/pkgs/database/types_api.go +++ b/pkgs/database/types_api.go @@ -30,48 +30,52 @@ type Amount struct { } type BankSend struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - FromAddress string `json:"from_address" doc:"From address (addresses)"` - ToAddress string `json:"to_address" doc:"To address (addresses)"` - Amount []Amount `json:"amount" doc:"Amount"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + FromAddress string `json:"from_address" doc:"From address (addresses)"` + ToAddress string `json:"to_address" doc:"To address (addresses)"` + Amount []Amount `json:"amount" doc:"Amount"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgCall struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Caller string `json:"caller" doc:"Caller address (addresses)"` - Send []Amount `json:"send" doc:"Send amount"` - PkgPath string `json:"pkg_path" doc:"Package path"` - FuncName string `json:"func_name" doc:"Function name"` - Args string `json:"args" doc:"Arguments"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + Send []Amount `json:"send" doc:"Send amount"` + PkgPath string `json:"pkg_path" doc:"Package path"` + FuncName string `json:"func_name" doc:"Function name"` + Args string `json:"args" doc:"Arguments"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgAddPackage struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Creator string `json:"creator" doc:"Creator address (addresses)"` - PkgPath string `json:"pkg_path" doc:"Package path"` - PkgName string `json:"pkg_name" doc:"Package name"` - PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` - Send []Amount `json:"send" doc:"Send amount"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Creator string `json:"creator" doc:"Creator address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgRun struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Caller string `json:"caller" doc:"Caller address (addresses)"` - PkgPath string `json:"pkg_path" doc:"Package path"` - PkgName string `json:"pkg_name" doc:"Package name"` - PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` - Send []Amount `json:"send" doc:"Send amount"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type Transaction struct { diff --git a/pkgs/generator/generator_test.go b/pkgs/generator/generator_test.go index 0b20f21..9029204 100644 --- a/pkgs/generator/generator_test.go +++ b/pkgs/generator/generator_test.go @@ -1,70 +1,45 @@ package generator import ( - "fmt" "log" "testing" + + "github.com/stretchr/testify/assert" ) // TestAuthenticGeneration demonstrates the new authentic key and address generation func TestAuthenticGeneration(t *testing.T) { - fmt.Println("Testing Authentic Address & PubKey Generation") - // Create a new data generator generator := NewDataGenerator(500) - fmt.Printf("Generated %d key pairs in the pool\n\n", len(generator.keyPairPool)) - // Test address generation - fmt.Println("Generated Addresses (from pool):") // test up to a 1k addresses - for i := range 500 { + for range 500 { addr := generator.GenerateAddress() - fmt.Printf("%d. %s (valid: %t)\n", i+1, addr, ValidateAddress(addr)) + assert.True(t, ValidateAddress(addr)) } - fmt.Println("\nGenerated Public Keys (from pool):") // test up to a 1k public keys - for i := range 500 { + for range 500 { pubkey := generator.GeneratePubKey() - fmt.Printf("%d. %s\n", i+1, pubkey[:50]+"...") - fmt.Printf(" Valid: %t\n", ValidatePubKey(pubkey)) + assert.True(t, ValidatePubKey(pubkey)) } // Test key pair access - fmt.Println("\nRandom Key Pairs with Full Details:") - for i := range 500 { + for range 500 { kp := generator.GetRandomKeyPair() - fmt.Printf("KeyPair %d:\n", i+1) - fmt.Printf(" Address: %s\n", kp.AddressBech32) - fmt.Printf(" PubKey: %s\n", kp.PubKeyBech32[:50]+"...") - fmt.Printf(" PrivKey: %s\n", kp.PrivKeyHex[:32]+"...") - fmt.Println() + assert.NotNil(t, kp) + assert.NotEmpty(t, kp.Address) + assert.NotEmpty(t, kp.PublicKey) } // Test synthetic transaction generation 500 times - fmt.Println("Sample Transaction with Authentic Addresses:") for range 500 { events, tx := generator.GenerateTransaction() - fmt.Printf("Transaction: %+v\n", tx) - for i := range events.Events { - event := &events.Events[i] - fmt.Printf("\tEvent %d: %s\n", i+1, event.Type) - for _, attr := range event.Attributes { - // Check if this attribute contains an address - if attr.Key == "from" || attr.Key == "to" || attr.Key == "Creator" || attr.Key == "Author" { - if ValidateAddress(attr.GetStringValue()) { - fmt.Printf("\t%s: %s (authentic)\n", attr.Key, attr.Value) - } else { - fmt.Printf("\t%s: %s (not authentic)\n", attr.Key, attr.Value) - } - } else { - fmt.Printf("\t%s: %s\n", attr.Key, attr.Value) - } - } - fmt.Println() - } + assert.NotNil(t, tx) + assert.NotNil(t, events.Events) } + } // RunAllTests runs all the test functions diff --git a/pkgs/sql_data_types/table.go b/pkgs/sql_data_types/table.go index 9057f92..0b8c21a 100644 --- a/pkgs/sql_data_types/table.go +++ b/pkgs/sql_data_types/table.go @@ -285,6 +285,7 @@ func (tg *TransactionGeneral) GetMessageTypes() []string { // - ToAddress (int32) // - Amount (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgSend struct { @@ -294,9 +295,10 @@ type MsgSend struct { // gno address, pull from the gno_addresses table FromAddress int32 `db:"from_address" dbtype:"INTEGER" nullable:"false" primary:"false"` // gno address, pull from the gno_addresses table - ToAddress int32 `db:"to_address" dbtype:"INTEGER" nullable:"true" primary:"false"` - Amount []Amount `db:"amount" dbtype:"amount[]" nullable:"false" primary:"false"` - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + ToAddress int32 `db:"to_address" dbtype:"INTEGER" nullable:"true" primary:"false"` + Amount []Amount `db:"amount" dbtype:"amount[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } // TableName returns the name of the table for the MsgSend struct @@ -352,6 +354,7 @@ func (ms *MsgSend) GetAllAddresses() *TxAddresses { // - Send (Amount[]) // - MaxDeposit (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgCall struct { @@ -359,13 +362,14 @@ type MsgCall struct { Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"` ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` // gno address, pull from the gno_addresses table - Caller int32 `db:"caller" dbtype:"INTEGER" nullable:"false" primary:"false"` - PkgPath string `db:"pkg_path" dbtype:"TEXT" nullable:"true" primary:"false"` - FuncName string `db:"func_name" dbtype:"TEXT" nullable:"true" primary:"false"` - Args string `db:"args" dbtype:"TEXT" nullable:"true" primary:"false"` - Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` - MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Caller int32 `db:"caller" dbtype:"INTEGER" nullable:"false" primary:"false"` + PkgPath string `db:"pkg_path" dbtype:"TEXT" nullable:"true" primary:"false"` + FuncName string `db:"func_name" dbtype:"TEXT" nullable:"true" primary:"false"` + Args string `db:"args" dbtype:"TEXT" nullable:"true" primary:"false"` + Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` + MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } func (mc MsgCall) TableName() string { @@ -417,6 +421,7 @@ func (mc *MsgCall) GetAllAddresses() *TxAddresses { // - MaxDeposit (Amount[]) // - Signers (int32[]) // - Timestamp (time.Time) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgAddPackage struct { @@ -431,7 +436,8 @@ type MsgAddPackage struct { Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` // signers are the addresses that signed the transaction - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } func (ma MsgAddPackage) TableName() string { @@ -483,6 +489,7 @@ func (ma *MsgAddPackage) GetAllAddresses() *TxAddresses { // - Send (Amount[]) // - MaxDeposit (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgRun struct { @@ -497,7 +504,8 @@ type MsgRun struct { Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` // signers are the addresses that signed the transaction - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } // A method to get the columns of the struct