From de407407e67ae588141b361307fae18215f53a18 Mon Sep 17 00:00:00 2001 From: NM Date: Mon, 24 Nov 2025 22:08:18 +0100 Subject: [PATCH 01/11] Bug fixes for the indexer and some minor improvments - Updated the Makefile to include test commands for the indexer. - Modified integration test configuration to use specific database user and name. - Refactored `ProcessValidatorSignings` to handle commits instead of blocks, updating related methods and tests accordingly more detail in the CHANGELOG once all has been implemented. - Implemented new methods for retrieving commits in the query operator. - Added to synthetic query operator a function to generate and return synthetic commits for testing. - Improved error handling and logging for commit retrieval. - Updated tests to cover new commit functionality and ensure proper behavior. --- .github/workflows/integration-tests.yml | 2 +- Makefile | 21 +++- indexer/data_processor/operator.go | 28 ++--- indexer/orchestrator/operator.go | 12 +- indexer/orchestrator/operator_test.go | 39 ++++-- indexer/orchestrator/types.go | 3 +- indexer/query/operator.go | 60 +++++++++ indexer/query/operator_test.go | 50 +++++--- indexer/query/types.go | 1 + indexer/rpc_client/client.go | 27 +++- indexer/rpc_client/commit.go | 64 ++++++++++ indexer/rpc_client/errors.go | 15 +++ indexer/rpc_client/rate_limited_client.go | 15 +++ indexer/rpc_client/types.go | 1 + integration/synthetic/query_operator.go | 63 ++++++++-- integration/synthetic/respone_maker.go | 142 +++++++++++++++------- integration/test_config.example.yml | 4 +- pkgs/generator/generator_test.go | 49 ++------ 18 files changed, 450 insertions(+), 146 deletions(-) create mode 100644 indexer/rpc_client/commit.go diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d284a02..d6ac1c1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -129,7 +129,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db + go run indexer/main.go setup create-db --db-user postgres --db-name postgres env: DB_HOST: localhost DB_PORT: 5432 diff --git a/Makefile b/Makefile index b69a6b4..38f45a8 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +######################################################## +# Build and install the indexer +######################################################## + .PHONY: build install clean build-experimental install-experimental build-api # Get git information @@ -27,4 +31,19 @@ build-experimental: # experimental install with greentea garbage collection # use at your own risk install-experimental: - cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" \ No newline at end of file + cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" + +######################################################## +# Test the indexer +######################################################## + +.PHONY: test-race integration-test test + +test: + go test -v ./... + +test-race: + go test -v -race ./... + +integration-test: + cd integration && go test -v -tags=integration -timeout=20m ./... \ No newline at end of file diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index abca23b..f39e1f8 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -459,27 +459,27 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e } func (d *DataProcessor) ProcessValidatorSignings( - blocks []*rpcClient.BlockResponse, + commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) { - validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, len(blocks)) + validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, len(commits)) wg := sync.WaitGroup{} - wg.Add(len(blocks)) + wg.Add(len(commits)) // Process blocks concurrently - for _, block := range blocks { - go func(block *rpcClient.BlockResponse) { + for _, commit := range commits { + go func(commit *rpcClient.CommitResponse) { defer wg.Done() signedVals := struct { Proposer int32 SignedVals []int32 }{ - Proposer: d.validatorCache.GetAddress(block.Result.Block.Header.ProposerAddress), + Proposer: d.validatorCache.GetAddress(commit.GetProposerAddress()), SignedVals: make([]int32, 0), } - precommits := block.Result.Block.LastCommit.Precommits + precommits := commit.GetSigners() for _, precommit := range precommits { if precommit != nil { signedVals.SignedVals = append( @@ -488,20 +488,20 @@ func (d *DataProcessor) ProcessValidatorSignings( } } - height, err := strconv.ParseUint(block.Result.Block.Header.Height, 10, 64) + height, err := commit.GetHeight() if err != nil { - log.Printf("Failed to parse block height %s: %v", block.Result.Block.Header.Height, err) + log.Printf("Failed to get commit height: %v", err) return } validatorChan <- &sqlDataTypes.ValidatorBlockSigning{ BlockHeight: height, - Timestamp: block.Result.Block.Header.Time, + Timestamp: commit.GetTimestamp(), Proposer: signedVals.Proposer, SignedVals: signedVals.SignedVals, ChainName: d.chainName, } - }(block) + }(commit) } // Close channel when all goroutines finish @@ -510,16 +510,16 @@ func (d *DataProcessor) ProcessValidatorSignings( close(validatorChan) }() - validatorData := make([]sqlDataTypes.ValidatorBlockSigning, 0, len(blocks)) + validatorData := make([]sqlDataTypes.ValidatorBlockSigning, 0, len(commits)) for validator := range validatorChan { validatorData = append(validatorData, *validator) } err := d.dbPool.InsertValidatorBlockSignings(validatorData) if err != nil { - log.Printf("Failed to insert validator block signings: %v", err) + log.Printf("Failed to insert validator commit signings: %v", err) } - log.Printf("Validator block signings processed from %d to %d", fromHeight, toHeight) + log.Printf("Validator commit signings processed from %d to %d", fromHeight, toHeight) } // createAddressTx is a private func that creates a slice of sqlDataTypes.AddressTx from a diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go index 3239294..bfb292a 100644 --- a/indexer/orchestrator/operator.go +++ b/indexer/orchestrator/operator.go @@ -73,7 +73,7 @@ func (or *Orchestrator) HistoricProcess( // Step 1: Get blocks concurrently blocks := or.queryOperator.GetFromToBlocks(startHeight, chunkEndHeight) - + commits := or.queryOperator.GetFromToCommits(startHeight, chunkEndHeight) if len(blocks) == 0 { log.Printf("No valid blocks in chunk %d-%d", startHeight, chunkEndHeight) } else { @@ -83,7 +83,7 @@ func (or *Orchestrator) HistoricProcess( log.Printf("Collected %d transactions from %d blocks in chunk", len(allTransactions), len(blocks)) // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, allTransactions, false, startHeight, chunkEndHeight); err != nil { + if err := or.processAllConcurrently(blocks, commits, allTransactions, false, startHeight, chunkEndHeight); err != nil { log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err) } else { // Progress logging @@ -205,8 +205,9 @@ func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { // Step 1: Get blocks concurrently blocks := or.queryOperator.GetFromToBlocks(chunkStart, chunkEnd) + commits := or.queryOperator.GetFromToCommits(chunkStart, chunkEnd) - if len(blocks) == 0 { + if len(blocks) == 0 && len(commits) == 0 { log.Printf("No valid blocks in live chunk %d-%d", chunkStart, chunkEnd) return nil } @@ -217,7 +218,7 @@ func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { log.Printf("Collected %d transactions from %d blocks in live chunk", len(allTransactions), len(blocks)) // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, allTransactions, false, chunkStart, chunkEnd); err != nil { + if err := or.processAllConcurrently(blocks, commits, allTransactions, false, chunkStart, chunkEnd); err != nil { return fmt.Errorf("failed to process live chunk %d-%d: %w", chunkStart, chunkEnd, err) } @@ -333,6 +334,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // The method will not throw an error if the data is not found, it will just return nil func (or *Orchestrator) processAllConcurrently( blocks []*rpcClient.BlockResponse, + commits []*rpcClient.CommitResponse, transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, @@ -409,7 +411,7 @@ func (or *Orchestrator) processAllConcurrently( go func() { defer wg2.Done() log.Printf("Phase 2: Starting ProcessValidatorSignings") - or.dataProcessor.ProcessValidatorSignings(blocks, fromHeight, toHeight) + or.dataProcessor.ProcessValidatorSignings(commits, fromHeight, toHeight) log.Printf("Phase 2: ProcessValidatorSignings completed") }() diff --git a/indexer/orchestrator/operator_test.go b/indexer/orchestrator/operator_test.go index 6e96d90..aa62f29 100644 --- a/indexer/orchestrator/operator_test.go +++ b/indexer/orchestrator/operator_test.go @@ -47,7 +47,7 @@ func (m *MockDataProcessor) ProcessMessages(transactions []dataprocessor.Trasnac } // Mock method for ProcessValidatorSignings -func (m *MockDataProcessor) ProcessValidatorSignings(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) { +func (m *MockDataProcessor) ProcessValidatorSignings(commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) { m.ProcessValidatorSigningsCalled = true } @@ -55,8 +55,9 @@ func (m *MockDataProcessor) ProcessValidatorSignings(blocks []*rpcClient.BlockRe // The idea should be to count the number of calls to the method // and check if the method is called with the correct parameters type MockQueryOperator struct { - ShouldReturnBlocks bool - CallCount int + ShouldReturnBlocks bool + CallCount int + ShouldReturnCommits bool } // Mock method for GetFromToBlocks @@ -70,6 +71,17 @@ func (m *MockQueryOperator) GetFromToBlocks(fromHeight uint64, toHeight uint64) return []*rpcClient.BlockResponse{{}} } +// Mock method for GetFromToCommits +func (m *MockQueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse { + m.CallCount++ + if !m.ShouldReturnCommits { + return []*rpcClient.CommitResponse{} // Return empty slice + } + + // Return a single empty commit + return []*rpcClient.CommitResponse{{}} +} + // Mock method for GetTransactions func (m *MockQueryOperator) GetTransactions(txs []string) []*rpcClient.TxResponse { return []*rpcClient.TxResponse{} // Empty transactions @@ -132,7 +144,8 @@ func TestOrchestrator_HistoricProcess_CallsAllProcessors(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} mockQueryOperator := &MockQueryOperator{ - ShouldReturnBlocks: true, // Return at least one block + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} @@ -179,7 +192,8 @@ func TestOrchestrator_HistoricProcess_SkipsProcessingWhenNoBlocks(t *testing.T) // Setup mocks - no blocks returned mockDataProcessor := &MockDataProcessor{} mockQueryOperator := &MockQueryOperator{ - ShouldReturnBlocks: false, // Return empty blocks + ShouldReturnBlocks: false, // Return empty blocks + ShouldReturnCommits: false, // Return empty commits } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} @@ -213,7 +227,10 @@ func TestOrchestrator_HistoricProcess_SkipsProcessingWhenNoBlocks(t *testing.T) func TestOrchestrator_LiveProcess_RespectsContext(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{HeightToReturn: 50} mockRPC := &MockGnolandRpcClient{HeightToReturn: 100} @@ -251,7 +268,10 @@ func TestOrchestrator_LiveProcess_RespectsContext(t *testing.T) { func TestOrchestrator_LiveProcess_SkipDbCheck(t *testing.T) { // Setup mocks mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{ShouldError: true} // Database will error mockRPC := &MockGnolandRpcClient{HeightToReturn: 100} @@ -284,7 +304,10 @@ func TestOrchestrator_Constructor_ValidatesMode(t *testing.T) { }() mockDataProcessor := &MockDataProcessor{} - mockQueryOperator := &MockQueryOperator{} + mockQueryOperator := &MockQueryOperator{ + ShouldReturnBlocks: true, // Return at least one block + ShouldReturnCommits: true, // Return at least one commit + } mockDB := &MockDatabaseHeight{} mockRPC := &MockGnolandRpcClient{} diff --git a/indexer/orchestrator/types.go b/indexer/orchestrator/types.go index 0264708..df59ff1 100644 --- a/indexer/orchestrator/types.go +++ b/indexer/orchestrator/types.go @@ -14,13 +14,14 @@ type DataProcessor interface { ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) ProcessTransactions(transactions []dataprocessor.TrasnactionsData, compressEvents bool, fromHeight uint64, toHeight uint64) ProcessMessages(transactions []dataprocessor.TrasnactionsData, fromHeight uint64, toHeight uint64) error - ProcessValidatorSignings(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) + ProcessValidatorSignings(commits []*rpcClient.CommitResponse, fromHeight uint64, toHeight uint64) } type QueryOperator interface { GetFromToBlocks(fromHeight uint64, toHeight uint64) []*rpcClient.BlockResponse GetTransactions(txs []string) []*rpcClient.TxResponse GetLatestBlockHeight() (uint64, error) + GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse } // Only needed for one opetaion diff --git a/indexer/query/operator.go b/indexer/query/operator.go index 958b0ba..623b261 100644 --- a/indexer/query/operator.go +++ b/indexer/query/operator.go @@ -140,6 +140,66 @@ func (q *QueryOperator) GetFromToBlocks(fromHeight uint64, toHeight uint64) []*r return blocks } +func (q *QueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rc.CommitResponse { + diff := toHeight - fromHeight + 1 + if diff < 1 { + return nil + } + + commits := make([]*rc.CommitResponse, diff) + var mu sync.Mutex + wg := sync.WaitGroup{} + wg.Add(int(diff)) + + // Launch goroutines to get the block signers + for i := range diff { + height := fromHeight + i + idx := i // Capture index + go func(height uint64, idx int) { + commit, err := q.rpcClient.GetCommit(height) + if err != nil { + // Use retry mechanism with callback pattern + retry.RetryWithContext( + q.retryAmount, + q.pause, + q.pauseTime, + q.exponentialBackoff, + func(args ...any) (*rc.CommitResponse, error) { + h := args[0].(uint64) + result, rpcErr := q.rpcClient.GetCommit(h) + if rpcErr != nil { + return nil, rpcErr + } + return result, nil + }, + func(result *rc.CommitResponse) { + mu.Lock() + commits[idx] = result + mu.Unlock() + wg.Done() + }, + func(retryErr error) { + log.Printf("failed to get commit %d after retries: %v", height, retryErr) + mu.Lock() + commits[idx] = nil + mu.Unlock() + wg.Done() + }, + height, + ) + return + } + mu.Lock() + commits[idx] = commit + mu.Unlock() + wg.Done() + }(height, int(idx)) + } + + wg.Wait() + return commits +} + // A swarm method to get transactions from a slice of tx hashes // This is a fan out method that lauches async workers for each tx and wait to get the resaults // the indexer should store them all together as one huge slice of transactions, diff --git a/indexer/query/operator_test.go b/indexer/query/operator_test.go index 6a16b38..df68d91 100644 --- a/indexer/query/operator_test.go +++ b/indexer/query/operator_test.go @@ -1,41 +1,61 @@ package query_test import ( + "sync" "testing" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/query" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" + "github.com/stretchr/testify/assert" ) // MockRpcClient - focuses on tracking what was called type MockRpcClient struct { + mu sync.Mutex GetBlockCalled bool GetLatestBlockHeightCalled bool GetTxCalled bool GetBlockCallCount int GetTxCallCount int + GetCommitCalled bool + GetCommitCallCount int } // Mock method for GetBlock func (m *MockRpcClient) GetBlock(height uint64) (*rpcClient.BlockResponse, *rpcClient.RpcHeightError) { + m.mu.Lock() m.GetBlockCalled = true m.GetBlockCallCount++ + m.mu.Unlock() return &rpcClient.BlockResponse{}, nil } // Mock method for GetLatestBlockHeight func (m *MockRpcClient) GetLatestBlockHeight() (uint64, *rpcClient.RpcHeightError) { + m.mu.Lock() m.GetLatestBlockHeightCalled = true + m.mu.Unlock() return 1, nil } // Mock method for GetTx func (m *MockRpcClient) GetTx(txHash string) (*rpcClient.TxResponse, *rpcClient.RpcStringError) { + m.mu.Lock() m.GetTxCalled = true m.GetTxCallCount++ + m.mu.Unlock() return &rpcClient.TxResponse{}, nil } +// Mock method for GetCommit +func (m *MockRpcClient) GetCommit(height uint64) (*rpcClient.CommitResponse, *rpcClient.RpcCommitError) { + m.mu.Lock() + m.GetCommitCalled = true + m.GetCommitCallCount++ + m.mu.Unlock() + return &rpcClient.CommitResponse{}, nil +} + // TestQueryOperator - tests the query operator func TestQueryOperator(t *testing.T) { mockRpcClient := &MockRpcClient{} @@ -43,32 +63,26 @@ func TestQueryOperator(t *testing.T) { // Test GetFromToBlocks - should call GetBlock multiple times (1 to 10 = 10 calls) queryOperator.GetFromToBlocks(1, 10) - if !mockRpcClient.GetBlockCalled { - t.Errorf("GetFromToBlocks should call GetBlock") - } - if mockRpcClient.GetBlockCallCount != 10 { - t.Errorf("GetFromToBlocks(1, 10) should call GetBlock 10 times, got %d", mockRpcClient.GetBlockCallCount) - } + assert.True(t, mockRpcClient.GetBlockCalled) + assert.Equal(t, 10, mockRpcClient.GetBlockCallCount) // Reset mock for next test mockRpcClient.GetLatestBlockHeightCalled = false // Test GetLatestBlockHeight _, err := queryOperator.GetLatestBlockHeight() - if err != nil { - t.Errorf("GetLatestBlockHeight should not return an error") - } - if !mockRpcClient.GetLatestBlockHeightCalled { - t.Errorf("GetLatestBlockHeight should be called") - } + assert.NoError(t, err) + assert.True(t, mockRpcClient.GetLatestBlockHeightCalled) // Test GetTransactions - should call GetTx for each transaction hash txHashes := []string{"txHash1", "txHash2", "txHash3"} queryOperator.GetTransactions(txHashes) - if !mockRpcClient.GetTxCalled { - t.Errorf("GetTransactions should call GetTx") - } - if mockRpcClient.GetTxCallCount != len(txHashes) { - t.Errorf("GetTransactions should call GetTx %d times, got %d", len(txHashes), mockRpcClient.GetTxCallCount) - } + assert.True(t, mockRpcClient.GetTxCalled) + assert.Equal(t, len(txHashes), mockRpcClient.GetTxCallCount) + + // Test GetFromToCommits - should call GetCommit multiple times (1 to 10 = 10 calls) + queryOperator.GetFromToCommits(1, 10) + assert.True(t, mockRpcClient.GetCommitCalled) + + assert.Equal(t, 10, mockRpcClient.GetCommitCallCount) } diff --git a/indexer/query/types.go b/indexer/query/types.go index 4aeca03..b19df5c 100644 --- a/indexer/query/types.go +++ b/indexer/query/types.go @@ -33,4 +33,5 @@ type RpcClient interface { GetBlock(height uint64) (*rpcClient.BlockResponse, *rpcClient.RpcHeightError) GetLatestBlockHeight() (uint64, *rpcClient.RpcHeightError) GetTx(txHash string) (*rpcClient.TxResponse, *rpcClient.RpcStringError) + GetCommit(height uint64) (*rpcClient.CommitResponse, *rpcClient.RpcCommitError) } diff --git a/indexer/rpc_client/client.go b/indexer/rpc_client/client.go index 999beda..4891f29 100644 --- a/indexer/rpc_client/client.go +++ b/indexer/rpc_client/client.go @@ -68,8 +68,9 @@ const ( Block = "block" AbciQuery = "abci_query" // might be useful for health check - Health = "health" - Tx = "tx" + Health = "health" + Tx = "tx" + RequestCommit = "commit" ) func (r *RpcGnoland) performRequest(method string, params map[string]any, result interface{}) error { @@ -274,3 +275,25 @@ func (r *RpcGnoland) GetAbciQuery(path string, data string, height *uint64, prov return response["result"], nil } + +func (r *RpcGnoland) GetCommit(height uint64) (*CommitResponse, *RpcCommitError) { + response := &CommitResponse{} + params := map[string]any{ + "height": strconv.FormatUint(height, 10), + } + if err := r.performRequest(RequestCommit, params, response); err != nil { + return nil, &RpcCommitError{ + Height: height, + HasHeight: true, + Err: err, + } + } + if response.Error != nil { + return nil, &RpcCommitError{ + Height: height, + HasHeight: true, + Err: fmt.Errorf("rpc error: %v, %s", response.Error.Code, response.Error.Message), + } + } + return response, nil +} diff --git a/indexer/rpc_client/commit.go b/indexer/rpc_client/commit.go new file mode 100644 index 0000000..7ad8352 --- /dev/null +++ b/indexer/rpc_client/commit.go @@ -0,0 +1,64 @@ +package rpcclient + +import ( + "fmt" + "strconv" + "time" +) + +// CommitResponse is the response from the commit endpoint +type CommitResponse struct { + Jsonrpc string `json:"jsonrpc"` + ID int `json:"id"` + Error *JsonRpcError `json:"error,omitempty"` + Result CommitResult `json:"result"` +} + +// CommitResult is the result from the commit endpoint +type CommitResult struct { + SignedHeader SignedHeader `json:"signed_header"` +} + +// SignedHeader is the struct for the signed header, which is a part of commit result +type SignedHeader struct { + Header BlockHeader `json:"header"` + Commit Commit `json:"commit"` +} + +// Commit is the struct for the commit, which is a part of signed header +type Commit struct { + BlockID BlockID `json:"block_id"` + Precommits []*Precommit `json:"precommits"` +} + +// GetHeight returns the height of the commit +func (cr *CommitResponse) GetHeight() (uint64, error) { + if cr == nil { + return 0, fmt.Errorf("CommitResponse Header Height is nil") + } + return strconv.ParseUint(cr.Result.SignedHeader.Header.Height, 10, 64) +} + +// GetTimestamp returns the timestamp of the commit +func (cr *CommitResponse) GetTimestamp() time.Time { + if cr == nil { + return time.Time{} + } + return cr.Result.SignedHeader.Header.Time +} + +// GetSigners returns the signers of the commit +func (cr *CommitResponse) GetSigners() []*Precommit { + if cr == nil { + return nil + } + return cr.Result.SignedHeader.Commit.Precommits +} + +// GetProposerAddress returns the proposer address of the commit +func (cr *CommitResponse) GetProposerAddress() string { + if cr == nil { + return "" + } + return cr.Result.SignedHeader.Header.ProposerAddress +} diff --git a/indexer/rpc_client/errors.go b/indexer/rpc_client/errors.go index dfb1520..71efaf7 100644 --- a/indexer/rpc_client/errors.go +++ b/indexer/rpc_client/errors.go @@ -31,3 +31,18 @@ func (e *RpcStringError) Error() string { } return fmt.Sprintf("rpc error: %v", e.Err) } + +// RpcCommitError represents an RPC error that includes the commit context for retry purposes +type RpcCommitError struct { + Height uint64 + HasHeight bool // indicates if Height field is valid + Err error +} + +// Error implements the error interface +func (e *RpcCommitError) Error() string { + if e.HasHeight { + return fmt.Sprintf("rpc error at height %d: %v", e.Height, e.Err) + } + return fmt.Sprintf("rpc error: %v", e.Err) +} diff --git a/indexer/rpc_client/rate_limited_client.go b/indexer/rpc_client/rate_limited_client.go index 1f9e1ba..680f835 100644 --- a/indexer/rpc_client/rate_limited_client.go +++ b/indexer/rpc_client/rate_limited_client.go @@ -79,6 +79,12 @@ func (r *RateLimitedRpcClient) GetAbciQuery(path string, data string, height *ui return r.client.GetAbciQuery(path, data, height, prove) } +// GetCommit method with rate limiting +func (r *RateLimitedRpcClient) GetCommit(height uint64) (*CommitResponse, *RpcCommitError) { + r.rateLimiter.Wait() + return r.client.GetCommit(height) +} + // TryHealth - non-blocking version that returns false if rate limited func (r *RateLimitedRpcClient) TryHealth() (error, bool) { if !r.rateLimiter.Allow() { @@ -132,6 +138,15 @@ func (r *RateLimitedRpcClient) TryGetAbciQuery(path string, data string, height return response, err, true } +// TryGetCommit - non-blocking version that returns false if rate limited +func (r *RateLimitedRpcClient) TryGetCommit(height uint64) (*CommitResponse, *RpcCommitError, bool) { + if !r.rateLimiter.Allow() { + return nil, nil, false // rate limited + } + response, err := r.client.GetCommit(height) + return response, err, true +} + // GetRateLimiterStatus returns information about the current rate limiter status func (r *RateLimitedRpcClient) GetRateLimiterStatus() rate_limit.ChannelRateLimiterStatus { return r.rateLimiter.GetStatus() diff --git a/indexer/rpc_client/types.go b/indexer/rpc_client/types.go index 765f6e0..40b1c94 100644 --- a/indexer/rpc_client/types.go +++ b/indexer/rpc_client/types.go @@ -34,6 +34,7 @@ type Client interface { GetLatestBlockHeight() (uint64, *RpcHeightError) GetTx(txHash string) (*TxResponse, *RpcStringError) GetAbciQuery(path string, data string, height *uint64, prove *bool) (any, error) + GetCommit(height uint64) (*CommitResponse, *RpcCommitError) } type RateLimiter interface { diff --git a/integration/synthetic/query_operator.go b/integration/synthetic/query_operator.go index 569217e..b79c085 100644 --- a/integration/synthetic/query_operator.go +++ b/integration/synthetic/query_operator.go @@ -30,6 +30,9 @@ type SyntheticQueryOperator struct { // Pre-generated data for consistent testing blocks map[uint64]*rpcClient.BlockResponse transactions map[string]*rpcClient.TxResponse + commits map[uint64]*rpcClient.CommitResponse + // response maker + responseMaker *ResponseMaker } // NewSyntheticQueryOperator creates a new synthetic query operator @@ -52,6 +55,8 @@ func NewSyntheticQueryOperator(chainID string, fromHeight uint64, maxHeight uint signedValidators: validators, blocks: make(map[uint64]*rpcClient.BlockResponse), transactions: make(map[string]*rpcClient.TxResponse), + commits: make(map[uint64]*rpcClient.CommitResponse), + responseMaker: NewResponseMaker(gen), } // Pre-generate some blocks and transactions for consistency @@ -92,17 +97,36 @@ func (sq *SyntheticQueryOperator) GetTransactions(txHashes []string) []*rpcClien return transactions } +// GetFromToCommits implements the QueryOperator interface by returning synthetic commits +func (sq *SyntheticQueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*rpcClient.CommitResponse { + diff := toHeight - fromHeight + 1 + if diff < 1 { + return nil + } + + commits := make([]*rpcClient.CommitResponse, 0, diff) + + for height := fromHeight; height <= toHeight; height++ { + commit := sq.getCommit(height) + if commit != nil { + commits = append(commits, commit) + } + } + return commits +} + // GetLatestBlockHeight implements the QueryOperator interface func (sq *SyntheticQueryOperator) GetLatestBlockHeight() (uint64, error) { return sq.currentHeight, nil } -// preGenerateData creates a consistent dataset of blocks and transactions +// preGenerateData creates a consistent dataset of blocks, transactions and commits func (sq *SyntheticQueryOperator) preGenerateData(fromHeight uint64, maxHeight uint64) { startTime := time.Now() // Generate blocks from height start to maxHeight for height := fromHeight; height <= maxHeight; height++ { sq.createSynthBlock(height) + sq.createCommit(height) // Log progress every 100 blocks if height%100 == 0 { @@ -111,6 +135,7 @@ func (sq *SyntheticQueryOperator) preGenerateData(fromHeight uint64, maxHeight u } log.Printf("Pre-generated data for blocks from %d to %d in %v", fromHeight, maxHeight, time.Since(startTime)) log.Printf("Total transactions generated: %d", len(sq.transactions)) + log.Printf("Total commits generated: %d", len(sq.commits)) } // getBlock returns existing block or creates a new one @@ -173,15 +198,13 @@ func (sq *SyntheticQueryOperator) createSynthBlock(height uint64) (*rpcClient.Bl // Create the block using existing synthetic response maker blockInput := GenBlockInput{ - Height: height, - ChainID: sq.chainID, - Timestamp: blockTimestamp, - ProposerAddress: sq.signedValidators[height%uint64(len(sq.signedValidators))], // random validator - SignedValidators: sq.signedValidators, - TxsRaw: txRaws, + Height: height, + ChainID: sq.chainID, + Timestamp: blockTimestamp, + TxsRaw: txRaws, } - block := GenerateBlockResponse(blockInput) + block := sq.responseMaker.GenerateBlockResponse(blockInput) sq.blocks[height] = block return block, txResponses @@ -211,12 +234,34 @@ func (sq *SyntheticQueryOperator) createTransaction( Events: txEvents, } - tx := GenerateTransactionResponse(txInput) + tx := sq.responseMaker.GenerateTransactionResponse(txInput) sq.transactions[txHash] = tx return tx } +// createCommit generates a synthetic commit for the given height +func (sq *SyntheticQueryOperator) createCommit(height uint64) *rpcClient.CommitResponse { + commitInput := GenCommitInput{ + Height: height, + ChainID: sq.chainID, + Timestamp: baseTimestamp.Add(time.Duration(height-1) * blockProductionRate), + ProposerAddress: sq.signedValidators[height%uint64(len(sq.signedValidators))], // random validator + SignedValidators: sq.signedValidators, + } + commit := sq.responseMaker.GenerateCommitResponse(commitInput) + sq.commits[height] = commit + return commit +} + +func (sq *SyntheticQueryOperator) getCommit(height uint64) *rpcClient.CommitResponse { + if commit, ok := sq.commits[height]; ok { + return commit + } + log.Fatal("commit not found") + return nil +} + // SetCurrentHeight allows updating the current height for live testing scenarios func (sq *SyntheticQueryOperator) SetCurrentHeight(height uint64) { sq.currentHeight = height diff --git a/integration/synthetic/respone_maker.go b/integration/synthetic/respone_maker.go index ff746dd..f111070 100644 --- a/integration/synthetic/respone_maker.go +++ b/integration/synthetic/respone_maker.go @@ -8,53 +8,39 @@ import ( "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/generator" ) +type ResponseMaker struct { + generator *generator.DataGenerator +} + +func NewResponseMaker(generator *generator.DataGenerator) *ResponseMaker { + return &ResponseMaker{ + generator: generator, + } +} + type GenBlockInput struct { - Height uint64 - ChainID string - Timestamp time.Time - ProposerAddress string - SignedValidators []string - TxsRaw []string + Height uint64 + ChainID string + Timestamp time.Time + TxsRaw []string } -func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { - gen := generator.NewDataGenerator(500) - - // here is the deal the progrma will just make some hash here - // usually there are 2 hashes, new hash and hash from the previous block - // however for this instance we will just ignore it because - // this level of precision is not needed since the purpose is to - // test the indexer - blockHash := gen.GenerateBlockHash() - partsHash := gen.GenerateBlockHash() - - // the program will generate some hash for the previous block - // so it has some similaraties to the real response - lastBlockHash := gen.GenerateBlockHash() - lastPartsHash := gen.GenerateBlockHash() - - // just make some random hash here - lastCommitHash := gen.GenerateBlockHash() - dataHash := gen.GenerateBlockHash() - validatorsHash := gen.GenerateBlockHash() - nextValidatorsHash := gen.GenerateBlockHash() - consensusHash := gen.GenerateBlockHash() - appHash := gen.GenerateBlockHash() - lastResultsHash := gen.GenerateBlockHash() +func (rm *ResponseMaker) GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { + // just generate the hash for block and last block + // for others just use empty strings since this is all just to mock the similar expereience + blockHash := rm.generator.GenerateBlockHash() + lastBlockHash := rm.generator.GenerateBlockHash() + partsHash := "" + lastPartsHash := "" + lastCommitHash := "" + dataHash := "" + validatorsHash := "" + nextValidatorsHash := "" + consensusHash := "" + appHash := "" + lastResultsHash := "" - precommits := make([]*rpcClient.Precommit, 0, len(input.SignedValidators)) - for i, validator := range input.SignedValidators { - precommits = append(precommits, &rpcClient.Precommit{ - ValidatorAddress: validator, - ValidatorIndex: strconv.Itoa(i), - Signature: gen.GenerateBlockHash(), - Type: 2, // not sure why 2 but that is what I saw in the real data - Height: strconv.FormatUint(input.Height, 10), - Round: "0", - BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, - Timestamp: input.Timestamp, - }) - } + precommits := make([]*rpcClient.Precommit, 0) // just mock it since we do not need it for the block block := rpcClient.BlockResponse{ Jsonrpc: "2.0", @@ -74,7 +60,7 @@ func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { NumTxs: "0", // it is not really important for the indexer TotalTxs: "0", // it is not really important for the indexer AppVersion: "1.0.0", - ProposerAddress: input.ProposerAddress, + ProposerAddress: "", // just mock it since we do not need it for the block LastBlockID: rpcClient.BlockID{Hash: lastBlockHash, Parts: rpcClient.Parts{Total: "1", Hash: lastPartsHash}}, // just make some random hash here LastCommitHash: lastCommitHash, @@ -95,7 +81,7 @@ func GenerateBlockResponse(input GenBlockInput) *rpcClient.BlockResponse { NumTxs: "0", // it is not really important for the indexer TotalTxs: "0", // it is not really important for the indexer AppVersion: "1.0.0", - ProposerAddress: input.ProposerAddress, + ProposerAddress: "", // just mock it since we do not need it for the block LastBlockID: rpcClient.BlockID{Hash: lastBlockHash, Parts: rpcClient.Parts{Total: "1", Hash: lastPartsHash}}, LastCommitHash: lastCommitHash, DataHash: dataHash, @@ -129,7 +115,7 @@ type GenTransactionInput struct { Events *generator.TxEvents } -func GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxResponse { +func (rm *ResponseMaker) GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxResponse { // because I pretty much complicated this part of the code the // program will now need to pull the data from proto events // and convert them to rpc client events @@ -179,3 +165,67 @@ func GenerateTransactionResponse(input GenTransactionInput) *rpcClient.TxRespons } return &txResponse } + +type GenCommitInput struct { + Height uint64 + ChainID string + Timestamp time.Time + ProposerAddress string + SignedValidators []string +} + +func (rm *ResponseMaker) GenerateCommitResponse(input GenCommitInput) *rpcClient.CommitResponse { + // just like in the gen block we need the data for the header but not really + // just generate the hash for block and last block + // for others just use empty strings since this is all just to mock the similar expereience + blockHash := rm.generator.GenerateBlockHash() + partsHash := "" + lastCommitHash := "" + dataHash := "" + validatorsHash := "" + nextValidatorsHash := "" + consensusHash := "" + appHash := "" + lastResultsHash := "" + + precommits := make([]*rpcClient.Precommit, 0, len(input.SignedValidators)) + for i, validator := range input.SignedValidators { + precommits = append(precommits, &rpcClient.Precommit{ + ValidatorAddress: validator, + ValidatorIndex: strconv.Itoa(i), + Signature: rm.generator.GenerateBlockHash(), + Type: 2, // not sure why 2 but that is what I saw in the real data + Height: strconv.FormatUint(input.Height, 10), + Round: "0", + BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, + Timestamp: input.Timestamp, + }) + } + + commitResponse := rpcClient.CommitResponse{ + Jsonrpc: "2.0", + ID: 1, + Result: rpcClient.CommitResult{ + SignedHeader: rpcClient.SignedHeader{ + Header: rpcClient.BlockHeader{ + Height: strconv.FormatUint(input.Height, 10), + Time: input.Timestamp, + ChainID: input.ChainID, + ProposerAddress: input.ProposerAddress, + LastCommitHash: lastCommitHash, + DataHash: dataHash, + ValidatorsHash: validatorsHash, + NextValidatorsHash: nextValidatorsHash, + ConsensusHash: consensusHash, + AppHash: appHash, + LastResultsHash: lastResultsHash, + }, + Commit: rpcClient.Commit{ + BlockID: rpcClient.BlockID{Hash: blockHash, Parts: rpcClient.Parts{Total: "1", Hash: partsHash}}, + Precommits: precommits, + }, + }, + }, + } + return &commitResponse +} diff --git a/integration/test_config.example.yml b/integration/test_config.example.yml index 506ca0c..27861f7 100644 --- a/integration/test_config.example.yml +++ b/integration/test_config.example.yml @@ -18,6 +18,6 @@ pool_max_conn_lifetime_jitter: 30s # synthetic test config chain_id: gnoland -max_height: 100000 +max_height: 5000 from_height: 1 -to_height: 100000 \ No newline at end of file +to_height: 5000 \ No newline at end of file diff --git a/pkgs/generator/generator_test.go b/pkgs/generator/generator_test.go index 0b20f21..4837304 100644 --- a/pkgs/generator/generator_test.go +++ b/pkgs/generator/generator_test.go @@ -1,69 +1,40 @@ package generator import ( - "fmt" "log" "testing" + + "github.com/stretchr/testify/assert" ) // TestAuthenticGeneration demonstrates the new authentic key and address generation func TestAuthenticGeneration(t *testing.T) { - fmt.Println("Testing Authentic Address & PubKey Generation") - // Create a new data generator generator := NewDataGenerator(500) - fmt.Printf("Generated %d key pairs in the pool\n\n", len(generator.keyPairPool)) - // Test address generation - fmt.Println("Generated Addresses (from pool):") // test up to a 1k addresses - for i := range 500 { + for range 500 { addr := generator.GenerateAddress() - fmt.Printf("%d. %s (valid: %t)\n", i+1, addr, ValidateAddress(addr)) + assert.True(t, ValidateAddress(addr)) } - fmt.Println("\nGenerated Public Keys (from pool):") // test up to a 1k public keys - for i := range 500 { + for range 500 { pubkey := generator.GeneratePubKey() - fmt.Printf("%d. %s\n", i+1, pubkey[:50]+"...") - fmt.Printf(" Valid: %t\n", ValidatePubKey(pubkey)) + assert.True(t, ValidatePubKey(pubkey)) } // Test key pair access - fmt.Println("\nRandom Key Pairs with Full Details:") - for i := range 500 { - kp := generator.GetRandomKeyPair() - fmt.Printf("KeyPair %d:\n", i+1) - fmt.Printf(" Address: %s\n", kp.AddressBech32) - fmt.Printf(" PubKey: %s\n", kp.PubKeyBech32[:50]+"...") - fmt.Printf(" PrivKey: %s\n", kp.PrivKeyHex[:32]+"...") - fmt.Println() + for range 500 { + generator.GetRandomKeyPair() } // Test synthetic transaction generation 500 times - fmt.Println("Sample Transaction with Authentic Addresses:") for range 500 { events, tx := generator.GenerateTransaction() - fmt.Printf("Transaction: %+v\n", tx) - for i := range events.Events { - event := &events.Events[i] - fmt.Printf("\tEvent %d: %s\n", i+1, event.Type) - for _, attr := range event.Attributes { - // Check if this attribute contains an address - if attr.Key == "from" || attr.Key == "to" || attr.Key == "Creator" || attr.Key == "Author" { - if ValidateAddress(attr.GetStringValue()) { - fmt.Printf("\t%s: %s (authentic)\n", attr.Key, attr.Value) - } else { - fmt.Printf("\t%s: %s (not authentic)\n", attr.Key, attr.Value) - } - } else { - fmt.Printf("\t%s: %s\n", attr.Key, attr.Value) - } - } - fmt.Println() - } + assert.NotNil(t, tx) + assert.NotNil(t, events) } } From d64790181c626e2ac0137dd9cce08d10ec2b6a7c Mon Sep 17 00:00:00 2001 From: NM Date: Tue, 25 Nov 2025 15:03:36 +0100 Subject: [PATCH 02/11] Modify message processing and database insertion - Added a `message_counter` field to message types (MsgSend, MsgCall, MsgAddPackage, MsgRun) to track message order and have unique message counter. - Updated the `GetMessageFromStdTx` method to include `message_counter` in the processed messages. - Modified error handling in the database insertion methods to log transaction hashes on failure. - Refactored the `Orchestrator` to streamline chunk processing and improve concurrency in block and commit retrieval. --- experiments/experiment1/enc_dec_test.go | 2 +- experiments/experiment4/encode_proto.go | 39 ++++++++-- indexer/data_processor/operator.go | 24 +++++-- indexer/decoder/decoder.go | 57 ++++++++------- indexer/decoder/product.go | 94 ++++++++++++++++--------- indexer/orchestrator/operator.go | 81 +++++++++++---------- pkgs/database/insert.go | 4 ++ pkgs/sql_data_types/table.go | 32 +++++---- 8 files changed, 214 insertions(+), 119 deletions(-) diff --git a/experiments/experiment1/enc_dec_test.go b/experiments/experiment1/enc_dec_test.go index 7ddf26e..922302e 100644 --- a/experiments/experiment1/enc_dec_test.go +++ b/experiments/experiment1/enc_dec_test.go @@ -16,7 +16,7 @@ import ( // This can be later used to query the tx from the rpc client via the tx method // Same data is pressent within the tx query method so this should be useful for the indexer func TestEncodeDecode(t *testing.T) { - raw_data := "CnQKDS9iYW5rLk1zZ1NlbmQSYwooZzE0ODU4M3Q1eDY2enM2cDkwZWhhZDZsNHFlZmV5YWY1NHM2OXdxbBIoZzFrM2NjcDA5NzdhajMyeXJqeHBtdjA5d2hxZ2w3bjgzcjZjc2pyYxoNMTAwMDAwMDB1Z25vdBIRCIDaxAkSCjEwMDAwdWdub3Qafgo6ChMvdG0uUHViS2V5U2VjcDI1NmsxEiMKIQMcQg5ueuQfcNtMh5w7PjWaTKO72epQaK9kax/g3+C9NRJAYMsAIGMPOzc964cBKZ49xg+KBO/5T3RJLhjDzO9x5a5Kr4sX+78p3QLvOV6a30htoJc11XYGV5tymCLgewf8lg==" + raw_data := "CpABCgovdm0ubV9jYWxsEoEBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIhZnbm8ubGFuZC9yL2dub3N3YXAvZ25zKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpwBCgovdm0ubV9jYWxsEo0BCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiJnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi91c2RjKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9iYXIqCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL2JheioIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMAqbAQoKL3ZtLm1fY2FsbBKMAQooZzE3MjkwY3d2bXJhcHZwODY5eGZuaGhhd2E4c205ZWRwdWZ6YXQ3ZCIhZ25vLmxhbmQvci9nbm9zd2FwL3Rlc3RfdG9rZW4vb2JsKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9mb28qCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL3F1eCoIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMBITCICEr18SDDEwMDAwMDB1Z25vdBp+CjoKEy90bS5QdWJLZXlTZWNwMjU2azESIwohAtGyA6l3UIrUup5z7yXo90bXDcXUOmLiK34YPffgQ6pAEkCg8pAtBv6Fhw98bKYdqrEX2UrcjYvIUpbGdxMyc5Zpfl1cMNo8G6vzpnFaQESX+7eZIFTfO5BCFjVJLfC5Ur5C" // decode the raw data from base64 to string base64Decoded, err := base64.StdEncoding.DecodeString(raw_data) diff --git a/experiments/experiment4/encode_proto.go b/experiments/experiment4/encode_proto.go index 7ee570e..c63ea38 100644 --- a/experiments/experiment4/encode_proto.go +++ b/experiments/experiment4/encode_proto.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/hex" "fmt" "log" @@ -82,17 +83,45 @@ func main() { fmt.Printf("Length: %d bytes\n", len(encoded)) // try different compressions - zstdOptions := zstandard.WithEncoderLevel(zstandard.SpeedBetterCompression) - zstdEncoder, err := zstandard.NewWriter(nil, zstdOptions) + // Test with different compression levels + for _, level := range []int{1, 2, 3, 10, 15, 18, 20} { + encoderLevel := zstandard.EncoderLevelFromZstd(level) + encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel)) + if err != nil { + log.Fatal(err) + } + + var buf bytes.Buffer + encoder.Reset(&buf) + _, err = encoder.Write(encoded) + if err != nil { + log.Fatal(err) + } + encoder.Close() + + compressed := buf.Bytes() + fmt.Printf("Compressed (level %d): %d bytes (%.1f%% of original)\n", + level, len(compressed), float64(len(compressed))/float64(len(encoded))*100) + } + + // Final compression with level 22 + encoderLevel := zstandard.EncoderLevelFromZstd(22) + encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel)) if err != nil { log.Fatal(err) } - compressed := zstdEncoder.EncodeAll(encoded, nil) + + var buf bytes.Buffer + encoder.Reset(&buf) + _, err = encoder.Write(encoded) if err != nil { log.Fatal(err) } - fmt.Println("Compressed bytes:", string(compressed)) - fmt.Printf("Length: %d bytes\n", len(compressed)) + encoder.Close() + + compressed := buf.Bytes() + fmt.Println("Final compressed bytes:", string(compressed)) + fmt.Printf("Final length: %d bytes\n", len(compressed)) decoded, err := DecodeProto(encoded) if err != nil { diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index f39e1f8..1ac9a28 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -421,28 +421,44 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e // Insert DbMsgSend messages with address IDs if len(groups.MsgSend) > 0 { if err := d.dbPool.InsertMsgSend(groups.MsgSend); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgSend: %w", err)) + hashes := make([]string, 0, len(groups.MsgSend)) + for _, msg := range groups.MsgSend { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgSend: %w, hashes: %v", err, hashes)) } } // Insert DbMsgCall messages with address IDs if len(groups.MsgCall) > 0 { if err := d.dbPool.InsertMsgCall(groups.MsgCall); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgCall: %w", err)) + hashes := make([]string, 0, len(groups.MsgCall)) + for _, msg := range groups.MsgCall { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgCall: %w, hashes: %v", err, hashes)) } } // Insert DbMsgAddPackage messages with address IDs if len(groups.MsgAddPkg) > 0 { if err := d.dbPool.InsertMsgAddPackage(groups.MsgAddPkg); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgAddPackage: %w", err)) + hashes := make([]string, 0, len(groups.MsgAddPkg)) + for _, msg := range groups.MsgAddPkg { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgAddPackage: %w, hashes: %v", err, hashes)) } } // Insert DbMsgRun messages with address IDs if len(groups.MsgRun) > 0 { if err := d.dbPool.InsertMsgRun(groups.MsgRun); err != nil { - insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgRun: %w", err)) + hashes := make([]string, 0, len(groups.MsgRun)) + for _, msg := range groups.MsgRun { + hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) + } + insertErrors = append(insertErrors, fmt.Errorf("failed to insert DbMsgRun: %w, hashes: %v", err, hashes)) } } diff --git a/indexer/decoder/decoder.go b/indexer/decoder/decoder.go index ed86815..e83ea47 100644 --- a/indexer/decoder/decoder.go +++ b/indexer/decoder/decoder.go @@ -108,7 +108,8 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { var messages []map[string]any // Process each message in the transaction - for _, msg := range tx.GetMsgs() { + for i, msg := range tx.GetMsgs() { + messageCounter := int16(i) switch m := msg.(type) { case bank.MsgSend: // amount should have something like 1000000 ugnot we just need to split it and convert it to uint64 @@ -117,10 +118,11 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { amount = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "bank_msg_send", - "from_address": m.FromAddress.String(), - "to_address": m.ToAddress.String(), - "amount": amount, + "msg_type": "bank_msg_send", + "from_address": m.FromAddress.String(), + "to_address": m.ToAddress.String(), + "amount": amount, + "message_counter": messageCounter, }) case vm.MsgCall: caller := m.Caller.String() @@ -139,13 +141,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { // combine the args into a string args := strings.Join(m.Args, ",") messages = append(messages, map[string]any{ - "msg_type": "vm_msg_call", - "caller": caller, - "pkg_path": pkgPath, - "func_name": funcName, - "args": args, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_call", + "caller": caller, + "pkg_path": pkgPath, + "func_name": funcName, + "args": args, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) case vm.MsgAddPackage: pkgPath := m.Package.Path @@ -161,13 +164,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { maxDeposit = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "vm_msg_add_package", - "pkg_path": pkgPath, - "pkg_name": pkgName, - "pkg_file_names": pkgFileNames, - "creator": creator, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_add_package", + "pkg_path": pkgPath, + "pkg_name": pkgName, + "pkg_file_names": pkgFileNames, + "creator": creator, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) case vm.MsgRun: @@ -186,13 +190,14 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { maxDeposit = []Coin{} } messages = append(messages, map[string]any{ - "msg_type": "vm_msg_run", - "caller": caller, - "pkg_path": pkgPath, - "pkg_name": pkgName, - "pkg_file_names": pkgFileNames, - "send": send, - "max_deposit": maxDeposit, + "msg_type": "vm_msg_run", + "caller": caller, + "pkg_path": pkgPath, + "pkg_name": pkgName, + "pkg_file_names": pkgFileNames, + "send": send, + "max_deposit": maxDeposit, + "message_counter": messageCounter, }) // case for AnyNewMessage add here: default: diff --git a/indexer/decoder/product.go b/indexer/decoder/product.go index 12cbd54..7a8add7 100644 --- a/indexer/decoder/product.go +++ b/indexer/decoder/product.go @@ -229,6 +229,11 @@ func (dm *DecodedMsg) convertToDbMsgSend( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgSend, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + fromAddress, ok := msgMap["from_address"].(string) if !ok { return nil, fmt.Errorf("missing from_address") @@ -255,13 +260,14 @@ func (dm *DecodedMsg) convertToDbMsgSend( } return &dataTypes.MsgSend{ - TxHash: txHash, - ChainName: chainName, - FromAddress: addressResolver.GetAddress(fromAddress), - ToAddress: addressResolver.GetAddress(toAddress), - Amount: amount, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + ChainName: chainName, + FromAddress: addressResolver.GetAddress(fromAddress), + ToAddress: addressResolver.GetAddress(toAddress), + Amount: amount, + Signers: signerIds, + Timestamp: timestamp, + MessageCounter: messageCounter, }, nil } @@ -274,6 +280,11 @@ func (dm *DecodedMsg) convertToDbMsgCall( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgCall, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + caller, ok := msgMap["caller"].(string) if !ok { return nil, fmt.Errorf("missing caller") @@ -325,16 +336,17 @@ func (dm *DecodedMsg) convertToDbMsgCall( } return &dataTypes.MsgCall{ - TxHash: txHash, - ChainName: chainName, - Caller: addressResolver.GetAddress(caller), - Send: send, - PkgPath: pkgPath, - FuncName: funcName, - Args: argsStr, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Caller: addressResolver.GetAddress(caller), + Send: send, + PkgPath: pkgPath, + FuncName: funcName, + Args: argsStr, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } @@ -347,6 +359,11 @@ func (dm *DecodedMsg) convertToDbMsgAddPackage( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgAddPackage, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + creator, ok := msgMap["creator"].(string) if !ok { return nil, fmt.Errorf("missing creator") @@ -393,15 +410,16 @@ func (dm *DecodedMsg) convertToDbMsgAddPackage( } return &dataTypes.MsgAddPackage{ - TxHash: txHash, - ChainName: chainName, - Creator: addressResolver.GetAddress(creator), - PkgPath: pkgPath, - PkgName: pkgName, - Send: send, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Creator: addressResolver.GetAddress(creator), + PkgPath: pkgPath, + PkgName: pkgName, + Send: send, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } @@ -414,6 +432,11 @@ func (dm *DecodedMsg) convertToDbMsgRun( timestamp time.Time, signerIds []int32, ) (*dataTypes.MsgRun, error) { + messageCounter, ok := msgMap["message_counter"].(int16) + if !ok { + return nil, fmt.Errorf("missing message_counter") + } + caller, ok := msgMap["caller"].(string) if !ok { return nil, fmt.Errorf("missing caller") @@ -460,14 +483,15 @@ func (dm *DecodedMsg) convertToDbMsgRun( } return &dataTypes.MsgRun{ - TxHash: txHash, - ChainName: chainName, - Caller: addressResolver.GetAddress(caller), - PkgPath: pkgPath, - PkgName: pkgName, - Send: send, - MaxDeposit: maxDeposit, - Signers: signerIds, - Timestamp: timestamp, + TxHash: txHash, + MessageCounter: messageCounter, + ChainName: chainName, + Caller: addressResolver.GetAddress(caller), + PkgPath: pkgPath, + PkgName: pkgName, + Send: send, + MaxDeposit: maxDeposit, + Signers: signerIds, + Timestamp: timestamp, }, nil } diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go index bfb292a..253ef48 100644 --- a/indexer/orchestrator/operator.go +++ b/indexer/orchestrator/operator.go @@ -65,33 +65,16 @@ func (or *Orchestrator) HistoricProcess( for startHeight := fromHeight; startHeight <= toHeight; { chunkEndHeight := min(startHeight+or.config.MaxBlockChunkSize-1, toHeight) - chunkStartTime := time.Now() log.Printf("Processing chunk from %d to %d", startHeight, chunkEndHeight) // Update current processing height or.currentProcessingHeight = startHeight - // Step 1: Get blocks concurrently - blocks := or.queryOperator.GetFromToBlocks(startHeight, chunkEndHeight) - commits := or.queryOperator.GetFromToCommits(startHeight, chunkEndHeight) - if len(blocks) == 0 { - log.Printf("No valid blocks in chunk %d-%d", startHeight, chunkEndHeight) - } else { - // Step 2: Collect all transactions from all blocks in this chunk - allTransactions := or.collectTransactionsFromBlocks(blocks) - - log.Printf("Collected %d transactions from %d blocks in chunk", len(allTransactions), len(blocks)) - - // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, commits, allTransactions, false, startHeight, chunkEndHeight); err != nil { - log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err) - } else { - // Progress logging - chunkDuration := time.Since(chunkStartTime) - totalDuration := time.Since(startTime) - log.Printf("Chunk %d-%d completed in %v, total time: %v", - startHeight, chunkEndHeight, chunkDuration, totalDuration) - } + // Process the chunk + err := or.processChunk(startHeight, chunkEndHeight) + if err != nil { + log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err) + } // Always advance to next chunk, regardless of whether blocks were found @@ -126,7 +109,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool log.Printf("Either there are no blocks in the database or the database is not properly configured.") log.Printf("Use skipInitialDbCheck=true if this is expected to run from the latest chain height without previous data.") log.Printf("Starting from height 1") - lastProcessedHeight = 1 + lastProcessedHeight = 0 } log.Printf("Retrieved last processed height from database: %d", lastProcessedHeight) } else { @@ -161,7 +144,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool continue } - blocksBehind := int64(latestHeight) - int64(lastProcessedHeight) + blocksBehind := latestHeight - lastProcessedHeight // If caught up, wait and continue if blocksBehind <= 0 { @@ -171,7 +154,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool } // Adjust chunk size based on how far behind we are - currentChunkSize := min(uint64(blocksBehind), or.config.MaxBlockChunkSize) + currentChunkSize := min(blocksBehind, or.config.MaxBlockChunkSize) chunkStart := lastProcessedHeight + 1 chunkEnd := min(chunkStart+currentChunkSize-1, latestHeight) @@ -182,7 +165,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool or.currentProcessingHeight = chunkStart // Process this chunk - err = or.processLiveChunk(chunkStart, chunkEnd) + err = or.processChunk(chunkStart, chunkEnd) if err != nil { log.Printf("Error processing live chunk %d-%d: %v", chunkStart, chunkEnd, err) time.Sleep(or.config.LivePooling) @@ -199,13 +182,27 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool } } -// processLiveChunk processes a single chunk of blocks for live processing -func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { +// processChunk processes a single chunk of blocks for live processing +func (or *Orchestrator) processChunk(chunkStart, chunkEnd uint64) error { chunkStartTime := time.Now() // Step 1: Get blocks concurrently - blocks := or.queryOperator.GetFromToBlocks(chunkStart, chunkEnd) - commits := or.queryOperator.GetFromToCommits(chunkStart, chunkEnd) + var wg sync.WaitGroup + wg.Add(2) + + var blocks []*rpcClient.BlockResponse + var commits []*rpcClient.CommitResponse + + go func() { + defer wg.Done() + blocks = or.queryOperator.GetFromToBlocks(chunkStart, chunkEnd) + }() + go func() { + defer wg.Done() + commits = or.queryOperator.GetFromToCommits(chunkStart, chunkEnd) + }() + + wg.Wait() if len(blocks) == 0 && len(commits) == 0 { log.Printf("No valid blocks in live chunk %d-%d", chunkStart, chunkEnd) @@ -218,18 +215,21 @@ func (or *Orchestrator) processLiveChunk(chunkStart, chunkEnd uint64) error { log.Printf("Collected %d transactions from %d blocks in live chunk", len(allTransactions), len(blocks)) // Step 3: Process all data concurrently - if err := or.processAllConcurrently(blocks, commits, allTransactions, false, chunkStart, chunkEnd); err != nil { + if err := or.processAll(blocks, commits, allTransactions, false, chunkStart, chunkEnd); err != nil { return fmt.Errorf("failed to process live chunk %d-%d: %w", chunkStart, chunkEnd, err) } chunkDuration := time.Since(chunkStartTime) - log.Printf("Live chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration) + log.Printf("Chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration) return nil } // updateProgressMetrics updates and logs progress metrics for live processing -func (or *Orchestrator) updateProgressMetrics(chunkStart, chunkEnd uint64, blocksBehind int64, lastProgressTime *time.Time) { +func (or *Orchestrator) updateProgressMetrics( + chunkStart, chunkEnd, blocksBehind uint64, + lastProgressTime *time.Time, +) { now := time.Now() timeSinceLastProgress := now.Sub(*lastProgressTime) @@ -242,8 +242,17 @@ func (or *Orchestrator) updateProgressMetrics(chunkStart, chunkEnd uint64, block } } -// collectTransactionsFromBlocks extracts all transactions from blocks and queries them concurrently -// This mimics the Python _process_historical_block_chunk behavior +/* + collectTransactionsFromBlocks extracts all transactions from blocks and queries them concurrently + +Parameters: + - blocks: a slice of blocks + +Returns: + - a slice of transactions + +The method will not throw an error if the transactions are not found, it will just return an empty slice. +*/ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockResponse) []dataprocessor.TrasnactionsData { // Collect all transaction hashes from all blocks var allTxHashes []string @@ -332,7 +341,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // - error: if processing fails // // The method will not throw an error if the data is not found, it will just return nil -func (or *Orchestrator) processAllConcurrently( +func (or *Orchestrator) processAll( blocks []*rpcClient.BlockResponse, commits []*rpcClient.CommitResponse, transactions []dataprocessor.TrasnactionsData, diff --git a/pkgs/database/insert.go b/pkgs/database/insert.go index de696d3..ee9cacb 100644 --- a/pkgs/database/insert.go +++ b/pkgs/database/insert.go @@ -208,6 +208,7 @@ func (t *TimescaleDb) InsertMsgSend(messages []sql_data_types.MsgSend) error { messages[i].ToAddress, makePgxArray(messages[i].Amount), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) @@ -241,6 +242,7 @@ func (t *TimescaleDb) InsertMsgCall(messages []sql_data_types.MsgCall) error { makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) @@ -274,6 +276,7 @@ func (t *TimescaleDb) InsertMsgAddPackage(messages []sql_data_types.MsgAddPackag makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) @@ -307,6 +310,7 @@ func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { makePgxArray(messages[i].Send), makePgxArray(messages[i].MaxDeposit), makePgxArray(messages[i].Signers), + messages[i].MessageCounter, }, nil }) diff --git a/pkgs/sql_data_types/table.go b/pkgs/sql_data_types/table.go index 9057f92..0b8c21a 100644 --- a/pkgs/sql_data_types/table.go +++ b/pkgs/sql_data_types/table.go @@ -285,6 +285,7 @@ func (tg *TransactionGeneral) GetMessageTypes() []string { // - ToAddress (int32) // - Amount (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgSend struct { @@ -294,9 +295,10 @@ type MsgSend struct { // gno address, pull from the gno_addresses table FromAddress int32 `db:"from_address" dbtype:"INTEGER" nullable:"false" primary:"false"` // gno address, pull from the gno_addresses table - ToAddress int32 `db:"to_address" dbtype:"INTEGER" nullable:"true" primary:"false"` - Amount []Amount `db:"amount" dbtype:"amount[]" nullable:"false" primary:"false"` - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + ToAddress int32 `db:"to_address" dbtype:"INTEGER" nullable:"true" primary:"false"` + Amount []Amount `db:"amount" dbtype:"amount[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } // TableName returns the name of the table for the MsgSend struct @@ -352,6 +354,7 @@ func (ms *MsgSend) GetAllAddresses() *TxAddresses { // - Send (Amount[]) // - MaxDeposit (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgCall struct { @@ -359,13 +362,14 @@ type MsgCall struct { Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"` ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` // gno address, pull from the gno_addresses table - Caller int32 `db:"caller" dbtype:"INTEGER" nullable:"false" primary:"false"` - PkgPath string `db:"pkg_path" dbtype:"TEXT" nullable:"true" primary:"false"` - FuncName string `db:"func_name" dbtype:"TEXT" nullable:"true" primary:"false"` - Args string `db:"args" dbtype:"TEXT" nullable:"true" primary:"false"` - Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` - MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Caller int32 `db:"caller" dbtype:"INTEGER" nullable:"false" primary:"false"` + PkgPath string `db:"pkg_path" dbtype:"TEXT" nullable:"true" primary:"false"` + FuncName string `db:"func_name" dbtype:"TEXT" nullable:"true" primary:"false"` + Args string `db:"args" dbtype:"TEXT" nullable:"true" primary:"false"` + Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` + MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } func (mc MsgCall) TableName() string { @@ -417,6 +421,7 @@ func (mc *MsgCall) GetAllAddresses() *TxAddresses { // - MaxDeposit (Amount[]) // - Signers (int32[]) // - Timestamp (time.Time) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgAddPackage struct { @@ -431,7 +436,8 @@ type MsgAddPackage struct { Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` // signers are the addresses that signed the transaction - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } func (ma MsgAddPackage) TableName() string { @@ -483,6 +489,7 @@ func (ma *MsgAddPackage) GetAllAddresses() *TxAddresses { // - Send (Amount[]) // - MaxDeposit (Amount[]) // - Signers (int32[]) +// - MessageCounter (int16) // // PRIMARY KEY (tx_hash, chain_name, timestamp) type MsgRun struct { @@ -497,7 +504,8 @@ type MsgRun struct { Send []Amount `db:"send" dbtype:"amount[]" nullable:"true" primary:"false"` MaxDeposit []Amount `db:"max_deposit" dbtype:"amount[]" nullable:"true" primary:"false"` // signers are the addresses that signed the transaction - Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + Signers []int32 `db:"signers" dbtype:"INTEGER[]" nullable:"false" primary:"false"` + MessageCounter int16 `db:"message_counter" dbtype:"smallint" nullable:"false" primary:"true"` } // A method to get the columns of the struct From 33ca7b44e0781ceaeb2146abdaec1db9e262a37e Mon Sep 17 00:00:00 2001 From: NM Date: Tue, 25 Nov 2025 15:32:43 +0100 Subject: [PATCH 03/11] Small fixes and linting errors addressed --- Makefile | 5 ++--- experiments/experiment1/enc_dec_test.go | 3 +-- experiments/experiment4/encode_proto.go | 22 ++++++++++++++-------- indexer/decoder/decoder.go | 3 +++ indexer/query/operator.go | 2 +- indexer/rpc_client/commit.go | 6 +++++- integration/synthetic/query_operator.go | 5 +---- integration/synthetic/respone_maker.go | 2 +- 8 files changed, 28 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 38f45a8..66f4d8b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ +.PHONY: build install clean build-experimental install-experimental build-api test-race integration-test test + ######################################################## # Build and install the indexer ######################################################## -.PHONY: build install clean build-experimental install-experimental build-api # Get git information GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") @@ -37,8 +38,6 @@ install-experimental: # Test the indexer ######################################################## -.PHONY: test-race integration-test test - test: go test -v ./... diff --git a/experiments/experiment1/enc_dec_test.go b/experiments/experiment1/enc_dec_test.go index 922302e..680abf6 100644 --- a/experiments/experiment1/enc_dec_test.go +++ b/experiments/experiment1/enc_dec_test.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "log" - "testing" ) // In Gnoland txs are marked with base64 not in hex like in other Cosmos SDK chains @@ -15,7 +14,7 @@ import ( // To get something that resembles txhash we need then to encode the data to sha256 and then to base64 // This can be later used to query the tx from the rpc client via the tx method // Same data is pressent within the tx query method so this should be useful for the indexer -func TestEncodeDecode(t *testing.T) { +func main() { raw_data := "CpABCgovdm0ubV9jYWxsEoEBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIhZnbm8ubGFuZC9yL2dub3N3YXAvZ25zKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpwBCgovdm0ubV9jYWxsEo0BCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiJnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi91c2RjKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9iYXIqCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL2JheioIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMAqbAQoKL3ZtLm1fY2FsbBKMAQooZzE3MjkwY3d2bXJhcHZwODY5eGZuaGhhd2E4c205ZWRwdWZ6YXQ3ZCIhZ25vLmxhbmQvci9nbm9zd2FwL3Rlc3RfdG9rZW4vb2JsKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9mb28qCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL3F1eCoIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMBITCICEr18SDDEwMDAwMDB1Z25vdBp+CjoKEy90bS5QdWJLZXlTZWNwMjU2azESIwohAtGyA6l3UIrUup5z7yXo90bXDcXUOmLiK34YPffgQ6pAEkCg8pAtBv6Fhw98bKYdqrEX2UrcjYvIUpbGdxMyc5Zpfl1cMNo8G6vzpnFaQESX+7eZIFTfO5BCFjVJLfC5Ur5C" // decode the raw data from base64 to string diff --git a/experiments/experiment4/encode_proto.go b/experiments/experiment4/encode_proto.go index c63ea38..082f1ae 100644 --- a/experiments/experiment4/encode_proto.go +++ b/experiments/experiment4/encode_proto.go @@ -97,13 +97,14 @@ func main() { if err != nil { log.Fatal(err) } - encoder.Close() + err = encoder.Close() + if err != nil { + log.Fatal(err) + } compressed := buf.Bytes() - fmt.Printf("Compressed (level %d): %d bytes (%.1f%% of original)\n", - level, len(compressed), float64(len(compressed))/float64(len(encoded))*100) + fmt.Printf("Compressed (level %d): %d bytes (%.1f%% of original)\n", level, len(compressed), float64(len(compressed))/float64(len(encoded))*100) } - // Final compression with level 22 encoderLevel := zstandard.EncoderLevelFromZstd(22) encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel)) @@ -117,12 +118,17 @@ func main() { if err != nil { log.Fatal(err) } - encoder.Close() - + err = encoder.Close() + if err != nil { + log.Fatal(err) + } + err = encoder.Close() + if err != nil { + log.Fatal(err) + } compressed := buf.Bytes() - fmt.Println("Final compressed bytes:", string(compressed)) + fmt.Println("Final compressed bytes:", hex.EncodeToString(compressed)) fmt.Printf("Final length: %d bytes\n", len(compressed)) - decoded, err := DecodeProto(encoded) if err != nil { log.Fatal(err) diff --git a/indexer/decoder/decoder.go b/indexer/decoder/decoder.go index e83ea47..c7b014a 100644 --- a/indexer/decoder/decoder.go +++ b/indexer/decoder/decoder.go @@ -109,6 +109,9 @@ func (d *Decoder) GetMessageFromStdTx() (BasicTxData, []map[string]any, error) { // Process each message in the transaction for i, msg := range tx.GetMsgs() { + if i > 32767 { + return BasicTxData{}, nil, fmt.Errorf("transaction message count exceeds maximum: %d", i) + } messageCounter := int16(i) switch m := msg.(type) { case bank.MsgSend: diff --git a/indexer/query/operator.go b/indexer/query/operator.go index 623b261..44fd597 100644 --- a/indexer/query/operator.go +++ b/indexer/query/operator.go @@ -151,7 +151,7 @@ func (q *QueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []* wg := sync.WaitGroup{} wg.Add(int(diff)) - // Launch goroutines to get the block signers + // Launch goroutines to get the commits for i := range diff { height := fromHeight + i idx := i // Capture index diff --git a/indexer/rpc_client/commit.go b/indexer/rpc_client/commit.go index 7ad8352..5df52d9 100644 --- a/indexer/rpc_client/commit.go +++ b/indexer/rpc_client/commit.go @@ -36,7 +36,11 @@ func (cr *CommitResponse) GetHeight() (uint64, error) { if cr == nil { return 0, fmt.Errorf("CommitResponse Header Height is nil") } - return strconv.ParseUint(cr.Result.SignedHeader.Header.Height, 10, 64) + height, err := strconv.ParseUint(cr.Result.SignedHeader.Header.Height, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse height: %v", err) + } + return height, nil } // GetTimestamp returns the timestamp of the commit diff --git a/integration/synthetic/query_operator.go b/integration/synthetic/query_operator.go index b79c085..c083fb7 100644 --- a/integration/synthetic/query_operator.go +++ b/integration/synthetic/query_operator.go @@ -107,10 +107,7 @@ func (sq *SyntheticQueryOperator) GetFromToCommits(fromHeight uint64, toHeight u commits := make([]*rpcClient.CommitResponse, 0, diff) for height := fromHeight; height <= toHeight; height++ { - commit := sq.getCommit(height) - if commit != nil { - commits = append(commits, commit) - } + commits = append(commits, sq.getCommit(height)) } return commits } diff --git a/integration/synthetic/respone_maker.go b/integration/synthetic/respone_maker.go index f111070..01e7284 100644 --- a/integration/synthetic/respone_maker.go +++ b/integration/synthetic/respone_maker.go @@ -177,7 +177,7 @@ type GenCommitInput struct { func (rm *ResponseMaker) GenerateCommitResponse(input GenCommitInput) *rpcClient.CommitResponse { // just like in the gen block we need the data for the header but not really // just generate the hash for block and last block - // for others just use empty strings since this is all just to mock the similar expereience + // for others just use empty strings since this is all just to mock the similar experience blockHash := rm.generator.GenerateBlockHash() partsHash := "" lastCommitHash := "" From d5dfb79ddcc67084dc4a1149910efd5ba4031806 Mon Sep 17 00:00:00 2001 From: NM Date: Tue, 25 Nov 2025 15:36:20 +0100 Subject: [PATCH 04/11] Added permissions to the workflow actions --- .github/workflows/golangci.yml | 4 ++++ .github/workflows/gotest.yml | 4 ++++ .github/workflows/integration-tests.yml | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/golangci.yml b/.github/workflows/golangci.yml index f0704b6..32614d1 100644 --- a/.github/workflows/golangci.yml +++ b/.github/workflows/golangci.yml @@ -1,4 +1,8 @@ name: Golangci-lint +permissions: + contents: read + packages: read + on: push: branches: [dev, release/**] diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 5bdaab1..a56871d 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -1,4 +1,8 @@ name: Go Test +permissions: + contents: read + packages: read + on: push: branches: [dev, release/**] diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d6ac1c1..c1cd640 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,5 +1,7 @@ name: Integration Tests - +permissions: + contents: read + packages: read on: pull_request: branches: [ main, dev ] From 89b5243f45163d25fa563e799673bf1236ac68a9 Mon Sep 17 00:00:00 2001 From: NM Date: Tue, 25 Nov 2025 16:23:18 +0100 Subject: [PATCH 05/11] Adjust version and commit representation for the indexer cli --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 66f4d8b..8cf8f58 100644 --- a/Makefile +++ b/Makefile @@ -8,14 +8,15 @@ # Get git information GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") GIT_TAG := $(shell git describe --tags --exact-match 2>/dev/null || echo "") -VERSION := $(if $(GIT_TAG),$(GIT_TAG),dev-$(GIT_COMMIT)) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "") +VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_BRANCH)-$(GIT_COMMIT)) build: mkdir -p build - go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer indexer/main.go + go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=d5dfb79 -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=release/0.4.0-d5dfb79" -o build/indexer indexer/main.go install: - cd indexer && go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" + go install -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" indexer/main.go build-api: mkdir -p build From dde82a4ecc9bce50963fbff3f4e13a3a20b47f9d Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 14:26:14 +0100 Subject: [PATCH 06/11] Refactor database methods to use context for improved concurrency - Updated all database interaction methods to accept a context parameter, allowing for better control over timeouts and cancellation. - Modified related methods in the address cache and data processor to utilize the new context-aware database methods. - Data processor now processes the data using map[string]struct{} and sync mux. - Database queries should now return the message counter for the api queries. --- indexer/address_cache/address.go | 37 +++-- indexer/address_cache/types.go | 8 +- indexer/cmd/setup.go | 6 +- indexer/context_hook/hook.go | 4 +- indexer/data_processor/event.go | 2 +- indexer/data_processor/operator.go | 157 +++++++++++++------- indexer/data_processor/operator_test.go | 23 +-- indexer/data_processor/types.go | 17 ++- indexer/db_init/hypertable.go | 14 +- indexer/db_init/tables.go | 20 +-- indexer/decoder/decoder.go | 6 +- indexer/decoder/product.go | 2 +- indexer/main_operator/operator.go | 4 +- indexer/orchestrator/operator.go | 8 +- indexer/orchestrator/operator_test.go | 2 +- indexer/orchestrator/types.go | 3 +- indexer/query/operator.go | 4 +- indexer/retry/worker.go | 2 +- indexer/rpc_client/client.go | 10 +- indexer/rpc_client/rate_limit/rate_limit.go | 2 +- indexer/rpc_client/rate_limited_client.go | 2 +- integration/synthetic/init.go | 3 +- pkgs/database/insert.go | 80 ++++++---- pkgs/database/pool.go | 10 +- pkgs/database/queries_api.go | 117 ++++++++++----- pkgs/database/queries_indexer.go | 38 +++-- pkgs/database/types_api.go | 70 +++++---- 27 files changed, 403 insertions(+), 248 deletions(-) diff --git a/indexer/address_cache/address.go b/indexer/address_cache/address.go index b0a4b9e..b8ca1ed 100644 --- a/indexer/address_cache/address.go +++ b/indexer/address_cache/address.go @@ -1,8 +1,10 @@ package addresscache import ( + "context" "log" "maps" + "time" ) // NewAddressCache is a constructor for the AddressCache struct @@ -10,7 +12,7 @@ import ( // it will also load the validator addresses if the loadVal is true // it will load the regular addresses if the loadVal is false // -// Args: +// Parameters: // - chainName: the name of the chain // - db: the database connection interface // - loadVal: whether to load the validator addresses @@ -50,7 +52,7 @@ func NewAddressCache(chainName string, db DatabaseForAddresses, loadVal bool) *A // This method is used to add addresses to the cache // It will add the addresses to the cache and update the highest index // -// Args: +// Parameters: // - newAddresses: the new addresses to add to the cache // // Returns: @@ -80,7 +82,7 @@ func (a *AddressCache) addAddresses(newAddresses map[string]int32) { // - add more documentation // - add more logging // -// Args: +// Parameters: // - address: the addresses to solve // - chainName: the chain name // - insertValidators: whether to insert validators @@ -110,7 +112,9 @@ func (a *AddressCache) AddressSolver( // chech if there is already recorded addresses in the db // technically this should be handled by LoadAddresses but let's make one more check // probably not needed but just in case - existingAddresses, err := a.db.FindExistingAccounts(address, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + existingAddresses, err := a.db.FindExistingAccounts(ctx, address, chainName, insertValidators) + cancel() if err != nil { return } @@ -128,13 +132,18 @@ func (a *AddressCache) AddressSolver( if len(addressToAdd) > 1 { for i := range retryAttempts { - loopErr := a.db.InsertAddresses(addressToAdd, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, addressToAdd, chainName, insertValidators) + cancel() + if loopErr != nil { // in the events the oneByOne is true the program will try to insert the addresses one by one // as a final resort with this some might be inserted but some might not if oneByOne != nil && *oneByOne && i == retryAttempts-1 { for _, addr := range addressToAdd { - loopErr := a.db.InsertAddresses([]string{addr}, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, []string{addr}, chainName, insertValidators) + cancel() if loopErr != nil { // this is a final resort, so we can log the error for debugging purposes log.Println("Error inserting address:", addr, "error:", loopErr) @@ -147,14 +156,18 @@ func (a *AddressCache) AddressSolver( } } else if len(addressToAdd) == 1 { // if there is only one address to insert, we can do it directly - loopErr := a.db.InsertAddresses([]string{addressToAdd[0]}, chainName, insertValidators) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + loopErr := a.db.InsertAddresses(ctx, []string{addressToAdd[0]}, chainName, insertValidators) + cancel() if loopErr != nil { return } } // at the end of the function we should add the addresses to the cache // we need to make a query of the added addresses to get the ids along with the addresses - newAddrMap, err := a.db.FindExistingAccounts(addressToAdd, chainName, insertValidators) + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + newAddrMap, err := a.db.FindExistingAccounts(ctx, addressToAdd, chainName, insertValidators) + cancel() if err != nil { log.Println("Error finding existing accounts:", err) return @@ -167,7 +180,7 @@ func (a *AddressCache) AddressSolver( // This method is used to get the address from the cache // If the address is not in the cache, it will return 0 // -// Args: +// Parameters: // - address: the address to get // // Returns: @@ -186,7 +199,7 @@ func (a *AddressCache) GetAddress(address string) int32 { // This method is called when the program starts and when the cache is empty // Should only be called once per program start // -// Args: +// Parameters: // - chainName: the name of the chain // - loadVal: whether to load the validator addresses // - db: the database connection interface @@ -195,7 +208,9 @@ func (a *AddressCache) GetAddress(address string) int32 { // - map[string]int32: the map of addresses and their ids // - error: if the query fails func loadAddresses(chainName string, loadVal bool, db DatabaseForAddresses) (map[string]int32, int32, error) { - addresses, maxIndex, err := db.GetAllAddresses(chainName, loadVal, nil) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + addresses, maxIndex, err := db.GetAllAddresses(ctx, chainName, loadVal, nil) if err != nil { return nil, 0, err } diff --git a/indexer/address_cache/types.go b/indexer/address_cache/types.go index d5407eb..33bee23 100644 --- a/indexer/address_cache/types.go +++ b/indexer/address_cache/types.go @@ -1,10 +1,12 @@ package addresscache +import "context" + // A database interface for what AddressCache needs from database type DatabaseForAddresses interface { - FindExistingAccounts(addresses []string, chainName string, searchValidators bool) (map[string]int32, error) - InsertAddresses(addresses []string, chainName string, insertValidators bool) error - GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) + FindExistingAccounts(ctx context.Context, addresses []string, chainName string, searchValidators bool) (map[string]int32, error) + InsertAddresses(ctx context.Context, addresses []string, chainName string, insertValidators bool) error + GetAllAddresses(ctx context.Context, chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) } // AddressCache is a map of addresses tied to their int32 index in the database diff --git a/indexer/cmd/setup.go b/indexer/cmd/setup.go index a192255..b48c856 100644 --- a/indexer/cmd/setup.go +++ b/indexer/cmd/setup.go @@ -1,7 +1,9 @@ package cmd import ( + "context" "log" + "time" dbinit "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/db_init" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" @@ -95,7 +97,9 @@ var createDbCmd = &cobra.Command{ // create a new database named "gnoland" // but check if the current database is "gnoland" - currentDb, err := db.CheckCurrentDatabaseName() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + currentDb, err := db.CheckCurrentDatabaseName(ctx) if err != nil { log.Fatalf("failed to check current database name: %v", err) } diff --git a/indexer/context_hook/hook.go b/indexer/context_hook/hook.go index 2fad928..981a91d 100644 --- a/indexer/context_hook/hook.go +++ b/indexer/context_hook/hook.go @@ -12,7 +12,7 @@ import ( // NewSignalHandler is a constructor function that creates a new signal handler with // cleanup and state dump functions // -// Args: +// Parameters: // - cleanup: the cleanup function // - stateDump: the state dump function // @@ -32,7 +32,7 @@ func NewSignalHandler(cleanup func() error, stateDump func() error) *SignalHandl // Context returns the context that will be cancelled on shutdown signals // -// Args: +// Parameters: // - sh: the signal handler // // Returns: diff --git a/indexer/data_processor/event.go b/indexer/data_processor/event.go index c4dfa0c..4bfabf6 100644 --- a/indexer/data_processor/event.go +++ b/indexer/data_processor/event.go @@ -62,7 +62,7 @@ func (er *EventResult) GetCompressedData() []byte { // 1. Native postgres format ([]sqlDataTypes.Event) // 2. Compressed protobuf format ([]byte) // -// Args: +// Parameters: // - txResponse: a transaction response // - useCompressed: if true, returns compressed format; otherwise native format // diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index 1ac9a28..08329cc 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -1,6 +1,7 @@ package dataprocessor import ( + "context" "crypto/sha256" "encoding/base64" "fmt" @@ -8,6 +9,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/decoder" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" @@ -16,7 +18,7 @@ import ( // Constructor function for the DataProcessor struct // -// Args: +// Parameters: // - db: the database connection interface // - addressCache: the address cache interface // - validatorCache: the validator cache interface @@ -40,10 +42,10 @@ func NewDataProcessor( } // ProcessValidatorAddresses is a method to process the validator addresses from a slice of blocks -// it will process the validator addresses from the blocks and store them in a sync.Map -// it will then extract the addresses from the sync.Map and insert them into the address cache +// it will process the validator addresses from the blocks and store them in a map[string]struct{} +// it will then extract the addresses from the map[string]struct{} and insert them into the address cache // -// Args: +// Parameters: // - blocks: a slice of blocks // - fromHeight: the start height // - toHeight: the end height @@ -57,9 +59,10 @@ func (d *DataProcessor) ProcessValidatorAddresses( fromHeight uint64, toHeight uint64, ) { - // sync.Map for thread-safe concurrent access and inser it as address/bool + // map[string]struct{} for thread-safe concurrent access and insert it as address/struct{} // the program should be able to avoid duplicates since it is a map - var addressesMap sync.Map + var mu sync.Mutex + addressesMap := make(map[string]struct{}) wg := sync.WaitGroup{} wg.Add(len(blocks)) @@ -72,24 +75,27 @@ func (d *DataProcessor) ProcessValidatorAddresses( precommits := block.Result.Block.LastCommit.Precommits for _, precommit := range precommits { if precommit != nil { - addressesMap.Store(precommit.ValidatorAddress, true) + mu.Lock() + addressesMap[precommit.ValidatorAddress] = struct{}{} + mu.Unlock() } } // Process proposer proposer := block.Result.Block.Header.ProposerAddress - addressesMap.Store(proposer, true) + mu.Lock() + addressesMap[proposer] = struct{}{} + mu.Unlock() }(block) } wg.Wait() - // Extract unique addresses from sync.Map + // Extract unique addresses from map[string]struct{} addresses := make([]string, 0) - addressesMap.Range(func(key, value interface{}) bool { - addresses = append(addresses, key.(string)) - return true - }) + for address := range addressesMap { + addresses = append(addresses, address) + } // retry 3 times just for the sake of it d.validatorCache.AddressSolver(addresses, d.chainName, true, 3, nil) @@ -100,7 +106,7 @@ func (d *DataProcessor) ProcessValidatorAddresses( // it will process the blocks using async workers and store them directly into a result slice // it will then insert the blocks into the database // -// Args: +// Parameters: // - blocks: a slice of blocks // - fromHeight: the start height // - toHeight: the end height @@ -111,10 +117,11 @@ func (d *DataProcessor) ProcessValidatorAddresses( // The method will not throw an error if the blocks are not found, it will just return nil func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHeight uint64, toHeight uint64) { // Preallocate slice to avoid growing allocations - blocksData := make([]sqlDataTypes.Blocks, len(blocks)) + blockAmount := len(blocks) + blocksData := make([]sqlDataTypes.Blocks, blockAmount) var mu sync.Mutex wg := sync.WaitGroup{} - wg.Add(len(blocks)) + wg.Add(blockAmount) for idx, block := range blocks { go func(idx int, block *rpcClient.BlockResponse) { @@ -168,7 +175,11 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei wg.Wait() - err := d.dbPool.InsertBlocks(blocksData) + // add multiplier for the timeout depending on the block amount + timeout := 10*time.Second + (time.Duration(blockAmount) / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := d.dbPool.InsertBlocks(ctx, blocksData) if err != nil { log.Printf("Failed to insert blocks: %v", err) } @@ -179,7 +190,7 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei // it will process the transactions using async workers and collect them in a preallocated slice // it will then insert the transactions into the database // -// Args: +// Parameters: // - transactions: a map of transactions and timestamps // - compressEvents: if true, compress the events // @@ -194,11 +205,12 @@ func (d *DataProcessor) ProcessTransactions( toHeight uint64) { // Preallocate slice to avoid growing allocations - transactionsData := make([]sqlDataTypes.TransactionGeneral, len(transactions)) + transactionAmount := len(transactions) + transactionsData := make([]sqlDataTypes.TransactionGeneral, transactionAmount) var mu sync.Mutex var validCount int wg := sync.WaitGroup{} - wg.Add(len(transactions)) + wg.Add(transactionAmount) for idx, transaction := range transactions { go func(idx int, transaction TrasnactionsData) { @@ -258,19 +270,24 @@ func (d *DataProcessor) ProcessTransactions( // It is more of a safety feature than nececity transactionsData = transactionsData[:validCount] - err := d.dbPool.InsertTransactionsGeneral(transactionsData) + // add multiplier for the timeout depending on the transaction amount + timeout := 10*time.Second + (time.Duration(transactionAmount) / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := d.dbPool.InsertTransactionsGeneral(ctx, transactionsData) if err != nil { log.Printf("Failed to insert transactions: %v", err) + return } log.Printf("Transactions processed from %d to %d", fromHeight, toHeight) } // ProcessMessages processes all messages from transactions using concurrent "swarm method" // This method uses a two-phase concurrent approach: -// 1. Collect and resolve all addresses to IDs using concurrent workers and sync.Map +// 1. Collect and resolve all addresses to IDs using concurrent workers and map[string]struct{} // 2. Convert messages to database-ready format with address IDs using concurrent processing // -// Args: +// Parameters: // - transactions: a map of transactions and timestamps // - fromHeight: the start height // - toHeight: the end height @@ -282,11 +299,13 @@ func (d *DataProcessor) ProcessMessages( fromHeight uint64, toHeight uint64) error { - // Phase 1: Concurrent address collection using sync.Map - var addressesMap sync.Map - addressCollectionChan := make(chan []*decoder.DecodedMsg, len(transactions)) + // Phase 1: Concurrent address collection using map[strinct]struct{} + var mu sync.Mutex + transactionAmount := len(transactions) + addressesMap := make(map[string]struct{}) + addressCollectionChan := make(chan []*decoder.DecodedMsg, transactionAmount) wg1 := sync.WaitGroup{} - wg1.Add(len(transactions)) + wg1.Add(transactionAmount) // Launch goroutines to collect addresses concurrently for _, transaction := range transactions { @@ -298,11 +317,13 @@ func (d *DataProcessor) ProcessMessages( return } - // Collect addresses from this transaction and store in sync.Map + // Collect addresses from this transaction and store in map[string]struct{} addresses := decodedMsg.CollectAllAddresses() + mu.Lock() for _, address := range addresses { - addressesMap.Store(address, true) // Thread-safe deduplication + addressesMap[address] = struct{}{} } + mu.Unlock() addressCollectionChan <- []*decoder.DecodedMsg{decodedMsg} }(transaction) @@ -320,12 +341,11 @@ func (d *DataProcessor) ProcessMessages( allDecodedMsgs = append(allDecodedMsgs, decodedMsgs[0]) } - // Extract addresses from sync.Map and resolve to IDs + // Extract addresses from map[string]struct{} and resolve to IDs allAddresses := make([]string, 0) - addressesMap.Range(func(key, value interface{}) bool { - allAddresses = append(allAddresses, key.(string)) - return true - }) + for address := range addressesMap { + allAddresses = append(allAddresses, address) + } if len(allAddresses) > 0 { d.addressCache.AddressSolver(allAddresses, d.chainName, false, 3, nil) @@ -343,7 +363,7 @@ func (d *DataProcessor) ProcessMessages( var aggregationMutex sync.Mutex wg2 := sync.WaitGroup{} - wg2.Add(len(transactions)) + wg2.Add(transactionAmount) // Launch goroutines to process messages concurrently for idx, transaction := range transactions { @@ -394,7 +414,11 @@ func (d *DataProcessor) ProcessMessages( // Create a slice of sqlDataType.AddressTx // we need to get all of the addresses from the aggregatedDbGroups addresses := createAddressTx(aggregatedDbGroups) - err := d.dbPool.InsertAddressTx(addresses) + // add multiplier for the timeout depending on the address amount + timeout := 10*time.Second + time.Duration(len(addresses)) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertAddressTx(ctx, addresses) + cancel() if err != nil { return fmt.Errorf("failed to insert address tx: %w", err) } @@ -418,9 +442,18 @@ func (d *DataProcessor) ProcessMessages( func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) error { var insertErrors []error + msgSendCount := len(groups.MsgSend) + msgCallCount := len(groups.MsgCall) + msgAddPkgCount := len(groups.MsgAddPkg) + msgRunCount := len(groups.MsgRun) + // Insert DbMsgSend messages with address IDs - if len(groups.MsgSend) > 0 { - if err := d.dbPool.InsertMsgSend(groups.MsgSend); err != nil { + if msgSendCount > 0 { + timeout := 10*time.Second + time.Duration(msgSendCount) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgSend(ctx, groups.MsgSend) + cancel() + if err != nil { hashes := make([]string, 0, len(groups.MsgSend)) for _, msg := range groups.MsgSend { hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) @@ -430,8 +463,12 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e } // Insert DbMsgCall messages with address IDs - if len(groups.MsgCall) > 0 { - if err := d.dbPool.InsertMsgCall(groups.MsgCall); err != nil { + if msgCallCount > 0 { + timeout := 10*time.Second + time.Duration(msgCallCount) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgCall(ctx, groups.MsgCall) + cancel() + if err != nil { hashes := make([]string, 0, len(groups.MsgCall)) for _, msg := range groups.MsgCall { hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) @@ -441,8 +478,12 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e } // Insert DbMsgAddPackage messages with address IDs - if len(groups.MsgAddPkg) > 0 { - if err := d.dbPool.InsertMsgAddPackage(groups.MsgAddPkg); err != nil { + if msgAddPkgCount > 0 { + timeout := 10*time.Second + time.Duration(msgAddPkgCount) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgAddPackage(ctx, groups.MsgAddPkg) + cancel() + if err != nil { hashes := make([]string, 0, len(groups.MsgAddPkg)) for _, msg := range groups.MsgAddPkg { hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) @@ -452,8 +493,12 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e } // Insert DbMsgRun messages with address IDs - if len(groups.MsgRun) > 0 { - if err := d.dbPool.InsertMsgRun(groups.MsgRun); err != nil { + if msgRunCount > 0 { + timeout := 10*time.Second + time.Duration(msgRunCount) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertMsgRun(ctx, groups.MsgRun) + cancel() + if err != nil { hashes := make([]string, 0, len(groups.MsgRun)) for _, msg := range groups.MsgRun { hashes = append(hashes, base64.StdEncoding.EncodeToString(msg.TxHash)) @@ -479,9 +524,10 @@ func (d *DataProcessor) ProcessValidatorSignings( fromHeight uint64, toHeight uint64) { - validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, len(commits)) + commitAmount := len(commits) + validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, commitAmount) wg := sync.WaitGroup{} - wg.Add(len(commits)) + wg.Add(commitAmount) // Process blocks concurrently for _, commit := range commits { @@ -531,7 +577,10 @@ func (d *DataProcessor) ProcessValidatorSignings( validatorData = append(validatorData, *validator) } - err := d.dbPool.InsertValidatorBlockSignings(validatorData) + timeout := 10*time.Second + (time.Duration(commitAmount) / 5) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := d.dbPool.InsertValidatorBlockSignings(ctx, validatorData) + cancel() if err != nil { log.Printf("Failed to insert validator commit signings: %v", err) } @@ -542,7 +591,11 @@ func (d *DataProcessor) ProcessValidatorSignings( // decoder.DbMessageGroups using concurrent workers with mutex-based aggregation // it should be used to create the data for the address_tx table func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { - txAmount := len(msg.MsgSend) + len(msg.MsgCall) + len(msg.MsgAddPkg) + len(msg.MsgRun) + msgSendCount := len(msg.MsgSend) + msgCallCount := len(msg.MsgCall) + msgAddPkgCount := len(msg.MsgAddPkg) + msgRunCount := len(msg.MsgRun) + txAmount := msgSendCount + msgCallCount + msgAddPkgCount + msgRunCount if txAmount == 0 { return []sqlDataTypes.AddressTx{} } @@ -552,7 +605,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { wg := sync.WaitGroup{} // Process MsgSend messages - wg.Add(len(msg.MsgSend)) + wg.Add(msgSendCount) for _, msgItem := range msg.MsgSend { go func(msgItem sqlDataTypes.MsgSend) { defer wg.Done() @@ -580,7 +633,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgCall messages - wg.Add(len(msg.MsgCall)) + wg.Add(msgCallCount) for _, msgItem := range msg.MsgCall { go func(msgItem sqlDataTypes.MsgCall) { defer wg.Done() @@ -606,7 +659,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgAddPkg messages - wg.Add(len(msg.MsgAddPkg)) + wg.Add(msgAddPkgCount) for _, msgItem := range msg.MsgAddPkg { go func(msgItem sqlDataTypes.MsgAddPackage) { defer wg.Done() @@ -632,7 +685,7 @@ func createAddressTx(msg *decoder.DbMessageGroups) []sqlDataTypes.AddressTx { } // Process MsgRun messages - wg.Add(len(msg.MsgRun)) + wg.Add(msgRunCount) for _, msgItem := range msg.MsgRun { go func(msgItem sqlDataTypes.MsgRun) { defer wg.Done() diff --git a/indexer/data_processor/operator_test.go b/indexer/data_processor/operator_test.go index c362891..0bfa462 100644 --- a/indexer/data_processor/operator_test.go +++ b/indexer/data_processor/operator_test.go @@ -1,6 +1,7 @@ package dataprocessor_test import ( + "context" "testing" dataProcessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor" @@ -14,37 +15,37 @@ type MockDatabase struct { LastInsertError error } -func (m *MockDatabase) InsertBlocks(blocks []sqlDataTypes.Blocks) error { +func (m *MockDatabase) InsertBlocks(ctx context.Context, blocks []sqlDataTypes.Blocks) error { m.InsertBlocksCalled = true return m.LastInsertError } -func (m *MockDatabase) InsertValidatorBlockSignings(signings []sqlDataTypes.ValidatorBlockSigning) error { +func (m *MockDatabase) InsertValidatorBlockSignings(ctx context.Context, signings []sqlDataTypes.ValidatorBlockSigning) error { return m.LastInsertError } -func (m *MockDatabase) InsertTransactionsGeneral(transactions []sqlDataTypes.TransactionGeneral) error { +func (m *MockDatabase) InsertTransactionsGeneral(ctx context.Context, transactions []sqlDataTypes.TransactionGeneral) error { m.InsertTransactionsCalled = true return m.LastInsertError } -func (m *MockDatabase) InsertMsgSend(messages []sqlDataTypes.MsgSend) error { +func (m *MockDatabase) InsertMsgSend(ctx context.Context, messages []sqlDataTypes.MsgSend) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgCall(messages []sqlDataTypes.MsgCall) error { +func (m *MockDatabase) InsertMsgCall(ctx context.Context, messages []sqlDataTypes.MsgCall) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgAddPackage(messages []sqlDataTypes.MsgAddPackage) error { +func (m *MockDatabase) InsertMsgAddPackage(ctx context.Context, messages []sqlDataTypes.MsgAddPackage) error { return m.LastInsertError } -func (m *MockDatabase) InsertMsgRun(messages []sqlDataTypes.MsgRun) error { +func (m *MockDatabase) InsertMsgRun(ctx context.Context, messages []sqlDataTypes.MsgRun) error { return m.LastInsertError } -func (m *MockDatabase) InsertAddressTx(addresses []sqlDataTypes.AddressTx) error { +func (m *MockDatabase) InsertAddressTx(ctx context.Context, addresses []sqlDataTypes.AddressTx) error { return m.LastInsertError } @@ -111,17 +112,17 @@ func TestDataProcessor_DatabaseInterface(t *testing.T) { var db dataProcessor.Database = &MockDatabase{} // Test interface methods - err := db.InsertBlocks([]sqlDataTypes.Blocks{}) + err := db.InsertBlocks(context.Background(), []sqlDataTypes.Blocks{}) if err != nil { t.Errorf("InsertBlocks should not return error with nil input, got: %v", err) } - err = db.InsertTransactionsGeneral([]sqlDataTypes.TransactionGeneral{}) + err = db.InsertTransactionsGeneral(context.Background(), []sqlDataTypes.TransactionGeneral{}) if err != nil { t.Errorf("InsertTransactionsGeneral should not return error with nil input, got: %v", err) } - err = db.InsertAddressTx([]sqlDataTypes.AddressTx{}) + err = db.InsertAddressTx(context.Background(), []sqlDataTypes.AddressTx{}) if err != nil { t.Errorf("InsertAddressTx should not return error with nil input, got: %v", err) } diff --git a/indexer/data_processor/types.go b/indexer/data_processor/types.go index 906f04c..7dc183d 100644 --- a/indexer/data_processor/types.go +++ b/indexer/data_processor/types.go @@ -1,6 +1,7 @@ package dataprocessor import ( + "context" "time" rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client" @@ -9,14 +10,14 @@ import ( // Define interface for what DataProcessor needs from database type Database interface { - InsertBlocks(blocks []sqlDataTypes.Blocks) error - InsertValidatorBlockSignings(validatorBlockSignings []sqlDataTypes.ValidatorBlockSigning) error - InsertTransactionsGeneral(transactionsGeneral []sqlDataTypes.TransactionGeneral) error - InsertMsgSend(messages []sqlDataTypes.MsgSend) error - InsertMsgCall(messages []sqlDataTypes.MsgCall) error - InsertMsgAddPackage(messages []sqlDataTypes.MsgAddPackage) error - InsertMsgRun(messages []sqlDataTypes.MsgRun) error - InsertAddressTx(addresses []sqlDataTypes.AddressTx) error + InsertBlocks(ctx context.Context, blocks []sqlDataTypes.Blocks) error + InsertValidatorBlockSignings(ctx context.Context, validatorBlockSignings []sqlDataTypes.ValidatorBlockSigning) error + InsertTransactionsGeneral(ctx context.Context, transactionsGeneral []sqlDataTypes.TransactionGeneral) error + InsertMsgSend(ctx context.Context, messages []sqlDataTypes.MsgSend) error + InsertMsgCall(ctx context.Context, messages []sqlDataTypes.MsgCall) error + InsertMsgAddPackage(ctx context.Context, messages []sqlDataTypes.MsgAddPackage) error + InsertMsgRun(ctx context.Context, messages []sqlDataTypes.MsgRun) error + InsertAddressTx(ctx context.Context, addresses []sqlDataTypes.AddressTx) error } // Define interface for what DataProcessor needs from AddressCache diff --git a/indexer/db_init/hypertable.go b/indexer/db_init/hypertable.go index 3d42939..aeeacca 100644 --- a/indexer/db_init/hypertable.go +++ b/indexer/db_init/hypertable.go @@ -19,20 +19,12 @@ import ( // - Community Edition 2.19.3+ (TimescaleDB) // - Cloud edition (Tiger Data) // -// Not Recommended: -// - Apache TimescaleDB 2.19.3+ (Apache TimescaleDB) -// To be clear it is not recommended to use the Apache TimescaleDB because it is not tested on and it won't -// be officially supported. The reason is most of the features that are available in the Community Edition are not -// available in the Apache TimescaleDB. However it might be possible to use it since we only need to use hypertables -// which should exist in the Apache TimescaleDB. -// -// For maximum compatibility, this implementation supports both approaches. // ConvertToHypertables is a method that converts the given table names to hypertables // // This function will only start this process however the whole process will run through the 3 steps // This is first step in the process -// Args: +// Parameters: // - tableNames: a slice of table names to convert to hypertables // // Returns: @@ -59,7 +51,7 @@ func (init *DBInitializer) ConvertToHypertables(tableNames []string) { // // This function will only start this process however the whole process will run through the 3 steps // This is second step in the process -// Args: +// Parameters: // - tables: a map of table names to their columns // // Returns: @@ -89,7 +81,7 @@ func (init *DBInitializer) AlterCompressionSegments(tables map[string][]string) // // This function will only start this process however the whole process will run through the 3 steps // This is third step in the process -// Args: +// Parameters: // - tableNames: a slice of table names to add the compression policy to // // Returns: diff --git a/indexer/db_init/tables.go b/indexer/db_init/tables.go index a50a16e..57d542d 100644 --- a/indexer/db_init/tables.go +++ b/indexer/db_init/tables.go @@ -41,7 +41,7 @@ type SpecialType struct { // GetTableInfo extracts database table information from a struct using reflection // This function reads the struct tags and converts them to database metadata // -// Args: +// Parameters: // - structType: the struct type to get the table info from // - tableName: the name of the table to get the table info from // @@ -113,7 +113,7 @@ func GetTableInfo(structType interface{}, tableName string) (*TableInfo, error) // GenerateCreateTableSQL generates a PostgreSQL CREATE TABLE statement from struct metadata // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // // Returns: @@ -174,7 +174,7 @@ func GenerateCreateTableSQL(tableInfo *TableInfo) string { // GenerateCreateHypertableSQL generates a PostgreSQL CREATE TABLE statement with modern TimescaleDB hypertable syntax // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // - chunkInterval: the interval to chunk the table by // - partitionColumn: the column to partition the table by @@ -234,7 +234,7 @@ func GenerateCreateHypertableSQL(tableInfo *TableInfo, chunkInterval string, par // GenerateSpecialTypeSQL generates a PostgreSQL CREATE TYPE statement from struct metadata // -// Args: +// Parameters: // - specialType: the special type to generate the SQL for // // Returns: @@ -260,7 +260,7 @@ func GenerateSpecialTypeSQL(specialType *SpecialType) string { // CreateTableSQL creates a table in the database based on struct metadata // -// Args: +// Parameters: // - t: the table info for the table to create // // Returns: @@ -350,7 +350,7 @@ func (db *DBInitializer) GetTimescaleDBVersion() (*TimescaleDBVersion, error) { // CreateTableFromStruct creates a database table based on struct tags // -// Args: +// Parameters: // - structType: the struct type to create the table from // - tableName: the name of the table to create // @@ -419,7 +419,7 @@ func GetSpecialTypeInfo(structType interface{}, typeName string) (*SpecialType, // CreateSpecialTypeFromStruct creates a database type based on struct tags // -// Args: +// Parameters: // - structType: the struct type to create the special type from // - typeName: the name of the special type to create // @@ -444,7 +444,7 @@ func (db *DBInitializer) CreateSpecialTypeFromStruct(structType interface{}, typ // CreateHypertableFromStruct creates a hypertable using the appropriate method based on TimescaleDB version // -// Args: +// Parameters: // - structType: the struct type to create the hypertable from // - tableName: the name of the table to create the hypertable from // - partitionColumn: the column to partition the table by @@ -493,7 +493,7 @@ func (db *DBInitializer) createHypertableModern(tableInfo *TableInfo, partitionC // createHypertableLegacy creates a hypertable using the legacy 3-step process // -// Args: +// Parameters: // - tableInfo: the table info for the table to create // - partitionColumn: the column to partition the table by // - chunkInterval: the interval to chunk the table by @@ -542,7 +542,7 @@ func (db *DBInitializer) CreateChainTypeEnum(enumValues []string) error { // CreateUser creates a user in the database // // The function will -// Args: +// Parameters: // - name: the name of the user to create // // Returns: diff --git a/indexer/decoder/decoder.go b/indexer/decoder/decoder.go index c7b014a..b2f80c6 100644 --- a/indexer/decoder/decoder.go +++ b/indexer/decoder/decoder.go @@ -23,7 +23,7 @@ type Decoder struct { // NewDecoder creates a new Decoder struct // -// Args: +// Parameters: // - encodedTx: base64 encoded stdTx // // Returns: @@ -39,7 +39,7 @@ func NewDecoder(encodedTx string) *Decoder { // The function decodes the data and unmarshals it into a std.Tx struct // The struct contains the transaction data and the messages // -// Args: +// Parameters: // - s: base64 encoded stdTx // // Returns: @@ -64,7 +64,7 @@ func (d *Decoder) DecodeStdTxFromBase64() (*std.Tx, error) { // to be reflected here, mostly... // Some will need to be updated and added manually in the future but for now this is good enough // -// Args: +// Parameters: // - none // // Returns: diff --git a/indexer/decoder/product.go b/indexer/decoder/product.go index 7a8add7..60e25bf 100644 --- a/indexer/decoder/product.go +++ b/indexer/decoder/product.go @@ -11,7 +11,7 @@ import ( // NewDecodedMsg creates a new DecodedMsg struct // -// Args: +// Parameters: // - encodedTx: the encoded transaction // // Returns: diff --git a/indexer/main_operator/operator.go b/indexer/main_operator/operator.go index 72d82ff..c737a45 100644 --- a/indexer/main_operator/operator.go +++ b/indexer/main_operator/operator.go @@ -111,7 +111,7 @@ func InitMainOperator( // initializeDatabase is a private function to initialize the database // it is used to initialize the database for the main operator // -// Args: +// Parameters: // - conf: the config // - env: the environment // @@ -174,7 +174,7 @@ func initializeDatabase(conf *config.Config, env *config.Environment) *database. // initializeMajorConstructors is a private function to initialize the major constructors // it is used to initialize the major constructors for the main operator // -// Args: +// Parameters: // - conf: the config // - env: the environment // - chainName: the chain name diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go index 253ef48..a895759 100644 --- a/indexer/orchestrator/operator.go +++ b/indexer/orchestrator/operator.go @@ -103,7 +103,9 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool // Initial setup - get starting height if !skipInitialDbCheck { - lastProcessedHeight, err = or.db.GetLastBlockHeight(or.chainName) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + lastProcessedHeight, err = or.db.GetLastBlockHeight(ctx, or.chainName) if err != nil { log.Printf("Failed to get last block height from database: %v", err) log.Printf("Either there are no blocks in the database or the database is not properly configured.") @@ -330,7 +332,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR // This function processes all data using optimized concurrent execution // -// Args: +// Parameters: // - blocks: a slice of blocks // - transactions: a map of transactions and timestamps // - compressEvents: if true, compress the events @@ -434,7 +436,7 @@ func (or *Orchestrator) processAll( // saveProcessingState is a private method that saves // the current processing state to a file // -// Args: +// Parameters: // - height: the height of the processing state // - reason: the reason for the processing state // diff --git a/indexer/orchestrator/operator_test.go b/indexer/orchestrator/operator_test.go index aa62f29..37cc34f 100644 --- a/indexer/orchestrator/operator_test.go +++ b/indexer/orchestrator/operator_test.go @@ -100,7 +100,7 @@ type MockDatabaseHeight struct { } // Mock method for GetLastBlockHeight -func (m *MockDatabaseHeight) GetLastBlockHeight(chainName string) (uint64, error) { +func (m *MockDatabaseHeight) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { if m.ShouldError { return 0, &DatabaseError{"database error"} } diff --git a/indexer/orchestrator/types.go b/indexer/orchestrator/types.go index df59ff1..8421fc7 100644 --- a/indexer/orchestrator/types.go +++ b/indexer/orchestrator/types.go @@ -1,6 +1,7 @@ package orchestrator import ( + "context" "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config" @@ -27,7 +28,7 @@ type QueryOperator interface { // Only needed for one opetaion // Part of the timescaledb interface type DatabaseHeight interface { - GetLastBlockHeight(chainName string) (uint64, error) + GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) } // Only needed for one opetaion diff --git a/indexer/query/operator.go b/indexer/query/operator.go index 44fd597..41ee6ce 100644 --- a/indexer/query/operator.go +++ b/indexer/query/operator.go @@ -51,7 +51,7 @@ func NewQueryOperator( // the indexer should store them all together as one huge slice of blocks, so the order is not important // the speed is what matters here. // -// Args: +// Parameters: // - fromHeight: the start height // - toHeight: the end height // @@ -204,7 +204,7 @@ func (q *QueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []* // This is a fan out method that lauches async workers for each tx and wait to get the resaults // the indexer should store them all together as one huge slice of transactions, // -// Args: +// Parameters: // - txs: a slice of tx hashes // // Returns: diff --git a/indexer/retry/worker.go b/indexer/retry/worker.go index d4f8dee..d5fa5e2 100644 --- a/indexer/retry/worker.go +++ b/indexer/retry/worker.go @@ -18,7 +18,7 @@ type RetryResult[T any] struct { // GenericRetryQuery is a generic retry wrapper that can work with any query function // It handles the retry logic and sends the result to a channel // -// Args: +// Parameters: // - retryAmount: the number of retry attempts, pulled from the query operator retry options // - pause: the number of attempts to pause after failing, pulled from the query operator retry options // - pauseTime: the time to pause after failing, pulled from the query operator retry options diff --git a/indexer/rpc_client/client.go b/indexer/rpc_client/client.go index 4891f29..33b8e04 100644 --- a/indexer/rpc_client/client.go +++ b/indexer/rpc_client/client.go @@ -27,7 +27,7 @@ import ( // - GetTx: call to get a tx from the rpc client // - GetAbciQuery: call to get a abci query from the rpc client // -// Args: +// Parameters: // // - rpcURL: the url of the rpc client // - timeout: the timeout for the rpc client(optional) @@ -119,7 +119,7 @@ func (r *RpcGnoland) Health() error { // GetValidators method to get validators from the rpc client. // -// Args: +// Parameters: // - height: the height of the block to get the validators for // // Returns: @@ -150,7 +150,7 @@ func (r *RpcGnoland) GetValidators(height uint64) (*ValidatorsResponse, *RpcHeig // GetBlock method to get a block from the rpc client. // -// Args: +// Parameters: // - height: the height of the block to get // // Returns: @@ -211,7 +211,7 @@ func (r *RpcGnoland) GetLatestBlockHeight() (uint64, *RpcHeightError) { // GetTx method to get a tx from the rpc client. // -// Args: +// Parameters: // - txHash: the base64 encoded string of the tx to get // // Returns: @@ -244,7 +244,7 @@ func (r *RpcGnoland) GetTx(txHash string) (*TxResponse, *RpcStringError) { // This method is used to get any kind of data from the rpc client. // This might not be used in the indexer but let's keep it here for now. // -// Args: +// Parameters: // - path: the path of the abci query // - data: the data of the abci query // - height: the height of the block to get the abci query for(optional, if not specified it will get the latest block) diff --git a/indexer/rpc_client/rate_limit/rate_limit.go b/indexer/rpc_client/rate_limit/rate_limit.go index 09f464b..9029e7d 100644 --- a/indexer/rpc_client/rate_limit/rate_limit.go +++ b/indexer/rpc_client/rate_limit/rate_limit.go @@ -14,7 +14,7 @@ type ChannelRateLimiter struct { // NewChannelRateLimiter creates a new ChannelRateLimiter // -// Args: +// Parameters: // - requestsAllowed: the number of requests allowed per time window // - timeWindow: the time window for rate limiting // diff --git a/indexer/rpc_client/rate_limited_client.go b/indexer/rpc_client/rate_limited_client.go index 680f835..8fddd34 100644 --- a/indexer/rpc_client/rate_limited_client.go +++ b/indexer/rpc_client/rate_limited_client.go @@ -8,7 +8,7 @@ import ( // NewRateLimitedRpcClient creates a new rate-limited RPC client wrapper // -// Args: +// Parameters: // - rpcURL: the url of the rpc client // - timeout: the timeout for the rpc client (optional) // - requestsPerWindow: number of requests allowed per time window diff --git a/integration/synthetic/init.go b/integration/synthetic/init.go index 09cf467..cf642dc 100644 --- a/integration/synthetic/init.go +++ b/integration/synthetic/init.go @@ -1,6 +1,7 @@ package synthetic import ( + "context" "log" addressCache "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/address_cache" @@ -81,7 +82,7 @@ type MockDatabaseHeight struct { lastHeight uint64 } -func (m *MockDatabaseHeight) GetLastBlockHeight(chainName string) (uint64, error) { +func (m *MockDatabaseHeight) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { return m.lastHeight, nil } diff --git a/pkgs/database/insert.go b/pkgs/database/insert.go index ee9cacb..ae1a74f 100644 --- a/pkgs/database/insert.go +++ b/pkgs/database/insert.go @@ -19,13 +19,19 @@ import ( // # Used inside of the address cache package to insert the addresses to the database // // Parameters: +// - ctx: the context to use for the insert // - addresses: a slice of addresses to insert // - chainName: the name of the chain to insert the addresses to // - insertValidators: a boolean to indicate if the addresses are validators or accounts // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, insertValidators bool) error { +func (t *TimescaleDb) InsertAddresses( + ctx context.Context, + addresses []string, + chainName string, + insertValidators bool, +) error { column_names := []string{"address", "chain_name"} var table_name string if insertValidators { @@ -38,7 +44,7 @@ func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, inse return []any{addresses[i], chainName}, nil }) // copy the addresses to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{table_name}, column_names, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{table_name}, column_names, pgxSlice) return err } @@ -49,12 +55,13 @@ func (t *TimescaleDb) InsertAddresses(addresses []string, chainName string, inse // // # Used for inserting a large number of blocks to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - blocks: a slice of blocks to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { +func (t *TimescaleDb) InsertBlocks(ctx context.Context, blocks []sql_data_types.Blocks) error { // Return early if no blocks to insert if len(blocks) == 0 { return nil @@ -75,7 +82,7 @@ func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { columns := blocks[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"blocks"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"blocks"}, columns, pgxSlice) return err } @@ -86,12 +93,16 @@ func (t *TimescaleDb) InsertBlocks(blocks []sql_data_types.Blocks) error { // // # Used for inserting a large number of validator block signings to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - validatorBlockSigning: a slice of validator block signings to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_data_types.ValidatorBlockSigning) error { +func (t *TimescaleDb) InsertValidatorBlockSignings( + ctx context.Context, + validatorBlockSigning []sql_data_types.ValidatorBlockSigning, +) error { // Return early if no validator block signings to insert if len(validatorBlockSigning) == 0 { return nil @@ -111,7 +122,7 @@ func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_d columns := validatorBlockSigning[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"validator_block_signing"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"validator_block_signing"}, columns, pgxSlice) return err } @@ -122,12 +133,16 @@ func (t *TimescaleDb) InsertValidatorBlockSignings(validatorBlockSigning []sql_d // // # Used for inserting a large number of transaction general data to the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - transactionsGeneral: a slice of transaction general data to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertTransactionsGeneral(transactionsGeneral []sql_data_types.TransactionGeneral) error { +func (t *TimescaleDb) InsertTransactionsGeneral( + ctx context.Context, + transactionsGeneral []sql_data_types.TransactionGeneral, +) error { // Return early if no transactions to insert if len(transactionsGeneral) == 0 { return nil @@ -154,18 +169,19 @@ func (t *TimescaleDb) InsertTransactionsGeneral(transactionsGeneral []sql_data_t columns := transactionsGeneral[0].TableColumns() // insert the data to the db - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"transaction_general"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"transaction_general"}, columns, pgxSlice) return err } // InsertAddressTx inserts a slice of AddressTx into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - addresses: a slice of AddressTx to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertAddressTx(addresses []sql_data_types.AddressTx) error { +func (t *TimescaleDb) InsertAddressTx(ctx context.Context, addresses []sql_data_types.AddressTx) error { // Return early if no addresses to insert if len(addresses) == 0 { return nil @@ -182,18 +198,19 @@ func (t *TimescaleDb) InsertAddressTx(addresses []sql_data_types.AddressTx) erro }) columns := addresses[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"address_tx"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"address_tx"}, columns, pgxSlice) return err } // InsertMsgSend inserts a slice of MsgSend messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgSend messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgSend(messages []sql_data_types.MsgSend) error { +func (t *TimescaleDb) InsertMsgSend(ctx context.Context, messages []sql_data_types.MsgSend) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -213,18 +230,19 @@ func (t *TimescaleDb) InsertMsgSend(messages []sql_data_types.MsgSend) error { }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"bank_msg_send"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"bank_msg_send"}, columns, pgxSlice) return err } // InsertMsgCall inserts a slice of MsgCall messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgCall messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgCall(messages []sql_data_types.MsgCall) error { +func (t *TimescaleDb) InsertMsgCall(ctx context.Context, messages []sql_data_types.MsgCall) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -247,18 +265,22 @@ func (t *TimescaleDb) InsertMsgCall(messages []sql_data_types.MsgCall) error { }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_call"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_call"}, columns, pgxSlice) return err } // InsertMsgAddPackage inserts a slice of MsgAddPackage messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgAddPackage messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgAddPackage(messages []sql_data_types.MsgAddPackage) error { +func (t *TimescaleDb) InsertMsgAddPackage( + ctx context.Context, + messages []sql_data_types.MsgAddPackage, +) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -281,18 +303,22 @@ func (t *TimescaleDb) InsertMsgAddPackage(messages []sql_data_types.MsgAddPackag }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_add_package"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_add_package"}, columns, pgxSlice) return err } // InsertMsgRun inserts a slice of MsgRun messages into the database // -// Args: +// Parameters: +// - ctx: the context to use for the insert // - messages: a slice of MsgRun messages to insert // // Returns: // - error: an error if the insertion fails -func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { +func (t *TimescaleDb) InsertMsgRun( + ctx context.Context, + messages []sql_data_types.MsgRun, +) error { // Return early if no messages to insert if len(messages) == 0 { return nil @@ -315,7 +341,7 @@ func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { }) columns := messages[0].TableColumns() - _, err := t.pool.CopyFrom(context.Background(), pgx.Identifier{"vm_msg_run"}, columns, pgxSlice) + _, err := t.pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_run"}, columns, pgxSlice) return err } @@ -325,7 +351,7 @@ func (t *TimescaleDb) InsertMsgRun(messages []sql_data_types.MsgRun) error { // bytearrays but to be sure it should be usable on any type that is supposed to be inserted into the database as // an array // -// Args: +// Parameters: // - v: a slice of any type // // Returns: diff --git a/pkgs/database/pool.go b/pkgs/database/pool.go index 9b6f9a6..3d31856 100644 --- a/pkgs/database/pool.go +++ b/pkgs/database/pool.go @@ -13,7 +13,7 @@ import ( // NewTimescaleDb is a constructor function that creates a new TimescaleDb instance // -// Args: +// Parameters: // - config: the database config, a struct that contains the necessary data from the config file // // Returns: @@ -44,7 +44,7 @@ func NewTimescaleDbSetup(config DatabasePoolConfig) *TimescaleDb { // connectToDb is a internal function that connects to the database for the indexer // -// Args: +// Parameters: // - config: the database config, a struct that contains the necessary data from the config file // // Returns: @@ -135,7 +135,7 @@ func setupConnection(config DatabasePoolConfig) (*pgxpool.Pool, error) { // create a new database with the given name // -// Args: +// Parameters: // - db: the database connection pool // - dbname: the name of the database to create // @@ -156,7 +156,7 @@ func CreateDatabase(db *TimescaleDb, dbname string) error { // most of the time when the postgres server is running, it will be in the "postgres" database // only to be used initiating command // -// Args: +// Parameters: // - db: the database connection pool // - config: the database config, a struct that contains the necessary data from the config file // @@ -186,7 +186,7 @@ func SwitchDatabase(db *TimescaleDb, config DatabasePoolConfig, dbname string) e // GetPool returns the database connection pool // -// Args: +// Parameters: // - db: the database connection pool // // Returns: diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go index 41203be..b980e0e 100644 --- a/pkgs/database/queries_api.go +++ b/pkgs/database/queries_api.go @@ -12,14 +12,14 @@ import ( // // # Used to get a block from the database for a given height and chain name // -// Args: +// Parameters: // - height: the height of the block // - chainName: the name of the chain // // Returns: // - *BlockData: the block data // - error: if the query fails -func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, error) { +func (t *TimescaleDb) GetBlock(ctx context.Context, height uint64, chainName string) (*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -32,7 +32,7 @@ func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, err WHERE height = $1 AND chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, height, chainName) + row := t.pool.QueryRow(ctx, query, height, chainName) var block BlockData err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) if err != nil { @@ -41,7 +41,7 @@ func (t *TimescaleDb) GetBlock(height uint64, chainName string) (*BlockData, err return &block, nil } -func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { +func (t *TimescaleDb) GetLatestBlock(ctx context.Context, chainName string) (*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -55,7 +55,7 @@ func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { ORDER BY height DESC LIMIT 1 ` - row := t.pool.QueryRow(context.Background(), query, chainName) + row := t.pool.QueryRow(ctx, query, chainName) var block BlockData err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs) if err != nil { @@ -70,14 +70,14 @@ func (t *TimescaleDb) GetLatestBlock(chainName string) (*BlockData, error) { // // # Used to get the last x blocks from the database for a given chain name // -// Args: +// Parameters: // - chainName: the name of the chain // - x: the number of blocks to get // // Returns: // - []*BlockData: the last x blocks // - error: if the query fails -func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, error) { +func (t *TimescaleDb) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -91,7 +91,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, ORDER BY height DESC LIMIT $2 ` - rows, err := t.pool.Query(context.Background(), query, chainName, x) + rows, err := t.pool.Query(ctx, query, chainName, x) if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, // // # Used to get a range of blocks from the database for a given height range and chain name // -// Args: +// Parameters: // - fromHeight: the starting height of the block // - toHeight: the ending height of the block (inclusive) // - chainName: the name of the chain @@ -125,7 +125,7 @@ func (t *TimescaleDb) GetLastXBlocks(chainName string, x uint64) ([]*BlockData, // Returns: // - []*BlockData: the range of block data // - error: if the query fails -func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) { +func (t *TimescaleDb) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) { query := ` SELECT encode(hash, 'base64'), height, @@ -138,7 +138,7 @@ func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainN WHERE height >= $1 AND height <= $2 AND chain_name = $3 ` - rows, err := t.pool.Query(context.Background(), query, fromHeight, toHeight, chainName) + rows, err := t.pool.Query(ctx, query, fromHeight, toHeight, chainName) if err != nil { return nil, err } @@ -161,14 +161,14 @@ func (t *TimescaleDb) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainN // // # Used to get all of the validators that signed that block + the proposer // -// Args: +// Parameters: // - chainName: the name of the chain // - blockHeight: the height of the block // // Returns: // - *BlockSigners: the block signers // - error: if the query fails -func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) (*BlockSigners, error) { +func (t *TimescaleDb) GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*BlockSigners, error) { query := ` SELECT vb.block_height, @@ -183,7 +183,7 @@ func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) ( WHERE vb.chain_name = $1 AND vb.block_height = $2 ` - row := t.pool.QueryRow(context.Background(), query, chainName, blockHeight) + row := t.pool.QueryRow(ctx, query, chainName, blockHeight) var blockSigners BlockSigners err := row.Scan(&blockSigners.BlockHeight, &blockSigners.Proposer, &blockSigners.SignedVals) if err != nil { @@ -198,14 +198,14 @@ func (t *TimescaleDb) GetAllBlockSigners(chainName string, blockHeight uint64) ( // // # Used to get the bank send message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *BankSend: the bank send message // - error: if the query fails -func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, error) { +func (t *TimescaleDb) GetBankSend(ctx context.Context, txHash string, chainName string) (*BankSend, error) { query := ` SELECT encode(bms.tx_hash, 'base64') AS tx_hash, @@ -224,7 +224,7 @@ func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, e WHERE bms.tx_hash = decode($1, 'base64') AND bms.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var bankSend BankSend err := row.Scan(&bankSend.TxHash, &bankSend.Timestamp, &bankSend.FromAddress, &bankSend.ToAddress, &bankSend.Amount, &bankSend.Signers) if err != nil { @@ -239,17 +239,18 @@ func (t *TimescaleDb) GetBankSend(txHash string, chainName string) (*BankSend, e // // # Used to get the msg call message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *MsgCall: the msg call message // - error: if the query fails -func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, error) { +func (t *TimescaleDb) GetMsgCall(ctx context.Context, txHash string, chainName string) (*MsgCall, error) { query := ` SELECT encode(vmc.tx_hash, 'base64') AS tx_hash, + vmc.message_counter, vmc.timestamp, gn.address AS caller, vmc.pkg_path, @@ -267,9 +268,20 @@ func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, err WHERE vmc.tx_hash = decode($1, 'base64') AND vmc.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var msgCall MsgCall - err := row.Scan(&msgCall.TxHash, &msgCall.Timestamp, &msgCall.Caller, &msgCall.PkgPath, &msgCall.FuncName, &msgCall.Args, &msgCall.Send, &msgCall.MaxDeposit, &msgCall.Signers) + err := row.Scan( + &msgCall.TxHash, + &msgCall.MessageCounter, + &msgCall.Timestamp, + &msgCall.Caller, + &msgCall.PkgPath, + &msgCall.FuncName, + &msgCall.Args, + &msgCall.Send, + &msgCall.MaxDeposit, + &msgCall.Signers, + ) if err != nil { return nil, err } @@ -282,17 +294,18 @@ func (t *TimescaleDb) GetMsgCall(txHash string, chainName string) (*MsgCall, err // // # Used to get the msg add package message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *MsgAddPackage: the msg add package message // - error: if the query fails -func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAddPackage, error) { +func (t *TimescaleDb) GetMsgAddPackage(ctx context.Context, txHash string, chainName string) (*MsgAddPackage, error) { query := ` SELECT encode(vmap.tx_hash, 'base64') AS tx_hash, + vmap.message_counter, vmap.timestamp, gn.address AS creator, vmap.pkg_path, @@ -310,9 +323,20 @@ func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAdd WHERE vmap.tx_hash = decode($1, 'base64') AND vmap.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var msgAddPackage MsgAddPackage - err := row.Scan(&msgAddPackage.TxHash, &msgAddPackage.Timestamp, &msgAddPackage.Creator, &msgAddPackage.PkgPath, &msgAddPackage.PkgName, &msgAddPackage.PkgFileNames, &msgAddPackage.Send, &msgAddPackage.MaxDeposit, &msgAddPackage.Signers) + err := row.Scan( + &msgAddPackage.TxHash, + &msgAddPackage.MessageCounter, + &msgAddPackage.Timestamp, + &msgAddPackage.Creator, + &msgAddPackage.PkgPath, + &msgAddPackage.PkgName, + &msgAddPackage.PkgFileNames, + &msgAddPackage.Send, + &msgAddPackage.MaxDeposit, + &msgAddPackage.Signers, + ) if err != nil { return nil, err } @@ -325,17 +349,18 @@ func (t *TimescaleDb) GetMsgAddPackage(txHash string, chainName string) (*MsgAdd // // # Used to get the msg run message for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *MsgRun: the msg run message // - error: if the query fails -func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error) { +func (t *TimescaleDb) GetMsgRun(ctx context.Context, txHash string, chainName string) (*MsgRun, error) { query := ` SELECT encode(vmr.tx_hash, 'base64') AS tx_hash, + vmr.message_counter, vmr.timestamp, gn.address AS caller, vmr.pkg_path, @@ -353,9 +378,20 @@ func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error WHERE vmr.tx_hash = decode($1, 'base64') AND vmr.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var msgRun MsgRun - err := row.Scan(&msgRun.TxHash, &msgRun.Timestamp, &msgRun.Caller, &msgRun.PkgPath, &msgRun.PkgName, &msgRun.PkgFileNames, &msgRun.Send, &msgRun.MaxDeposit, &msgRun.Signers) + err := row.Scan( + &msgRun.TxHash, + &msgRun.MessageCounter, + &msgRun.Timestamp, + &msgRun.Caller, + &msgRun.PkgPath, + &msgRun.PkgName, + &msgRun.PkgFileNames, + &msgRun.Send, + &msgRun.MaxDeposit, + &msgRun.Signers, + ) if err != nil { log.Println("error getting msg run", err) return nil, err @@ -369,14 +405,14 @@ func (t *TimescaleDb) GetMsgRun(txHash string, chainName string) (*MsgRun, error // // # Used to get the transaction for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - *Transaction: the transaction // - error: if the query fails -func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transaction, error) { +func (t *TimescaleDb) GetTransaction(ctx context.Context, txHash string, chainName string) (*Transaction, error) { query := ` SELECT encode(tx.tx_hash, 'base64') AS tx_hash, @@ -391,7 +427,7 @@ func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transact WHERE tx.tx_hash = decode($1, 'base64') AND tx.chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var transaction Transaction err := row.Scan( &transaction.TxHash, @@ -416,10 +452,10 @@ func (t *TimescaleDb) GetTransaction(txHash string, chainName string) (*Transact // // # Used to get the last x transactions from the database for a given chain name // -// Args: +// Parameters: // - chainName: the name of the chain // - x: the number of transactions to get -func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Transaction, error) { +func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*Transaction, error) { query := ` SELECT encode(tx.tx_hash, 'base64') AS tx_hash, @@ -435,7 +471,7 @@ func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Trans ORDER BY tx.timestamp DESC LIMIT $2 ` - rows, err := t.pool.Query(context.Background(), query, chainName, x) + rows, err := t.pool.Query(ctx, query, chainName, x) if err != nil { return nil, err } @@ -461,21 +497,21 @@ func (t *TimescaleDb) GetLastXTransactions(chainName string, x uint64) ([]*Trans // // # Used to get the message type for a given transaction hash // -// Args: +// Parameters: // - txHash: the hash of the transaction // - chainName: the name of the chain // // Returns: // - string: the message type // - error: if the query fails -func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error) { +func (t *TimescaleDb) GetMsgType(ctx context.Context, txHash string, chainName string) (string, error) { query := ` SELECT msg_types FROM transaction_general WHERE tx_hash = decode($1, 'base64') AND chain_name = $2 ` - row := t.pool.QueryRow(context.Background(), query, txHash, chainName) + row := t.pool.QueryRow(ctx, query, txHash, chainName) var msgType []string // to future me // in the events the transactions can harbor more transaction types this @@ -498,7 +534,7 @@ func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error // // # Used to get the transactions for a given address for a certain time period // -// Args: +// Parameters: // - address: the address // - chainName: the name of the chain // - fromTimestamp: the starting timestamp @@ -508,6 +544,7 @@ func (t *TimescaleDb) GetMsgType(txHash string, chainName string) (string, error // - []*AddressTx: the transactions // - error: if the query fails func (t *TimescaleDb) GetAddressTxs( + ctx context.Context, address string, chainName string, fromTimestamp time.Time, @@ -527,7 +564,7 @@ func (t *TimescaleDb) GetAddressTxs( addressTxs := make([]AddressTx, 0) - rows, err := t.pool.Query(context.Background(), query, address, chainName, fromTimestamp, toTimestamp) + rows, err := t.pool.Query(ctx, query, address, chainName, fromTimestamp, toTimestamp) if err != nil { return nil, err } diff --git a/pkgs/database/queries_indexer.go b/pkgs/database/queries_indexer.go index ec486f6..6f9dd02 100644 --- a/pkgs/database/queries_indexer.go +++ b/pkgs/database/queries_indexer.go @@ -11,8 +11,9 @@ import ( // Used within the account cache package to get query about the existing accounts // and then we can know which ones to insert // -// Args: +// Parameters: // +// - ctx: the context to use for the query // - addresses: the addresses to check // - chainName: the name of the chain // @@ -20,7 +21,12 @@ import ( // // - map[string]int32: the map of existing addresses and their ids // - error: if the query fails -func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, searchValidators bool) (map[string]int32, error) { +func (t *TimescaleDb) FindExistingAccounts( + ctx context.Context, + addresses []string, + chainName string, + searchValidators bool, +) (map[string]int32, error) { addressesMap := make(map[string]int32) // we need to check if the addresses are already in the map // so we make this query to the db to get the addresses that are already in the map @@ -40,7 +46,7 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, AND address = ANY($2) ` } - rows, err := t.pool.Query(context.Background(), query, chainName, addresses) + rows, err := t.pool.Query(ctx, query, chainName, addresses) if err != nil { return nil, err } @@ -65,8 +71,9 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, // // # Only used when the program is initializing to get all the accounts with their ids // -// Args: +// Parameters: // +// - ctx: the context to use for the query // - chainName: the name of the chain // - searchValidators: whether to search for validators or accounts // - highestIndex: the highest index of the addresses already recorded or it could be a 0 @@ -75,7 +82,12 @@ func (t *TimescaleDb) FindExistingAccounts(addresses []string, chainName string, // // - map[string]int32: the map of all accounts and their ids // - error: if the query fails -func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, highestIndex *int32) (map[string]int32, int32, error) { +func (t *TimescaleDb) GetAllAddresses( + ctx context.Context, + chainName string, + searchValidators bool, + highestIndex *int32, +) (map[string]int32, int32, error) { addressesMap := make(map[string]int32) var maxIndex int32 = 0 if highestIndex != nil { @@ -98,7 +110,7 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, h AND id > $2 ` } - rows, err := t.pool.Query(context.Background(), query, chainName, highestIndex) + rows, err := t.pool.Query(ctx, query, chainName, highestIndex) if err != nil { return nil, 0, err } @@ -125,15 +137,18 @@ func (t *TimescaleDb) GetAllAddresses(chainName string, searchValidators bool, h // // Used to check if the current database is "gnoland" // +// Parameters: +// - ctx: the context to use for the query +// // Returns: // // - string: the name of the current database // - error: if the query fails -func (t *TimescaleDb) CheckCurrentDatabaseName() (string, error) { +func (t *TimescaleDb) CheckCurrentDatabaseName(ctx context.Context) (string, error) { query := ` SELECT current_database() ` - row := t.pool.QueryRow(context.Background(), query) + row := t.pool.QueryRow(ctx, query) var currentDb string err := row.Scan(¤tDb) if err != nil { @@ -148,19 +163,20 @@ func (t *TimescaleDb) CheckCurrentDatabaseName() (string, error) { // // # Used to get the last block height from the database for a given chain // -// Args: +// Parameters: +// - ctx: the context to use for the query // - chainName: the name of the chain // // Returns: // - uint64: the last block height // - error: if the query fails -func (t *TimescaleDb) GetLastBlockHeight(chainName string) (uint64, error) { +func (t *TimescaleDb) GetLastBlockHeight(ctx context.Context, chainName string) (uint64, error) { query := ` SELECT MAX(height) FROM blocks WHERE chain_name = $1 ` - row := t.pool.QueryRow(context.Background(), query, chainName) + row := t.pool.QueryRow(ctx, query, chainName) var lastBlockHeight uint64 err := row.Scan(&lastBlockHeight) if err != nil { diff --git a/pkgs/database/types_api.go b/pkgs/database/types_api.go index 34782dc..7824e54 100644 --- a/pkgs/database/types_api.go +++ b/pkgs/database/types_api.go @@ -30,48 +30,52 @@ type Amount struct { } type BankSend struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - FromAddress string `json:"from_address" doc:"From address (addresses)"` - ToAddress string `json:"to_address" doc:"To address (addresses)"` - Amount []Amount `json:"amount" doc:"Amount"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + FromAddress string `json:"from_address" doc:"From address (addresses)"` + ToAddress string `json:"to_address" doc:"To address (addresses)"` + Amount []Amount `json:"amount" doc:"Amount"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgCall struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Caller string `json:"caller" doc:"Caller address (addresses)"` - Send []Amount `json:"send" doc:"Send amount"` - PkgPath string `json:"pkg_path" doc:"Package path"` - FuncName string `json:"func_name" doc:"Function name"` - Args string `json:"args" doc:"Arguments"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + Send []Amount `json:"send" doc:"Send amount"` + PkgPath string `json:"pkg_path" doc:"Package path"` + FuncName string `json:"func_name" doc:"Function name"` + Args string `json:"args" doc:"Arguments"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgAddPackage struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Creator string `json:"creator" doc:"Creator address (addresses)"` - PkgPath string `json:"pkg_path" doc:"Package path"` - PkgName string `json:"pkg_name" doc:"Package name"` - PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` - Send []Amount `json:"send" doc:"Send amount"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Creator string `json:"creator" doc:"Creator address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type MsgRun struct { - TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` - Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` - Caller string `json:"caller" doc:"Caller address (addresses)"` - PkgPath string `json:"pkg_path" doc:"Package path"` - PkgName string `json:"pkg_name" doc:"Package name"` - PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` - Send []Amount `json:"send" doc:"Send amount"` - MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` - Signers []string `json:"signers" doc:"Signers (addresses)"` + MessageCounter int16 `json:"message_counter" doc:"Transaction order integer, starts from 0"` + TxHash string `json:"tx_hash" doc:"Transaction hash (base64 encoded)"` + Timestamp time.Time `json:"timestamp" doc:"Transaction timestamp"` + Caller string `json:"caller" doc:"Caller address (addresses)"` + PkgPath string `json:"pkg_path" doc:"Package path"` + PkgName string `json:"pkg_name" doc:"Package name"` + PkgFileNames []string `json:"pkg_file_names" doc:"Package file names"` + Send []Amount `json:"send" doc:"Send amount"` + MaxDeposit []Amount `json:"max_deposit" doc:"Max deposit"` + Signers []string `json:"signers" doc:"Signers (addresses)"` } type Transaction struct { From 74a35b301070bb1dab5cbadec6fe64b16ad7eb3b Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 17:56:05 +0100 Subject: [PATCH 07/11] Adjusted the API and database interactions with context support - Updated API handlers to pass context to database methods. - Refactored transaction handling to support multiple message types within a single transaction, including adjustments to the response structure. - Added favicon support to the API. - Improved OpenAPI documentation with detailed descriptions for various endpoints. --- Makefile | 6 +- api/handlers/address.go | 8 +- api/handlers/blocks.go | 10 +- api/handlers/database_test.go | 45 ++++--- api/handlers/interface.go | 27 ++-- api/handlers/transactions.go | 163 ++++++++++++---------- api/huma-types/transaction.go | 4 +- api/main.go | 91 +++++++++++-- api/public/favicon.ico | Bin 0 -> 15086 bytes indexer/data_processor/operator.go | 8 +- pkgs/database/queries_api.go | 210 +++++++++++++++++++---------- pkgs/database/queries_indexer.go | 2 +- 12 files changed, 363 insertions(+), 211 deletions(-) create mode 100644 api/public/favicon.ico diff --git a/Makefile b/Makefile index 8cf8f58..6dc0139 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_BRANCH)-$(GIT_COMMIT)) build: mkdir -p build - go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=d5dfb79 -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=release/0.4.0-d5dfb79" -o build/indexer indexer/main.go + go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer indexer/main.go install: go install -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" indexer/main.go @@ -28,12 +28,12 @@ clean: # experimental build with greentea garbage collection # use at your own risk build-experimental: - GOEXPERIMENT=greenteagc go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer-tea indexer/main.go + GOEXPERIMENT=greenteagc go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer-tea indexer/main.go # experimental install with greentea garbage collection # use at your own risk install-experimental: - cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" + cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" ######################################################## # Test the indexer diff --git a/api/handlers/address.go b/api/handlers/address.go index 0983cf8..6d12638 100644 --- a/api/handlers/address.go +++ b/api/handlers/address.go @@ -20,7 +20,13 @@ func (h *AddressHandler) GetAddressTxs( ctx context.Context, input *humatypes.AddressGetInput, ) (*humatypes.AddressGetOutput, error) { - address, err := h.db.GetAddressTxs(input.Address, h.chainName, input.FromTimestamp, input.ToTimestamp) + address, err := h.db.GetAddressTxs( + ctx, + input.Address, + h.chainName, + input.FromTimestamp, + input.ToTimestamp, + ) if err != nil { return nil, huma.Error404NotFound("Address not found", err) } diff --git a/api/handlers/blocks.go b/api/handlers/blocks.go index f92dced..43e60eb 100644 --- a/api/handlers/blocks.go +++ b/api/handlers/blocks.go @@ -23,7 +23,7 @@ func NewBlocksHandler(db DatabaseHandler, chainName string) *BlocksHandler { // GetBlock retrieves a block by height func (h *BlocksHandler) GetBlock(ctx context.Context, input *humatypes.BlockGetInput) (*humatypes.BlockGetOutput, error) { // Fetch from database - block, err := h.db.GetBlock(input.Height, h.chainName) + block, err := h.db.GetBlock(ctx, input.Height, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Block at height %d not found", input.Height), err) } @@ -51,7 +51,7 @@ func (h *BlocksHandler) GetFromToBlocks( if input.FromHeight > input.ToHeight { return nil, huma.Error400BadRequest("From height must be less than to height", nil) } - blocks, err := h.db.GetFromToBlocks(input.FromHeight, input.ToHeight, h.chainName) + blocks, err := h.db.GetFromToBlocks(ctx, input.FromHeight, input.ToHeight, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Blocks from height %d to height %d not found", input.FromHeight, input.ToHeight), err) } @@ -76,7 +76,7 @@ func (h *BlocksHandler) GetAllBlockSigners( input *humatypes.AllBlockSignersGetInput, ) (*humatypes.AllBlockSignersGetOutput, error) { // Fetch from database - blockSigners, err := h.db.GetAllBlockSigners(h.chainName, input.BlockHeight) + blockSigners, err := h.db.GetAllBlockSigners(ctx, h.chainName, input.BlockHeight) if err != nil { return nil, huma.Error404NotFound("Block signers not found", err) } @@ -92,7 +92,7 @@ func (h *BlocksHandler) GetAllBlockSigners( // Get latest block height func (h *BlocksHandler) GetLatestBlock(ctx context.Context, _ *humatypes.LatestBlockHeightGetInput) (*humatypes.LatestBlockHeightGetOutput, error) { - block, err := h.db.GetLatestBlock(h.chainName) + block, err := h.db.GetLatestBlock(ctx, h.chainName) if err != nil { return nil, huma.Error404NotFound("Latest block height not found", err) } @@ -112,7 +112,7 @@ func (h *BlocksHandler) GetLatestBlock(ctx context.Context, _ *humatypes.LatestB // Get last x blocks func (h *BlocksHandler) GetLastXBlocks(ctx context.Context, input *humatypes.LastXBlocksGetInput) (*humatypes.LastXBlocksGetOutput, error) { // Fetch from database - blocks, err := h.db.GetLastXBlocks(h.chainName, input.Amount) + blocks, err := h.db.GetLastXBlocks(ctx, h.chainName, input.Amount) if err != nil { return nil, huma.Error404NotFound("Last x blocks not found", err) } diff --git a/api/handlers/database_test.go b/api/handlers/database_test.go index 2aeb6b8..c8488ae 100644 --- a/api/handlers/database_test.go +++ b/api/handlers/database_test.go @@ -1,6 +1,7 @@ package handlers_test import ( + "context" "fmt" "time" @@ -18,13 +19,13 @@ type MockDatabase struct { msgCall map[string]*database.MsgCall msgAddPackage map[string]*database.MsgAddPackage msgRun map[string]*database.MsgRun - msgTypes map[string]string + msgTypes map[string][]string shouldError bool errorMsg string } -func (m *MockDatabase) GetBlock(height uint64, chainName string) (*database.BlockData, error) { +func (m *MockDatabase) GetBlock(ctx context.Context, height uint64, chainName string) (*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -35,7 +36,7 @@ func (m *MockDatabase) GetBlock(height uint64, chainName string) (*database.Bloc return block, nil } -func (m *MockDatabase) GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) { +func (m *MockDatabase) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -50,7 +51,7 @@ func (m *MockDatabase) GetFromToBlocks(fromHeight uint64, toHeight uint64, chain return result, nil } -func (m *MockDatabase) GetAllBlockSigners(chainName string, blockHeight uint64) (*database.BlockSigners, error) { +func (m *MockDatabase) GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*database.BlockSigners, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -61,7 +62,7 @@ func (m *MockDatabase) GetAllBlockSigners(chainName string, blockHeight uint64) return blockSigners, nil } -func (m *MockDatabase) GetTransaction(txHash string, chainName string) (*database.Transaction, error) { +func (m *MockDatabase) GetTransaction(ctx context.Context, txHash string, chainName string) (*database.Transaction, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -72,7 +73,7 @@ func (m *MockDatabase) GetTransaction(txHash string, chainName string) (*databas return transaction, nil } -func (m *MockDatabase) GetAddressTxs(address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) { +func (m *MockDatabase) GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -83,7 +84,7 @@ func (m *MockDatabase) GetAddressTxs(address string, chainName string, fromTimes return addressTxs, nil } -func (m *MockDatabase) GetLatestBlock(chainName string) (*database.BlockData, error) { +func (m *MockDatabase) GetLatestBlock(ctx context.Context, chainName string) (*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -93,7 +94,7 @@ func (m *MockDatabase) GetLatestBlock(chainName string) (*database.BlockData, er return m.latestBlock, nil } -func (m *MockDatabase) GetLastXBlocks(chainName string, x uint64) ([]*database.BlockData, error) { +func (m *MockDatabase) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*database.BlockData, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -104,7 +105,7 @@ func (m *MockDatabase) GetLastXBlocks(chainName string, x uint64) ([]*database.B return blocks, nil } -func (m *MockDatabase) GetLastXTransactions(chainName string, x uint64) ([]*database.Transaction, error) { +func (m *MockDatabase) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*database.Transaction, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -115,18 +116,18 @@ func (m *MockDatabase) GetLastXTransactions(chainName string, x uint64) ([]*data return transactions, nil } -func (m *MockDatabase) GetMsgType(txHash string, chainName string) (string, error) { +func (m *MockDatabase) GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) { if m.shouldError { - return "", fmt.Errorf("%s", m.errorMsg) + return nil, fmt.Errorf("%s", m.errorMsg) } - msgType, ok := m.msgTypes[txHash] + msgTypes, ok := m.msgTypes[txHash] if !ok { - return "", fmt.Errorf("message type not found") + return nil, fmt.Errorf("message type not found") } - return msgType, nil + return msgTypes, nil } -func (m *MockDatabase) GetBankSend(txHash string, chainName string) (*database.BankSend, error) { +func (m *MockDatabase) GetBankSend(ctx context.Context, txHash string, chainName string) ([]*database.BankSend, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -134,10 +135,10 @@ func (m *MockDatabase) GetBankSend(txHash string, chainName string) (*database.B if !ok { return nil, fmt.Errorf("bank send not found") } - return bankSend, nil + return []*database.BankSend{bankSend}, nil } -func (m *MockDatabase) GetMsgCall(txHash string, chainName string) (*database.MsgCall, error) { +func (m *MockDatabase) GetMsgCall(ctx context.Context, txHash string, chainName string) ([]*database.MsgCall, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -145,10 +146,10 @@ func (m *MockDatabase) GetMsgCall(txHash string, chainName string) (*database.Ms if !ok { return nil, fmt.Errorf("message call not found") } - return msgCall, nil + return []*database.MsgCall{msgCall}, nil } -func (m *MockDatabase) GetMsgAddPackage(txHash string, chainName string) (*database.MsgAddPackage, error) { +func (m *MockDatabase) GetMsgAddPackage(ctx context.Context, txHash string, chainName string) ([]*database.MsgAddPackage, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -156,10 +157,10 @@ func (m *MockDatabase) GetMsgAddPackage(txHash string, chainName string) (*datab if !ok { return nil, fmt.Errorf("message add package not found") } - return msgAddPackage, nil + return []*database.MsgAddPackage{msgAddPackage}, nil } -func (m *MockDatabase) GetMsgRun(txHash string, chainName string) (*database.MsgRun, error) { +func (m *MockDatabase) GetMsgRun(ctx context.Context, txHash string, chainName string) ([]*database.MsgRun, error) { if m.shouldError { return nil, fmt.Errorf("%s", m.errorMsg) } @@ -167,5 +168,5 @@ func (m *MockDatabase) GetMsgRun(txHash string, chainName string) (*database.Msg if !ok { return nil, fmt.Errorf("message run not found") } - return msgRun, nil + return []*database.MsgRun{msgRun}, nil } diff --git a/api/handlers/interface.go b/api/handlers/interface.go index 739fc7c..962bca1 100644 --- a/api/handlers/interface.go +++ b/api/handlers/interface.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "time" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database" @@ -9,21 +10,21 @@ import ( // DatabaseHandler interface for the database type DatabaseHandler interface { // Addresses - GetAddressTxs(address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) + GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) // Blocks - GetBlock(height uint64, chainName string) (*database.BlockData, error) - GetFromToBlocks(fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) - GetAllBlockSigners(chainName string, blockHeight uint64) (*database.BlockSigners, error) - GetLatestBlock(chainName string) (*database.BlockData, error) - GetLastXBlocks(chainName string, x uint64) ([]*database.BlockData, error) + GetBlock(ctx context.Context, height uint64, chainName string) (*database.BlockData, error) + GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*database.BlockData, error) + GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*database.BlockSigners, error) + GetLatestBlock(ctx context.Context, chainName string) (*database.BlockData, error) + GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*database.BlockData, error) // Transactions - GetTransaction(txHash string, chainName string) (*database.Transaction, error) - GetLastXTransactions(chainName string, x uint64) ([]*database.Transaction, error) - GetMsgType(txHash string, chainName string) (string, error) - GetBankSend(txHash string, chainName string) (*database.BankSend, error) - GetMsgCall(txHash string, chainName string) (*database.MsgCall, error) - GetMsgAddPackage(txHash string, chainName string) (*database.MsgAddPackage, error) - GetMsgRun(txHash string, chainName string) (*database.MsgRun, error) + GetTransaction(ctx context.Context, txHash string, chainName string) (*database.Transaction, error) + GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*database.Transaction, error) + GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) + GetBankSend(ctx context.Context, txHash string, chainName string) ([]*database.BankSend, error) + GetMsgCall(ctx context.Context, txHash string, chainName string) ([]*database.MsgCall, error) + GetMsgAddPackage(ctx context.Context, txHash string, chainName string) ([]*database.MsgAddPackage, error) + GetMsgRun(ctx context.Context, txHash string, chainName string) ([]*database.MsgRun, error) } diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go index cb72d92..08e0d08 100644 --- a/api/handlers/transactions.go +++ b/api/handlers/transactions.go @@ -20,6 +20,7 @@ func NewTransactionsHandler(db DatabaseHandler, chainName string) *TransactionsH return &TransactionsHandler{db: db, chainName: chainName} } +// GetTransactionBasic retrieves basic transaction details by tx hash func (h *TransactionsHandler) GetTransactionBasic( ctx context.Context, input *humatypes.TransactionGetInput, @@ -30,7 +31,7 @@ func (h *TransactionsHandler) GetTransactionBasic( if err != nil { return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) } - transaction, err := h.db.GetTransaction(txHashBase64, h.chainName) + transaction, err := h.db.GetTransaction(ctx, txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } @@ -39,6 +40,7 @@ func (h *TransactionsHandler) GetTransactionBasic( }, nil } +// GetTransactionMessage retrieves all messages within a transaction by tx hash func (h *TransactionsHandler) GetTransactionMessage( ctx context.Context, input *humatypes.TransactionGetInput, @@ -46,93 +48,106 @@ func (h *TransactionsHandler) GetTransactionMessage( input.TxHash = strings.Trim(input.TxHash, " ") txHash, err := base64.URLEncoding.DecodeString(input.TxHash) txHashBase64 := base64.StdEncoding.EncodeToString(txHash) + response := make(map[int16]humatypes.TransactionMessage) if err != nil { return nil, huma.Error400BadRequest("Transaction hash is not valid base64url encoded", err) } - msgType, err := h.db.GetMsgType(txHashBase64, h.chainName) + msgTypes, err := h.db.GetMsgTypes(ctx, txHashBase64, h.chainName) if err != nil { return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) } - var message humatypes.TransactionMessage - - switch msgType { - case "bank_msg_send": - data, err := h.db.GetBankSend(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - FromAddress: data.FromAddress, - ToAddress: data.ToAddress, - Amount: data.Amount, - } - case "vm_msg_call": - data, err := h.db.GetMsgCall(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Caller: data.Caller, - Send: data.Send, - PkgPath: data.PkgPath, - FuncName: data.FuncName, - Args: data.Args, - MaxDeposit: data.MaxDeposit, - } - case "vm_msg_add_package": - data, err := h.db.GetMsgAddPackage(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + for _, msgType := range msgTypes { + switch msgType { + case "bank_msg_send": + data, err := h.db.GetBankSend(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + FromAddress: data.FromAddress, + ToAddress: data.ToAddress, + Amount: data.Amount, + } + } + case "vm_msg_call": + data, err := h.db.GetMsgCall(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + Send: data.Send, + PkgPath: data.PkgPath, + FuncName: data.FuncName, + Args: data.Args, + MaxDeposit: data.MaxDeposit, + } + } + case "vm_msg_add_package": + data, err := h.db.GetMsgAddPackage(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Creator: data.Creator, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } + } + case "vm_msg_run": + data, err := h.db.GetMsgRun(ctx, txHashBase64, h.chainName) + if err != nil { + return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + } + for _, data := range data { + index := data.MessageCounter + response[index] = humatypes.TransactionMessage{ + MessageType: msgType, + TxHash: data.TxHash, + Timestamp: data.Timestamp, + Signers: data.Signers, + Caller: data.Caller, + PkgPath: data.PkgPath, + PkgName: data.PkgName, + PkgFileNames: data.PkgFileNames, + Send: data.Send, + MaxDeposit: data.MaxDeposit, + } + } + default: + return nil, huma.Error400BadRequest("Transaction message type not found", nil) } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Creator: data.Creator, - PkgPath: data.PkgPath, - PkgName: data.PkgName, - PkgFileNames: data.PkgFileNames, - Send: data.Send, - MaxDeposit: data.MaxDeposit, - } - case "vm_msg_run": - data, err := h.db.GetMsgRun(txHashBase64, h.chainName) - if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) - } - message = humatypes.TransactionMessage{ - MessageType: msgType, - TxHash: data.TxHash, - Timestamp: data.Timestamp, - Signers: data.Signers, - Caller: data.Caller, - PkgPath: data.PkgPath, - PkgName: data.PkgName, - PkgFileNames: data.PkgFileNames, - Send: data.Send, - MaxDeposit: data.MaxDeposit, - } - default: - return nil, huma.Error400BadRequest("Transaction message type not found", nil) } - return &humatypes.TransactionMessageGetOutput{ - Body: message, + Body: response, }, nil } +// GetLastXTransactions retrieves the last X transactions func (h *TransactionsHandler) GetLastXTransactions(ctx context.Context, input *humatypes.TransactionGeneralListGetInput) (*humatypes.TransactionGeneralListGetOutput, error) { - transactions, err := h.db.GetLastXTransactions(h.chainName, input.Amount) + transactions, err := h.db.GetLastXTransactions(ctx, h.chainName, input.Amount) if err != nil { return nil, huma.Error404NotFound("Last x transactions not found", err) } diff --git a/api/huma-types/transaction.go b/api/huma-types/transaction.go index 68996fa..137e1d9 100644 --- a/api/huma-types/transaction.go +++ b/api/huma-types/transaction.go @@ -11,6 +11,7 @@ type TransactionGetInput struct { TxHash string `path:"tx_hash" minLength:"44" maxLength:"44" doc:"Transaction hash (base64url encoded)" required:"true"` } +// TransactionBasicGetOutput represents the response for basic transaction details type TransactionBasicGetOutput struct { Body database.Transaction } @@ -48,8 +49,9 @@ type TransactionMessage struct { MaxDeposit []database.Amount `json:"max_deposit,omitempty" doc:"Max deposit (for vm_msg_call, vm_msg_add_package, and vm_msg_run)"` } +// TransactionMessageGetOutput represents the response containing all messages within a transaction type TransactionMessageGetOutput struct { - Body TransactionMessage + Body map[int16]TransactionMessage } type TransactionGeneralListGetInput struct { diff --git a/api/main.go b/api/main.go index b1fc333..cf2d733 100644 --- a/api/main.go +++ b/api/main.go @@ -1,9 +1,11 @@ package main import ( + _ "embed" "fmt" "log" "net/http" + "strings" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/config" "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/handlers" @@ -17,6 +19,9 @@ import ( // "github.com/go-chi/httprate" ) +//go:embed public/favicon.ico +var favicon []byte + var ( Commit = "unknown" // Set via ldflags at build time Version = "unknown" // Set via ldflags at build time @@ -103,9 +108,24 @@ var rootCmd = &cobra.Command{ api := humachi.New(router, huma.DefaultConfig("Spectra Gnoland Indexer API", Version)) - api.OpenAPI().Info.Title = "Spectra Gnoland Indexer API" - api.OpenAPI().Info.Version = Version - api.OpenAPI().Info.Description = "API for the Spectra Gnoland Indexer" + openApi := api.OpenAPI() + openApi.Info = &huma.Info{ + Title: "Spectra Gnoland Indexer API", + Version: strings.TrimPrefix(Version, "v"), + Description: "API for the Spectra Gnoland Indexer", + Contact: &huma.Contact{ + Name: "Cogwheel Validator", + URL: "https://cogwheel.zone", + Email: "info@cogwheel.zone", + }, + License: &huma.License{ + Name: "Apache 2.0", + URL: "https://github.com/Cogwheel-Validator/spectra-gnoland-indexer?tab=Apache-2.0-1-ov-file#readme", + }, + } + openApi.ExternalDocs = &huma.ExternalDocs{ + URL: "https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/tree/main/docs", + } // Initialize database connection from environment variables db := database.NewTimescaleDb(database.DatabasePoolConfig{ @@ -128,20 +148,66 @@ var rootCmd = &cobra.Command{ transactionsHandler := handlers.NewTransactionsHandler(db, conf.ChainName) addressHandler := handlers.NewAddressHandler(db, conf.ChainName) + // favicon route + router.Get("/favicon.ico", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/x-icon") + w.Write(favicon) + }) + // Register Block API routes - huma.Get(api, "/block/{height}", blocksHandler.GetBlock) - huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks) - huma.Get(api, "/blocks/{block_height}/signers", blocksHandler.GetAllBlockSigners) - huma.Get(api, "/blocks/latest", blocksHandler.GetLatestBlock) - huma.Get(api, "/blocks", blocksHandler.GetLastXBlocks) + huma.Get(api, "/block/{height}", blocksHandler.GetBlock, + func(op *huma.Operation) { + op.Summary = "Get Block Height" + op.Description = "Retrieve block data by its height" + }) + huma.Get(api, "/blocks/{from_height}/{to_height}", blocksHandler.GetFromToBlocks, + func(op *huma.Operation) { + op.Summary = "Get From To Blocks" + op.Description = "Retrieve blocks data by its height range" + }) + huma.Get(api, "/blocks/{block_height}/signers", blocksHandler.GetAllBlockSigners, + func(op *huma.Operation) { + op.Summary = "Get All Block Signers" + op.Description = "Retrieve all validators that signed a block by its height" + }) + huma.Get(api, "/blocks/latest", blocksHandler.GetLatestBlock, + func(op *huma.Operation) { + op.Summary = "Get Latest Block" + op.Description = "Retrieve the latest block data" + }) + huma.Get(api, "/blocks", blocksHandler.GetLastXBlocks, + func(op *huma.Operation) { + op.Summary = "Get Last X Blocks" + op.Description = "Retrieve the last X blocks data" + }) // Register Transaction API routes - huma.Get(api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic) - huma.Get(api, "/transaction/{tx_hash}/message", transactionsHandler.GetTransactionMessage) - huma.Get(api, "/transactions", transactionsHandler.GetLastXTransactions) + huma.Get( + api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic, + func(op *huma.Operation) { + op.Summary = "Get Transaction Basic" + op.Description = "Retrieve basic transaction data by its hash" + }) + huma.Get( + api, + "/transaction/{tx_hash}/message", + transactionsHandler.GetTransactionMessage, + func(op *huma.Operation) { + op.Summary = "Get All Transaction Messages" + op.Description = "Retrieve all messages contained within a transaction by its hash" + }) + huma.Get(api, "/transactions", transactionsHandler.GetLastXTransactions, + func(op *huma.Operation) { + op.Summary = "Get Last X Transactions" + op.Description = "Retrieve the last X transactions data" + }) // Register Address API routes - huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs) + huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs, + func(op *huma.Operation) { + op.Summary = "Get Address Transactions" + op.Description = "Retrieve all transactions for a given address for a certain time period" + }) // Start server using config values addr := fmt.Sprintf("%s:%d", conf.Host, conf.Port) @@ -169,7 +235,6 @@ func init() { rootCmd.PersistentFlags().StringP("config", "c", "config-api.yml", "config file path") rootCmd.PersistentFlags().StringP("cert-file", "t", "", "certificate file path") rootCmd.PersistentFlags().StringP("key-file", "k", "", "key file path") - } func main() { diff --git a/api/public/favicon.ico b/api/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..f0e5dff91e544704ccd4942bc3720301cdb20c51 GIT binary patch literal 15086 zcmd5@d32OjmajA}ZMW@moYUhR=eTuydX7EQ+LIB?w^S9P4UtWOu#0FLc2ES_AwiHG zSrj1zNN{hov9%p^Y!~$CjJCUEU#cppRBA~=fItXIRlWJ$`@Q$oR|P@4=byL{?tjfj<4lSKM^_*y?HegvwHFLdBQ*#LA=kr0S%8f0fWD zR}1}tYGFKBebRWSx^I`u zf7bFUmg-|FzSYN8^yuR&1@H@C7y5m`Jqg12F+K?VzlS_jBaA6E!kk(o^y#$;eMVhC zpINuP@6_sBZXQ*3k*lEKN8hWmAN+<4{8=LF=Df0ta`MW5spXXyLE7}u6$03WJ`R}2 zVN`37_>IX`!gv6}_<>#GH>TAJV>)C;oiJwA34M0mDdXX~H*@CHY2L~HA7`$^LUQ)M z_EIvS+ONw8d3mBw-+L>s${AI$Tpv~5tml;r#%|-6YasoTNc;n**NEI%z^U-(V-3)~ zN%&_#X4VOFHr6l)@^HN{=GGrK=lV*$i~QF!w|eD9Cc^aR&gGEKwY;)hwUHH{=p)N} z^igHvy!a=AFN7c58NHxhOjzmzcbvM%W70jalkN?L{}IM7jCsDKG2d4M8Jb<#{1cb! z?>`^eL%;ZmKB8=xKBBx4*adWNiPHZV=$%Ay2mH|p|0JAQ;GeQ8AZBl9g#I(|OWj-8 zftz&y2-f*1WIp6Ezc3c~+qDJ0rQQPn618*$`5;BHUTJK@iRzKQoFqGam(&(mHl}J2xYBYqb_aeNJJyh4 zcXYNp)ndnisCe=1R<@b8%~8CkGYjnITySR|gn2{QC4OKxmjr~l6tX;UT3;4e{+s#V z{?t0D3T^tiC0_*0zlLq}+gw+i!3cYwu6xOQ&B-0;UivHX!dtE4ollRk&2idG=DlTa zzYyh?ki*_?-+C*701nNlkvZC#?fyGINnoQxf3md>lPHPTL+i-FPR3?36- zNEGf>jGM3v;|a)`rgwfhyY^zW2ibY`mjh#ijdLuxp@cA};l?^9R0%CDMuWXnK8mr| z2sikCx~oTY_r%56RoHj3^~@XKhpjQ;qfFAfgEPi*iMtSZ8GBHej2+K`8+sS!T9(u1 z`XGhjFhn9t8&@?Ec#~EfCh-zxS_A1?>6`ch>=Ykp(#{j-jR&OODLzAdV@yNbGru0X zk2C)1Tbt6blZ|BTbY3&{Zey1^cj8_L?CY9@xt=6g%;y+9(#KaEvT-WBYGlmNxr&P& zdN(MpC%!Oez;SSf?L2fpW@Uh}_dx$k_J$ptIR|#)I@=gvCXOq$IS)gxcT!6TSd@Exo&N#7CTqa|){Edw^{tfVB?4Ok0 zm2YC(j&sXg#~Pg26;}wm#B6Q^c9KmYVQvm3jg3KX&V;G~X;?|W4q=`!SHKzgRppx) zH}OTD)lC2T*shQu-IJ8Q(}tJ~XPEi!;LHjqcHmcfx3LpXEPY$Jq4!OAz8UzRL|nDG zX*JH_QYT)1On7Assi_>w*16;iV~4((?-(f;al{6t|JBOa-F!4$MihjKc2Z%xYj(=QNWq7FR6nzzxg}2?Y_S ze@yYBE#&`|0P{KdGL^7Hi0c#=sCW`_0mWl~IDAx`f$dTFi9_FD4}J$bN`A>;?BF%~ zBZ-~tqEqKi+|apui^RPRqmu36IB-`hynH4}&q|zmH|rewR(zQbzR3Jl#baK?1I340 zMd#^m#-5P+r*)ixAA1+^L@wbZ+pg@o;yQQ^n_{0&H#qKQ+*0R+nRLHh%2OC2#)DOX zv+$x;a0gylRFvN%VA`cw7$R|cF_K7ho{9;@|K#Yfdob*J4n6$Q0Jb)Z)<+Bm- zZT+b;0wkEN1U$fW-Teu%_@#`PA zi;tksBj80-OPA<6(`}zug}1kn)Z4s^#laRaWE;-&25^b(c@y}Z&T$X%gmg`qfs^GKj2WU&srisG z+qjA2Da@wA?XZn@{LZ-(?7GUGVBd%0j8+9Z&)N$*&mUEt6n8%fj?)<+yH40QLx|_Z zZQ?lNmU<@KPP{wu+?EI)M|yyDKh?zeT&lC_)VIQ|>Kronu<|}&mocyvi&?(u;eBCo z`b>|sKMBzT-TT3lAzL!B!v`sw$k>7JsSNB2@2;qD?}`Wy#9aizXck3$+O(Pl%ngN^ z^(}3SjU6$girp!Oq25PAW@F&?8l09_#(bqGDeayAyWSJ7O_eC$Qs4j>Q7vzrhvKJMsQ#N4L0( zcw%#gu!Hl8GftipUkE$St-A<(B7GB93ol_#$?nK9$^+o{kJ`Rp`6A{EVTX{;&x>8f z1n^JNufF#Aafv?x`-1pEVTWJxuw4g!#dmc^74{;j^GwXJuWpv8FpHxJy*P3^TcUjOO@$Gj$DFWV`fPQZ#0lyBv6YGS;oxjd$rkh2|LG2(>ZWw)Hn>v`y4T_L3IMs zyUJDJTUWns#cyQmcO7m=%u9Ag;#Yd7v#R(GoqLGmtaFLkiF+SjBT?(v7ul+6a^4wb zmxAZj%oV53o#zyMCmUnw9eBC61k4nVDeQz_#szaR~uDCV_e7-icTJ23Br z*x2znQ|F8wud6lei+1RxZNFAkEUJ&|kND>6bFl}&Z&H4(MA>%IxuZT~IO9mxJFu&G z3^C39hzlt9C7JVwSemV2-$cA&zGq^`b6dw!=Y+dNKDNe#Hy5Da;rQ!2v*{{huCIc* zqHLoBJJljtNbkfKt}S5%c7tPh@Lt8i@J~awQ~qM*@8kEuZd3fIY`o$;+nEgfPV6O8 z=Y*La(^wiiYLqr-XV$R11>yqeems0>r{zzfZ{iDc2Ha76S228A?xf;L_*IH05yN}n zTRS?tIqn=t@eA8Z=-kP9;*GM8WEU0R6;Bwq)IA=XrO_nTup@W7;M$4ekJ<)(LDkRn z$9%7%wvpsB3hcIhC*0@4PBFbR?^F2)@cR+hwVdd-VjFd4d(}N~D*R5|rKEG#JK-Pq9kUNgi&e}=&<51Q~B~ms)K6t135UWp|dz+*O!1@ zasCNl=iCpNDSwjrsvCal&);>3cj1Rqe#(L0Y3m7p8t)}%N?R266J|*y-&=y--VgZ8 zC7-+~ILKHUh}fJV>>L|7uoG_xKXV?~$;T3Y@>9US=aXaN`M(}_;8!*Wqr~rV>Ymo1 zYz^&!rTdd+No z&F{6#m>9aJbL_O4g#A4D&C=L$qoj3Vc2WK1XL0^~i;@3F``}(0(99KqD%i=S)4w`x zCD|L)^HTUJRzqAo4|&Q$_+cxicX9`~i=bcl+H>lk_aGKFN?Rv+55zA0A=oqZua?K> zVOirZ^umTef%B(RH8RN?*m|z-N!=S%H{kdzBqlu@6%V`soaApw_v-Ab7+%F^%I;YF ziJSYvU+N{1+j9>k)9qw$AQe~|&-EVIrG6`FFVj{w+z;N=ppKoec#kzW>?OxIvNq;M zemEHZ>28W8o%ql4Sy*2xUWmB&MOJIiMt<%s_`tF=PHaE$@wt{jA0;2B71saUSlze` zHpgeI#X4-81J0;ikaH@;JFxk79;n%)Kr+-dG)!5*X1-G zcDee{ScvbVk=|Y4a^Y<*m&fIb(|DcB)kWjwE*Cz=V}M4SHXi%9T!-m7#zGn~QQAna z{XhFFBs>nBhvjn~v5o*eS0mP`*3Ei=K9pXRo&>!g|JZsANWCgOOTE)Qt?MCP$ocPS z;SKahn|7{0yq-Pv%L~oCvYWMW$4({YdK3~Eg^`qb)|wQ2SFIn!!? z%lCa8U)tN(r=~pn9c1H;d&_P?zxp8hr%mX4b>dD5-Kk1oe!d50^;=L|%&$%0{!|ZY z7G1bI6~W!3_q0dq?jTHmTM_b#JT2kkIGWA4A@s6na`e%HXrb>)OzCUppatYM1O;vZN+7iuAxi;mj^ZapRzTek(q3@Qw z0|HD_y1=?KK+xQn&&1q5%$?+W0yIDMdm-orTRlhWBccB?eGTfjo8f1v#z-|fwRX<) zk!#xZAmj~;18v4)AFfg)`#5M}t(~(XWANPmGk?tX+d+Md2 zkFsP(lUQAXTC;QQs{X2SMXre;&sq{lAkJ^omNea(+6$?8%+L4sMwNZ8jVeppcRbV` z1Jb?f70b0#|6|j0VNrB2#(h<;k5OHWd%;*c<*3N5IZs00#=38USgRo5cR$|vZ&}y} zrGAcOMprC@j=Qb98JN#8H?3V8kJqU0Nc}JB!=Q(;?~NAm#}AHLdY^^4N$>Lhf~oR) z?&HcB+scU($Zv{s(c@HyUK?}d7nr-5_gMA#72i~^S#r*r8@*0&?ltstKKSambyp6x zDXzWBb#qN&3Ff7G0@t4?AGhP<+L+3H`k2bNb;kf}qm2LmnjzNA^%KO=DwpKg zE)><*H~)t=v0^aRcPN7w^K^{;KArs76hGDEdC!yGI?zQ0wNo zp?}V$xyyGaH+_^bMg|+QiByz^iV%m#x<8?5{fSsFIC?jTwx&Dg&J&i2C`c zHTYiib4=QARkNc!ROWGr2UC4h>J3oMZc|g-+|v9f^m*TcEsfjvVx9A1L%b2mU*k{ZF9Xw)@;n} z%}Ly&4P)*k-$i|}N(`Vo8d%>@aC-P-=;dlj#}Jj z+VEzpn2i41MC#cgmd=CRiM7rrG6pRc~s7Y*k8-9t|Z@W-G7FTl`WTho$kwg;trg({HH^r?f4ng z@vQl&2X*M16Ji9ux4+^S=Ng!c>Px7Naj%c!bQP;3-!ONEY2G(6XPoDS?$tdLuDfF& zOxus*Y*VclyeF=$d##1<0Z|>gyy>*ar5f~laEt5FvhT$`QOv1w9GV+BriXe)MUf=L z51&~GZgsQo2bex&kJUN8R7S;JPfjp9g`z9%|WpHp_NF{`#7`b1Ujn;PLz zHCk1lrMbzb(iw+c8?-4XzH96hmB28BcrEM5HqWrf9xI1U)h|(RMjlSJs08Gl+(TBq z754_O!Q2fzFZ(cx9U$PE?MtXG?65EW(Ifs0ef=*RI>nvDGs$!28t2lOht_@W`rT2j zq}f#d-@5`e>Us6^Al=M0u1%|$1@o%8sirG!8}aQg^X>D*ALj!pp$mEv)_r+{O9GSQ+TH z>uM@?Av+__`sC-MBL5}Sf>r$+eFy4;TeUx_gMG0cvm{z(>^*j4D*sS(tI^hg^#B-d zEAU^fuM9qBt_lXhxj60?BvWfg{DAX57+f1nwuk%)>7In@kHyiXXHO)KJ!o<7jjYp3 zqrafJdzmdYdBz*OsB7u#Lf0CrgS#=e4|fDRz`JhD9Y-xCi8GhP+LQ2UN#mJt!m~5n z1MZ!2?+ypeJ<(TN!@mzC9lEWw|K(Y#@ zxc<&R_4&j29!Q@6z8htVi)k*Jljf#6#TAh2AwQ%}(ck<21^sesFT?MZa*-qKM(kXV zyIu+81<3zv8}R$7{o0(`;l}iaEB~E-2d4h8aQY1b<$1_&4;vC&b`JFxtJ+x8f2XGJH`Yh#F$TLjja`aoC2`dp- z;7$eq?gD;~?js^!O`x{@7h_GsmB#3bJM}T;M}gVOk0?h$9_&VJoR7G0=9Wg`p}sW5 zD3k+8n+@)sFxNESgB)@za-2B*CIP>t!@Wc+m+~UcdKB^0%I6}&%kc&5KJp2U-EroP zb&UygL+E*ZV)Zv%bHV2bGsTM@=xhMugrYyUh*v&p7k3o~I0kmFZV(=_RTPWiKB{L+ zs0lIdF^aL>h^g;H96k8aIxzxqE!`(=M+`yt*;gEhi7`(#iDBr`4n%y@AGL1sYX;7k znP1bc=DZVg4hGf{xa+nK_r^}5A4>7(YhN4}vhCCq>R^0@dKxlAtR^CE6c>tpXiZ?q^JMBcFpaao-2V9@@up8etu z#N|)o-rP>yMHqk_fa80%L(&fJjmBY*_Um)%`lB|`#(iweYc7F6&;8NoTKGy#EPta_ z4A>6)MVQHk!+v|~?zVA76p+8RCpwCi&+lzaC58{5=LoZ?}$L}55kVD7a8=I29!(I)9E%xkePQw1jJ$s_<=qJCS z6@{*)I)h#4z*(f)i?<+lse4t^9q7+&Ma=pgdKxET!%yLUPlZ_&-f8ZM4eIxN+ht^b Uv*d>lQ_UCW@NfO2&(LT87i6u7lmGw# literal 0 HcmV?d00001 diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index 08329cc..ffbe966 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -176,7 +176,7 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei wg.Wait() // add multiplier for the timeout depending on the block amount - timeout := 10*time.Second + (time.Duration(blockAmount) / 5) + timeout := 10*time.Second + (time.Duration(blockAmount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() err := d.dbPool.InsertBlocks(ctx, blocksData) @@ -271,7 +271,7 @@ func (d *DataProcessor) ProcessTransactions( transactionsData = transactionsData[:validCount] // add multiplier for the timeout depending on the transaction amount - timeout := 10*time.Second + (time.Duration(transactionAmount) / 5) + timeout := 10*time.Second + (time.Duration(transactionAmount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() err := d.dbPool.InsertTransactionsGeneral(ctx, transactionsData) @@ -415,7 +415,7 @@ func (d *DataProcessor) ProcessMessages( // we need to get all of the addresses from the aggregatedDbGroups addresses := createAddressTx(aggregatedDbGroups) // add multiplier for the timeout depending on the address amount - timeout := 10*time.Second + time.Duration(len(addresses)) + timeout := 10*time.Second + (time.Duration(len(addresses)) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertAddressTx(ctx, addresses) cancel() @@ -577,7 +577,7 @@ func (d *DataProcessor) ProcessValidatorSignings( validatorData = append(validatorData, *validator) } - timeout := 10*time.Second + (time.Duration(commitAmount) / 5) + timeout := 10*time.Second + (time.Duration(commitAmount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertValidatorBlockSignings(ctx, validatorData) cancel() diff --git a/pkgs/database/queries_api.go b/pkgs/database/queries_api.go index b980e0e..216907a 100644 --- a/pkgs/database/queries_api.go +++ b/pkgs/database/queries_api.go @@ -168,7 +168,11 @@ func (t *TimescaleDb) GetFromToBlocks(ctx context.Context, fromHeight uint64, to // Returns: // - *BlockSigners: the block signers // - error: if the query fails -func (t *TimescaleDb) GetAllBlockSigners(ctx context.Context, chainName string, blockHeight uint64) (*BlockSigners, error) { +func (t *TimescaleDb) GetAllBlockSigners( + ctx context.Context, + chainName string, + blockHeight uint64, +) (*BlockSigners, error) { query := ` SELECT vb.block_height, @@ -203,9 +207,13 @@ func (t *TimescaleDb) GetAllBlockSigners(ctx context.Context, chainName string, // - chainName: the name of the chain // // Returns: -// - *BankSend: the bank send message +// - []*BankSend: the bank send messages // - error: if the query fails -func (t *TimescaleDb) GetBankSend(ctx context.Context, txHash string, chainName string) (*BankSend, error) { +func (t *TimescaleDb) GetBankSend( + ctx context.Context, + txHash string, + chainName string, +) ([]*BankSend, error) { query := ` SELECT encode(bms.tx_hash, 'base64') AS tx_hash, @@ -224,13 +232,31 @@ func (t *TimescaleDb) GetBankSend(ctx context.Context, txHash string, chainName WHERE bms.tx_hash = decode($1, 'base64') AND bms.chain_name = $2 ` - row := t.pool.QueryRow(ctx, query, txHash, chainName) - var bankSend BankSend - err := row.Scan(&bankSend.TxHash, &bankSend.Timestamp, &bankSend.FromAddress, &bankSend.ToAddress, &bankSend.Amount, &bankSend.Signers) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &bankSend, nil + defer rows.Close() + bankSends := make([]*BankSend, 0) + for rows.Next() { + bankSend := &BankSend{} + err := rows.Scan( + &bankSend.TxHash, + &bankSend.Timestamp, + &bankSend.FromAddress, + &bankSend.ToAddress, + &bankSend.Amount, + &bankSend.Signers, + ) + if err != nil { + return nil, err + } + bankSends = append(bankSends, bankSend) + } + if err := rows.Err(); err != nil { + return nil, err + } + return bankSends, nil } // GetMsgCall gets the msg call message for a given transaction hash @@ -244,9 +270,13 @@ func (t *TimescaleDb) GetBankSend(ctx context.Context, txHash string, chainName // - chainName: the name of the chain // // Returns: -// - *MsgCall: the msg call message +// - []*MsgCall: the msg call messages // - error: if the query fails -func (t *TimescaleDb) GetMsgCall(ctx context.Context, txHash string, chainName string) (*MsgCall, error) { +func (t *TimescaleDb) GetMsgCall( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgCall, error) { query := ` SELECT encode(vmc.tx_hash, 'base64') AS tx_hash, @@ -268,24 +298,35 @@ func (t *TimescaleDb) GetMsgCall(ctx context.Context, txHash string, chainName s WHERE vmc.tx_hash = decode($1, 'base64') AND vmc.chain_name = $2 ` - row := t.pool.QueryRow(ctx, query, txHash, chainName) - var msgCall MsgCall - err := row.Scan( - &msgCall.TxHash, - &msgCall.MessageCounter, - &msgCall.Timestamp, - &msgCall.Caller, - &msgCall.PkgPath, - &msgCall.FuncName, - &msgCall.Args, - &msgCall.Send, - &msgCall.MaxDeposit, - &msgCall.Signers, - ) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &msgCall, nil + defer rows.Close() + msgCalls := make([]*MsgCall, 0) + for rows.Next() { + msgCall := &MsgCall{} + err := rows.Scan( + &msgCall.TxHash, + &msgCall.MessageCounter, + &msgCall.Timestamp, + &msgCall.Caller, + &msgCall.PkgPath, + &msgCall.FuncName, + &msgCall.Args, + &msgCall.Send, + &msgCall.MaxDeposit, + &msgCall.Signers, + ) + if err != nil { + return nil, err + } + msgCalls = append(msgCalls, msgCall) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgCalls, nil } // GetMsgAddPackage gets the msg add package message for a given transaction hash @@ -299,9 +340,13 @@ func (t *TimescaleDb) GetMsgCall(ctx context.Context, txHash string, chainName s // - chainName: the name of the chain // // Returns: -// - *MsgAddPackage: the msg add package message +// - []*MsgAddPackage: the msg add package messages // - error: if the query fails -func (t *TimescaleDb) GetMsgAddPackage(ctx context.Context, txHash string, chainName string) (*MsgAddPackage, error) { +func (t *TimescaleDb) GetMsgAddPackage( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgAddPackage, error) { query := ` SELECT encode(vmap.tx_hash, 'base64') AS tx_hash, @@ -323,24 +368,35 @@ func (t *TimescaleDb) GetMsgAddPackage(ctx context.Context, txHash string, chain WHERE vmap.tx_hash = decode($1, 'base64') AND vmap.chain_name = $2 ` - row := t.pool.QueryRow(ctx, query, txHash, chainName) - var msgAddPackage MsgAddPackage - err := row.Scan( - &msgAddPackage.TxHash, - &msgAddPackage.MessageCounter, - &msgAddPackage.Timestamp, - &msgAddPackage.Creator, - &msgAddPackage.PkgPath, - &msgAddPackage.PkgName, - &msgAddPackage.PkgFileNames, - &msgAddPackage.Send, - &msgAddPackage.MaxDeposit, - &msgAddPackage.Signers, - ) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { return nil, err } - return &msgAddPackage, nil + defer rows.Close() + msgAddPackages := make([]*MsgAddPackage, 0) + for rows.Next() { + msgAddPackage := &MsgAddPackage{} + err := rows.Scan( + &msgAddPackage.TxHash, + &msgAddPackage.MessageCounter, + &msgAddPackage.Timestamp, + &msgAddPackage.Creator, + &msgAddPackage.PkgPath, + &msgAddPackage.PkgName, + &msgAddPackage.PkgFileNames, + &msgAddPackage.Send, + &msgAddPackage.MaxDeposit, + &msgAddPackage.Signers, + ) + if err != nil { + return nil, err + } + msgAddPackages = append(msgAddPackages, msgAddPackage) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgAddPackages, nil } // GetMsgRun gets the msg run message for a given transaction hash @@ -354,9 +410,13 @@ func (t *TimescaleDb) GetMsgAddPackage(ctx context.Context, txHash string, chain // - chainName: the name of the chain // // Returns: -// - *MsgRun: the msg run message +// - []*MsgRun: the msg run messages // - error: if the query fails -func (t *TimescaleDb) GetMsgRun(ctx context.Context, txHash string, chainName string) (*MsgRun, error) { +func (t *TimescaleDb) GetMsgRun( + ctx context.Context, + txHash string, + chainName string, +) ([]*MsgRun, error) { query := ` SELECT encode(vmr.tx_hash, 'base64') AS tx_hash, @@ -378,25 +438,35 @@ func (t *TimescaleDb) GetMsgRun(ctx context.Context, txHash string, chainName st WHERE vmr.tx_hash = decode($1, 'base64') AND vmr.chain_name = $2 ` - row := t.pool.QueryRow(ctx, query, txHash, chainName) - var msgRun MsgRun - err := row.Scan( - &msgRun.TxHash, - &msgRun.MessageCounter, - &msgRun.Timestamp, - &msgRun.Caller, - &msgRun.PkgPath, - &msgRun.PkgName, - &msgRun.PkgFileNames, - &msgRun.Send, - &msgRun.MaxDeposit, - &msgRun.Signers, - ) + rows, err := t.pool.Query(ctx, query, txHash, chainName) if err != nil { - log.Println("error getting msg run", err) return nil, err } - return &msgRun, nil + defer rows.Close() + msgRuns := make([]*MsgRun, 0) + for rows.Next() { + msgRun := &MsgRun{} + err := rows.Scan( + &msgRun.TxHash, + &msgRun.MessageCounter, + &msgRun.Timestamp, + &msgRun.Caller, + &msgRun.PkgPath, + &msgRun.PkgName, + &msgRun.PkgFileNames, + &msgRun.Send, + &msgRun.MaxDeposit, + &msgRun.Signers, + ) + if err != nil { + return nil, err + } + msgRuns = append(msgRuns, msgRun) + } + if err := rows.Err(); err != nil { + return nil, err + } + return msgRuns, nil } // GetTransaction gets the transaction for a given transaction hash @@ -491,7 +561,7 @@ func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string return transactions, nil } -// GetMsgType gets the message type for a given transaction hash +// GetMsgTypes gets the message type for a given transaction hash // // Usage: // @@ -502,9 +572,9 @@ func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string // - chainName: the name of the chain // // Returns: -// - string: the message type +// - []string: the message types // - error: if the query fails -func (t *TimescaleDb) GetMsgType(ctx context.Context, txHash string, chainName string) (string, error) { +func (t *TimescaleDb) GetMsgTypes(ctx context.Context, txHash string, chainName string) ([]string, error) { query := ` SELECT msg_types FROM transaction_general @@ -512,20 +582,12 @@ func (t *TimescaleDb) GetMsgType(ctx context.Context, txHash string, chainName s AND chain_name = $2 ` row := t.pool.QueryRow(ctx, query, txHash, chainName) - var msgType []string - // to future me - // in the events the transactions can harbor more transaction types this - // will not work, for now I only have seen with one message type per transaction - // if this happens at least throw some log warning - if len(msgType) >= 2 { - log.Println("warning: transaction has more than one message type", msgType) - return msgType[0], nil - } - err := row.Scan(&msgType) + var msgTypes []string + err := row.Scan(&msgTypes) if err != nil { - return "", err + return nil, err } - return msgType[0], nil + return msgTypes, nil } // GetAddressTxs gets the transactions for a given address for a certain time period diff --git a/pkgs/database/queries_indexer.go b/pkgs/database/queries_indexer.go index 6f9dd02..18870cb 100644 --- a/pkgs/database/queries_indexer.go +++ b/pkgs/database/queries_indexer.go @@ -110,7 +110,7 @@ func (t *TimescaleDb) GetAllAddresses( AND id > $2 ` } - rows, err := t.pool.Query(ctx, query, chainName, highestIndex) + rows, err := t.pool.Query(ctx, query, chainName, maxIndex) if err != nil { return nil, 0, err } From 1d4ef8cacc7356f2924dca8e51b488d453dd94f0 Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 19:00:27 +0100 Subject: [PATCH 08/11] Fixed some linting warnings and cleared the error for the transaction messages --- api/handlers/transactions.go | 8 ++++---- api/main.go | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go index 08e0d08..1083e3c 100644 --- a/api/handlers/transactions.go +++ b/api/handlers/transactions.go @@ -62,7 +62,7 @@ func (h *TransactionsHandler) GetTransactionMessage( case "bank_msg_send": data, err := h.db.GetBankSend(ctx, txHashBase64, h.chainName) if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) } for _, data := range data { index := data.MessageCounter @@ -79,7 +79,7 @@ func (h *TransactionsHandler) GetTransactionMessage( case "vm_msg_call": data, err := h.db.GetMsgCall(ctx, txHashBase64, h.chainName) if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) } for _, data := range data { index := data.MessageCounter @@ -99,7 +99,7 @@ func (h *TransactionsHandler) GetTransactionMessage( case "vm_msg_add_package": data, err := h.db.GetMsgAddPackage(ctx, txHashBase64, h.chainName) if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) } for _, data := range data { index := data.MessageCounter @@ -119,7 +119,7 @@ func (h *TransactionsHandler) GetTransactionMessage( case "vm_msg_run": data, err := h.db.GetMsgRun(ctx, txHashBase64, h.chainName) if err != nil { - return nil, huma.Error404NotFound(fmt.Sprintf("Transaction with hash %s not found", input.TxHash), err) + return nil, huma.Error404NotFound(fmt.Sprintf("Failed to fetch %s data for transaction %s", msgType, input.TxHash), err) } for _, data := range data { index := data.MessageCounter diff --git a/api/main.go b/api/main.go index cf2d733..667f53e 100644 --- a/api/main.go +++ b/api/main.go @@ -151,7 +151,10 @@ var rootCmd = &cobra.Command{ // favicon route router.Get("/favicon.ico", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "image/x-icon") - w.Write(favicon) + _, err := w.Write(favicon) + if err != nil { + log.Fatalf("failed to write favicon: %v", err) + } }) // Register Block API routes From aa4953673d0ad3709d5b0e07aa51dafb42e51869 Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 20:39:55 +0100 Subject: [PATCH 09/11] Add changelog and update go version used --- CHANGELOG.md | 16 ++++++++++++++++ go.mod | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d3805..09a747e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2025-11-26 + +Mostly it has some bug fixes. + +### Fixed + +- The indexer would go into the blocks data and store the signers from the last commit, which is actually all of the block signers from the previous block. So it woult insert it like it was meant for that block height. From now on the indexer will fetch data from the /commit method and insert it properly. [de40740](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/de407407e67ae588141b361307fae18215f53a18) +- Gnoland can indeed execute multiple message types in the same transaction. The indexer wasn't able to hold this data properly and would cause and error because in the postgres primary key was attached making each message type unique. Now each message type has a message_counter which is a smallint(int16) which is used as a index for that transaction. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) + +### Changes + +- The REST API now returns a map of int16 and message data for the `/transaction/{tx_hash}/message` route. [74a35b30](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/74a35b301070bb1dab5cbadec6fe64b16ad7eb3b) +- The indexer should stop using sync.Map and use sync.Mutex with regular map to store the addresses before handing the operation to the AddressSolver function. [dde82a4](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/dde82a4ecc9bce50963fbff3f4e13a3a20b47f9d) +- The Orchestrator needs to make a fetch to the commit RPC method. This operation is done side by side with fetching the blocks method. A side effect of this is that indexer at that moment will fecth 2X times more request to the RPC, and if the Gnoland node has a limit on the amount of RPC clients that can use it, it might cause the indexer to slow down and throw errors on this requests. In future releases this should be improved. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) +- Updated the go version to 1.25.4 + ## [0.3.0] - 2025-11-10 In this release there are some fixes and improvements. The live process should work properly now and the REST API has some new routes. CLI commands are now combined with the ones from the setup cli. Some processes have been improved to use less memory. diff --git a/go.mod b/go.mod index 898a65b..26eb2d0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Cogwheel-Validator/spectra-gnoland-indexer -go 1.25.1 +go 1.25.4 require ( github.com/caarlos0/env/v6 v6.10.1 From 833025013a2ae3925352b441eae20e184ef366ea Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 21:12:08 +0100 Subject: [PATCH 10/11] Corrected grammer, simplifed the makefile so it installs only the main binary, removed expermintal-install option from the make file --- CHANGELOG.md | 6 +++--- Makefile | 11 +++-------- api/main.go | 4 +++- indexer/data_processor/operator.go | 8 ++++---- indexer/{main.go => indexer.go} | 0 pkgs/generator/generator_test.go | 8 ++++++-- 6 files changed, 19 insertions(+), 18 deletions(-) rename indexer/{main.go => indexer.go} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a747e..e953e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,14 +11,14 @@ Mostly it has some bug fixes. ### Fixed -- The indexer would go into the blocks data and store the signers from the last commit, which is actually all of the block signers from the previous block. So it woult insert it like it was meant for that block height. From now on the indexer will fetch data from the /commit method and insert it properly. [de40740](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/de407407e67ae588141b361307fae18215f53a18) -- Gnoland can indeed execute multiple message types in the same transaction. The indexer wasn't able to hold this data properly and would cause and error because in the postgres primary key was attached making each message type unique. Now each message type has a message_counter which is a smallint(int16) which is used as a index for that transaction. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) +- The indexer would go into the blocks data and store the signers from the last commit, which is actually all of the block signers from the previous block. So it would insert it like it was meant for that block height. From now on the indexer will fetch data from the /commit method and insert it properly. [de40740](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/de407407e67ae588141b361307fae18215f53a18) +- Gnoland can indeed execute multiple message types in the same transaction. The indexer wasn't able to hold this data properly and would cause an error because in the postgres primary key was attached making each message type unique. Now each message type has a message_counter which is a smallint(int16) which is used as a index for that transaction. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) ### Changes - The REST API now returns a map of int16 and message data for the `/transaction/{tx_hash}/message` route. [74a35b30](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/74a35b301070bb1dab5cbadec6fe64b16ad7eb3b) - The indexer should stop using sync.Map and use sync.Mutex with regular map to store the addresses before handing the operation to the AddressSolver function. [dde82a4](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/dde82a4ecc9bce50963fbff3f4e13a3a20b47f9d) -- The Orchestrator needs to make a fetch to the commit RPC method. This operation is done side by side with fetching the blocks method. A side effect of this is that indexer at that moment will fecth 2X times more request to the RPC, and if the Gnoland node has a limit on the amount of RPC clients that can use it, it might cause the indexer to slow down and throw errors on this requests. In future releases this should be improved. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) +- The Orchestrator needs to make a fetch to the commit RPC method. This operation is done side by side with fetching the blocks method. A side effect of this is that indexer at that moment will fetch 2 times more request to the RPC, and if the Gnoland node has a limit on the amount of RPC clients that can use it, it might cause the indexer to slow down and throw errors on this requests. In future releases this should be improved. [d647901](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d64790181c626e2ac0137dd9cce08d10ec2b6a7c) - Updated the go version to 1.25.4 ## [0.3.0] - 2025-11-10 diff --git a/Makefile b/Makefile index 6dc0139..e77b38f 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,10 @@ VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_BRANCH)-$(GIT_COMMIT)) build: mkdir -p build - go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer indexer/main.go + go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer indexer/indexer.go install: - go install -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" indexer/main.go + go install -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" indexer/indexer.go build-api: mkdir -p build @@ -28,12 +28,7 @@ clean: # experimental build with greentea garbage collection # use at your own risk build-experimental: - GOEXPERIMENT=greenteagc go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer-tea indexer/main.go - -# experimental install with greentea garbage collection -# use at your own risk -install-experimental: - cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" + GOEXPERIMENT=greenteagc go build -ldflags="-X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=$(GIT_COMMIT) -X github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=$(VERSION)" -o build/indexer-tea indexer/indexer.go ######################################################## # Test the indexer diff --git a/api/main.go b/api/main.go index 667f53e..10728ce 100644 --- a/api/main.go +++ b/api/main.go @@ -153,7 +153,9 @@ var rootCmd = &cobra.Command{ w.Header().Set("Content-Type", "image/x-icon") _, err := w.Write(favicon) if err != nil { - log.Fatalf("failed to write favicon: %v", err) + log.Printf("failed to write favicon: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return } }) diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go index ffbe966..b5b38bc 100644 --- a/indexer/data_processor/operator.go +++ b/indexer/data_processor/operator.go @@ -449,7 +449,7 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e // Insert DbMsgSend messages with address IDs if msgSendCount > 0 { - timeout := 10*time.Second + time.Duration(msgSendCount) + timeout := 10*time.Second + (time.Duration(msgSendCount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertMsgSend(ctx, groups.MsgSend) cancel() @@ -464,7 +464,7 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e // Insert DbMsgCall messages with address IDs if msgCallCount > 0 { - timeout := 10*time.Second + time.Duration(msgCallCount) + timeout := 10*time.Second + (time.Duration(msgCallCount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertMsgCall(ctx, groups.MsgCall) cancel() @@ -479,7 +479,7 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e // Insert DbMsgAddPackage messages with address IDs if msgAddPkgCount > 0 { - timeout := 10*time.Second + time.Duration(msgAddPkgCount) + timeout := 10*time.Second + (time.Duration(msgAddPkgCount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertMsgAddPackage(ctx, groups.MsgAddPkg) cancel() @@ -494,7 +494,7 @@ func (d *DataProcessor) insertDbMessageGroups(groups *decoder.DbMessageGroups) e // Insert DbMsgRun messages with address IDs if msgRunCount > 0 { - timeout := 10*time.Second + time.Duration(msgRunCount) + timeout := 10*time.Second + (time.Duration(msgRunCount) * time.Second / 5) ctx, cancel := context.WithTimeout(context.Background(), timeout) err := d.dbPool.InsertMsgRun(ctx, groups.MsgRun) cancel() diff --git a/indexer/main.go b/indexer/indexer.go similarity index 100% rename from indexer/main.go rename to indexer/indexer.go diff --git a/pkgs/generator/generator_test.go b/pkgs/generator/generator_test.go index 4837304..9029204 100644 --- a/pkgs/generator/generator_test.go +++ b/pkgs/generator/generator_test.go @@ -27,15 +27,19 @@ func TestAuthenticGeneration(t *testing.T) { // Test key pair access for range 500 { - generator.GetRandomKeyPair() + kp := generator.GetRandomKeyPair() + assert.NotNil(t, kp) + assert.NotEmpty(t, kp.Address) + assert.NotEmpty(t, kp.PublicKey) } // Test synthetic transaction generation 500 times for range 500 { events, tx := generator.GenerateTransaction() assert.NotNil(t, tx) - assert.NotNil(t, events) + assert.NotNil(t, events.Events) } + } // RunAllTests runs all the test functions From 89e1d72564fc40caa2b9542140df62195cd0d662 Mon Sep 17 00:00:00 2001 From: NM Date: Wed, 26 Nov 2025 21:19:44 +0100 Subject: [PATCH 11/11] Fix workflow integration test --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c1cd640..a298fe5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -57,7 +57,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db --db-user postgres --db-name postgres + go run indexer/indexer.go setup create-db --db-user postgres --db-name postgres env: DB_HOST: localhost DB_PORT: 5432 @@ -131,7 +131,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db --db-user postgres --db-name postgres + go run indexer/indexer.go setup create-db --db-user postgres --db-name postgres env: DB_HOST: localhost DB_PORT: 5432 @@ -205,7 +205,7 @@ jobs: - name: Initialize database schema run: | - go run indexer/main.go setup create-db + go run indexer/indexer.go setup create-db env: DB_HOST: localhost DB_PORT: 5432