diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0b66969..1bbf67c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -115,10 +115,11 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- - name: Build and push Docker image
+ - name: Build and push Docker Indexer image
uses: docker/build-push-action@v5
with:
context: .
+ file: Dockerfile
push: true
platforms: linux/amd64
tags: |
@@ -130,4 +131,22 @@ jobs:
GIT_TAG=${{ steps.gitmeta.outputs.git_tag }}
GIT_BRANCH=${{ steps.gitmeta.outputs.git_branch }}
cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Build and push Docker API image
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ file: Dockerfile-api
+ push: true
+ platforms: linux/amd64
+ tags: |
+ ghcr.io/cogwheel-validator/spectra-gnoland-api:${{ steps.version.outputs.version }}
+ ghcr.io/cogwheel-validator/spectra-gnoland-api:${{ steps.version.outputs.docker_tag }}
+ build-args: |
+ VERSION=${{ steps.version.outputs.version }}
+ GIT_COMMIT=${{ steps.gitmeta.outputs.git_commit }}
+ GIT_TAG=${{ steps.gitmeta.outputs.git_tag }}
+ GIT_BRANCH=${{ steps.gitmeta.outputs.git_branch }}
+ cache-from: type=gha
cache-to: type=gha,mode=max
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index f772e8f..aa95985 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ config.yml
test_config.yml
.env
config-api.yml
+training-config.yml
# State dumps and diagnostics directories
state_dumps/
@@ -48,4 +49,8 @@ server.crt
server.key
# Jet Brains
-.idea/
\ No newline at end of file
+.idea/
+
+# Docker
+.dockerignore
+docker-compose-local.yml
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 19f19c6..973723c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.25.4-trixie AS builder
+FROM golang:1.25 AS builder
WORKDIR /app
@@ -14,14 +14,18 @@ RUN if [ -z "$VERSION" ]; then \
else VERSION="${GIT_BRANCH}-${GIT_COMMIT}"; \
fi; \
fi && \
- go build -ldflags="-X 'github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=${GIT_COMMIT}' -X 'github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=${VERSION}'" -o indexer ./indexer
+ go build -ldflags="-s -w -X 'github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Commit=${GIT_COMMIT}' -X 'github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd.Version=${VERSION}'" -o indexer ./indexer
RUN chmod +x indexer
+RUN touch config.yml
-FROM debian:trixie-slim
+FROM gcr.io/distroless/base-debian13:latest
WORKDIR /app
-COPY --from=builder /app/indexer /app/indexer
+COPY --from=builder /app/indexer .
+COPY --from=builder --chown=nonroot:nonroot /app/config.yml .
+
+USER nonroot
ENTRYPOINT ["/app/indexer"]
\ No newline at end of file
diff --git a/Dockerfile-api b/Dockerfile-api
new file mode 100644
index 0000000..758c44d
--- /dev/null
+++ b/Dockerfile-api
@@ -0,0 +1,30 @@
+FROM golang:1.25 AS builder
+
+WORKDIR /app
+
+COPY . .
+
+ARG GIT_COMMIT=""
+ARG GIT_TAG=""
+ARG GIT_BRANCH=""
+ARG VERSION=""
+
+RUN if [ -z "$VERSION" ]; then \
+ if [ -n "$GIT_TAG" ]; then VERSION="$GIT_TAG"; \
+ else VERSION="${GIT_BRANCH}-${GIT_COMMIT}"; \
+ fi; \
+ fi && \
+ go build -ldflags="-s -w -X main.Commit=${GIT_COMMIT} -X main.Version=${VERSION}" -o build/api ./api
+
+RUN touch config-api.yml
+RUN chmod +x build/api
+
+FROM gcr.io/distroless/base-debian13:latest
+WORKDIR /app
+
+COPY --from=builder --chown=nonroot:nonroot /app/build/api .
+COPY --from=builder --chown=nonroot:nonroot /app/config-api.yml .
+
+USER nonroot
+
+ENTRYPOINT ["/app/api"]
\ No newline at end of file
diff --git a/Makefile b/Makefile
index e77b38f..03e5b02 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,15 @@
-.PHONY: build install clean build-experimental install-experimental build-api test-race integration-test test
+.PHONY: build
+ install
+ clean
+ build-experimental
+ install-experimental
+ build-api
+ integration-test
+ test
+ vulnerability-scan
+ snyk
+ semgrep
+ code-quality
########################################################
# Build and install the indexer
@@ -37,8 +48,37 @@ build-experimental:
test:
go test -v ./...
-test-race:
- go test -v -race ./...
-
integration-test:
- cd integration && go test -v -tags=integration -timeout=20m ./...
\ No newline at end of file
+ cd integration && go test -v -tags=integration -timeout=20m ./...
+
+########################################################
+# Vulnerability scanning
+########################################################
+
+vulnerability-scan:
+ govulncheck ./...
+
+snyk:
+ snyk test
+
+semgrep:
+ semgrep ci
+
+########################################################
+# Code quality
+########################################################
+
+code-quality:
+ golangci-lint run
+
+########################################################
+# Train the zstd dictionary
+########################################################
+
+.PHONY: train-zstd
+
+train-zstd:
+ @echo "Training the zstd dictionary"
+ @read -p "Enter the amount of events to collect (default: 10000): " amount; \
+ amount=$${amount:-10000}; \
+ go run compression/cmd/main.go --config training-config.yml --amount $$amount --chain-name gnoland --dict-path ./pkgs/dict_loader/events.zstd.bin
\ No newline at end of file
diff --git a/README.md b/README.md
index e1716dc..8420484 100644
--- a/README.md
+++ b/README.md
@@ -4,99 +4,109 @@
This project is still in development. Future releases might have some breaking changes.
-The Spectra Gnoland Indexer(SGI) is a tool that records the data from the Gnoland blockchain and stores the data
-in the timeseries database, Timescale DB. This indexer will be a part of the Spectra explorer and will be used to
+The Spectra Gnoland Indexer(SGI) is a tool that records the data from the Gnoland blockchain and stores the data
+in the timeseries database, Timescale DB. This indexer will be a part of the Spectra explorer and will be used to
store the data for the explorer. This program can be used for any kind of data/analytics program or app.
-The biggest problem when dealing with the blockchain data is how to store the data in a way that is easy to query
-and analyze. Some projects rely on the direct access to the data from the blockchain nodes and the problem in this
-case is that the nodes are meant for multiple purposes.
+The biggest problem when dealing with the blockchain data is how to store the data in a way that is easy to query
+and analyze. Some projects rely on the direct access to the data from the blockchain nodes and the problem in this
+case is that the nodes are meant for multiple purposes.
-They usually provide at least one, sometimes even more, endpoints for querying the data, then this node need to be
-able to work with other nodes and to acquire the block data from other nodes via P2P network. Most nodes can be
+They usually provide at least one, sometimes even more, endpoints for querying the data, then this node need to be
+able to work with other nodes and to acquire the block data from other nodes via P2P network. Most nodes can be
used as a miners/validators that need to sign the blocks. Even if these nodes are only to be used as a RPC nodes
-they can still provide the data but depending on the underlying SDK, tech stack and other factors it might be
+they can still provide the data but depending on the underlying SDK, tech stack and other factors it might be
not the best choice. And finally the nodes are not the best when it comes to storing the data and scalability.
## Table of content
+
- [Table of content](#table-of-content)
- [The solution](#the-solution)
- [Why TimescaleDB? And can it work on other SQL databases?](#why-timescaledb-and-can-it-work-on-other-sql-databases)
- [How does the indexer work?](#how-does-the-indexer-work)
- [What data is stored in the database?](#what-data-is-stored-in-the-database)
- - [Blocks:](#blocks)
- - [Validator signings:](#validator-signings)
- - [Transactions:](#transactions)
+ - [Blocks](#blocks)
+ - [Validator signings](#validator-signings)
+ - [Transactions](#transactions)
- [Pros and cons of the SGI](#pros-and-cons-of-the-sgi)
- - [🦾 Pros:](#-pros)
- - [🐞 Cons:](#-cons)
+ - [🦾 Pros](#-pros)
+ - [🐞 Cons](#-cons)
- [In depth documentation](#in-depth-documentation)
## The solution
-The data in the blockchain is mostly tied to blocks, however when any kind of analytics is needed we need some
+The data in the blockchain is mostly tied to blocks, however when any kind of analytics is needed we need some
parameter that falls familiar to us from the daily life. So it comes more natural to compare the data with some
-sort of time parameter then to a block height. Then the data needs to be stored in a way that is easy to query and
-analyze. And since we are dealing with the time parameter, we need to have the ability to aggregate the data over
+sort of time parameter then to a block height. Then the data needs to be stored in a way that is easy to query and
+analyze. And since we are dealing with the time parameter, we need to have the ability to aggregate the data over
some time period so we can timeseries and easily plot the data.
-A lot of indexers use NoSQL databases for this use case. However they usually have their own problems and
+A lot of indexers use NoSQL databases for this use case. However they usually have their own problems and
limitations. They are not as flexible as the SQL databases when it comes to the data types and the query language.
that is the reason why the TimescaleDB is a perfect fit for this use case.
Anyone with some experience with the SQL, especially with the PostgreSQL, can easily understand the TimescaleDB.
-It is a extension of the PostgreSQL that adds the ability to store the time series data in a way that is easy to
+It is a extension of the PostgreSQL that adds the ability to store the time series data in a way that is easy to
query and analyze.
## Why TimescaleDB? And can it work on other SQL databases?
-TimescaleDB (Tiger Data is their commercial version) is a extension of the PostgreSQL that adds the ability to
-store the time series data in a way that is easy to query and analyze. It also has a lot of features that can
+TimescaleDB (Tiger Data is their commercial version) is a extension of the PostgreSQL that adds the ability to
+store the time series data in a way that is easy to query and analyze. It also has a lot of features that can
extend the capabilities of the PostgreSQL and make it more powerful.
+The TimescaleDB sits between the OLTP and the OLAP databases. It is a hybrid database that can be used for both.
+It is a good fit for the time series data and it is a good fit for the analytics and the exploration of the data.
+
The TimescaleDB has also feature of it's own that is not present in the Postgres. The user could extend the indexer
by adding the data aggregation features, automatic jobs scheduling, hyperfunctions and more.
-This database extension also has a data compression feature that can be used to reduce the storage space and
+This database extension also has a data compression feature that can be used to reduce the storage space and
segment the data into smaller chunks by using time based intervals making the queries faster.
-Technically speaking the indexer can work on Postgres database, however you would need to create the tables and
-types manually. This might be added later in the future if there is a demand for it but for now the focus is on the
+Technically speaking the indexer can work on Postgres database, however you would need to create the tables and
+types manually. This might be added later in the future if there is a demand for it but for now the focus is on the
TimescaleDB.
There are some other SQL databases that could work in theory. For them to work they would need to have:
+
- The postgres wire protocol
- Equivalent of Numeric type ( some databases might have it as DECIMAL, but not all of them )
- The ability to create custom types
-If all of the above are met then the indexer can work on them. It might work on CockroachDB for example but this is
+If all of the above are met then the indexer can work on them. It might work on CockroachDB for example but this is
out of the scope of this project. Maybe in the future it might be an interesting idea to support it.
## How does the indexer work?
The indexer has 2 main modes of operation:
+
- Live mode
- Historic mode
-Live mode is the mode that is used to index the data in the real time. It will sync up the database to the latest
-block height and will continue to index the data in the real time.
+Live mode is the mode that is used to index the data in the real time. It will sync up the database to the latest
+block height and will continue to index the data in the real time. It works by using near-real-time batch processing method. It will collect all of the data up to the latest block height and then it will process it in
+batches. So the process is not instant but it is very fast and it is able to keep up with the latest block height.
-Historic mode is the mode that allows the user to index a certain range of blocks. For example you might not need
-the whole chain history or just need a part of it. This mode is useful for the testing, partial indexing of the
+Historic mode is the mode that allows the user to index a certain range of blocks. For example you might not need
+the whole chain history or just need a part of it. This mode is useful for the testing, partial indexing of the
chain or gradual indexing of the chain.
-Durring both modes the indexer will use fan out method to send the requests to the RPC node. Then all of the data
-is collected and processed and decoded in parallel. All of the addresses are collected and stored in the database
-and the indexer stores them in it's own cache. Then any address that is found in the transaction is referanced by
-the integer value in the transaction tables. At the end of the processing the data is inserted in batches into the
+During both modes the indexer will use fan out method to send the requests to the RPC node. Then all of the data
+is collected and processed and decoded in parallel. All of the addresses are collected and stored in the database
+and the indexer stores them in it's own cache. Then any address that is found in the transaction is referenced by
+the integer value in the transaction tables. At the end of the processing the data is inserted in batches into the
database.
## What data is stored in the database?
-The indexer stores the esential data that is related to the blocks, transactions, messages, and accounts.
+The indexer stores the essential data that is related to the blocks, transactions, messages, and accounts.
It also stores the validator block signings and the validator addresses.
-This way it is able to provide a lot of data for the analytics and the explorer.
+The data schema is denormalized to decrease the space needed and to provide the easy and fast access for the
+explorer and any other visualizations and analytics tools.
+
The data is stored in the following tables:
+
- blocks
- transactions general data
- transaction messages (each message type has its own table)
@@ -107,8 +117,10 @@ The data is stored in the following tables:
Some of the data is not indexed and it is not planned to be indexed in the future. Such as:
-### Blocks:
+### Blocks
+
Stored data:
+
- Basic block hash
- Block height
- Block timestamp
@@ -117,6 +129,7 @@ Stored data:
- Block transactions hashes
Not stored:
+
- Last commit hash
- App hash
- Data hash
@@ -124,40 +137,44 @@ Not stored:
- Next validators hash
- Consensus hash
-### Validator signings:
+### Validator signings
+
Stored data:
+
- Validator block signing height
- Validator block signing timestamp
- Validator block signing signed validators
- Proposer address
Not stored:
+
- Missed validators
- Precommits and all of the hashes and other data, so only a confirmation that the validator signed the block
-### Transactions:
-Almost all of the data regarding to the transaction and the messages are stored with the exception for the
+### Transactions
+
+Almost all of the data regarding to the transaction and the messages are stored with the exception for the
VM message Add Package and Call where in theory one could extract even the body of the smart contract. So this is unnecessary to store for explorers and other analytics tools. This might be added in the future if there is a demand for it.
## Pros and cons of the SGI
-### 🦾 Pros:
+### 🦾 Pros
- The indexer process the data using goroutines and channels, which can provide a faster processing of the data.
- Fast data processing. [see benchmarks](./docs/benchmarks.md)
- The program has 2 modes that can be used for the indexing of the data. This can be useful for the testing, partial indexing of the chain or gradual indexing of the chain.
- The data is stored ready to be used for any kind of analytics and visualization with any programming language.
-- No need to deal with Amino encoding and decoding for the messagess as the indexer decodes the messages and stores them in the database.
+- No need to deal with Amino encoding and decoding for the messages as the indexer decodes the messages and stores them in the database.
- It comes with a REST API to get you started quickly.
- It relies on a SQL database, which can provide a easier experience for any user that is familiar with the SQL.
-- It uses TimescaleDB, a PostgresSQL extenstion that can be extended with any other extensions, plus the TimescaleDB has a lot of features that are not present in the Postgres.
+- It uses TimescaleDB, a PostgresSQL extension that can be extended with any other extensions, plus the TimescaleDB has a lot of features that are not present in the Postgres.
-### 🐞 Cons:
+### 🐞 Cons
-- The indexer has a address cache of all of the addresses that were ever used in the transactions. This gives the indexer ability to swap the addresses with the their integer index in the database. However this intorduces complexity. Anyone who plans to use the indexer and plans to make some custom solution on working with the data will need to fully understand the data structure and how to use it. The REST API provides a easy way to interact with the data and to get the data in a readable format.
+- The indexer has a address cache of all of the addresses that were ever used in the transactions. This gives the indexer ability to swap the addresses with the their integer index in the database. However this introduces complexity. Anyone who plans to use the indexer and plans to make some custom solution on working with the data will need to fully understand the data structure and how to use it. The REST API provides a easy way to interact with the data and to get the data in a readable format.
- The indexer relies on the RPC node for the data. If the RPC node is not available the indexer will not be able to index the data. ( although in the future the indexer might be able to use multiple RPC nodes )
- Technically the indexer has a limit of 2 billion addresses. If at any point the Gnoland grows to that size the indexer would need to be updated to support it. It is not a problem for now but it is something to keep in mind.
## In depth documentation
-For more detailed documentation, please refer to the [docs](./docs/README.md) directory.
\ No newline at end of file
+For more detailed documentation, please refer to the [docs](./docs/README.md) directory.
diff --git a/api/handlers/address.go b/api/handlers/address.go
index 6d12638..a22e6e4 100644
--- a/api/handlers/address.go
+++ b/api/handlers/address.go
@@ -2,6 +2,7 @@ package handlers
import (
"context"
+ "time"
humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types"
"github.com/danielgtaylor/huma/v2"
@@ -20,17 +21,42 @@ func (h *AddressHandler) GetAddressTxs(
ctx context.Context,
input *humatypes.AddressGetInput,
) (*humatypes.AddressGetOutput, error) {
- address, err := h.db.GetAddressTxs(
+ var fromTs, toTs *time.Time
+ if !input.FromTimestamp.IsZero() {
+ fromTs = &input.FromTimestamp
+ }
+ if !input.ToTimestamp.IsZero() {
+ toTs = &input.ToTimestamp
+ }
+ var limit, page *uint64
+ if input.Limit != 0 {
+ limit = &input.Limit
+ }
+ if input.Page != 0 {
+ page = &input.Page
+ }
+ var cursor *string
+ if input.Cursor != "" {
+ cursor = &input.Cursor
+ }
+ addressTxs, nextCursor, txCount, err := h.db.GetAddressTxs(
ctx,
input.Address,
h.chainName,
- input.FromTimestamp,
- input.ToTimestamp,
+ fromTs,
+ toTs,
+ limit,
+ page,
+ cursor,
)
if err != nil {
return nil, huma.Error404NotFound("Address not found", err)
}
return &humatypes.AddressGetOutput{
- Body: *address,
+ Body: humatypes.AddressTxsBody{
+ AddressTxs: *addressTxs,
+ TxCount: txCount,
+ NextCursor: nextCursor,
+ },
}, nil
}
diff --git a/api/handlers/address_test.go b/api/handlers/address_test.go
index 14fc7dd..680c172 100644
--- a/api/handlers/address_test.go
+++ b/api/handlers/address_test.go
@@ -27,12 +27,18 @@ func TestAddressHandler_GetAddressTxs_Success(t *testing.T) {
},
}
handler := handlers.NewAddressHandler(&db, "gnoland")
- response, err := handler.GetAddressTxs(context.Background(), &humatypes.AddressGetInput{Address: "gno_address_1"})
+ response, err := handler.GetAddressTxs(context.Background(), &humatypes.AddressGetInput{
+ Address: "gno_address_1",
+ Limit: 10,
+ Page: 0,
+ })
require.NoError(t, err)
require.NotNil(t, response)
- assert.Equal(t, 3, len(response.Body))
- assert.Equal(t, "tx_hash_1", response.Body[0].Hash)
+ assert.Equal(t, 3, len(response.Body.AddressTxs))
+ assert.Equal(t, "tx_hash_1", response.Body.AddressTxs[0].Hash)
+ assert.Equal(t, uint64(3), response.Body.TxCount)
+ assert.Equal(t, "", response.Body.NextCursor)
}
func TestAddressHandler_GetAddressTxs_Fail(t *testing.T) {
@@ -41,7 +47,11 @@ func TestAddressHandler_GetAddressTxs_Fail(t *testing.T) {
errorMsg: "error getting address transactions",
}
handler := handlers.NewAddressHandler(&db, "gnoland")
- response, err := handler.GetAddressTxs(context.Background(), &humatypes.AddressGetInput{Address: "gno_address_1"})
+ response, err := handler.GetAddressTxs(context.Background(), &humatypes.AddressGetInput{
+ Address: "gno_address_1",
+ Limit: 10,
+ Page: 0,
+ })
assert.Error(t, err)
assert.Nil(t, response)
diff --git a/api/handlers/convert_base64.go b/api/handlers/convert_base64.go
new file mode 100644
index 0000000..f145769
--- /dev/null
+++ b/api/handlers/convert_base64.go
@@ -0,0 +1,41 @@
+package handlers
+
+import (
+ "context"
+ "encoding/base64"
+
+ humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types"
+ "github.com/danielgtaylor/huma/v2"
+)
+
+func ConvertFromBase64toBase64Url(ctx context.Context, input *humatypes.ConvertFromBase64toBase64UrlInput) (*humatypes.ConvertFromBase64toBase64UrlOutput, error) {
+ if len(input.TxHash64) != 44 {
+ return nil, huma.Error400BadRequest("Input is not a valid tx hash")
+ }
+ decoded, err := base64.StdEncoding.DecodeString(input.TxHash64)
+ if err != nil {
+ return nil, huma.Error400BadRequest("Input is not valid base64 encoded", err)
+ }
+ response := humatypes.ConvertFromBase64toBase64UrlOutput{
+ Body: humatypes.ConvertFromBase64toBase64UrlBody{
+ TxHash64Url: base64.URLEncoding.EncodeToString(decoded),
+ },
+ }
+ return &response, nil
+}
+
+func ConvertFromBase64UrlToBase64(ctx context.Context, input *humatypes.ConvertFromBase64UrlToBase64Input) (*humatypes.ConvertFromBase64UrlToBase64Output, error) {
+ if len(input.TxHash64Url) != 44 {
+ return nil, huma.Error400BadRequest("Input is not a valid tx hash")
+ }
+ decoded, err := base64.URLEncoding.DecodeString(input.TxHash64Url)
+ if err != nil {
+ return nil, huma.Error400BadRequest("Input is not valid base64url encoded", err)
+ }
+ response := humatypes.ConvertFromBase64UrlToBase64Output{
+ Body: humatypes.ConvertFromBase64UrlToBase64Body{
+ TxHash64: base64.StdEncoding.EncodeToString(decoded),
+ },
+ }
+ return &response, nil
+}
diff --git a/api/handlers/database_test.go b/api/handlers/database_test.go
index c8488ae..89ef5f5 100644
--- a/api/handlers/database_test.go
+++ b/api/handlers/database_test.go
@@ -73,15 +73,25 @@ func (m *MockDatabase) GetTransaction(ctx context.Context, txHash string, chainN
return transaction, nil
}
-func (m *MockDatabase) GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error) {
+func (m *MockDatabase) GetAddressTxs(
+ ctx context.Context,
+ address string,
+ chainName string,
+ fromTimestamp *time.Time,
+ toTimestamp *time.Time,
+ limit *uint64,
+ page *uint64,
+ cursor *string,
+) (*[]database.AddressTx, string, uint64, error) {
if m.shouldError {
- return nil, fmt.Errorf("%s", m.errorMsg)
+ return nil, "", 0, fmt.Errorf("%s", m.errorMsg)
}
addressTxs, ok := m.addressTxs[address]
if !ok {
- return nil, fmt.Errorf("address transactions not found")
+ return nil, "", 0, fmt.Errorf("address transactions not found")
}
- return addressTxs, nil
+ txCount := uint64(len(*addressTxs))
+ return addressTxs, "", txCount, nil
}
func (m *MockDatabase) GetLatestBlock(ctx context.Context, chainName string) (*database.BlockData, error) {
@@ -170,3 +180,14 @@ func (m *MockDatabase) GetMsgRun(ctx context.Context, txHash string, chainName s
}
return []*database.MsgRun{msgRun}, nil
}
+
+func (m *MockDatabase) GetTransactionsByCursor(ctx context.Context, chainName string, cursor string, limit uint64) ([]*database.Transaction, error) {
+ if m.shouldError {
+ return nil, fmt.Errorf("%s", m.errorMsg)
+ }
+ transactions := make([]*database.Transaction, 0, len(m.transactions))
+ for _, transaction := range m.transactions {
+ transactions = append(transactions, transaction)
+ }
+ return transactions, nil
+}
diff --git a/api/handlers/interface.go b/api/handlers/interface.go
index 962bca1..3701c38 100644
--- a/api/handlers/interface.go
+++ b/api/handlers/interface.go
@@ -10,7 +10,16 @@ import (
// DatabaseHandler interface for the database
type DatabaseHandler interface {
// Addresses
- GetAddressTxs(ctx context.Context, address string, chainName string, fromTimestamp time.Time, toTimestamp time.Time) (*[]database.AddressTx, error)
+ GetAddressTxs(
+ ctx context.Context,
+ address string,
+ chainName string,
+ fromTimestamp *time.Time,
+ toTimestamp *time.Time,
+ limit *uint64,
+ page *uint64,
+ cursor *string,
+ ) (*[]database.AddressTx, string, uint64, error)
// Blocks
GetBlock(ctx context.Context, height uint64, chainName string) (*database.BlockData, error)
@@ -27,4 +36,5 @@ type DatabaseHandler interface {
GetMsgCall(ctx context.Context, txHash string, chainName string) ([]*database.MsgCall, error)
GetMsgAddPackage(ctx context.Context, txHash string, chainName string) ([]*database.MsgAddPackage, error)
GetMsgRun(ctx context.Context, txHash string, chainName string) ([]*database.MsgRun, error)
+ GetTransactionsByCursor(ctx context.Context, chainName string, cursor string, limit uint64) ([]*database.Transaction, error)
}
diff --git a/api/handlers/transactions.go b/api/handlers/transactions.go
index 1083e3c..681ceb5 100644
--- a/api/handlers/transactions.go
+++ b/api/handlers/transactions.go
@@ -7,7 +7,6 @@ import (
"strings"
humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types"
- "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
"github.com/danielgtaylor/huma/v2"
)
@@ -145,17 +144,13 @@ func (h *TransactionsHandler) GetTransactionMessage(
}, nil
}
-// GetLastXTransactions retrieves the last X transactions
-func (h *TransactionsHandler) GetLastXTransactions(ctx context.Context, input *humatypes.TransactionGeneralListGetInput) (*humatypes.TransactionGeneralListGetOutput, error) {
- transactions, err := h.db.GetLastXTransactions(ctx, h.chainName, input.Amount)
+// Get tx by limit and limit and cursor
+func (h *TransactionsHandler) GetTransactionsByCursor(ctx context.Context, input *humatypes.TransactionGeneralListByCursorGetInput) (*humatypes.TransactionGeneralListByCursorGetOutput, error) {
+ transactions, err := h.db.GetTransactionsByCursor(ctx, h.chainName, input.Cursor, input.Limit)
if err != nil {
- return nil, huma.Error404NotFound("Last x transactions not found", err)
+ return nil, huma.Error404NotFound("Transactions by cursor not found", err)
}
- response := &humatypes.TransactionGeneralListGetOutput{
- Body: make([]database.Transaction, 0, len(transactions)),
- }
- for _, transaction := range transactions {
- response.Body = append(response.Body, *transaction)
- }
- return response, nil
+ return &humatypes.TransactionGeneralListByCursorGetOutput{
+ Body: transactions,
+ }, nil
}
diff --git a/api/handlers/transactions_test.go b/api/handlers/transactions_test.go
index 538ff09..7bcbce5 100644
--- a/api/handlers/transactions_test.go
+++ b/api/handlers/transactions_test.go
@@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/require"
)
-func TestTransactionsHandler_GetLastXTransactions_Success(t *testing.T) {
+func TestTransactionsHandler_GetTransactionsByCursor_Success(t *testing.T) {
fixedTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
db := MockDatabase{
@@ -32,7 +32,7 @@ func TestTransactionsHandler_GetLastXTransactions_Success(t *testing.T) {
}
handler := handlers.NewTransactionsHandler(&db, "gnoland")
- response, err := handler.GetLastXTransactions(context.Background(), &humatypes.TransactionGeneralListGetInput{Amount: 1})
+ response, err := handler.GetTransactionsByCursor(context.Background(), &humatypes.TransactionGeneralListByCursorGetInput{Cursor: "", Limit: 1})
require.NoError(t, err)
require.NotNil(t, response)
@@ -40,17 +40,17 @@ func TestTransactionsHandler_GetLastXTransactions_Success(t *testing.T) {
assert.Equal(t, "tx_hash_1", response.Body[0].TxHash)
}
-func TestTransactionsHandler_GetLastXTransactions_Fail(t *testing.T) {
+func TestTransactionsHandler_GetTransactionsByCursor_Fail(t *testing.T) {
db := MockDatabase{
shouldError: true,
- errorMsg: "error getting last x transactions",
+ errorMsg: "error getting transactions by cursor",
}
handler := handlers.NewTransactionsHandler(&db, "gnoland")
- response, err := handler.GetLastXTransactions(context.Background(), &humatypes.TransactionGeneralListGetInput{Amount: 1})
+ response, err := handler.GetTransactionsByCursor(context.Background(), &humatypes.TransactionGeneralListByCursorGetInput{Cursor: "", Limit: 1})
assert.Error(t, err)
assert.Nil(t, response)
- assert.Contains(t, err.Error(), "Last x transactions not found")
+ assert.Contains(t, err.Error(), "Transactions by cursor not found")
}
func TestTransactionsHandler_GetTransactionBasic_Success(t *testing.T) {
diff --git a/api/huma-types/address.go b/api/huma-types/address.go
index ad5ec42..2083894 100644
--- a/api/huma-types/address.go
+++ b/api/huma-types/address.go
@@ -7,11 +7,20 @@ import (
)
type AddressGetInput struct {
- Address string `path:"address" doc:"Address" required:"true"`
- FromTimestamp time.Time `query:"from_timestamp" doc:"From timestamp" format:"2025-01-01T00:00:00+00:00 or 2025-01-01T00:00:00Z" required:"true"`
- ToTimestamp time.Time `query:"to_timestamp" doc:"To timestamp" format:"2025-01-01T00:00:00+00:00 or 2025-01-01T00:00:00Z" required:"true"`
+ Address string `path:"address" doc:"Gno address you want to query" required:"true"`
+ FromTimestamp time.Time `query:"from_timestamp" doc:"From timestamp (inclusive)" format:"date-time"`
+ ToTimestamp time.Time `query:"to_timestamp" doc:"To timestamp (inclusive)" format:"date-time"`
+ Limit uint64 `query:"limit" doc:"Limit of transactions to return" min:"1" max:"100" default:"10"`
+ Page uint64 `query:"page" doc:"Page of transactions to return"`
+ Cursor string `query:"cursor" doc:"Cursor to continue from"`
}
type AddressGetOutput struct {
- Body []database.AddressTx
+ Body AddressTxsBody
+}
+
+type AddressTxsBody struct {
+ AddressTxs []database.AddressTx `json:"address_txs" doc:"Data about address transactions"`
+ TxCount uint64 `json:"tx_count" doc:"Total number of transactions"`
+ NextCursor string `json:"next_cursor" doc:"Next cursor that can be used in the query"`
}
diff --git a/api/huma-types/convert.go b/api/huma-types/convert.go
new file mode 100644
index 0000000..34f50ba
--- /dev/null
+++ b/api/huma-types/convert.go
@@ -0,0 +1,23 @@
+package humatypes
+
+type ConvertFromBase64toBase64UrlInput struct {
+ TxHash64 string `query:"tx_hash_64" doc:"Input to convert" required:"true" minLength:"44" maxLength:"44"`
+}
+
+type ConvertFromBase64UrlToBase64Input struct {
+ TxHash64Url string `query:"tx_hash_64_url" doc:"Input to convert" required:"true" minLength:"44" maxLength:"44"`
+}
+
+type ConvertFromBase64toBase64UrlBody struct {
+ TxHash64Url string `json:"tx_hash_64_url" doc:"Output of base64 to base64url conversion" required:"true"`
+}
+type ConvertFromBase64toBase64UrlOutput struct {
+ Body ConvertFromBase64toBase64UrlBody
+}
+
+type ConvertFromBase64UrlToBase64Body struct {
+ TxHash64 string `json:"tx_hash_64" doc:"Output of base64url to base64 conversion" required:"true"`
+}
+type ConvertFromBase64UrlToBase64Output struct {
+ Body ConvertFromBase64UrlToBase64Body
+}
diff --git a/api/huma-types/transaction.go b/api/huma-types/transaction.go
index 137e1d9..077ed30 100644
--- a/api/huma-types/transaction.go
+++ b/api/huma-types/transaction.go
@@ -54,10 +54,11 @@ type TransactionMessageGetOutput struct {
Body map[int16]TransactionMessage
}
-type TransactionGeneralListGetInput struct {
- Amount uint64 `query:"amount" doc:"Amount of transactions to get" required:"true" min:"1" max:"100" default:"10"`
+type TransactionGeneralListByCursorGetInput struct {
+ Cursor string `query:"cursor" doc:"Cursor to get the next set of transactions in form of timestamp|tx_hash(base64url encoded)" required:"false"`
+ Limit uint64 `query:"limit" doc:"Limit of transactions to get" required:"true" min:"1" max:"100" default:"10"`
}
-type TransactionGeneralListGetOutput struct {
- Body []database.Transaction
+type TransactionGeneralListByCursorGetOutput struct {
+ Body []*database.Transaction
}
diff --git a/api/main.go b/api/main.go
index 10728ce..3b6f4a0 100644
--- a/api/main.go
+++ b/api/main.go
@@ -160,7 +160,7 @@ var rootCmd = &cobra.Command{
})
// Register Block API routes
- huma.Get(api, "/block/{height}", blocksHandler.GetBlock,
+ huma.Get(api, "/blocks/{height}", blocksHandler.GetBlock,
func(op *huma.Operation) {
op.Summary = "Get Block Height"
op.Description = "Retrieve block data by its height"
@@ -188,30 +188,59 @@ var rootCmd = &cobra.Command{
// Register Transaction API routes
huma.Get(
- api, "/transaction/{tx_hash}", transactionsHandler.GetTransactionBasic,
+ api, "/transactions/{tx_hash}", transactionsHandler.GetTransactionBasic,
func(op *huma.Operation) {
op.Summary = "Get Transaction Basic"
op.Description = "Retrieve basic transaction data by its hash"
})
huma.Get(
api,
- "/transaction/{tx_hash}/message",
+ "/transactions/{tx_hash}/messages",
transactionsHandler.GetTransactionMessage,
func(op *huma.Operation) {
op.Summary = "Get All Transaction Messages"
op.Description = "Retrieve all messages contained within a transaction by its hash"
})
- huma.Get(api, "/transactions", transactionsHandler.GetLastXTransactions,
+ huma.Get(api, "/transactions", transactionsHandler.GetTransactionsByCursor,
func(op *huma.Operation) {
- op.Summary = "Get Last X Transactions"
- op.Description = "Retrieve the last X transactions data"
+ op.Summary = "Get Transactions"
+ op.Description = `Retrieve transactions by setting the limit and using cursor.
+ To fetch multiple transaction you can use this endpoint. Without cursor you will
+ fetch latest data. However if you need to acquire older data you can use cursor.
+ The cursor is a string in the form of timestamp|tx_hash(base64url encoded).
+ The timestamp is the timestamp of the transaction and the tx_hash is the hash of the transaction.
+ The tx_hash is base64url encoded to be able to query safely via API.
+ `
})
// Register Address API routes
huma.Get(api, "/address/{address}/txs", addressHandler.GetAddressTxs,
func(op *huma.Operation) {
op.Summary = "Get Address Transactions"
- op.Description = "Retrieve all transactions for a given address for a certain time period"
+ op.Description = `Retrieve all transactions for a given address.
+ There are 3 ways to query the transactions:
+
+ 1. by timestamp range
+ 2. by cursor
+ 3. by limit and page
+
+ For the timestamp range, you can specify the from and to timestamps.
+ For the cursor, just make the first query without any parameters besides the address.
+ The query will contain the data alongside the next cursor that can be used as a query to get the data needed.
+ For the limit and page, you can specify the limit and page to get the next set of transactions.
+ `
+ })
+
+ // Register Convert Base64 API routes
+ huma.Post(api, "/convert/base64-to-base64url", handlers.ConvertFromBase64toBase64Url,
+ func(op *huma.Operation) {
+ op.Summary = "Convert Base64 to Base64Url"
+ op.Description = "Convert a base64 encoded tx hash to a base64url encoded tx hash"
+ })
+ huma.Post(api, "/convert/base64url-to-base64", handlers.ConvertFromBase64UrlToBase64,
+ func(op *huma.Operation) {
+ op.Summary = "Convert Base64Url to Base64"
+ op.Description = "Convert a base64url encoded tx hash to a base64 encoded tx hash"
})
// Start server using config values
diff --git a/compression/cmd/main.go b/compression/cmd/main.go
new file mode 100644
index 0000000..b786237
--- /dev/null
+++ b/compression/cmd/main.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+ "log"
+ "os"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/compression/train"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+ "github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+ Use: "train",
+ Short: "Train",
+ Long: `Train tool for Zstd dictionary training the Spectra Gnoland Indexer.
+
+ Please note that the training process will only ever be done prior to the indexer being deployed and only be
+ done once, or if you want to re-train the dictionary.
+
+ Purpose of this cli is to collect the data from the database where it would gather all of the transaction events
+ and then build the zstd dictionary from the events. By combining protobuf serialization and zstd dictionary,
+ you can achieve a very efficient compression of the data. In theory you could do this manually by using zstd cli,
+ however then you would need to create some logic to pull the data from the RPC endpoint and serialize the data into protobuf format.
+ And then you would need to use the zstd cli to build the dictionary from the serialized data.
+
+ However this cli is a lot more efficient since it is doing all of the work in a single step
+ and it is a lot more convenient to use.
+ `,
+ Run: func(cmd *cobra.Command, args []string) {
+ log.Println("Training Zstd dictionary")
+ configPath, err := cmd.Flags().GetString("config")
+ if err != nil {
+ log.Fatalf("failed to get config path: %v", err)
+ }
+ amount, err := cmd.Flags().GetUint64("amount")
+ if err != nil {
+ log.Fatalf("failed to get amount: %v", err)
+ }
+ chainName, err := cmd.Flags().GetString("chain-name")
+ if err != nil {
+ log.Fatalf("failed to get chain name: %v", err)
+ }
+ dictPath, err := cmd.Flags().GetString("dict-path")
+ if err != nil {
+ log.Fatalf("failed to get dict path: %v", err)
+ }
+
+ loadedConfig, err := train.LoadTrainingConfig(&configPath)
+ if err != nil {
+ log.Fatalf("failed to load config: %v", err)
+ }
+ dbConfig := database.DatabasePoolConfig{
+ Host: loadedConfig.Host,
+ Port: loadedConfig.Port,
+ User: loadedConfig.User,
+ Password: loadedConfig.Password,
+ Dbname: loadedConfig.Dbname,
+ Sslmode: loadedConfig.Sslmode,
+ PoolMaxConns: loadedConfig.PoolMaxConns,
+ PoolMinConns: loadedConfig.PoolMinConns,
+ PoolMaxConnLifetime: loadedConfig.PoolMaxConnLifetime,
+ PoolMaxConnIdleTime: loadedConfig.PoolMaxConnIdleTime,
+ PoolHealthCheckPeriod: loadedConfig.PoolHealthCheckPeriod,
+ PoolMaxConnLifetimeJitter: loadedConfig.PoolMaxConnLifetimeJitter,
+ }
+ db := train.InitDatabase(dbConfig)
+
+ events, err := train.CollectEvents(db, chainName, amount)
+ if err != nil {
+ log.Fatalf("failed to collect events: %v", err)
+ }
+ dict, err := train.BuildZstdDict(events)
+ if err != nil {
+ log.Fatalf("failed to build zstd dictionary: %v", err)
+ }
+ err = os.WriteFile(dictPath, dict, 0644)
+ if err != nil {
+ log.Fatalf("failed to write zstd dictionary: %v", err)
+ }
+ log.Println("Zstd dictionary built")
+ log.Println("Zstd dictionary written to: ", dictPath)
+ },
+}
+
+func init() {
+ rootCmd.Flags().StringP("config", "c", "", "the path to the config file")
+ rootCmd.Flags().Uint64P("amount", "a", 1000, "the amount of events to collect")
+ rootCmd.Flags().StringP("chain-name", "n", "gnoland", "the name of the chain")
+ rootCmd.Flags().StringP("dict-path", "d", "events.zstd.bin", "the path to the zstd dictionary")
+}
+
+func main() {
+ if err := rootCmd.Execute(); err != nil {
+ log.Fatalf("failed to execute command: %v", err)
+ }
+}
diff --git a/compression/train/init.go b/compression/train/init.go
new file mode 100644
index 0000000..bfe253e
--- /dev/null
+++ b/compression/train/init.go
@@ -0,0 +1,76 @@
+package train
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+ "go.yaml.in/yaml/v4"
+)
+
+// It is identical to the database.DatabasePoolConfig struct.
+// However, it is used for the training process and used very rarely.
+type TrainingConfig struct {
+ User string `yaml:"user"`
+ Password string `yaml:"password"`
+ Host string `yaml:"host"`
+ Port int `yaml:"port"`
+ Dbname string `yaml:"dbname"`
+ Sslmode string `yaml:"sslmode"`
+
+ PoolMaxConns int `yaml:"pool_max_conns"`
+ PoolMinConns int `yaml:"pool_min_conns"`
+ PoolMaxConnLifetime time.Duration `yaml:"pool_max_conn_lifetime"`
+ PoolMaxConnIdleTime time.Duration `yaml:"pool_max_conn_idle_time"`
+ PoolHealthCheckPeriod time.Duration `yaml:"pool_health_check_period"`
+ PoolMaxConnLifetimeJitter time.Duration `yaml:"pool_max_conn_lifetime_jitter"`
+}
+
+// LoadTrainingConfig loads the training config from the config file
+//
+// Usage:
+//
+// # Used to load the training config from the config file
+//
+// Parameters:
+// - configPath: the path to the config file
+//
+// Returns:
+// - TrainingConfig: the training config
+// - error: if the training config fails to load
+func LoadTrainingConfig(configPath *string) (TrainingConfig, error) {
+ var config TrainingConfig
+ if configPath == nil {
+ return config, fmt.Errorf("config path is required")
+ }
+ if !strings.HasSuffix(*configPath, ".yml") && !strings.HasSuffix(*configPath, ".yaml") {
+ return config, fmt.Errorf("config path must end with .yml")
+ }
+ yamlFile, err := os.ReadFile(*configPath)
+ if err != nil {
+ return config, err
+ }
+ err = yaml.Unmarshal(yamlFile, &config)
+ if err != nil {
+ return config, err
+ }
+ return config, nil
+}
+
+// Init initializes the database connection pool
+//
+// Usage:
+//
+// # Used to initialize the database connection pool
+//
+// Parameters:
+// - config: the configuration for the database connection pool
+//
+// Returns:
+// - *database.TimescaleDb: the database connection pool
+// - error: if the database connection pool fails to initialize
+func InitDatabase(config database.DatabasePoolConfig) *database.TimescaleDb {
+ return database.NewTimescaleDb(config)
+}
diff --git a/compression/train/process.go b/compression/train/process.go
new file mode 100644
index 0000000..ff00fac
--- /dev/null
+++ b/compression/train/process.go
@@ -0,0 +1,131 @@
+package train
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "math"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/events_proto"
+ "github.com/klauspost/compress/zstd"
+ "google.golang.org/protobuf/proto"
+)
+
+// Collect collects the transactions from the database
+//
+// Usage:
+//
+// # Used to collect the transactions from the database
+//
+// Parameters:
+// - db: the database connection pool
+// - chainName: the name of the chain
+// - amount: the amount of transactions to collect
+//
+// Returns:
+// - [][]byte: the transactions events in serialized protobuf format
+// - error: if the transactions fail to collect
+func CollectEvents(db *database.TimescaleDb, chainName string, amount uint64) ([][]byte, error) {
+ // define the limits and offset
+ if amount > 250000 {
+ return nil, fmt.Errorf("amount cannot be greater than 250000")
+ }
+ if amount == 0 {
+ return nil, fmt.Errorf("amount cannot be 0")
+ }
+ var limit uint64
+ var goroutines int
+ limit = min(amount, 100)
+
+ goroutines = int(math.Ceil(float64(amount) / 100))
+ transactions := make([]*database.Transaction, 0)
+ wg := sync.WaitGroup{}
+ wg.Add(goroutines)
+ mu := sync.Mutex{}
+ for i := 0; i < goroutines; i++ {
+ go func(i int) {
+ defer wg.Done()
+ offset := uint64(i) * limit
+ log.Printf("getting the transactions from %s with limit %d and offset %d", chainName, limit, offset)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ txs, err := db.GetTransactionsByOffset(ctx, chainName, limit, offset)
+ if err != nil {
+ log.Printf("failed to get transactions from %s with limit %d and offset %d: %v", chainName, limit, offset, err)
+ return
+ }
+ log.Printf("got %d transactions from %s with limit %d and offset %d", len(txs), chainName, limit, offset)
+ mu.Lock()
+ transactions = append(transactions, txs...)
+ mu.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ events := make([][]byte, 0)
+ for _, transaction := range transactions {
+ txEvents := transaction.TxEvents
+ protoTxEvents := &events_proto.TxEvents{
+ Events: make([]*events_proto.Event, 0),
+ }
+ if len(txEvents) > 0 {
+ for _, event := range txEvents {
+ protoAttrs := make([]*events_proto.Attribute, 0)
+ for _, attribute := range event.Attributes {
+ protoAttrs = append(protoAttrs, events_proto.NewAttributeFromString(attribute.Key, attribute.Value))
+ }
+ protoEv := &events_proto.Event{
+ AtType: event.AtType,
+ Type: event.Type,
+ Attributes: protoAttrs,
+ PkgPath: &event.PkgPath,
+ }
+ protoTxEvents.Events = append(protoTxEvents.Events, protoEv)
+ }
+ }
+ if len(protoTxEvents.Events) == 0 {
+ continue
+ }
+ bs, err := proto.Marshal(protoTxEvents)
+ if err != nil {
+ log.Printf("failed to marshal tx events: %v", err)
+ continue
+ }
+ events = append(events, bs)
+ }
+ log.Printf("All events collected")
+ log.Printf("Collected %d events", len(events))
+ return events, nil
+}
+
+// BuildZstdDict builds the zstd dict from the events
+func BuildZstdDict(events [][]byte) ([]byte, error) {
+ const maxHistorySize = 112 << 10
+
+ var history []byte
+ for _, e := range events {
+ history = append(history, e...)
+ if len(history) >= maxHistorySize {
+ break
+ }
+ }
+ if len(history) > maxHistorySize {
+ history = history[:maxHistorySize]
+ }
+
+ lvl := zstd.SpeedBestCompression
+ dict, err := zstd.BuildDict(zstd.BuildDictOptions{
+ ID: 1,
+ Contents: events,
+ History: history,
+ Level: lvl,
+ DebugOut: os.Stdout,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return dict, nil
+}
diff --git a/config-api.yml.example b/config-api.yml.example
index b432523..89aac0a 100644
--- a/config-api.yml.example
+++ b/config-api.yml.example
@@ -1,4 +1,4 @@
-host: 127.0.0.1
+host: 0.0.0.0
port: 8080
cors_allowed_origins:
- "*"
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
new file mode 100644
index 0000000..7959cde
--- /dev/null
+++ b/docker-compose-dev.yml
@@ -0,0 +1,80 @@
+services:
+
+ timescaledb:
+ image: timescale/timescaledb-ha:pg17
+ command: -c 'max_connections=500'
+ container_name: timescale-gnoland
+ restart: always
+ ports:
+ - "127.0.0.1:5432:5432"
+ environment:
+ POSTGRES_PASSWORD: 12345678
+ POSTGRES_DB: gnoland
+ volumes:
+ - spectra-data:/home/postgres/pgdata/data
+ networks:
+ - spectra
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres -d gnoland"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+ indexer:
+ build:
+ context: .
+ args:
+ GIT_COMMIT: dev
+ GIT_TAG: dev
+ GIT_BRANCH: dev
+ VERSION: dev
+ dockerfile: Dockerfile
+ container_name: indexer
+ restart: always
+ environment:
+ DB_HOST: timescaledb
+ DB_PORT: 5432
+ DB_USER: postgres
+ DB_PASSWORD: 12345678
+ DB_NAME: gnoland
+ networks:
+ - spectra
+ depends_on:
+ - timescaledb
+ volumes:
+ - ./config.yml:/app/config.yml:ro,Z
+ command: ["run", "live", "--config", "/app/config.yml"]
+ api:
+ build:
+ context: .
+ args:
+ GIT_COMMIT: dev
+ GIT_TAG: dev
+ GIT_BRANCH: dev
+ VERSION: dev
+ dockerfile: Dockerfile-api
+ container_name: api
+ restart: always
+ ports:
+ - "127.0.0.1:8080:8080"
+ environment:
+ DB_HOST: timescaledb
+ DB_PORT: 5432
+ DB_USER: postgres
+ DB_PASSWORD: 12345678
+ DB_NAME: gnoland
+ networks:
+ - spectra
+ depends_on:
+ - timescaledb
+ volumes:
+ - ./config-api.yml:/app/config-api.yml:ro,Z
+ command: ["-c", "/app/config-api.yml"]
+
+networks:
+ spectra:
+ driver: bridge
+
+volumes:
+ spectra-data:
+ driver: local
diff --git a/docker-compose.yml b/docker-compose.yml
index ec45fab..71c2125 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,43 +11,56 @@ services:
POSTGRES_PASSWORD: 12345678
POSTGRES_DB: gnoland
volumes:
- - timescale_data:/home/postgres/pgdata
+ - spectra-data:/home/postgres/pgdata/data
networks:
- - gnoland
+ - spectra
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres -d gnoland"]
interval: 10s
timeout: 5s
retries: 5
- # first build the docker image
- # all of the envs you will need to change to values
- # as for the indexer mode the default here is live however look at deployment.md
- # for more information
- #indexer:
- # image: gnoland-indexer:latest
- # container_name: indexer-gnoland
- # restart: always
- # environment:
- # DB_HOST: timescale-gnoland
- # DB_PORT: 5432
- # DB_USER: postgres
- # DB_PASSWORD: 12345678
- # DB_NAME: gnoland
- # networks:
- # - gnoland
- # depends_on:
- # timescaledb:
- # condition: service_healthy
- #
- # volumes:
- # - ./config.yml:/app/config.yml
- # command: ["app/indexer", "live", "--config", "app/config.yml"]
-
+ indexer:
+ image: ghcr.io/cogwheel-validator/spectra-gnoland-indexer:latest
+ container_name: indexer
+ restart: always
+ environment:
+ DB_HOST: timescaledb
+ DB_PORT: 5432
+ DB_USER: postgres
+ DB_PASSWORD: 12345678
+ DB_NAME: gnoland
+ networks:
+ - spectra
+ depends_on:
+ - timescaledb
+ volumes:
+ - ./config.yml:/app/config.yml:ro,Z
+ command: ["run", "live", "--config", "/app/config.yml"]
+ api:
+ image: ghcr.io/cogwheel-validator/spectra-gnoland-api:latest
+ container_name: api
+ restart: always
+ ports:
+ - "127.0.0.1:8080:8080"
+ environment:
+ DB_HOST: timescaledb
+ DB_PORT: 5432
+ DB_USER: postgres
+ DB_PASSWORD: 12345678
+ DB_NAME: gnoland
+ networks:
+ - spectra
+ depends_on:
+ - timescaledb
+ volumes:
+ - ./config-api.yml:/app/config-api.yml:ro,Z
+ command: ["-c", "/app/config-api.yml"]
networks:
- gnoland:
+ spectra:
driver: bridge
volumes:
- timescale_data:
+ spectra-data:
+ driver: local
diff --git a/docs/README.md b/docs/README.md
index b51f794..0c7a9bf 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -9,4 +9,5 @@ This documentation is a work in progress and will be updated as the indexer is u
- [Integration Testing](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/blob/main/integration)
- [Scaling](./scaling.md)
- [Benchmarks](./benchmarks.md)
-- [API](./api.md)
\ No newline at end of file
+- [API](./api.md)
+- [Data Model](./data-model.md)
diff --git a/docs/api.md b/docs/api.md
index 7f1f826..b36e552 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -2,8 +2,8 @@
With this indexer you can use the REST API from the api directory. It is built with chi framework and huma.
-Out of the box you get a [Spotlight UI](https://stoplight.io/) to interact with the API on the /docs route.
-You can also use some other API UIs but you will need to make the changes yourself. See docs about changing the UI [here](https://huma.rocks/features/api-docs/).
+Out of the box you get a [Spotlight UI](https://stoplight.io/) to interact with the API on the /docs route.
+You can also use some other API UIs but you will need to make the changes yourself. See docs about changing the [UI docs](https://huma.rocks/features/api-docs/).
The huma is framework agnostic and you could modify the API to use some other framework, use another middleware
or maybe use the stdlib http package. This API provides the most basic and necessary features for querying the database.
@@ -14,22 +14,33 @@ There are total of 5 routes available. This are the basic routes that are needed
### Blocks
-- /block/{height} - Get a specific block data by height
+- /blocks/{height} - Get a specific block data by height
- /blocks/{from_height}/{to_height} - Get a range of blocks data by height range
- /blocks/{block_height}/signers - Get all of the validators that signed that block + the proposer
+- /blocks/latest - Get the latest block data
+- /blocks - Get a list of blocks by setting the limit and using cursor.
### Transactions
-- /transaction/{tx_hash} - Get a specific basic transaction data by hash, this gives the basic data about the transaction like hash, timestamp, block height, gas used, gas wanted, fee and more.
-- /transaction/{tx_hash}/message - Get a specific transaction message data by hash, this gives more detailed data about type of transaction, specific data for that message type and more.
+- /transactions/{tx_hash} - Get a specific basic transaction data by hash, this gives the basic data about the transaction like hash, timestamp, block height, gas used, gas wanted, fee and more.
+- /transactions/{tx_hash}/message - Get a specific transaction message data by hash, this gives more detailed data about type of transaction, specific data for that message type and more.
+- /transactions - Get a list of transactions by setting the limit and using cursor.
### Addresses
- /address/{address}/txs?from_timestamp={from_timestamp}&to_timestamp={to_timestamp} - Get all of the transactions for a given address for a certain time period
+### Utilities
+
+These endpoints can be queried via POST method.
+
+- /convert/base64-to-base64url - Convert a base64 encoded tx hash to a base64url encoded tx hash
+- /convert/base64url-to-base64 - Convert a base64url encoded tx hash to a base64 encoded tx hash
+
## Setup API
To setup the API you can use the config file. The example config file is in the root under config-api.yml.example.
+
```yaml
# Example config file for the API
host: 127.0.0.1
@@ -73,6 +84,7 @@ You can make the API by running the following command from the project root:
```bash
make build-api
```
+
This command will build the API and it will be located in the build directory.
To run the API you can use the following command:
@@ -88,4 +100,3 @@ You can also use the following command to run the API with HTTPS if you have the
```
The docker image doesn't exist for now and it will be added in the future.
-
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index c679ee2..17ef62e 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -3,7 +3,7 @@
## Date 04/10/2025
The benchmarks were ran on different machines with different max block chunk size in the settings. It was done on
-the old Gnoland 7.2 testnet from block height 950866 to 960866. The chunk size declares how much blocks can be
+the old Gnoland 7.2 testnet from block height 950866 to 960866. The chunk size declares how much blocks can be
processed at the same time. The max transaction chunk was left at 100 as it is the default value.
| Cpu Count | 50 chunks | 100 chunks | 200 chunks | 500 chunks |
@@ -11,13 +11,22 @@ processed at the same time. The max transaction chunk was left at 100 as it is t
| 4 vCPUs AMD Epyc | 1m30s | 1m05s | 37s | 18s |
| 16 core/32 threads AMD Ryzen 9 7900x3d | 24s | 16s | 10s | 10-5.4s* |
-* The 16 core/32 threads benchmark was ran 2/4 times the resault was at 6.5s however at this point the RPC node might be a bottleneck. One time it got 10 seconds and another time it was 5.4 seconds.
+* The 16 core/32 threads benchmark was ran 2/4 times the result was at 6.5s however at this point the RPC node might be a bottleneck. One time it got 10 seconds and another time it was 5.4 seconds.
This blocks are mostly empty of transactions so it might not be the best benchmark. But it is a good indication of the performance and can be used as a reference.
-However for the safety of the indexer I would recommend to use at the 50 - 100 chunks for now. Having 200 is a bit risky at this point untill more testing is done.
-
-
+## Recommendations
+For normal processing of the data, the recommended block chunk should be anywhere between 50-150. The transaction
+chunk should be anywhere between 50-300. This is a good balance between the performance and the not causing the RPC
+node to be overloaded.
+If you are using your own RPC node, only dedicated for the indexer by default settings it should be able to handle
+up to 900 requests concurrently. If you run the node with this settings you could in theory push the indexer to use
+block chunk to size of 450 and transaction chunk to size of 900. However only do this if you are the only one using
+the node because the node will simply block requests from other users, and even the indexer. The indexer will still
+try to record the data and it will probably be successful, but it will actually be slower because it needs to retry
+the missed requests. So unless you are the only one using the node, it is not recommended to use these settings.
+Or if you are the owner of this node, you can adjust this parameter in the settings and allow for more requests.
+If you plan to alter the node configurations better to consult the Gnoland documentation.
diff --git a/docs/compression.md b/docs/compression.md
new file mode 100644
index 0000000..689a7ad
--- /dev/null
+++ b/docs/compression.md
@@ -0,0 +1,55 @@
+# Event Compression
+
+WARNING: This feature is still experimental and it is not recommended for production use. It might even be removed
+in the future if there is no use for it.
+
+The Spectra Gnoland Indexer does allow to compress the transaction events to reduce the size. This is done by using
+the zstandard compression algorithm combined with protobuf serialization.
+
+## Why this combination?
+
+This is a mechanism that is used for the Spectra indexer for Cosmos SDK chains. The idea is to reduce the total size
+as much as possible. The protobuf can serialize the data into very compact format and it uses logic that "has
+repetition". Every part of the event has it's own slot in protobuf that is usually marked using numbers, which are
+later serialized into bytes. This allows for additional compression by the zstandard algorithm. The zstandard
+can train the dictionary to allow further compression on small data. The events are usually not very long but some
+transactions can harbor a lot of events and each event can have a lot of attributes. This can add up to a lot of
+data. But if the data from the event is serialized with protobuf and then compressed with zstandard, the final
+size can be reduced by a lot.
+
+## For what use cases this might work?
+
+Depending on what you plan to do with this data this might not work for your use case. If the purpose is simply to
+present the data as it is then you should be fine such as blockchain explorer, visual dashboards, etc.
+If you plan to do heavy analytics on the data such as aggregating data, this can be a big
+problem. The compressed data can't be queried over SQL in the database so any data you need you will need to
+decompress and deserialize the data into the native format. This can be a big performance hit if you are doing
+heavy analytics on the compressed data.
+
+## How to use this feature?
+
+At the current stage the trained zstandard dictionary exists. It should be possible to add `-e or --compress-events`
+flag when running the indexer in live or historic mode. This will tell the indexer to compress the events.
+However this feature is not fully ready.
+
+The current dictionary is not production ready. It was trained from the real data but the sample was small. To
+gain any significant compression the sample needs to be much larger.
+
+## How to train the dictionary?
+
+WARNING this process produces totally new zstd dictionary. You must keep this file safe and not lose it. Any
+compression done with this dictionary will only work with this dictionary. IF YOU LOSE IT, YOU WON'T BE ABLE TO
+TO RECREATE THE DICTIONARY AND THE COMPRESSED DATA WON'T BE ABLE TO BE DECOMPRESSED! Always try to use the official
+dictionary unless you really know what you are doing. Always commit this dictionary to the repository, that way you
+can always recreate the dictionary if you need to. More on zstd dictionary can be read at this [repo](https://github.com/facebook/zstd#the-case-for-small-data-compression).
+
+To simplify the process run the indexer first and collect as much data as possible. The training program will
+collect the data from the database and then build the zstd dictionary from the events.
+
+In the project root directory you will find the `training-config.example.yml` file. This file is used to configure
+the training process. Copy it to `training-config.yml` and configure it to your needs.
+
+The dictionary can be trained by using the `make train-zstd` command.
+
+There is a hard cap on amount of transaction that can be used to train the dictionary is set to 250000. The
+dictionary will be placed in `pkgs/dict_loader/events.zstd.bin` file.
diff --git a/docs/data-model.md b/docs/data-model.md
new file mode 100644
index 0000000..1bc4149
--- /dev/null
+++ b/docs/data-model.md
@@ -0,0 +1,216 @@
+# Database data model and schema
+
+This file will outline how the data is gathered, stored and processed by the indexer.
+
+## Data flow from the RPC node
+
+The indexer will connect to the RPC node and will start to gather the data from the node.
+It collects and processes data by using batch processing. For live mode the indexer will collect the data up to the
+latest block height and then it will process it in batches.
+
+The data is gathered in the following way:
+
+```mermaid
+flowchart TD
+ RPC[\RPC Node/]
+
+ subgraph TX Pipeline
+ FT([Fetch Transactions])
+ D[Transactions]
+ E[Transaction Messages]
+ F[Gno Regular Addresses]
+ end
+
+ subgraph Validator Pipeline
+ C{Validator Block Signings}
+ G[Gno Validator Addresses]
+ end
+
+ BH[Block Height]
+ I[(TimescaleDB)]
+
+ RPC --> |Gather Block Height| BH
+ RPC --> |Gather Validator Block Signings| C
+ RPC --> FT
+ BH --> |Gather TX Hashes| FT
+
+ FT --> D
+ D --> |Process Transactions| E
+ E --> |Process Transaction Messages| F
+ C --> |Process Validator Block Signings| G
+
+ BH --> I
+ D --> I
+ E --> I
+ F --> I
+ C --> I
+ G --> I
+```
+
+First the indexer gathers block data and validator signing for those blocks. If there are transactions the
+tx hashes are gathered from the block height data and are queried from the RPC. At that moment the transaction
+data is gathered and processes and all of the transaction general data and messages contained in the transaction
+are stored in the database. The regular and validator addresses are processed in that way that the addresses are
+stored as unique int32 ids and then referenced by the integer value in the transaction tables.
+
+## Database schema
+
+```mermaid
+erDiagram
+ blocks {
+ BYTEA hash
+ BIGINT height PK
+ TIMESTAMPTZ timestamp PK
+ INTEGER proposer
+ chain_name chain_name
+ BYTEA[] txs
+ }
+ validator_block_signings {
+ BIGINT height FK
+ TIMESTAMPTZ timestamp
+ INTEGER proposer FK
+ INTEGER[] signed_vals
+ chain_name chain_name
+ }
+ transactions_general {
+ BYTEA tx_hash PK
+ chain_name chain_name PK
+ TIMESTAMPTZ timestamp PK
+ BIGINT block_height
+ TEXT[] msg_types
+ Event[] tx_events
+ BYTEA tx_events_compressed
+ BOOLEAN compression_on
+ BIGINT gas_used
+ BIGINT gas_wanted
+ Amount fee
+ }
+ gno_addresses {
+ INTEGER GENERATED ALWAYS AS IDENTITY id PK
+ TEXT address UNIQUE
+ chain_name chain_name UNIQUE
+ }
+ gno_validator_addresses {
+ INTEGER GENERATED ALWAYS AS IDENTITY id PK
+ TEXT address UNIQUE
+ chain_name chain_name UNIQUE
+ }
+ address_tx {
+ INTEGER address PK
+ BYTEA tx_hash
+ chain_name chain_name
+ TIMESTAMPTZ timestamp PK
+ TEXT[] msg_types
+ }
+ msg_send {
+ BYTEA tx_hash PK
+ SMALLINT message_counter PK
+ TIMESTAMPTZ timestamp PK
+ INTEGER from_address
+ INTEGER to_address
+ INTEGER[] signers
+ Amount[] amount
+ }
+ msg_call {
+ BYTEA tx_hash PK
+ SMALLINT message_counter PK
+ TIMESTAMPTZ timestamp PK
+ INTEGER caller
+ TEXT pkg_path
+ TEXT func_name
+ TEXT args
+ INTEGER[] signers
+ Amount[] send
+ Amount[] max_deposit
+ }
+ msg_add_package {
+ BYTEA tx_hash PK
+ SMALLINT message_counter PK
+ TIMESTAMPTZ timestamp PK
+ INTEGER creator
+ TEXT pkg_path
+ TEXT pkg_name
+ TEXT[] pkg_file_names
+ INTEGER[] signers
+ Amount[] send
+ Amount[] max_deposit
+ }
+ msg_run {
+ BYTEA tx_hash PK
+ SMALLINT message_counter PK
+ TIMESTAMPTZ timestamp PK
+ INTEGER caller
+ TEXT pkg_path
+ TEXT pkg_name
+ TEXT[] pkg_file_names
+ INTEGER[] signers
+ Amount[] send
+ Amount[] max_deposit
+ }
+
+ blocks ||--o{ transactions_general : "contains"
+ blocks ||--o{ validator_block_signings : "has"
+ gno_validator_addresses ||--o{ validator_block_signings : "signs"
+ gno_validator_addresses ||--o{ blocks : "proposes"
+
+ transactions_general ||--o{ address_tx : "involves"
+ gno_addresses ||--o{ address_tx : "participates"
+
+ transactions_general ||--o{ msg_send : "contains"
+ transactions_general ||--o{ msg_call : "contains"
+ transactions_general ||--o{ msg_add_package : "contains"
+ transactions_general ||--o{ msg_run : "contains"
+
+ gno_addresses ||--o{ msg_send : "from/to"
+ gno_addresses ||--o{ msg_call : "caller"
+ gno_addresses ||--o{ msg_add_package : "creator"
+ gno_addresses ||--o{ msg_run : "caller"
+```
+
+And custom types:
+
+```mermaid
+erDiagram
+ amount {
+ NUMERIC amount
+ TEXT denom
+ }
+ attribute {
+ TEXT key
+ TEXT value
+ }
+ event {
+ TEXT at_type
+ TEXT type
+ Attribute[] attributes
+ TEXT pkg_path
+ }
+ transaction_general {
+ Event[] tx_events
+ }
+ msg_send {
+ Amount[] amount
+ }
+ msg_call {
+ Amount[] send
+ Amount[] max_deposit
+ }
+ msg_add_package {
+ Amount[] send
+ Amount[] max_deposit
+ }
+ msg_run {
+ Amount[] send
+ Amount[] max_deposit
+ }
+
+ amount ||--o{ transaction_general : "fee"
+ amount ||--o{ msg_send : "amount"
+ amount ||--o{ msg_call : "send"
+ amount ||--o{ msg_add_package : "send"
+ amount ||--o{ msg_add_package : "max_deposit"
+ amount ||--o{ msg_run : "send"
+ amount ||--o{ msg_run : "max_deposit"
+ attribute ||--o{ event : "attributes"
+ transaction_general ||--o{ event : "contains"
+```
diff --git a/docs/requirements.md b/docs/requirements.md
index da6555e..b7d4bf0 100644
--- a/docs/requirements.md
+++ b/docs/requirements.md
@@ -1,7 +1,7 @@
# Requirements
-The indexer could can run on one machine while the database is located on another machine. You can also use the
-Tiger Data (Timescaledb cloud edition) which is a managed service by Tiger Data which is a company that builds the
+The indexer could can run on one machine while the database is located on another machine. You can also use the
+Tiger Data (Timescaledb cloud edition) which is a managed service by Tiger Data which is a company that builds the
Timescaledb.
However for the ease of understanding through the documentation will refer that the indexer and the database are located on the same machine.
@@ -20,24 +20,23 @@ Recommended system requirements:
- 4vCPUs
- 16GB RAM
-The indexer could probably run on ARM64 architecture but it is not tested yet. So stick with the x86_64
-architecture. There shouldn't be any major difference but if you really want to run it on ARM64 you might need to compile
-the indexer from the source code.
+The indexer could probably run on ARM64 architecture but it is not tested yet. So stick with the x86_64
+architecture. There shouldn't be any major difference but if you really want to run it on ARM64 you might need to compile the indexer from the source code.
-For storage HDD should be enough. Durring the whole time of the development it was tested on old smal laptop HDD
-with 5400 RPM i think. So it should work for the most part. But for any bigger projects where write and read
-speeds are important you might needs SSD depenging on what you are doing. The SATA SSD is a good choice for the
+For storage HDD should be enough. During the whole time of the development it was tested on old laptop HDD
+with 5400 RPM. So it should work for the most part. But for any bigger projects where write and read
+speeds are important you might needs SSD depending on what you are doing. The SATA SSD is a good choice for the
most part but you can also use the NVMe SSD for better performance.
-As for the size it depends on the amount of the data that you are indexing.
-For example the integration test did about 400K blocks and 1 million
-transactions. So it took around 2.2 GB of disk space. There are a lot of things that can affect the size of the
+As for the size it depends on the amount of the data that you are indexing.
+For example the integration test did about 400K blocks and 1 million
+transactions. So it took around 2.2 GB of disk space. There are a lot of things that can affect the size of the
database but at least use this as some reference point to how much space you might need.
-For the RAM and CPU it kinda depends but for now this is a good starting point. As the database size grows, you
+For the RAM and CPU it kinda depends but for now this is a good starting point. As the database size grows, you
might need to increase the RAM and CPU.
-Aditional info for RAM. It is not that simple for more info look [here](https://docs.tigerdata.com/use-timescale/latest/hypertables/improve-query-performance/).
+Additional info for RAM. It is not that simple for more info look at [tiger data documentation](https://docs.tigerdata.com/use-timescale/latest/hypertables/improve-query-performance/).
To make queries and compression efficient you need to have the necessary amount of RAM. This is kinda hard to calculate but according to the Tiger Data documentation:
@@ -48,25 +47,25 @@ the indexes for chunks currently being ingested into fit within 25% of main memo
growth is around 10 GB per day, use a 1-day interval.
```
-So this is a bit hard to calculate. For example the Spectra cosmos indexer works in a similar way. At the testing
-phase on Osmosis the 100K of block data took about 400MB-700MB of storage space. Now not all of the data is indexed
-but for the sake of example let's say it is 700MB. The Osmosis has around 1s block production rate so this is 2
-days and maybe around 2/3 of one day. Let's assume that for the week it will take arround 2GB. If Gnoland ever
-reached these amount of data you might need to increase the RAM size or modify the chunk interval. But this is if
-the indexed data was 2GB. Realistically this is not the case and the indexer data would need a lot of indexed
+So this is a bit hard to calculate. For example the Spectra cosmos indexer works in a similar way. At the testing
+phase on Osmosis the 100K of block data took about 400MB-700MB of storage space. Now not all of the data is indexed
+but for the sake of example let's say it is 700MB. The Osmosis has around 1s block production rate so this is 2
+days and maybe around 2/3 of one day. Let's assume that for the week it will take arround 2GB. If Gnoland ever
+reached these amount of data you might need to increase the RAM size or modify the chunk interval. But this is if
+the indexed data was 2GB. Realistically this is not the case and the indexer data would need a lot of indexed
data.
-Good rule of thumb is if the queries are slow increase the RAM size or modify the chunk interval or look at the
-queries and see if they are efficient.
+Good rule of thumb is if the queries are slow increase the RAM size or modify the chunk interval or look at the
+queries and see if they are efficient.
-8 GB of RAM is a good starting point. 16 GB should be enough for most cases. Depending on how popular the chain is
+8 GB of RAM is a good starting point. 16 GB should be enough for most cases. Depending on how popular the chain is
and how much data is indexed you might need to increase the RAM size.
## Software and OS requirements
The following software and OS requirements are required to run the indexer:
-- go 1.25.4
+- go 1.25.7
- TimescaleDB 2.18 or higher but with PostgresSQL 16 or higher
- OS: Linux, anything based on Debian(Ubuntu, Mint, etc.) or RHEL(CentOS stream, Rocky Linux, etc.) should work, openSUSE also ok
-- Docker (optional)
\ No newline at end of file
+- Docker (optional)
diff --git a/docs/scaling.md b/docs/scaling.md
index 3aa9486..751fec8 100644
--- a/docs/scaling.md
+++ b/docs/scaling.md
@@ -1,32 +1,32 @@
# Scailing the indexer
-If you use the Tiger Data (TimescaleDB cloud edition) they have some internal methods of scaling the database that
+If you use the Tiger Data (TimescaleDB cloud edition) they have some internal methods of scaling the database that
are not present for the self hosted version.
Regardless the Timescaledb is still a Postgres database and there are some ways to scale the indexer.
I will just mention some of the methods that are possible, I will not go into the details of how to do it. I might make some notes in the future for this section.
The Postgres database can scale vertically and horizontally(although the horizontal scaling is more complex).
-With the vertical scaling you would need to increase the resources of the database server. With the horizontal
+With the vertical scaling you would need to increase the resources of the database server. With the horizontal
scaling you would need to increase the number of database servers. The horizontal scaling is more complex because
you would need to handle the way that data is extracted and inserted into the database.
## Read replicas
-The Postgres has a feature of read replicas. This can be used to scale the indexer if there are a lot of read
+The Postgres has a feature of read replicas. This can be used to scale the indexer if there are a lot of read
operations. The read replicas are a copy of the database that is updated asynchronously. You would need to set up
all of the read replicas to gather the data from the master node. This is a bit out of scope for this project.
## Sharding
-Sharding is also possible but it is a bit more complex for the indexer. There are a lot of methods to do this, I
-would recommend to do application level sharding. The proxy sharding is also a valid option. The
-catch with the indexer is that you would need to have some stop point at which you would split the data. The best
-would be either by time or by block height. The indexer doesn't have a stop option for the live indexing so you
-would need to have some other way to stop the indexing. This might be changed in the future but for now there is no
-such option. The indexer historic mode has a stop option by height but this can only be done for the blocks that
+Sharding is also possible but it is a bit more complex for the indexer. There are a lot of methods to do this, I
+would recommend to do application level sharding. The proxy sharding is also a valid option. The
+catch with the indexer is that you would need to have some stop point at which you would split the data. The best
+would be either by time or by block height. The indexer doesn't have a stop option for the live indexing so you
+would need to have some other way to stop the indexing. This might be changed in the future but for now there is no
+such option. The indexer historic mode has a stop option by height but this can only be done for the blocks that
have already been produced by the blockchain.
The thing that you would need to pay most attention is to the address cache. The indexer has in memory cache that
-ties the address to the integer value and are mapped everywhere where some sort of adderess is stored.
-So for this to work you might need to copy all of the addresses from one database to another. I guess you could
-also skip this part but then this would require some sophisticated way to query from all of the shards.
\ No newline at end of file
+ties the address to the integer value and are mapped everywhere where some sort of address is stored.
+So for this to work you might need to copy all of the addresses from one database to another. I guess you could
+also skip this part but then this would require some sophisticated way to query from all of the shards.
diff --git a/docs/setup.md b/docs/setup.md
index 3494d20..1d9410a 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -31,15 +31,15 @@ docker build -t gnoland-indexer:latest .
The recommended way to do this would be to use docker for the database. You can also use any linux package to install the database. For this section it is better to head over to the official documentation of the database.
-For linux https://docs.tigerdata.com/self-hosted/latest/install/installation-linux/.
-For docker https://docs.tigerdata.com/self-hosted/latest/install/installation-docker/.
+For linux .
+For docker .
If you plan to use docker it would be better to use the timescaledb-ha image.
You can also use the docker-compose.yml file to setup the database. It is already configured to use the timescaledb-ha image.
The important thing to setup is the database connections. The indexer can do good job even with only 50 connections.
-However if you are planning to add a lot of services a top of the database maybe increase it. The docker compose is
+However if you are planning to add a lot of services a top of the database maybe increase it. The docker compose is
set to 500 connections.
For creating all of the necessary tables and types you can use the indexer executable and the setup command.
@@ -70,7 +70,7 @@ To create the database you can use the following command:
indexer setup create-db --db-host localhost --db-port 5432 --db-user postgres --db-name postgres --ssl-mode disable --new-db-name gnoland --chain-name gnoland
```
-To create the users for the database you can use the following command.
+To create the users for the database you can use the following command.
The user is the name of the user to create and the privilege is the privilege level for the user.
The program will ask for admin password, and later it will ask for the password of the new user.
The privilege level can be "reader" or "writer". The reader should have only the select privileges.
@@ -87,26 +87,25 @@ The indexer can be ran in 2 modes: live and historic.
Now when you have the database running you can actually run the indexer. The indexer has a lot of flags that can be used to configure it:
```bash
-Run the indexer in either live or historic mode.
+A blockchain indexer for Gnoland that processes blocks and transactions.
Usage:
- indexer run [command]
+ indexer [command]
Available Commands:
- historic Run the indexer in historic mode
- live Run the indexer in live mode
+ completion Generate the autocompletion script for the specified shell
+ help Help about any command
+ run Run the indexer
+ setup Database setup tools
Flags:
- -e, --compress-events compress events
- -c, --config string config file path (default "config.yml")
- -h, --help help for run
- -m, --max-req-per-window int max requests per window (default 10000000)
- -r, --rate-limit-window duration rate limit window (default 1m0s)
- -t, --timeout duration timeout (default 20s)
+ -h, --help help for indexer
+ -v, --version version for indexer
+Use "indexer [command] --help" for more information about a command.
```
-Compress events doesn't work at the moment so do not use it!
+Compress events doesn't work at the moment so do not use it!
The max request per window is rated to the max request that can be made to the RPC per rate limit window. So depending on the RPC rate limit option you can decrease the rate limit window and set up any desired value for the maximum requests per window.
@@ -175,14 +174,17 @@ exponential_backoff: 2s
```
To run the indexer in historic mode you can use the following command:
+
```bash
indexer run historic --config config.yml --from-height 1000 --to-height 2000
```
+
The from height is the starting height of the block to index. The to height is the ending height of the block to index.
The indexer will index the blocks from the from height to the to height inclusive. You can also add the other flags
such as the max request per window, the rate limit window, the timeout, etc.
Historic mode flags:
+
```bash
Runs the spectra indexer in historic mode, processing blocks from a given height to a given height.
The historic mode takes in starting height point and a finishing height. It should be used to
@@ -208,10 +210,13 @@ Global Flags:
```
To run the indexer in live mode you can use the following command:
+
```bash
indexer run live --config config.yml
```
+
Live mode flags:
+
```bash
Runs the spectra indexer, listening to any new blocks and processing them.
It will check the database for the last processed height and start from there
@@ -241,20 +246,20 @@ The skip db check is a flag that will skip the initial database check. You can u
### When to use each mode and how to run it in the production
-These mods can be used differently together. For example you might get access to the archive RPC node. But you
+These mods can be used differently together. For example you might get access to the archive RPC node. But you
want to run the indexer in segments. So you can use the historic mode to index the data in the database and then
when you reach some height that normal RPC node has access to you can switch to the live mode.
-Maybe you need to view at some segment of the blockchain individually. You can set up different database within the
-same database server. From there you can instruct the indexer to index the data from a certain height to a certain
+Maybe you need to view at some segment of the blockchain individually. You can set up different database within the
+same database server. From there you can instruct the indexer to index the data from a certain height to a certain
height. And then you can look at the data from there and do any kind of testing or analytics for example.
-If you started the indexer with the live mode without the prior data it will try to start the indexer from the
-height 1. If the RPC node doesn't have that height it will fail. So you can use the skip-db-check flag so that the
-live mode skips the check of what was the last processed height and will start from the latest chain height. This
+If you started the indexer with the live mode without the prior data it will try to start the indexer from the
+height 1. If the RPC node doesn't have that height it will fail. So you can use the skip-db-check flag so that the
+live mode skips the check of what was the last processed height and will start from the latest chain height. This
mode with skip db check is also useful for the testing of the indexer.
-So the best way to have any kind of good setup would be to run the indexer in historic mode to index the data in
+So the best way to have any kind of good setup would be to run the indexer in historic mode to index the data in
segments. Then switch to the live mode to index the data in the real time. To be clear the live mode can process and
index the data the same as the historic mode, you just gain more control over the flow of the indexer.
@@ -263,6 +268,7 @@ index the data the same as the historic mode, you just gain more control over th
Like mentioned above you can use the docker-compose.yml file to setup the database and the indexer.
If you plan to run the indexer over docker you can use this commands:
+
```bash
# Live mode with custom config
docker run gnoland-indexer run live --config /path/to/config.yml --skip-db-check
@@ -278,6 +284,7 @@ docker run gnoland-indexer run live --help
```
You can also run the indexer with something like systemd:
+
```bash
[Unit]
Description=Gnoland Indexer
@@ -307,5 +314,3 @@ RestrictNamespaces=yes
[Install]
WantedBy=multi-user.target
```
-
-You can ovbiusly use some kind of orchestrator like nomad or kuberneties to run the indexer but going in depth is out of scope of this documentation.
diff --git a/experiments/experiment1/enc_dec_test.go b/experiments/experiment1/enc_dec_test.go
deleted file mode 100644
index 680abf6..0000000
--- a/experiments/experiment1/enc_dec_test.go
+++ /dev/null
@@ -1,44 +0,0 @@
-package main
-
-import (
- "crypto/sha256"
- "encoding/base64"
- "encoding/hex"
- "fmt"
- "log"
-)
-
-// In Gnoland txs are marked with base64 not in hex like in other Cosmos SDK chains
-// To get the txhash we need to query the block method and under the data field we can find the txs
-// This data reveals basic info such as tx type, action happening, which package was used and what occured
-// To get something that resembles txhash we need then to encode the data to sha256 and then to base64
-// This can be later used to query the tx from the rpc client via the tx method
-// Same data is pressent within the tx query method so this should be useful for the indexer
-func main() {
- raw_data := "CpABCgovdm0ubV9jYWxsEoEBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIhZnbm8ubGFuZC9yL2dub3N3YXAvZ25zKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpwBCgovdm0ubV9jYWxsEo0BCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiJnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi91c2RjKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9iYXIqCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL2JheioIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMAqbAQoKL3ZtLm1fY2FsbBKMAQooZzE3MjkwY3d2bXJhcHZwODY5eGZuaGhhd2E4c205ZWRwdWZ6YXQ3ZCIhZ25vLmxhbmQvci9nbm9zd2FwL3Rlc3RfdG9rZW4vb2JsKghUcmFuc2ZlcjIoZzEyejc3bHk0aHdyeWtqMnphZmNoZmR2bGxmZm5wbDU1d2dzcHVjbDIJMTAwMDAwMDAwCpsBCgovdm0ubV9jYWxsEowBCihnMTcyOTBjd3ZtcmFwdnA4Njl4Zm5oaGF3YThzbTllZHB1ZnphdDdkIiFnbm8ubGFuZC9yL2dub3N3YXAvdGVzdF90b2tlbi9mb28qCFRyYW5zZmVyMihnMTJ6NzdseTRod3J5a2oyemFmY2hmZHZsbGZmbnBsNTV3Z3NwdWNsMgkxMDAwMDAwMDAKmwEKCi92bS5tX2NhbGwSjAEKKGcxNzI5MGN3dm1yYXB2cDg2OXhmbmhoYXdhOHNtOWVkcHVmemF0N2QiIWduby5sYW5kL3IvZ25vc3dhcC90ZXN0X3Rva2VuL3F1eCoIVHJhbnNmZXIyKGcxMno3N2x5NGh3cnlrajJ6YWZjaGZkdmxsZmZucGw1NXdnc3B1Y2wyCTEwMDAwMDAwMBITCICEr18SDDEwMDAwMDB1Z25vdBp+CjoKEy90bS5QdWJLZXlTZWNwMjU2azESIwohAtGyA6l3UIrUup5z7yXo90bXDcXUOmLiK34YPffgQ6pAEkCg8pAtBv6Fhw98bKYdqrEX2UrcjYvIUpbGdxMyc5Zpfl1cMNo8G6vzpnFaQESX+7eZIFTfO5BCFjVJLfC5Ur5C"
-
- // decode the raw data from base64 to string
- base64Decoded, err := base64.StdEncoding.DecodeString(raw_data)
- if err != nil {
- log.Fatal(err)
- }
-
- // print the decoded string
- fmt.Println(string(base64Decoded))
-
- // encode to sha256
- hash := sha256.Sum256(base64Decoded)
- fmt.Println(hex.EncodeToString(hash[:]))
-
- // encode the encoded sha256 to base64
- base64Encoded := base64.StdEncoding.EncodeToString(hash[:])
- fmt.Println(base64Encoded)
-
- // decode the last base64 to the sha256
- rawByte, err := base64.StdEncoding.DecodeString(base64Encoded)
- if err != nil {
- log.Fatal(err)
- }
- fmt.Printf("The final base64 decoded has %v bytes", len(rawByte))
- fmt.Printf("The first decoded base64 has %v bytes", len(base64Decoded))
-}
diff --git a/experiments/experiment10/main.go b/experiments/experiment10/main.go
deleted file mode 100644
index d8908ad..0000000
--- a/experiments/experiment10/main.go
+++ /dev/null
@@ -1,257 +0,0 @@
-package main
-
-import (
- "context"
- "encoding/base64"
- "fmt"
- "log"
- "math/big"
- "time"
-
- dataTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types"
- "github.com/jackc/pgx/v5"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/jackc/pgx/v5/pgxpool"
-)
-
-func setupConnection() (*pgxpool.Pool, error) {
- host := "localhost"
- port := 6543
- user := "postgres"
- password := "12345678"
- dbname := "gnoland"
- sslmode := "disable"
-
- config, err := pgxpool.ParseConfig(
- fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
- host, port, user, password, dbname, sslmode),
- )
- if err != nil {
- return nil, err
- }
-
- config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
- dataTypeNames := dataTypes.CustomTypeNames()
-
- for _, typeName := range dataTypeNames {
- dataType, err := conn.LoadType(ctx, typeName)
- if err != nil {
- return err
- }
- conn.TypeMap().RegisterType(dataType)
- }
-
- return nil
- }
-
- return pgxpool.NewWithConfig(context.Background(), config)
-}
-
-func makePgxArray[T any](v []T) pgtype.Array[T] {
- if v == nil {
- return pgtype.Array[T]{Valid: false}
- }
-
- return pgtype.Array[T]{
- Elements: v,
- Dims: []pgtype.ArrayDimension{{Length: int32(len(v)), LowerBound: 1}},
- Valid: true,
- }
-}
-
-func main() {
- // let's try to insert real data
-
- // block
- hash1, err := base64.StdEncoding.DecodeString("FGYVJEJQlox8AMRKnhClsIUa/TrFmWFr9SmNVJDLSIg=")
- if err != nil {
- log.Fatalf("Error decoding the hash1: %s", err)
- }
- hash2, err := base64.StdEncoding.DecodeString("QcG+qinbhad+1OTIch2H6tR/cYnKEEh4MXInmGDv940=")
- if err != nil {
- log.Fatalf("Error decoding the hash1: %s", err)
- }
- timestamp1, err := time.Parse(time.RFC3339Nano, "2025-09-05T07:30:58.618646057Z")
- if err != nil {
- log.Fatalf("Error parsing the timestamp1, %s", err)
- }
- timestamp2, err := time.Parse(time.RFC3339Nano, "2025-09-05T07:30:53.054040872Z")
- if err != nil {
- log.Fatalf("Error parsing the timestamp2, %s", err)
- }
- txHash1, err := base64.StdEncoding.DecodeString("gupsX4SH279MJ2xuosYlRwhHDKv/s9xco7BWnbBPTmE=")
- if err != nil {
- log.Fatalf("Error decoding the txHash1, %s", err)
- }
- txHash2, err := base64.StdEncoding.DecodeString("KClwYV7356OEfzdvahXE3/d+pRUPthPxo8Cy33R8E2o=")
- if err != nil {
- log.Fatalf("Error decoding the txHash2, %s", err)
- }
- blocks := []dataTypes.Blocks{
- {
- Hash: hash1,
- Height: 970866,
- Timestamp: timestamp1,
- ChainID: "gnoalnd-testnet-7",
- Txs: [][]byte{txHash1, txHash2},
- ChainName: "gnoland",
- },
- {
- Hash: hash2,
- Height: 970865,
- Timestamp: timestamp2,
- ChainID: "gnoalnd-testnet-7",
- Txs: nil, // empty
- ChainName: "gnoland",
- },
- }
- transactionGeneral := []dataTypes.TransactionGeneral{
- {
- TxHash: txHash1,
- ChainName: "gnoland",
- Timestamp: timestamp1,
- MsgTypes: []string{"bank_msg_send"},
- TxEvents: []dataTypes.Event{}, // null
- TxEventsCompressed: nil,
- CompressionOn: false,
- GasUsed: 100000,
- GasWanted: 90000,
- Fee: dataTypes.Amount{
- Amount: pgtype.Numeric{Int: big.NewInt(1000000)},
- Denom: "ugnot",
- },
- },
- {
- TxHash: txHash2,
- ChainName: "gnoland",
- Timestamp: timestamp2,
- MsgTypes: []string{"vm_msg_call"},
- TxEvents: []dataTypes.Event{
- {
- // it is fake but just to test it out
- AtType: "tm.Gno.MsgCall",
- Type: "vm_msg_call",
- Attributes: []dataTypes.Attribute{
- {Key: "caller", Value: "gno1234567890"},
- {Key: "func_name", Value: "test_func"},
- },
- PkgPath: "gno.land/r/tests",
- },
- },
- TxEventsCompressed: nil,
- CompressionOn: false,
- GasUsed: 100000,
- GasWanted: 90000,
- Fee: dataTypes.Amount{
- Amount: pgtype.Numeric{Int: big.NewInt(1000000)},
- Denom: "ugnot",
- },
- },
- }
- // treat it like an array because that is how the indexer should do it even for one message
- bankMsgSend := []dataTypes.MsgSend{
- {
- TxHash: txHash1,
- Timestamp: timestamp1,
- ChainName: "gnoland",
- FromAddress: 1,
- ToAddress: 2,
- Amount: []dataTypes.Amount{
- {
- Amount: pgtype.Numeric{Int: big.NewInt(1000000)},
- Denom: "ugnot",
- },
- },
- Signers: []int32{1, 2},
- },
- }
- vmMsgCall := []dataTypes.MsgCall{
- {
- TxHash: txHash2,
- Timestamp: timestamp2,
- ChainName: "gnoland",
- Caller: 1,
- PkgPath: "gno.land/r/tests",
- FuncName: "test_func",
- Args: "test_args",
- MaxDeposit: []dataTypes.Amount{
- {
- Amount: pgtype.Numeric{Int: big.NewInt(1000000)},
- Denom: "ugnot",
- },
- },
- Send: []dataTypes.Amount{
- {
- Amount: pgtype.Numeric{Int: big.NewInt(1000000)},
- Denom: "ugnot",
- },
- },
- Signers: []int32{1, 2},
- },
- }
-
- // let's insert the data into the database
- pool, err := setupConnection()
- if err != nil {
- panic(err)
- }
- defer pool.Close()
-
- ctx := context.Background()
-
- // insert the blocks
- pgxSlice := pgx.CopyFromSlice(len(blocks), func(i int) ([]any, error) {
- return []any{
- blocks[i].Hash,
- blocks[i].Height,
- blocks[i].Timestamp,
- blocks[i].ChainID,
- makePgxArray(blocks[i].Txs),
- blocks[i].ChainName,
- }, nil
- })
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"blocks"}, []string{"hash", "height", "timestamp", "chain_id", "proposer_address", "txs", "chain_name"}, pgxSlice)
- if err != nil {
- panic(err)
- }
-
- // insert the transaction general
- pgxSlice = pgx.CopyFromSlice(len(transactionGeneral), func(i int) ([]any, error) {
- return []any{
- transactionGeneral[i].TxHash,
- transactionGeneral[i].ChainName,
- transactionGeneral[i].Timestamp,
- makePgxArray(transactionGeneral[i].MsgTypes),
- transactionGeneral[i].TxEvents,
- transactionGeneral[i].TxEventsCompressed,
- transactionGeneral[i].CompressionOn,
- transactionGeneral[i].GasUsed,
- transactionGeneral[i].GasWanted,
- transactionGeneral[i].Fee,
- }, nil
- })
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"transaction_general"}, []string{
- "tx_hash", "chain_name", "timestamp", "msg_types", "tx_events", "tx_events_compressed", "compression_on", "gas_used", "gas_wanted", "fee"}, pgxSlice)
- if err != nil {
- panic(err)
- }
-
- // insert the bank msg send
- pgxSlice = pgx.CopyFromSlice(len(bankMsgSend), func(i int) ([]any, error) {
- return []any{bankMsgSend[i].TxHash, bankMsgSend[i].ChainName, bankMsgSend[i].FromAddress, bankMsgSend[i].ToAddress, bankMsgSend[i].Amount, bankMsgSend[i].Timestamp, makePgxArray(bankMsgSend[i].Signers)}, nil
- })
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"bank_msg_send"}, []string{"tx_hash", "chain_name", "from_address", "to_address", "amount", "timestamp", "signers"}, pgxSlice)
- if err != nil {
- panic(err)
- }
-
- // insert the vm msg call
- pgxSlice = pgx.CopyFromSlice(len(vmMsgCall), func(i int) ([]any, error) {
- return []any{vmMsgCall[i].TxHash, vmMsgCall[i].ChainName, vmMsgCall[i].Caller, vmMsgCall[i].PkgPath, vmMsgCall[i].FuncName, vmMsgCall[i].Args, vmMsgCall[i].Send, vmMsgCall[i].MaxDeposit, vmMsgCall[i].Timestamp, makePgxArray(vmMsgCall[i].Signers)}, nil
- })
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"vm_msg_call"}, []string{"tx_hash", "chain_name", "caller", "pkg_path", "func_name", "args", "send", "max_deposit", "timestamp", "signers"}, pgxSlice)
- if err != nil {
- panic(err)
- }
-
-}
diff --git a/experiments/experiment11/main.go b/experiments/experiment11/main.go
deleted file mode 100644
index fa423fc..0000000
--- a/experiments/experiment11/main.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package main
-
-import (
- "encoding/base64"
- "fmt"
- "log"
-)
-
-func main() {
- fmt.Println("Enter base64 to encode to base64url: ")
- var input string
- _, err := fmt.Scanln(&input)
- if err != nil {
- log.Fatal(err)
- }
- base64Decoded, err := base64.StdEncoding.DecodeString(input)
- if err != nil {
- log.Fatal(err)
- }
- base64url := base64.URLEncoding.EncodeToString(base64Decoded)
- fmt.Println("Base64url: ", base64url)
-}
diff --git a/experiments/experiment2/main.go b/experiments/experiment2/main.go
deleted file mode 100644
index 8f4f28d..0000000
--- a/experiments/experiment2/main.go
+++ /dev/null
@@ -1,88 +0,0 @@
-package main
-
-import (
- "encoding/base64"
- "encoding/hex"
- "fmt"
- "log"
- "strings"
-
- "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
- "github.com/gnolang/gno/tm2/pkg/amino"
- "github.com/gnolang/gno/tm2/pkg/sdk/bank"
- "github.com/gnolang/gno/tm2/pkg/std"
-)
-
-func DecodeStdTxFromBase64(s string) (*std.Tx, error) {
- bz, err := base64.StdEncoding.DecodeString(s)
- if err != nil {
- return nil, err
- }
- var tx std.Tx
- if err := amino.Unmarshal(bz, &tx); err != nil {
- return nil, err
- }
- return &tx, nil
-}
-
-func GetDataFromStdTx(tx *std.Tx) (string, error) {
- fees := fmt.Sprint(tx.Fee.GasFee.Amount) + " " + string(tx.Fee.GasFee.Denom)
- signatures := tx.GetSignatures()
- for _, signature := range signatures {
- pubKey := signature.PubKey.String()
- signatureHex := hex.EncodeToString(signature.Signature)
-
- fmt.Printf("signature: %v %v \n", pubKey, signatureHex)
- }
- signers := tx.GetSigners()
-
- for _, msg := range tx.GetMsgs() {
- switch m := msg.(type) {
- case bank.MsgSend:
- fromAddress := m.FromAddress.String()
- toAddress := m.ToAddress.String()
- amount := m.Amount.String()
- return fmt.Sprintf("bank.MsgSend: %s -> %s, amount: %s, fees: %s, signatures: %s, signers: %s \n", fromAddress, toAddress, amount, fees, signatures[0], signers), nil
- case vm.MsgCall:
- caller := m.Caller.String()
- pkgPath := m.PkgPath
- funcName := m.Func
- args := strings.Join(m.Args, ",")
- return fmt.Sprintf("vm.MsgCall, caller: %s, pkgPath: %s, funcName: %s, args: %s, fees: %s, signatures: %s, signers: %s \n", caller, pkgPath, funcName, args, fees, signatures[0], signers), nil
- case vm.MsgAddPackage:
- creator := m.Creator.String()
- pkgPath := m.Package.Path
- pkgName := m.Package.Name
- pkgFiles := m.Package.FileNames()
- pkgType := m.Package.Type
- return fmt.Sprintf(
- "vm.MsgAddPackage, creator: %s, pkgPath: %s, pkgName: %s, pkgFiles: %s, pkgType: %s, fees: %s, signatures: %s, signers: %s \n", creator, pkgPath, pkgName, pkgFiles, pkgType, fees, signatures[0], signers), nil
- case vm.MsgRun:
- caller := m.Caller.String()
- pkgPath := m.Package.Path
- pkgName := m.Package.Name
- pkgFiles := m.Package.FileNames()
- pkgType := m.Package.Type
- return fmt.Sprintf("vm.MsgRun, caller: %s, pkgPath: %s, pkgName: %s, pkgFiles: %s, pkgType: %s, fees: %s, signatures: %s, signers: %s \n", caller, pkgPath, pkgName, pkgFiles, pkgType, fees, signatures[0], signers), nil
- default:
- return "", fmt.Errorf("unknown or unsupported message type: %T", m)
- }
- }
- return "", fmt.Errorf("no message found in transaction")
-}
-
-func main() {
- data := "CrQFCgwvdm0ubV9hZGRwa2cSowUKKGcxamc4bXR1dHU5a2hoZndjNG54bXVoY3BmdGYwcGFqZGhmdnNxZjUS9gQKF21hdHRoZXdfc3RvcmFnZV9kZXBvc2l0Ektnbm8ubGFuZC9yL2cxamc4bXR1dHU5a2hoZndjNG54bXVoY3BmdGYwcGFqZGhmdnNxZjUvbWF0dGhld19zdG9yYWdlX2RlcG9zaXQacQoLZ25vbW9kLnRvbWwSYm1vZHVsZSA9ICJnbm8ubGFuZC9yL2cxamc4bXR1dHU5a2hoZndjNG54bXVoY3BmdGYwcGFqZGhmdnNxZjUvbWF0dGhld19zdG9yYWdlX2RlcG9zaXQiCmdubyA9ICIwLjkiGvYCCgtwYWNrYWdlLmdubxLmAnBhY2thZ2UgbWF0dGhld19zdG9yYWdlX2RlcG9zaXQKCnZhciBkYXRhID0gbWFrZShtYXBbc3RyaW5nXXN0cmluZykKCmZ1bmMgRGVwb3NpdChjdXIgcmVhbG0sIGtleSBzdHJpbmcpIHsKCWRhdGFba2V5XSA9ICIiCglmb3IgaSA6PSAwOyBpIDwgMTAwMDA7IGkrKyB7CgkJZGF0YVtrZXldID0gZGF0YVtrZXldICsgInRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3R0ZXN0dGVzdHRlc3QiCgl9Cn0KCmZ1bmMgUmVsZWFzZShjdXIgcmVhbG0sIGtleSBzdHJpbmcpIHsKCWRlbGV0ZShkYXRhLCBrZXkpCn0iIgoTL2duby5NZW1QYWNrYWdlVHlwZRILCglNUFVzZXJBbGwSEwiAwtcvEgwxMDAwMDAwdWdub3Qafgo6ChMvdG0uUHViS2V5U2VjcDI1NmsxEiMKIQPhYTbbFx4y30iZNZQfBW4i+Jhj43OdCrfNSexCg5ydshJAeKL4UNZFxab5c3gSFfFIh6supP4w2K2kQlfOPe6z1wA49aQz81Y4ZbwwiFulqXusxR62nuA6pjSX9UpGN1oU/w=="
- tx, err := DecodeStdTxFromBase64(data)
- if err != nil {
- log.Fatal(err)
- }
- fmt.Println(GetDataFromStdTx(tx))
-
- data2 := "Cn4KCi92bS5tX2NhbGwScAooZzFxMmhza3NuaDVxNTV4a20zZDh0dWwyOGdnOXV1d2wyZW04dmVyNiIXZ25vLmxhbmQvci9kZW1vL3Byb2ZpbGUqDlNldFN0cmluZ0ZpZWxkMgtEaXNwbGF5TmFtZTIObm9kZXJ1bm5lcmluZG8SEwiAh6cOEgwxMDAwMDAwdWdub3Qafgo6ChMvdG0uUHViS2V5U2VjcDI1NmsxEiMKIQNHbE/VfTVWkgdRu7mmAbi2MQuJrkFiIM9xHCA0e9/V+RJAVSpcFFNgOO8K/XrXfdUOW9SpDTwhsnYsWpWEWcOo0wVJRFtDVzTb2qkkm3vao2uteVkXIKrLh2vle2yGY5v8bA=="
- tx2, err := DecodeStdTxFromBase64(data2)
- if err != nil {
- log.Fatal(err)
- }
- fmt.Println(GetDataFromStdTx(tx2))
-}
diff --git a/experiments/experiment3/rate_limited_rpc_example.go b/experiments/experiment3/rate_limited_rpc_example.go
deleted file mode 100644
index 48d2564..0000000
--- a/experiments/experiment3/rate_limited_rpc_example.go
+++ /dev/null
@@ -1,183 +0,0 @@
-package main
-
-import (
- "fmt"
- "sync"
- "time"
-
- rpcclient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
-)
-
-func main() {
- fmt.Println("Rate Limited RPC Client Examples")
-
- // Example 1: Basic usage with blocking calls
- fmt.Println("1. Basic Rate Limited RPC Client:")
- basicExample()
-
- // Example 2: Non-blocking calls (Try methods)
- fmt.Println("\n2. Non-blocking Rate Limited Calls:")
- nonBlockingExample()
-
- // Example 3: Concurrent usage demonstration
- fmt.Println("\n3. Concurrent Rate Limited Usage:")
- concurrentExample()
-
- // Example 4: Rate limiter status monitoring
- fmt.Println("\n4. Rate Limiter Status Monitoring:")
- statusMonitoringExample()
-}
-
-func basicExample() {
- // Create a rate-limited client: 5 requests per 3 seconds
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil, // use default timeout
- 5, // 5 requests
- 3*time.Second, // per 3 seconds
- )
- if err != nil {
- fmt.Printf(" Failed to create client: %v\n", err)
- return
- }
- defer client.Close()
-
- // Make several requests - should be rate limited
- for i := 0; i < 7; i++ {
- start := time.Now()
-
- err := client.Health()
- elapsed := time.Since(start)
-
- if err != nil {
- fmt.Printf(" Request %d: FAILED (%v) after %v\n", i+1, err, elapsed.Round(time.Millisecond))
- } else {
- fmt.Printf(" Request %d: SUCCESS after %v\n", i+1, elapsed.Round(time.Millisecond))
- }
- }
-}
-
-func nonBlockingExample() {
- // Create a rate-limited client: 3 requests per 2 seconds
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 3, // 3 requests
- 2*time.Second, // per 2 seconds
- )
- if err != nil {
- fmt.Printf(" Failed to create client: %v\n", err)
- return
- }
- defer client.Close()
-
- // Try making more requests than allowed - non-blocking
- successCount := 0
- rateLimitedCount := 0
-
- for i := 0; i < 6; i++ {
- err, allowed := client.TryHealth()
-
- if !allowed {
- rateLimitedCount++
- fmt.Printf(" Request %d: RATE LIMITED (would block)\n", i+1)
- } else if err != nil {
- fmt.Printf(" Request %d: FAILED (%v)\n", i+1, err)
- } else {
- successCount++
- fmt.Printf(" Request %d: SUCCESS (non-blocking)\n", i+1)
- }
- }
-
- fmt.Printf(" Summary: %d successful, %d rate limited\n", successCount, rateLimitedCount)
-}
-
-func concurrentExample() {
- // Create a rate-limited client: 10 requests per 2 seconds
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 10, // 10 requests
- 2*time.Second, // per 2 seconds
- )
- if err != nil {
- fmt.Printf(" Failed to create client: %v\n", err)
- return
- }
- defer client.Close()
-
- var wg sync.WaitGroup
- const numWorkers = 5
- const requestsPerWorker = 4
-
- // Start multiple goroutines making concurrent requests
- for worker := 0; worker < numWorkers; worker++ {
- wg.Add(1)
- go func(workerID int) {
- defer wg.Done()
-
- for req := 0; req < requestsPerWorker; req++ {
- start := time.Now()
-
- err := client.Health()
- elapsed := time.Since(start)
-
- if err != nil {
- fmt.Printf(" Worker %d, Request %d: FAILED (%v) after %v\n",
- workerID, req+1, err, elapsed.Round(time.Millisecond))
- } else {
- fmt.Printf(" Worker %d, Request %d: SUCCESS after %v\n",
- workerID, req+1, elapsed.Round(time.Millisecond))
- }
- }
- }(worker)
- }
-
- wg.Wait()
- fmt.Printf(" All %d workers completed (%d total requests)\n",
- numWorkers, numWorkers*requestsPerWorker)
-}
-
-func statusMonitoringExample() {
- // Create a rate-limited client: 4 requests per 1 second
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 4, // 4 requests
- 1*time.Second, // per 1 second
- )
- if err != nil {
- fmt.Printf(" Failed to create client: %v\n", err)
- return
- }
- defer client.Close()
-
- // Monitor status while making requests
- printStatus := func(label string) {
- status := client.GetRateLimiterStatus()
- fmt.Printf(" %s: %d/%d tokens available (Empty: %v, Full: %v)\n",
- label, status.TokensAvailable, status.Capacity, status.IsEmpty, status.IsFull)
- }
-
- printStatus("Initial state")
-
- // Use up all tokens with non-blocking calls
- for i := 0; i < 6; i++ {
- _, allowed := client.TryHealth()
- if allowed {
- fmt.Printf(" Request %d: Allowed\n", i+1)
- } else {
- fmt.Printf(" Request %d: Rate limited\n", i+1)
- }
- printStatus(fmt.Sprintf("After request %d", i+1))
- }
-
- // Wait a bit and check status again
- fmt.Println(" Waiting 500ms for token refill...")
- time.Sleep(500 * time.Millisecond)
- printStatus("After 500ms wait")
-
- fmt.Println(" Waiting another 600ms for more tokens...")
- time.Sleep(600 * time.Millisecond)
- printStatus("After total 1100ms wait")
-}
diff --git a/experiments/experiment3/rate_limited_rpc_test.go b/experiments/experiment3/rate_limited_rpc_test.go
deleted file mode 100644
index 2e1c8f1..0000000
--- a/experiments/experiment3/rate_limited_rpc_test.go
+++ /dev/null
@@ -1,396 +0,0 @@
-package main
-
-import (
- "sync"
- "sync/atomic"
- "testing"
- "time"
-
- rpcclient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
- "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client/rate_limit"
-)
-
-func TestRateLimitedRpcClient(t *testing.T) {
- t.Run("BasicRateLimiting", testBasicRateLimiting)
- t.Run("NonBlockingMethods", testNonBlockingMethods)
- t.Run("ConcurrentAccess", testConcurrentAccess)
- t.Run("StatusMonitoring", testStatusMonitoring)
- t.Run("TokenRefillBehavior", testTokenRefillBehavior)
-}
-
-// Test the ChannelRateLimiter directly (no network calls) to demonstrate token bucket behavior
-func TestChannelRateLimiterTokenBucket(t *testing.T) {
- // Create a rate limiter: 3 tokens per 300ms (100ms per token)
- limiter := rate_limit.NewChannelRateLimiter(3, 300*time.Millisecond)
- defer limiter.Close()
-
- // Initial state: should have 3 tokens
- if !limiter.Allow() {
- t.Error("Expected first token to be available")
- }
- if !limiter.Allow() {
- t.Error("Expected second token to be available")
- }
- if !limiter.Allow() {
- t.Error("Expected third token to be available")
- }
-
- // Fourth token should not be available
- if limiter.Allow() {
- t.Error("Expected fourth token to be blocked (bucket empty)")
- }
-
- // Wait for one token refill (120ms > 100ms per token)
- time.Sleep(120 * time.Millisecond)
-
- // Should have exactly 1 token now
- if !limiter.Allow() {
- t.Error("Expected one token after 120ms wait")
- }
-
- // Should be empty again
- if limiter.Allow() {
- t.Error("Expected to be empty after using the refilled token")
- }
-
- // Wait for two tokens (220ms > 200ms for 2 tokens)
- time.Sleep(220 * time.Millisecond)
-
- // Should have 2 tokens available
- tokensUsed := 0
- for i := 0; i < 4; i++ { // Try to use more than available
- if limiter.Allow() {
- tokensUsed++
- }
- }
-
- if tokensUsed != 2 {
- t.Errorf("Expected exactly 2 tokens after 220ms, got %d", tokensUsed)
- }
-
- t.Log("Token bucket behavior working correctly")
-}
-
-func testBasicRateLimiting(t *testing.T) {
- // Create client with very restrictive limits for testing
- // 2 requests per 1 second = 500ms per token refill
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone", // Using real endpoint for integration test
- nil,
- 2, // 2 requests
- 1*time.Second, // per second (500ms per token)
- )
- if err != nil {
- t.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- // Use non-blocking calls to test token bucket behavior without network latency
- // First two requests should succeed immediately (bucket starts full)
- successCount := 0
- start := time.Now()
- for i := 0; i < 2; i++ {
- if _, allowed := client.TryHealth(); allowed {
- successCount++
- }
- }
- elapsed := time.Since(start)
- t.Logf("First 2 requests took %v", elapsed)
-
- if successCount != 2 {
- t.Errorf("Expected 2 immediate successes, got %d", successCount)
- }
-
- // Check status before third request
- status := client.GetRateLimiterStatus()
- t.Logf("Status before third request: %d tokens available", status.TokensAvailable)
-
- // NOTE: With network latency, tokens may have been refilled during the first 2 requests
- // This is CORRECT token bucket behavior - tokens refill continuously
- if status.TokensAvailable > 0 {
- t.Logf("Tokens were refilled during network calls (correct token bucket behavior)")
- } else {
- t.Log("No tokens available immediately after consumption")
- }
-
- // Wait for token refill and try again
- time.Sleep(600 * time.Millisecond) // Wait longer than refill interval (500ms)
- if _, allowed := client.TryHealth(); !allowed {
- t.Error("Request should succeed after token refill")
- }
-
- t.Log("Rate limiting working correctly with token bucket behavior")
-}
-
-func testNonBlockingMethods(t *testing.T) {
- // Use a slower refill rate for predictable testing
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 2, // 2 requests
- 2*time.Second, // per 2 seconds (1 second per token)
- )
- if err != nil {
- t.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- // Should have 2 tokens initially
- allowedCount := 0
- blockedCount := 0
-
- // Try 4 requests immediately - only 2 should be allowed
- for i := 0; i < 4; i++ {
- _, allowed := client.TryHealth()
- if allowed {
- allowedCount++
- } else {
- blockedCount++
- }
- }
-
- if allowedCount != 2 {
- t.Errorf("Expected 2 allowed requests initially, got %d", allowedCount)
- }
- if blockedCount != 2 {
- t.Errorf("Expected 2 blocked requests initially, got %d", blockedCount)
- }
-
- // Wait for one token refill (1 second) and try again
- time.Sleep(1100 * time.Millisecond)
-
- if _, allowed := client.TryHealth(); !allowed {
- t.Error("Request should be allowed after token refill")
- } else {
- allowedCount++
- }
-
- t.Logf("Non-blocking methods working: %d total allowed, %d initially blocked", allowedCount, blockedCount)
-}
-
-func testConcurrentAccess(t *testing.T) {
- // Use modest rate limiting for concurrent test
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 5, // 5 requests
- 1*time.Second, // per second (200ms per token)
- )
- if err != nil {
- t.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- const numGoroutines = 3
- const requestsPerGoroutine = 2 // total 6 requests (more than initial capacity)
-
- var wg sync.WaitGroup
- var successCount int64
- var errorCount int64
-
- start := time.Now()
-
- for i := 0; i < numGoroutines; i++ {
- wg.Add(1)
- go func(goroutineID int) {
- defer wg.Done()
-
- for j := 0; j < requestsPerGoroutine; j++ {
- // Use Try method first to avoid blocking in concurrent test
- if _, allowed := client.TryHealth(); allowed {
- atomic.AddInt64(&successCount, 1)
- } else {
- // If not allowed, use blocking method (will wait for token)
- if err := client.Health(); err != nil {
- atomic.AddInt64(&errorCount, 1)
- t.Logf("Goroutine %d, request %d failed: %v", goroutineID, j+1, err)
- } else {
- atomic.AddInt64(&successCount, 1)
- }
- }
- }
- }(i)
- }
-
- wg.Wait()
- elapsed := time.Since(start)
-
- totalRequests := int64(numGoroutines * requestsPerGoroutine)
- t.Logf("Concurrent test completed in %v", elapsed)
- t.Logf("Total requests: %d, Success: %d, Errors: %d",
- totalRequests, successCount, errorCount)
-
- // Most requests should succeed (either immediately or after waiting)
- if successCount < totalRequests-1 {
- t.Errorf("Expected most requests to succeed, got %d successes out of %d",
- successCount, totalRequests)
- }
-
- // Test should demonstrate some rate limiting effect
- if elapsed < 200*time.Millisecond {
- t.Logf("Note: Test completed quickly (%v), rate limiting may not have been needed", elapsed)
- } else {
- t.Logf("Rate limiting working in concurrent scenario (took %v)", elapsed)
- }
-}
-
-func testStatusMonitoring(t *testing.T) {
- // Use very slow refill rate for predictable status testing
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 3, // 3 requests
- 3*time.Second, // per 3 seconds (1 second per token)
- )
- if err != nil {
- t.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- // Initial status should show full capacity
- status := client.GetRateLimiterStatus()
- if status.Capacity != 3 {
- t.Errorf("Expected capacity 3, got %d", status.Capacity)
- }
- if status.TokensAvailable != 3 {
- t.Errorf("Expected 3 tokens initially, got %d", status.TokensAvailable)
- }
- if !status.IsFull {
- t.Error("Expected rate limiter to be full initially")
- }
- if status.IsEmpty {
- t.Error("Expected rate limiter to not be empty initially")
- }
-
- // Use all tokens quickly (before any can refill)
- for i := 0; i < 3; i++ {
- _, allowed := client.TryHealth()
- if !allowed {
- t.Errorf("Expected request %d to be allowed", i+1)
- }
- }
-
- // Check status immediately (should be empty)
- status = client.GetRateLimiterStatus()
- if status.TokensAvailable != 0 {
- t.Errorf("Expected 0 tokens after using all, got %d", status.TokensAvailable)
- }
- if !status.IsEmpty {
- t.Error("Expected rate limiter to be empty after using all tokens")
- }
- if status.IsFull {
- t.Error("Expected rate limiter to not be full after using all tokens")
- }
-
- // Wait for partial refill
- time.Sleep(1100 * time.Millisecond) // Should get 1 token back
-
- status = client.GetRateLimiterStatus()
- if status.TokensAvailable == 0 {
- t.Error("Expected at least 1 token after 1 second")
- }
- if status.TokensAvailable > 2 {
- t.Errorf("Expected at most 2 tokens after 1 second, got %d", status.TokensAvailable)
- }
-
- t.Logf("Status monitoring working correctly")
-}
-
-func testTokenRefillBehavior(t *testing.T) {
- // Use predictable refill timing: 2 requests per 1 second = 500ms per token
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 2, // 2 requests
- 1*time.Second, // per 1 second (500ms per token)
- )
- if err != nil {
- t.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- // Use all tokens quickly
- usedTokens := 0
- start := time.Now()
- for i := 0; i < 2; i++ {
- if _, allowed := client.TryHealth(); allowed {
- usedTokens++
- }
- }
- consumeTime := time.Since(start)
- t.Logf("Consumed %d tokens in %v", usedTokens, consumeTime)
-
- if usedTokens != 2 {
- t.Errorf("Expected to use 2 tokens, actually used %d", usedTokens)
- }
-
- // Verify empty immediately
- status := client.GetRateLimiterStatus()
- t.Logf("Status after consuming all tokens: %d available", status.TokensAvailable)
- if !status.IsEmpty {
- t.Error("Expected empty bucket immediately after using all tokens")
- }
-
- // Wait for one token refill (600ms > 500ms per token)
- t.Logf("Waiting 600ms for token refill...")
- time.Sleep(600 * time.Millisecond)
-
- // Check status after wait
- status = client.GetRateLimiterStatus()
- t.Logf("Status after 600ms wait: %d tokens available", status.TokensAvailable)
-
- // Should have exactly 1 token available
- allowedCount := 0
- for i := 0; i < 3; i++ { // Try more than available to test limits
- if _, allowed := client.TryHealth(); allowed {
- allowedCount++
- }
- }
-
- // Calculate expected tokens: (consumeTime + waitTime) / tokenInterval
- // Total elapsed time for refill calculation
- totalElapsedTime := consumeTime + 600*time.Millisecond
- expectedTokens := int(totalElapsedTime / (500 * time.Millisecond))
- if expectedTokens > 2 { // Cap at bucket capacity
- expectedTokens = 2
- }
-
- t.Logf("Total elapsed time: %v, expected tokens: %d", totalElapsedTime, expectedTokens)
-
- if allowedCount < 1 {
- t.Errorf("Expected at least 1 token after %v total time, got %d", totalElapsedTime, allowedCount)
- }
- if allowedCount > 2 {
- t.Errorf("Expected at most 2 tokens (bucket capacity), got %d", allowedCount)
- }
-
- t.Logf("Token refill working correctly: %d tokens available (expected %d)", allowedCount, expectedTokens)
-
- // Test demonstrates that token bucket refills correctly over time
- t.Log("Token refill behavior is mathematically correct for token bucket algorithm")
-}
-
-// Benchmark the rate-limited client
-func BenchmarkRateLimitedRpcClient(b *testing.B) {
- client, err := rpcclient.NewRateLimitedRpcClient(
- "https://gnoland-testnet-rpc.cogwheel.zone",
- nil,
- 1000, // High limit to avoid blocking in benchmark
- 1*time.Second,
- )
- if err != nil {
- b.Fatalf("Failed to create client: %v", err)
- }
- defer client.Close()
-
- b.ResetTimer()
- b.RunParallel(func(pb *testing.PB) {
- for pb.Next() {
- // Use TryHealth to avoid blocking on rate limits
- _, allowed := client.TryHealth()
- if !allowed {
- b.Errorf("Expected request to be allowed")
- }
- }
- })
-}
diff --git a/experiments/experiment4/encode_proto.go b/experiments/experiment4/encode_proto.go
deleted file mode 100644
index 082f1ae..0000000
--- a/experiments/experiment4/encode_proto.go
+++ /dev/null
@@ -1,137 +0,0 @@
-package main
-
-import (
- "bytes"
- "encoding/hex"
- "fmt"
- "log"
-
- "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/events_proto"
- zstandard "github.com/klauspost/compress/zstd"
- "google.golang.org/protobuf/proto"
-)
-
-var EventsProto events_proto.TxEvents = events_proto.TxEvents{
- Events: []*events_proto.Event{
- {
- AtType: "/tm.GnoEvent",
- Type: "ProfileFieldCreated",
- Attributes: []*events_proto.Attribute{
- {
- Key: "FieldType",
- Value: &events_proto.Attribute_StringValue{
- StringValue: "StringField",
- },
- },
- },
- PkgPath: proto.String("gno.land/r/demo/profile"),
- },
- {
- AtType: "at_type",
- Type: "StorageDeposit",
- Attributes: []*events_proto.Attribute{
- {
- Key: "Deposit",
- Value: &events_proto.Attribute_Int64Value{
- Int64Value: 214600, // ugnot just for test this is a string
- },
- },
- },
- PkgPath: proto.String("gno.land/r/demo/profile"),
- },
- {
- AtType: "/tm.GnoEvent",
- Type: "StorageDeposit",
- Attributes: []*events_proto.Attribute{
- {
- Key: "Deposit",
- Value: &events_proto.Attribute_DoubleValue{
- DoubleValue: 214600.05111, // ugnot just for test this is a string
- },
- },
- },
- PkgPath: proto.String("gno.land/r/demo/profile"),
- },
- },
-}
-
-func EncodeProto(txEvents *events_proto.TxEvents) ([]byte, error) {
- return proto.Marshal(txEvents)
-}
-
-func DecodeProto(data []byte) (*events_proto.TxEvents, error) {
- var txEvents events_proto.TxEvents
- err := proto.Unmarshal(data, &txEvents)
- if err != nil {
- return nil, err
- }
- return &txEvents, nil
-}
-
-func main() {
- encoded, err := EncodeProto(&EventsProto)
- if err != nil {
- log.Fatal(err)
- }
-
- hexEncoded := hex.EncodeToString(encoded)
-
- // Different ways to display the encoded data:
- fmt.Println("Raw bytes (decimal):", string(encoded))
- fmt.Println("Raw bytes (hex):", hexEncoded)
- fmt.Printf("Raw bytes (hex with spaces): %x\n", hexEncoded)
- fmt.Printf("Length: %d bytes\n", len(encoded))
-
- // try different compressions
- // Test with different compression levels
- for _, level := range []int{1, 2, 3, 10, 15, 18, 20} {
- encoderLevel := zstandard.EncoderLevelFromZstd(level)
- encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel))
- if err != nil {
- log.Fatal(err)
- }
-
- var buf bytes.Buffer
- encoder.Reset(&buf)
- _, err = encoder.Write(encoded)
- if err != nil {
- log.Fatal(err)
- }
- err = encoder.Close()
- if err != nil {
- log.Fatal(err)
- }
-
- compressed := buf.Bytes()
- fmt.Printf("Compressed (level %d): %d bytes (%.1f%% of original)\n", level, len(compressed), float64(len(compressed))/float64(len(encoded))*100)
- }
- // Final compression with level 22
- encoderLevel := zstandard.EncoderLevelFromZstd(22)
- encoder, err := zstandard.NewWriter(nil, zstandard.WithEncoderLevel(encoderLevel))
- if err != nil {
- log.Fatal(err)
- }
-
- var buf bytes.Buffer
- encoder.Reset(&buf)
- _, err = encoder.Write(encoded)
- if err != nil {
- log.Fatal(err)
- }
- err = encoder.Close()
- if err != nil {
- log.Fatal(err)
- }
- err = encoder.Close()
- if err != nil {
- log.Fatal(err)
- }
- compressed := buf.Bytes()
- fmt.Println("Final compressed bytes:", hex.EncodeToString(compressed))
- fmt.Printf("Final length: %d bytes\n", len(compressed))
- decoded, err := DecodeProto(encoded)
- if err != nil {
- log.Fatal(err)
- }
- fmt.Println("decoded: ", decoded.String())
-}
diff --git a/experiments/experiment6/main.go b/experiments/experiment6/main.go
deleted file mode 100644
index 761fb0c..0000000
--- a/experiments/experiment6/main.go
+++ /dev/null
@@ -1,106 +0,0 @@
-package main
-
-import (
- "encoding/base64"
- "fmt"
- "log"
-
- "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
- "github.com/gnolang/gno/tm2/pkg/amino"
- "github.com/gnolang/gno/tm2/pkg/crypto"
- "github.com/gnolang/gno/tm2/pkg/sdk/bank"
- "github.com/gnolang/gno/tm2/pkg/std"
-)
-
-func DecodeStdTxFromBase64(s string) (*std.Tx, error) {
- bz, err := base64.StdEncoding.DecodeString(s)
- if err != nil {
- return nil, err
- }
- var tx std.Tx
- if err := amino.Unmarshal(bz, &tx); err != nil {
- return nil, err
- }
- return &tx, nil
-}
-
-func printAllData(tx *std.Tx) {
- fmt.Printf("Transaction contains %d messages:\n", len(tx.Msgs))
- fmt.Printf("Basic tx data: %v, %v, %v, %v\n", tx.Fee, tx.Memo, tx.GetSigners(), tx.Signatures)
- fmt.Println(tx.Msgs)
- for i, msg := range tx.Msgs {
- fmt.Printf("Message %d: %T\n", i+1, msg)
- switch m := msg.(type) {
- case vm.MsgCall:
- fmt.Printf(" - vm.MsgCall: caller=%s, pkg=%s, func=%s\n", m.Caller, m.PkgPath, m.Func)
- case vm.MsgAddPackage:
- fmt.Printf(" - vm.MsgAddPackage: creator=%s, pkg=%s\n", m.Creator, m.Package.Path)
- case vm.MsgRun:
- fmt.Printf(" - vm.MsgRun: caller=%s, pkg=%s\n", m.Caller, m.Package.Path)
- case bank.MsgSend:
- fmt.Printf(" - bank.MsgSend: from=%s, to=%s, amount=%s\n", m.FromAddress, m.ToAddress, m.Amount)
- }
- }
-}
-
-func encodeData() {
- // try to recreate and encode the data
- address, err := crypto.AddressFromString("g1q2hsksnh5q55xkm3d8tul28gg9uuwl2em8ver6")
- if err != nil {
- log.Fatalf("Error creating address: %v", err)
- }
- // lets use real pubkey for now
- pubkey, err := crypto.PubKeyFromBech32("gpub1pgfj7ard9eg82cjtv4u4xetrwqer2dntxyfzxz3pqdrkcn740564dys82xamnfsphzmrzzuf4eqkygx0wywzqdrmml2lj6rxjtk")
- if err != nil {
- log.Fatalf("Error creating pubkey: %v", err)
- }
- msgs := []std.Msg{
- vm.MsgCall{
- Caller: address,
- PkgPath: "gno.land/r/demo/profile",
- Func: "SetStringField",
- MaxDeposit: []std.Coin{
- {
- Amount: 1000000,
- Denom: "ugnot",
- },
- },
- Args: []string{"Ready for lift off"},
- },
- }
- tx := std.Tx{
- Msgs: msgs,
- }
- tx.Fee = std.Fee{
- GasFee: std.Coin{
- Amount: 1000000,
- Denom: "ugnot",
- },
- GasWanted: 1000000,
- }
- tx.Memo = "Ready for lift off"
- tx.Signatures = []std.Signature{
- {PubKey: pubkey, Signature: []byte("signature")},
- }
-
- // now it should encode the data and then the bytes need to be encoded to base64
- bz, err := amino.Marshal(tx)
- if err != nil {
- log.Fatal(err)
- }
- base64Encoded := base64.StdEncoding.EncodeToString(bz)
- fmt.Println("base64Encoded: ", base64Encoded)
-}
-
-func main() {
- data := "Cn4KCi92bS5tX2NhbGwScAooZzFxMmhza3NuaDVxNTV4a20zZDh0dWwyOGdnOXV1d2wyZW04dmVyNiIXZ25vLmxhbmQvci9kZW1vL3Byb2ZpbGUqDlNldFN0cmluZ0ZpZWxkMgtEaXNwbGF5TmFtZTIObm9kZXJ1bm5lcmluZG8SEwiAh6cOEgwxMDAwMDAwdWdub3Qafgo6ChMvdG0uUHViS2V5U2VjcDI1NmsxEiMKIQNHbE/VfTVWkgdRu7mmAbi2MQuJrkFiIM9xHCA0e9/V+RJAVSpcFFNgOO8K/XrXfdUOW9SpDTwhsnYsWpWEWcOo0wVJRFtDVzTb2qkkm3vao2uteVkXIKrLh2vle2yGY5v8bA=="
- tx, err := DecodeStdTxFromBase64(data)
- if err != nil {
- log.Fatal(err)
- }
- printAllData(tx)
-
- encodeData()
- // print the first encoded data it will not match this but i think there should be some similaraties
- fmt.Println("first encoded data: ", data)
-}
diff --git a/experiments/experiment7/main.go b/experiments/experiment7/main.go
deleted file mode 100644
index 618cd76..0000000
--- a/experiments/experiment7/main.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package main
-
-import (
- "encoding/hex"
- "fmt"
-
- "github.com/gnolang/gno/tm2/pkg/crypto"
- "github.com/gnolang/gno/tm2/pkg/crypto/secp256k1"
-)
-
-func generateKeyPair() (crypto.PrivKey, crypto.PubKey, crypto.Address) {
- // Generate a new private key using secp256k1
- privKey := secp256k1.GenPrivKey()
-
- // Derive the public key from the private key
- pubKey := privKey.PubKey()
-
- // Derive the address from the public key
- address := pubKey.Address()
-
- return privKey, pubKey, address
-}
-
-func testKeyGeneration() {
- fmt.Println("=== Testing Key Generation ===")
-
- // Generate 5 key pairs for testing
- for i := 0; i < 5; i++ {
- privKey, pubKey, address := generateKeyPair()
-
- fmt.Printf("Key Pair %d:\n", i+1)
- fmt.Printf(" Private Key: %s\n", hex.EncodeToString(privKey.Bytes()))
- fmt.Printf(" Public Key: %s\n", pubKey.String())
- fmt.Printf(" Address: %s\n", address.String())
- fmt.Printf(" Address Bech32: %s\n", crypto.AddressToBech32(address))
- fmt.Printf(" PubKey Bech32: %s\n", crypto.PubKeyToBech32(pubKey))
- fmt.Println()
- }
-}
-
-func main() {
- testKeyGeneration()
-}
diff --git a/experiments/experiment8/main.go b/experiments/experiment8/main.go
deleted file mode 100644
index fe3f900..0000000
--- a/experiments/experiment8/main.go
+++ /dev/null
@@ -1,123 +0,0 @@
-package main
-
-import (
- "context"
- "fmt"
-
- "github.com/jackc/pgx/v5"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/jackc/pgx/v5/pgxpool"
- "github.com/shopspring/decimal"
-)
-
-// Experiment 8
-// Trying to insert the data using the pgx package
-// using the COPY FROM method
-// to use any custom type I need to reguster it first
-// then I can copy into it,
-// for numeric use directly the pgtype.Numeric type
-
-type Balance struct {
- Currency string
- Amount pgtype.Numeric
-}
-
-type Account struct {
- ID int `db:"id"`
- Balances []Balance `db:"balances"`
-}
-
-func setupConnection() (*pgxpool.Pool, error) {
- host := "localhost"
- port := 6543
- user := "postgres"
- password := "12345678"
- dbname := "gnoland"
- sslmode := "disable"
-
- config, err := pgxpool.ParseConfig(
- fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
- host, port, user, password, dbname, sslmode),
- )
- if err != nil {
- return nil, err
- }
-
- config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
- dataTypeNames := []string{"balance", "_balance"}
-
- for _, typeName := range dataTypeNames {
- dataType, err := conn.LoadType(ctx, typeName)
- if err != nil {
- return err
- }
- conn.TypeMap().RegisterType(dataType)
- }
-
- return nil
- }
-
- return pgxpool.NewWithConfig(context.Background(), config)
-}
-
-func main() {
- pool, err := setupConnection()
- if err != nil {
- panic(err)
- }
- defer pool.Close()
-
- ctx := context.Background()
-
- // Create account with multiple balances
- accounts := []Account{
- {ID: 1,
- Balances: []Balance{
- {Currency: "USD", Amount: numericFromDecimal(decimal.NewFromFloat(100.50))},
- {Currency: "EUR", Amount: numericFromDecimal(decimal.NewFromFloat(85.25))},
- {Currency: "GBP", Amount: numericFromDecimal(decimal.NewFromFloat(75.00))},
- },
- },
- {ID: 2,
- Balances: []Balance{
- {Currency: "USD", Amount: numericFromDecimal(decimal.NewFromFloat(83.50))},
- {Currency: "EUR", Amount: numericFromDecimal(decimal.NewFromFloat(50.25))},
- {Currency: "GBP", Amount: numericFromDecimal(decimal.NewFromFloat(25.00))},
- },
- },
- }
-
- // Copy from slice
- pgxSlice := pgx.CopyFromSlice(len(accounts), func(i int) ([]any, error) {
- return []any{accounts[i].ID, accounts[i].Balances}, nil
- })
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"account"}, []string{"id", "balances"}, pgxSlice)
- if err != nil {
- fmt.Println(err)
- }
-
- // Query back
- var retrieved Account
- err = pool.QueryRow(ctx, `
- SELECT id, balances
- FROM account WHERE id = $1`, 1).
- Scan(&retrieved.ID, &retrieved.Balances)
- if err != nil {
- fmt.Printf("Error scanning: %v\n", err)
- }
- fmt.Printf("Account ID: %d\n", retrieved.ID)
- fmt.Printf("Balances: %v\n", retrieved.Balances)
-
- fmt.Println("All Balances:")
- for _, bal := range retrieved.Balances {
- fmt.Printf(" %s %v, %s\n", bal.Amount.Int.String(), bal.Amount.Exp, bal.Currency)
- }
-}
-
-func numericFromDecimal(d decimal.Decimal) pgtype.Numeric {
- if d.Exponent() == 0 {
- return pgtype.Numeric{Int: d.Coefficient(), Exp: 0, Valid: true}
- }
- n := pgtype.Numeric{Int: d.Coefficient(), Exp: d.Exponent(), Valid: true}
- return n
-}
diff --git a/experiments/experiment9/main.go b/experiments/experiment9/main.go
deleted file mode 100644
index 4f474f7..0000000
--- a/experiments/experiment9/main.go
+++ /dev/null
@@ -1,155 +0,0 @@
-package main
-
-import (
- "context"
- "fmt"
-
- "github.com/jackc/pgx/v5"
- "github.com/jackc/pgx/v5/pgxpool"
-)
-
-// Experiment 9
-// Trying to insert the data
-
-type Attribute struct {
- Key string
- Value string
-}
-
-type Event struct {
- AtType string `db:"at_type" dbtype:"TEXT"`
- Type string `db:"type" dbtype:"TEXT"`
- Attributes []Attribute `db:"attributes" dbtype:"attribute[]"`
- PkgPath string `db:"pkg_path" dbtype:"TEXT"`
-}
-
-// table for testing
-/*
-CREATE TYPE attribute AS (
- key TEXT,
- value TEXT
-);
-
-CREATE TYPE event AS (
- at_type TEXT,
- type TEXT,
- attributes attribute[],
- pkg_path TEXT
-);
-
-CREATE TABLE account (
- id INTEGER,
- height BIGINT,
- events event[]
-);
-*/
-type Account struct {
- ID int `db:"id"`
- Height uint64 `db:"height"`
- Events []Event `db:"events"`
-}
-
-func setupConnection() (*pgxpool.Pool, error) {
- host := "localhost"
- port := 6543
- user := "postgres"
- password := "12345678"
- dbname := "gnoland"
- sslmode := "disable"
-
- config, err := pgxpool.ParseConfig(
- fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
- host, port, user, password, dbname, sslmode),
- )
- if err != nil {
- return nil, err
- }
-
- config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
- dataTypeNames := []string{"attribute", "_attribute", "event", "_event"}
-
- for _, typeName := range dataTypeNames {
- dataType, err := conn.LoadType(ctx, typeName)
- if err != nil {
- return err
- }
- conn.TypeMap().RegisterType(dataType)
- }
-
- return nil
- }
-
- return pgxpool.NewWithConfig(context.Background(), config)
-}
-
-func main() {
- pool, err := setupConnection()
- if err != nil {
- panic(err)
- }
- defer pool.Close()
-
- ctx := context.Background()
-
- // Create a synthetic data for testing
- accounts := []Account{
- {
- ID: 1,
- Height: 1,
- Events: []Event{
- {
- AtType: "tm.GnoEvent",
- Type: "send",
- PkgPath: "gno.land/r/bank",
- Attributes: []Attribute{
- {Key: "sender", Value: "gno1..."},
- {Key: "recipient", Value: "gno2..."},
- {Key: "amount", Value: "1000ugnot"},
- },
- },
- {
- AtType: "tm.GnoEvent",
- Type: "receive",
- // Let's try without the pkgpath
- Attributes: []Attribute{
- {Key: "sender", Value: "gno2..."},
- {Key: "recipient", Value: "gno1..."},
- {Key: "amount", Value: "500ugnot"},
- },
- },
- },
- },
- {
- ID: 2,
- Height: 2,
- Events: []Event{}, // empty events
- },
- }
-
- // Copy from slice
- pgxSlice := pgx.CopyFromSlice(len(accounts), func(i int) ([]any, error) {
- return []any{accounts[i].ID, accounts[i].Height, accounts[i].Events}, nil
- })
-
- _, err = pool.CopyFrom(ctx, pgx.Identifier{"account"}, []string{"id", "height", "events"}, pgxSlice)
- if err != nil {
- panic(fmt.Errorf("failed to copy data: %w", err))
- }
-
- fmt.Println("Successfully inserted account data.")
-
- // Query back to verify
- var retrieved Account
- err = pool.QueryRow(ctx, `SELECT id, events FROM account WHERE id = $1`, 1).Scan(&retrieved.ID, &retrieved.Events)
- if err != nil {
- panic(fmt.Errorf("failed to query data: %w", err))
- }
-
- fmt.Printf("Retrieved Account ID: %d\n", retrieved.ID)
- for i, event := range retrieved.Events {
- fmt.Printf(" Event %d: Type=%s, PkgPath=%s\n", i+1, event.Type, event.PkgPath)
- for _, attr := range event.Attributes {
- fmt.Printf(" Attribute: %s = %s\n", attr.Key, attr.Value)
- }
- }
-}
diff --git a/go.mod b/go.mod
index 26eb2d0..dbdb81d 100644
--- a/go.mod
+++ b/go.mod
@@ -1,42 +1,57 @@
module github.com/Cogwheel-Validator/spectra-gnoland-indexer
-go 1.25.4
+go 1.25.7
require (
github.com/caarlos0/env/v6 v6.10.1
- github.com/danielgtaylor/huma/v2 v2.34.1
- github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd
- github.com/go-chi/chi/v5 v5.2.3
+ github.com/danielgtaylor/huma/v2 v2.37.2
+ github.com/gnolang/gno v0.0.0-20260227152025-7ffdf321db6e
+ github.com/go-chi/chi/v5 v5.2.5
github.com/go-chi/cors v1.2.2
- github.com/jackc/pgx/v5 v5.7.6
+ github.com/jackc/pgx/v5 v5.8.0
github.com/joho/godotenv v1.5.1
- github.com/klauspost/compress v1.18.0
- github.com/shopspring/decimal v1.4.0
- github.com/spf13/cobra v1.10.1
+ github.com/rs/zerolog v1.34.0
+ github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
- go.yaml.in/yaml/v4 v4.0.0-rc.2
- golang.org/x/term v0.35.0
- google.golang.org/protobuf v1.36.9
+ github.com/valyala/gozstd v1.24.0
+ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0
+ go.opentelemetry.io/otel/log v0.16.0
+ go.opentelemetry.io/otel/sdk/log v0.16.0
+ go.yaml.in/yaml/v4 v4.0.0-rc.4
+ golang.org/x/term v0.40.0
+ google.golang.org/protobuf v1.36.11
)
+require github.com/klauspost/compress v1.18.4 // indirect
+
require (
- github.com/btcsuite/btcd/btcec/v2 v2.3.5 // indirect
+ github.com/btcsuite/btcd/btcec/v2 v2.3.6 // indirect
github.com/btcsuite/btcd/btcutil v1.1.6 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
+ github.com/cosmos/gogoproto v1.7.2 // indirect
+ github.com/cosmos/ics23/go v0.11.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect
+ github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
+ github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/emicklei/dot v1.11.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
- github.com/gofrs/flock v0.12.1 // indirect
+ github.com/gofrs/flock v0.13.0 // indirect
+ github.com/google/go-cmp v0.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
- github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
+ github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+ github.com/mattn/go-colorable v0.1.14 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
@@ -44,24 +59,27 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
- go.opentelemetry.io/otel v1.38.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 // indirect
- go.opentelemetry.io/otel/metric v1.38.0 // indirect
- go.opentelemetry.io/otel/sdk v1.38.0 // indirect
- go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
- go.opentelemetry.io/otel/trace v1.38.0 // indirect
- go.opentelemetry.io/proto/otlp v1.8.0 // indirect
+ go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240809024635-0c3fcdf3c470
+ go.opentelemetry.io/otel v1.40.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.40.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect
+ go.opentelemetry.io/otel/metric v1.40.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.40.0 // indirect
+ go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
+ go.opentelemetry.io/otel/trace v1.40.0 // indirect
+ go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/crypto v0.42.0 // indirect
- golang.org/x/mod v0.28.0 // indirect
- golang.org/x/net v0.44.0 // indirect
- golang.org/x/sync v0.17.0 // indirect
- golang.org/x/sys v0.36.0 // indirect
- golang.org/x/text v0.29.0 // indirect
- golang.org/x/tools v0.37.0 // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 // indirect
- google.golang.org/grpc v1.75.1 // indirect
+ golang.org/x/crypto v0.48.0 // indirect
+ golang.org/x/mod v0.33.0 // indirect
+ golang.org/x/net v0.51.0 // indirect
+ golang.org/x/sync v0.19.0 // indirect
+ golang.org/x/sys v0.41.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
+ golang.org/x/tools v0.42.0 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
+ google.golang.org/grpc v1.79.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index 9f7acd6..c0613e3 100644
--- a/go.sum
+++ b/go.sum
@@ -12,8 +12,8 @@ github.com/btcsuite/btcd v0.24.2 h1:aLmxPguqxza+4ag8R1I2nnJjSu2iFn/kqtHTIImswcY=
github.com/btcsuite/btcd v0.24.2/go.mod h1:5C8ChTkl5ejr3WHj8tkQSCmydiMEPB0ZhQhehpq7Dgg=
github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA=
github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
-github.com/btcsuite/btcd/btcec/v2 v2.3.5 h1:dpAlnAwmT1yIBm3exhT1/8iUSD98RDJM5vqJVQDQLiU=
-github.com/btcsuite/btcd/btcec/v2 v2.3.5/go.mod h1:m22FrOAiuxl/tht9wIqAoGHcbnCCaPWyauO8y2LGGtQ=
+github.com/btcsuite/btcd/btcec/v2 v2.3.6 h1:IzlsEr9olcSRKB/n7c4351F3xHKxS2lma+1UFGCYd4E=
+github.com/btcsuite/btcd/btcec/v2 v2.3.6/go.mod h1:m22FrOAiuxl/tht9wIqAoGHcbnCCaPWyauO8y2LGGtQ=
github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A=
github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE=
github.com/btcsuite/btcd/btcutil v1.1.5/go.mod h1:PSZZ4UitpLBWzxGd5VGOrLnmOjtPP/a6HaFo12zMs00=
@@ -52,9 +52,14 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cosmos/gogoproto v1.7.2 h1:5G25McIraOC0mRFv9TVO139Uh3OklV2hczr13KKVHCA=
+github.com/cosmos/gogoproto v1.7.2/go.mod h1:8S7w53P1Y1cHwND64o0BnArT6RmdgIvsBuco6uTllsk=
+github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU=
+github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/danielgtaylor/huma/v2 v2.34.1 h1:EmOJAbzEGfy0wAq/QMQ1YKfEMBEfE94xdBRLPBP0gwQ=
-github.com/danielgtaylor/huma/v2 v2.34.1/go.mod h1:ynwJgLk8iGVgoaipi5tgwIQ5yoFNmiu+QdhU7CEEmhk=
+github.com/danielgtaylor/huma/v2 v2.37.2 h1:Nf9vjy2sxBJFaupPlthXL/Hy2+LurfVbaKHmCMEI7xE=
+github.com/danielgtaylor/huma/v2 v2.37.2/go.mod h1:95S04G/lExFRYlBkKaBaZm9lVmxRmqX9f2CgoOZ11AM=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -63,19 +68,29 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
+github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
+github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/emicklei/dot v1.11.0 h1:zsrhCuFHAJge/aZIC4N4LdHy5tqYu4tWEaUzIwdYj4Y=
+github.com/emicklei/dot v1.11.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
-github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd h1:r/uiCyhbsQU+cs51/nfFE3kXSvMEUjUlv+lMObsBD0I=
-github.com/gnolang/gno v0.0.0-20250926165528-791c114918cd/go.mod h1:j9wKq29meqwktEj2ReqPbSkeYUwoisfxHHVeV20lhtw=
-github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
-github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/gnolang/gno v0.0.0-20260227152025-7ffdf321db6e h1:LzbVbR3XelGTivsxPPeLnZrKS0tHKuYj9jdD8m6plZA=
+github.com/gnolang/gno v0.0.0-20260227152025-7ffdf321db6e/go.mod h1:nbafBiUL0yLiPYnJwPG+lSRqanlP4dsYyFNWZcdM6Rg=
+github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
+github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -83,10 +98,13 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
-github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
+github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -110,8 +128,10 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -119,8 +139,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk=
-github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
+github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
+github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -129,8 +149,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -139,6 +159,13 @@ github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -168,13 +195,13 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
-github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b h1:oV47z+jotrLVvhiLRNzACVe7/qZ8DcRlMlDucR/FARo=
github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b/go.mod h1:JprPCeMgYyLKJoAy9nxpVScm7NwFSwpibdrUKm4kcw0=
-github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
-github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -192,49 +219,70 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/gozstd v1.24.0 h1:M/9L3h7bVwbj2gZwrmuoaxzwVrmBUvos2jG9cZtuhlc=
+github.com/valyala/gozstd v1.24.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE=
-go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0=
+go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240809024635-0c3fcdf3c470 h1:sWl5wVvtCqwmq6E4hMwTvYOZ+2KK/mobiYUKGtDPhq8=
+go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240809024635-0c3fcdf3c470/go.mod h1:19QonlFAPKOV21UMTHW8yWjGBZ2ilfhSAyU5LKPA5HE=
+go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
+go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 h1:djrxvDxAe44mJUrKataUbOhCKhR3F8QCyWucO16hTQs=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0/go.mod h1:dt3nxpQEiSoKvfTVxp3TUg5fHPLhKtbcnN3Z1I1ePD0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.40.0 h1:NOyNnS19BF2SUDApbOKbDtWZ0IK7b8FJ2uAGdIWOGb0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.40.0/go.mod h1:VL6EgVikRLcJa9ftukrHu/ZkkhFBSo1lzvdBC9CF1ss=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0 h1:9y5sHvAxWzft1WQ4BwqcvA+IFVUJ1Ya75mSAUnFEVwE=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0/go.mod h1:eQqT90eR3X5Dbs1g9YSM30RavwLF725Ris5/XSXWvqE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
+go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4=
+go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes=
+go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
+go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
+go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
+go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI=
+go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4=
+go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y=
+go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
+go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
+go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
+go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s=
-go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
+go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
-golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
-golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
-golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
-golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -244,36 +292,39 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
-golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
-golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
-golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
-golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9 h1:jm6v6kMRpTYKxBRrDkYAitNJegUeO1Mf3Kt80obv0gg=
-google.golang.org/genproto/googleapis/api v0.0.0-20250922171735-9219d122eba9/go.mod h1:LmwNphe5Afor5V3R5BppOULHOnt2mCIf+NxMd4XiygE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 h1:V1jCN2HBa8sySkR5vLcCSqJSTMv093Rw9EJefhQGP7M=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4=
+google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
+google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
-google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
diff --git a/indexer/cmd/func.go b/indexer/cmd/func.go
index e6cfe86..614195d 100644
--- a/indexer/cmd/func.go
+++ b/indexer/cmd/func.go
@@ -2,7 +2,6 @@ package cmd
import (
"fmt"
- "log"
"os"
"path/filepath"
"slices"
@@ -61,7 +60,7 @@ func parseCommonFlags(cmd *cobra.Command, defaultDbName string) (*dbParams, erro
params.port = 5432
}
if params.user == "" {
- params.user = "postgres"
+ params.user = defaultDBUser
}
if params.name == "" {
params.name = defaultDbName
@@ -113,17 +112,17 @@ func (p *dbParams) createDatabaseConfig() database.DatabasePoolConfig {
}
}
-func createConfig(overwrite bool, fileName string) {
+func createConfig(overwrite bool, fileName string) error {
if fileName == "" {
fileName = "config.yml"
}
absolutePath, err := filepath.Abs(fileName)
if err != nil {
- log.Fatalf("failed to get absolute path: %v", err)
+ return fmt.Errorf("failed to get absolute path: %w", err)
}
fileName = absolutePath
- // config file
- config := config.Config{
+
+ cfg := config.Config{
RpcUrl: "http://localhost:26657",
PoolMaxConns: 50,
PoolMinConns: 10,
@@ -140,25 +139,27 @@ func createConfig(overwrite bool, fileName string) {
PauseTime: &[]time.Duration{15 * time.Second}[0],
ExponentialBackoff: &[]time.Duration{2 * time.Second}[0],
}
- // marshal the config to yaml
- yamlFile, err := yaml.Marshal(config)
+
+ yamlFile, err := yaml.Marshal(cfg)
if err != nil {
- log.Fatalf("failed to marshal config: %v", err)
+ return fmt.Errorf("failed to marshal config: %w", err)
}
- // check if the config file exists
+
if _, err = os.Stat(fileName); err == nil {
if overwrite {
if writeErr := os.WriteFile(fileName, yamlFile, 0644); writeErr != nil {
- log.Fatalf("failed to overwrite config file: %v", writeErr)
+ return fmt.Errorf("failed to overwrite config file: %w", writeErr)
}
} else {
- log.Fatalf("config file already exists, use --overwrite to overwrite it")
+ return fmt.Errorf("config file already exists, use --overwrite to overwrite it")
}
} else if os.IsNotExist(err) {
if writeErr := os.WriteFile(fileName, yamlFile, 0644); writeErr != nil {
- log.Fatalf("failed to create config file: %v", writeErr)
+ return fmt.Errorf("failed to create config file: %w", writeErr)
}
} else {
- log.Fatalf("failed to stat config file: %v", err)
+ return fmt.Errorf("failed to stat config file: %w", err)
}
+
+ return nil
}
diff --git a/indexer/cmd/historic.go b/indexer/cmd/historic.go
index 7ab5b3d..fb2282b 100644
--- a/indexer/cmd/historic.go
+++ b/indexer/cmd/historic.go
@@ -1,10 +1,11 @@
package cmd
import (
- "log"
+ "fmt"
mainOperator "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/main_operator"
mainTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/main_types"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
"github.com/spf13/cobra"
)
@@ -14,47 +15,52 @@ var historicCmd = &cobra.Command{
Long: `Runs the spectra indexer in historic mode, processing blocks from a given height to a given height.
The historic mode takes in starting height point and a finishing height. It should be used to
sync up the database to the latest block height.
-
+
It can also be useful if you want to index blockchain partially and work with data for any kind of testing
or partial scan of the chain where you want to index from a certain height to a certain height.
`,
- Run: func(cmd *cobra.Command, args []string) {
- log.Println("Running in historic mode")
+ RunE: func(cmd *cobra.Command, args []string) error {
+ l := logger.Get()
+ l.Info().Msg("running in historic mode")
configPath, err := cmd.Flags().GetString("config")
if err != nil {
- log.Fatalf("failed to get config path: %v", err)
+ l.Error().Err(err).Msg("failed to get config path")
+ return err
}
- // Parse RPC flags from command
maxRequestsPerWindow, err := cmd.Flags().GetInt("max-req-per-window")
if err != nil {
- log.Fatalf("failed to get max requests per window: %v", err)
+ l.Error().Err(err).Msg("failed to get max requests per window")
+ return err
}
rateLimitWindow, err := cmd.Flags().GetDuration("rate-limit-window")
if err != nil {
- log.Fatalf("failed to get rate limit window: %v", err)
+ l.Error().Err(err).Msg("failed to get rate limit window")
+ return err
}
timeout, err := cmd.Flags().GetDuration("timeout")
if err != nil {
- log.Fatalf("failed to get timeout: %v", err)
+ l.Error().Err(err).Msg("failed to get timeout")
+ return err
}
compressEvents, err := cmd.Flags().GetBool("compress-events")
if err != nil {
- log.Fatalf("failed to get compress events: %v", err)
+ l.Error().Err(err).Msg("failed to get compress events")
+ return err
}
- // Parse historic-specific flags
fromHeight, err := cmd.Flags().GetUint64("from-height")
if err != nil {
- log.Fatalf("failed to get from height: %v", err)
+ l.Error().Err(err).Msg("failed to get from height")
+ return err
}
toHeight, err := cmd.Flags().GetUint64("to-height")
if err != nil {
- log.Fatalf("failed to get to height: %v", err)
+ l.Error().Err(err).Msg("failed to get to height")
+ return err
}
- // Build flag structs
rateLimitFlags := mainTypes.RpcFlags{
RequestsPerWindow: maxRequestsPerWindow,
TimeWindow: rateLimitWindow,
@@ -63,27 +69,29 @@ var historicCmd = &cobra.Command{
runningFlags := mainTypes.RunningFlags{
RunningMode: "historic",
- SkipInitialDbCheck: false, // Not applicable for historic mode
+ SkipInitialDbCheck: false,
CompressEvents: compressEvents,
FromHeight: fromHeight,
ToHeight: toHeight,
}
- log.Println("Indexer started")
+ l.Info().Msg("indexer started")
+ if compressEvents {
+ l.Warn().Msg("compress events is enabled, this is experimental and it might slow down the data processing speed")
+ }
mainOperator.InitMainOperator(configPath, ".", rateLimitFlags, runningFlags)
+ return nil
},
}
func init() {
- // Historic-specific flags
historicCmd.Flags().Uint64P("from-height", "f", 1, "starting block height")
historicCmd.Flags().Uint64P("to-height", "o", 1000, "ending block height")
- // Mark required flags for historic mode
if err := historicCmd.MarkFlagRequired("from-height"); err != nil {
- log.Fatalf("failed to mark from height as required: %v", err)
+ panic(fmt.Sprintf("failed to mark from-height as required: %v", err))
}
if err := historicCmd.MarkFlagRequired("to-height"); err != nil {
- log.Fatalf("failed to mark to height as required: %v", err)
+ panic(fmt.Sprintf("failed to mark to-height as required: %v", err))
}
}
diff --git a/indexer/cmd/live.go b/indexer/cmd/live.go
index 55f306b..1cee7e8 100644
--- a/indexer/cmd/live.go
+++ b/indexer/cmd/live.go
@@ -1,10 +1,11 @@
package cmd
import (
- "log"
+ "fmt"
mainOperator "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/main_operator"
mainTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/main_types"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
"github.com/spf13/cobra"
)
@@ -20,39 +21,43 @@ var liveCmd = &cobra.Command{
However if you do not need previous data, you can run the live mode with the skip-db-check flag set to true.
Afterwards you can run live mode normal without the skip-db-check flag.
`,
- Run: func(cmd *cobra.Command, args []string) {
- log.Println("Running in live mode")
+ RunE: func(cmd *cobra.Command, args []string) error {
+ l := logger.Get()
+ l.Info().Msg("running in live mode")
configPath, err := cmd.Flags().GetString("config")
if err != nil {
- log.Fatalf("failed to get config path: %v", err)
+ l.Error().Err(err).Msg("failed to get config path")
+ return err
}
- // Parse RPC flags from command
maxRequestsPerWindow, err := cmd.Flags().GetInt("max-req-per-window")
if err != nil {
- log.Fatalf("failed to get max requests per window: %v", err)
+ l.Error().Err(err).Msg("failed to get max requests per window")
+ return err
}
rateLimitWindow, err := cmd.Flags().GetDuration("rate-limit-window")
if err != nil {
- log.Fatalf("failed to get rate limit window: %v", err)
+ l.Error().Err(err).Msg("failed to get rate limit window")
+ return err
}
timeout, err := cmd.Flags().GetDuration("timeout")
if err != nil {
- log.Fatalf("failed to get timeout: %v", err)
+ l.Error().Err(err).Msg("failed to get timeout")
+ return err
}
compressEvents, err := cmd.Flags().GetBool("compress-events")
if err != nil {
- log.Fatalf("failed to get compress events: %v", err)
+ l.Error().Err(err).Msg("failed to get compress events")
+ return err
}
- // Parse live-specific flags
skipDbCheck, err := cmd.Flags().GetBool("skip-db-check")
if err != nil {
- log.Fatalf("failed to get skip db check: %v", err)
+ l.Error().Err(err).Msg("failed to get skip db check")
+ return err
}
- // Build flag structs
rateLimitFlags := mainTypes.RpcFlags{
RequestsPerWindow: maxRequestsPerWindow,
TimeWindow: rateLimitWindow,
@@ -67,12 +72,16 @@ var liveCmd = &cobra.Command{
ToHeight: 0,
}
- log.Println("Indexer started")
+ l.Info().Msg("indexer started")
+ if compressEvents {
+ l.Warn().Msg("compress events is enabled, this is experimental and it might slow down the data processing speed")
+ }
+ fmt.Println(compressEvents)
mainOperator.InitMainOperator(configPath, ".", rateLimitFlags, runningFlags)
+ return nil
},
}
func init() {
- // Live-specific flags
liveCmd.Flags().BoolP("skip-db-check", "s", false, "skip initial database check")
}
diff --git a/indexer/cmd/root.go b/indexer/cmd/root.go
index 9feab5a..71cd4c7 100644
--- a/indexer/cmd/root.go
+++ b/indexer/cmd/root.go
@@ -10,14 +10,15 @@ var (
)
var RootCmd = &cobra.Command{
- Use: "indexer",
- Short: "Spectra Gnoland Indexer",
- Long: "A blockchain indexer for Gnoland that processes blocks and transactions.",
- Version: Version + " (commit: " + Commit + ")",
+ Use: "indexer",
+ Short: "Spectra Gnoland Indexer",
+ Long: "A blockchain indexer for Gnoland that processes blocks and transactions.",
+ Version: Version + " (commit: " + Commit + ")",
+ SilenceErrors: true,
+ SilenceUsage: true,
}
func init() {
- // Add parent commands to root
RootCmd.AddCommand(runCmd)
RootCmd.AddCommand(setupCmd)
}
diff --git a/indexer/cmd/setup.go b/indexer/cmd/setup.go
index 8fa4566..226f06c 100644
--- a/indexer/cmd/setup.go
+++ b/indexer/cmd/setup.go
@@ -2,15 +2,17 @@ package cmd
import (
"context"
- "log"
"time"
dbinit "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/db_init"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types"
"github.com/spf13/cobra"
)
+const defaultDBUser = "postgres"
+
var allowedSslModes = []string{"disable", "require", "verify-ca", "verify-full", "allow", "prefer"}
// dbParams holds common database connection parameters
@@ -61,14 +63,16 @@ var createDbCmd = &cobra.Command{
Use: "create-db",
Short: "Create a new database named gnoland",
Long: `Create a new database named gnoland for the indexer. It goes\n
- throught a lot of steps to create the database and insert the tables and data.`,
+ through a lot of steps to create the database and insert the tables and data.`,
RunE: func(cmd *cobra.Command, args []string) error {
- log.Printf("Initiating the cmd to set up the database for the indexer...")
+ l := logger.Get()
+ l.Info().Msg("initiating database setup for the indexer")
// Parse and validate common database flags
params, err := parseCommonFlags(cmd, "postgres")
if err != nil {
- log.Fatalf("failed to parse flags: %v", err)
+ l.Error().Err(err).Msg("failed to parse flags")
+ return err
}
// get the new database name from the flags
@@ -86,7 +90,8 @@ var createDbCmd = &cobra.Command{
// Prompt for password
params.password, err = promptPassword()
if err != nil {
- log.Fatalf("failed to read password: %v", err)
+ l.Error().Err(err).Msg("failed to read password")
+ return err
}
// Create database config
@@ -101,26 +106,28 @@ var createDbCmd = &cobra.Command{
defer cancel()
currentDb, err := db.CheckCurrentDatabaseName(ctx)
if err != nil {
- log.Fatalf("failed to check current database name: %v", err)
+ l.Error().Err(err).Msg("failed to check current database name")
+ return err
}
- log.Printf("Logged in into the database %s", currentDb)
+ l.Info().Str("db", currentDb).Msg("logged into database")
// if the current database is not "gnoland", create a new database named "gnoland"
// and insert all of the tables and data from the "gnoland" database
if currentDb != newDbName {
- // create a new database named "gnoland"
- log.Printf("Creating a new database named %s", newDbName)
+ l.Info().Str("db", newDbName).Msg("creating new database")
err = database.CreateDatabase(db, newDbName)
if err != nil {
- log.Fatalf("failed to create database: %v", err)
+ l.Error().Err(err).Msg("failed to create database")
+ return err
}
- // switch to the new database
- log.Printf("Switching to the new database %s", newDbName)
- // only for now later add dbName value, it is only for the testing now
+
+ l.Info().Str("db", newDbName).Msg("switching to new database")
err = database.SwitchDatabase(db, dbConfig, newDbName)
if err != nil {
- log.Fatalf("failed to switch database: %v", err)
+ l.Error().Err(err).Msg("failed to switch database")
+ return err
}
+
// insert all of the tables and data from the new database
// First create special types (custom postgres types that tables depend on)
// and type enums
@@ -137,23 +144,25 @@ var createDbCmd = &cobra.Command{
dbInit := dbinit.NewDBInitializer(db.GetPool())
// Create special types first (they need to exist before tables that use them)
- log.Printf("Inserting all of the special types into the %s database", chainName)
+ l.Info().Str("chain", chainName).Msg("inserting special types")
for _, specialType := range specialTypes {
err = dbInit.CreateSpecialTypeFromStruct(specialType, specialType.TypeName())
if err != nil {
- log.Fatalf("failed to create special type %s: %v", specialType.TypeName(), err)
+ l.Error().Err(err).Str("type", specialType.TypeName()).Msg("failed to create special type")
+ return err
}
}
// Create type enums
- log.Printf("Inserting all of the type enums into the %s database", chainName)
+ l.Info().Str("chain", chainName).Msg("inserting type enums")
err = dbInit.CreateChainTypeEnum(typeEnums)
if err != nil {
- log.Fatalf("failed to create type enum %s: %v", typeEnums, err)
+ l.Error().Err(err).Strs("enums", typeEnums).Msg("failed to create type enum")
+ return err
}
// Create regular tables (non-time-series tables)
- log.Printf("Inserting all of the regular tables into the %s database", chainName)
+ l.Info().Str("chain", chainName).Msg("inserting regular tables")
regularTables := []sql_data_types.DBTable{
sql_data_types.GnoAddress{},
sql_data_types.GnoValidatorAddress{},
@@ -162,12 +171,13 @@ var createDbCmd = &cobra.Command{
for _, dataType := range regularTables {
err = dbInit.CreateTableFromStruct(dataType, dataType.TableName())
if err != nil {
- log.Fatalf("failed to create table %s: %v", dataType.TableName(), err)
+ l.Error().Err(err).Str("table", dataType.TableName()).Msg("failed to create table")
+ return err
}
}
// Create hypertables (time-series tables with timestamp columns)
- log.Printf("Inserting all of the hypertables into the %s database", chainName)
+ l.Info().Str("chain", chainName).Msg("inserting hypertables")
hypertables := []struct {
table sql_data_types.DBTable
partitionColumn string
@@ -186,12 +196,13 @@ var createDbCmd = &cobra.Command{
for _, ht := range hypertables {
err = dbInit.CreateHypertableFromStruct(ht.table, ht.table.TableName(), ht.partitionColumn, ht.chunkInterval)
if err != nil {
- log.Fatalf("failed to create hypertable %s: %v", ht.table.TableName(), err)
+ l.Error().Err(err).Str("table", ht.table.TableName()).Msg("failed to create hypertable")
+ return err
}
}
- log.Printf("Successfully created all of the hypertables into the %s database", chainName)
+ l.Info().Str("chain", chainName).Msg("successfully created all hypertables")
} else {
- log.Printf("The current database is %s, and it already exists", currentDb)
+ l.Info().Str("db", currentDb).Msg("database already exists, skipping creation")
// TODO else if the current database is "gnoland" then we need to check if the tables exist
// and if they don't exist then we need to create them
// also any kind of future updates to the database should be done here
@@ -205,30 +216,37 @@ var createUserCmd = &cobra.Command{
Short: "Create a new user for the database",
Long: `Create a new user for the database. It will ask for the password and create the user.`,
RunE: func(cmd *cobra.Command, args []string) error {
+ l := logger.Get()
+
// Parse and validate common database flags
params, err := parseCommonFlags(cmd, "gnoland")
if err != nil {
- log.Fatalf("failed to parse flags: %v", err)
+ l.Fatal().Err(err).Msg("failed to parse flags")
+ return err
}
// Get privilege flag
privilege, _ := cmd.Flags().GetString("privilege")
if privilege == "" {
- log.Fatalf("privilege is required")
+ l.Fatal().Msg("privilege is required")
+ return cmd.Usage()
} else if privilege != "reader" && privilege != "writer" {
- log.Fatalf("invalid privilege: %s", privilege)
+ l.Fatal().Str("privilege", privilege).Msg("invalid privilege")
+ return cmd.Usage()
}
// get the user name from the flags
userName, _ := cmd.Flags().GetString("user")
if userName == "" {
- log.Fatalf("user name is required")
+ l.Fatal().Msg("user name is required")
+ return cmd.Usage()
}
// Prompt for password
params.password, err = promptPassword()
if err != nil {
- log.Fatalf("failed to read password: %v", err)
+ l.Fatal().Err(err).Msg("failed to read password")
+ return err
}
// Create database config and connection
@@ -239,16 +257,18 @@ var createUserCmd = &cobra.Command{
// Create a new user
err = dbInit.CreateUser(userName)
if err != nil {
- log.Fatalf("failed to create user: %v", err)
+ l.Fatal().Err(err).Str("user", userName).Msg("failed to create user")
+ return err
}
// Appoint privileges to the user
- err = dbInit.AppointPrivileges(params.user, privilege, []string{})
+ err = dbInit.AppointPrivileges(userName, privilege, []string{})
if err != nil {
- log.Fatalf("failed to appoint privileges to user: %v", err)
+ l.Fatal().Err(err).Str("user", userName).Str("privilege", privilege).Msg("failed to appoint privileges")
+ return err
}
- log.Printf("Successfully created user %s with privilege %s", params.user, privilege)
+ l.Info().Str("user", userName).Str("privilege", privilege).Msg("successfully created user")
return nil
},
}
@@ -259,14 +279,18 @@ var createConfigCmd = &cobra.Command{
Long: `Generate a config with default values. It will make a config file with default values.
You can add --overwrite to overwrite the existing config file. And you can use --config to specifly the path`,
RunE: func(cmd *cobra.Command, args []string) error {
+ l := logger.Get()
+
// get the config file name from the flags
configFileName, _ := cmd.Flags().GetString("config")
if configFileName == "" {
configFileName = "config.yaml"
}
overwrite, _ := cmd.Flags().GetBool("overwrite")
- createConfig(overwrite, configFileName)
- log.Printf("Successfully created config file %s", configFileName)
+ if err := createConfig(overwrite, configFileName); err != nil {
+ return err
+ }
+ l.Info().Str("file", configFileName).Msg("successfully created config file")
return nil
},
}
diff --git a/indexer/context_hook/hook.go b/indexer/context_hook/hook.go
index 981a91d..1ef5be5 100644
--- a/indexer/context_hook/hook.go
+++ b/indexer/context_hook/hook.go
@@ -2,13 +2,16 @@ package contexthook
import (
"context"
- "log"
"os"
"os/signal"
"syscall"
"time"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
+var l = logger.Get()
+
// NewSignalHandler is a constructor function that creates a new signal handler with
// cleanup and state dump functions
//
@@ -63,14 +66,14 @@ func (sh *SignalHandler) StartListening() {
go func() {
sig := <-signalChan
- log.Printf("Received signal: %v", sig)
+ l.Info().Msgf("Received signal: %v", sig)
switch sig {
case os.Interrupt, syscall.SIGTERM:
- log.Printf("Starting graceful shutdown...")
+ l.Info().Msg("Starting graceful shutdown...")
sh.gracefulShutdown()
case syscall.SIGQUIT:
- log.Printf("Emergency shutdown requested, dumping state...")
+ l.Info().Msg("Emergency shutdown requested, dumping state...")
sh.emergencyShutdown()
}
}()
@@ -89,7 +92,7 @@ func (sh *SignalHandler) gracefulShutdown() {
// Give operations time to finish gracefully
shutdownTimeout := 30 * time.Second
- log.Printf("Waiting up to %v for operations to complete...", shutdownTimeout)
+ l.Info().Msgf("Waiting up to %v for operations to complete...", shutdownTimeout)
done := make(chan struct{})
go func() {
@@ -99,22 +102,22 @@ func (sh *SignalHandler) gracefulShutdown() {
select {
case <-done:
- log.Printf("All operations completed successfully")
+ l.Info().Msg("All operations completed successfully")
case <-time.After(shutdownTimeout):
- log.Printf("Timeout reached, forcing shutdown")
+ l.Info().Msg("Timeout reached, forcing shutdown")
}
// Run cleanup function if provided
if sh.cleanup != nil {
- log.Printf("Running cleanup operations...")
+ l.Info().Msg("Running cleanup operations...")
if err := sh.cleanup(); err != nil {
- log.Printf("Error during cleanup: %v", err)
+ l.Error().Caller().Stack().Err(err).Msgf("Error during cleanup")
} else {
- log.Printf("Cleanup completed successfully")
+ l.Info().Msg("Cleanup completed successfully")
}
}
- log.Printf("Graceful shutdown complete")
+ l.Info().Msg("Graceful shutdown complete")
os.Exit(0)
}
@@ -130,15 +133,15 @@ func (sh *SignalHandler) emergencyShutdown() {
// Dump state if function is provided
if sh.stateDump != nil {
- log.Printf("Dumping application state...")
+ l.Info().Msg("Dumping application state...")
if err := sh.stateDump(); err != nil {
- log.Printf("Error dumping state: %v", err)
+ l.Error().Caller().Stack().Err(err).Msgf("Error dumping state")
} else {
- log.Printf("State dump completed")
+ l.Info().Msg("State dump completed")
}
}
- log.Printf("Emergency shutdown complete")
+ l.Info().Msg("Emergency shutdown complete")
os.Exit(1)
}
diff --git a/indexer/data_processor/event.go b/indexer/data_processor/event.go
index 4bfabf6..80cf4c2 100644
--- a/indexer/data_processor/event.go
+++ b/indexer/data_processor/event.go
@@ -1,10 +1,12 @@
package dataprocessor
import (
- "fmt"
-
rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
+ dictloader "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/dict_loader"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/events_proto"
sqlDataTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types"
+ "github.com/cosmos/gogoproto/proto"
+ "github.com/klauspost/compress/zstd"
)
// EventFormat represents the format type of the returned data
@@ -15,6 +17,19 @@ const (
CompressedFormat
)
+var dictBytes = dictloader.LoadDict()
+var zstdDict = zstd.WithEncoderDict(dictBytes)
+var zstdLvl = zstd.WithEncoderLevel(zstd.SpeedBestCompression)
+var zstdWriter *zstd.Encoder
+
+func init() {
+ var err error
+ zstdWriter, err = zstd.NewWriter(nil, zstdDict, zstdLvl)
+ if err != nil {
+ l.Fatal().Err(err).Msg("failed to initialize zstd writer")
+ }
+}
+
// EventResult holds the result of EventSolver with type discrimination
type EventResult struct {
Format EventFormat
@@ -48,13 +63,6 @@ func (er *EventResult) GetCompressedData() []byte {
return nil
}
-// TODO: when the dict training is done this will need to hold
-// 2 types of storing the data
-// 1. native postgres format
-// 2. protobuf format
-//
-// Untill the training is done and safe to use focus on the native postgres format.
-
// EventSolver is a function that solves the event of a transaction
// it will solve the event of a transaction and return the event
//
@@ -70,17 +78,30 @@ func (er *EventResult) GetCompressedData() []byte {
// - *EventResult: contains either native events or compressed data
// - error: an error if the event solving fails
func EventSolver(txResponse *rpcClient.TxResponse, useCompressed bool) (*EventResult, error) {
- if useCompressed {
- // TODO: Implement compressed format when protobuf training is complete
- // This would involve:
- // 1. Converting to protobuf format
- // 2. Compressing with zstandard
- // 3. Returning the compressed bytes
- return nil, fmt.Errorf("compressed format not yet implemented")
+ events := &txResponse.Result.TxResult.ResponseBase.Events
+ evCount := len(*events)
+
+ /*
+ The reason why the program only compresses events with more than 2 elements is because of the size.
+ Even with the trained zstd dictionary in theory we could compress the data but unless it is more than 70 bytes
+ altogether in it's raw form then it will just add compression overhead.
+
+ Until the dictionary is improved and trained on a larger set of data for now it will work like this.
+ */
+ if useCompressed && evCount >= 2 {
+ protoSerializedEv, err := serializeEvent(events)
+ if err != nil {
+ return nil, err
+ }
+ compressed := zstdWriter.EncodeAll(protoSerializedEv, nil)
+ return &EventResult{
+ Format: CompressedFormat,
+ CompressedData: compressed,
+ }, nil
}
// Native format implementation
- events := make([]sqlDataTypes.Event, 0, len(txResponse.Result.TxResult.ResponseBase.Events))
+ nativeEvents := make([]sqlDataTypes.Event, 0, len(txResponse.Result.TxResult.ResponseBase.Events))
for _, event := range txResponse.Result.TxResult.ResponseBase.Events {
attributes := make([]sqlDataTypes.Attribute, 0, len(event.Attrs))
for _, attribute := range event.Attrs {
@@ -89,7 +110,7 @@ func EventSolver(txResponse *rpcClient.TxResponse, useCompressed bool) (*EventRe
Value: attribute.Value,
})
}
- events = append(events, sqlDataTypes.Event{
+ nativeEvents = append(nativeEvents, sqlDataTypes.Event{
AtType: event.AtType,
Type: event.Type,
Attributes: attributes,
@@ -99,6 +120,31 @@ func EventSolver(txResponse *rpcClient.TxResponse, useCompressed bool) (*EventRe
return &EventResult{
Format: NativeFormat,
- NativeEvents: events,
+ NativeEvents: nativeEvents,
}, nil
}
+
+func serializeEvent(events *[]rpcClient.Event) ([]byte, error) {
+ protoTxEvents := &events_proto.TxEvents{
+ Events: make([]*events_proto.Event, 0),
+ }
+ for _, event := range *events {
+ protoAttrs := make([]*events_proto.Attribute, 0)
+ for _, attribute := range event.Attrs {
+ protoAttrs = append(protoAttrs, events_proto.NewAttributeFromString(attribute.Key, attribute.Value))
+ }
+ pkgPath := event.PkgPath
+ protoEv := &events_proto.Event{
+ AtType: event.AtType,
+ Type: event.Type,
+ Attributes: protoAttrs,
+ PkgPath: &pkgPath,
+ }
+ protoTxEvents.Events = append(protoTxEvents.Events, protoEv)
+ }
+ bs, err := proto.Marshal(protoTxEvents)
+ if err != nil {
+ return nil, err
+ }
+ return bs, nil
+}
diff --git a/indexer/data_processor/operator.go b/indexer/data_processor/operator.go
index b5b38bc..29525c8 100644
--- a/indexer/data_processor/operator.go
+++ b/indexer/data_processor/operator.go
@@ -13,9 +13,12 @@ import (
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/decoder"
rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
sqlDataTypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/sql_data_types"
)
+var l = logger.Get()
+
// Constructor function for the DataProcessor struct
//
// Parameters:
@@ -99,7 +102,10 @@ func (d *DataProcessor) ProcessValidatorAddresses(
// retry 3 times just for the sake of it
d.validatorCache.AddressSolver(addresses, d.chainName, true, 3, nil)
- log.Printf("Validator addresses processed from %d to %d", fromHeight, toHeight)
+ l.Info().
+ Msgf(
+ "Validator addresses processed from %d to %d", fromHeight, toHeight,
+ )
}
// ProcessBlocks is a "swarm" method to process the blocks from a slice of blocks
@@ -152,7 +158,12 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei
// match the transaction hash
txHashSha256 := sha256.Sum256(txHash)
if err != nil {
- log.Printf("Failed to decode tx hash %s: %v", tx, err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to decode tx hash %s: %v", tx, err,
+ )
continue
}
txs = append(txs, txHashSha256[:])
@@ -181,9 +192,17 @@ func (d *DataProcessor) ProcessBlocks(blocks []*rpcClient.BlockResponse, fromHei
defer cancel()
err := d.dbPool.InsertBlocks(ctx, blocksData)
if err != nil {
- log.Printf("Failed to insert blocks: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to insert blocks: %v", err,
+ )
}
- log.Printf("Blocks processed from %d to %d", fromHeight, toHeight)
+ l.Info().
+ Msgf(
+ "Blocks processed from %d to %d", fromHeight, toHeight,
+ )
}
// ProcessTransactions is a swarm method to process the transactions from a map of transactions and timestamps
@@ -245,6 +264,10 @@ func (d *DataProcessor) ProcessTransactions(
}
// Use mutex only when writing to shared slice
+ txEvents := events.GetNativeEvents()
+ txEventsCompressed := events.GetCompressedData()
+ compressionOn := events.IsCompressed()
+
mu.Lock()
transactionsData[idx] = sqlDataTypes.TransactionGeneral{
TxHash: txHash,
@@ -252,9 +275,9 @@ func (d *DataProcessor) ProcessTransactions(
Timestamp: transaction.Timestamp,
BlockHeight: transaction.BlockHeight,
MsgTypes: msgTypes,
- TxEvents: events.GetNativeEvents(),
- TxEventsCompressed: events.GetCompressedData(),
- CompressionOn: compressEvents,
+ TxEvents: txEvents,
+ TxEventsCompressed: txEventsCompressed,
+ CompressionOn: compressionOn,
GasUsed: gasUsed,
GasWanted: gasWanted,
Fee: fee,
@@ -276,10 +299,18 @@ func (d *DataProcessor) ProcessTransactions(
defer cancel()
err := d.dbPool.InsertTransactionsGeneral(ctx, transactionsData)
if err != nil {
- log.Printf("Failed to insert transactions: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to insert transactions: %v", err,
+ )
return
}
- log.Printf("Transactions processed from %d to %d", fromHeight, toHeight)
+ l.Info().
+ Msgf(
+ "Transactions processed from %d to %d", fromHeight, toHeight,
+ )
}
// ProcessMessages processes all messages from transactions using concurrent "swarm method"
@@ -303,17 +334,23 @@ func (d *DataProcessor) ProcessMessages(
var mu sync.Mutex
transactionAmount := len(transactions)
addressesMap := make(map[string]struct{})
- addressCollectionChan := make(chan []*decoder.DecodedMsg, transactionAmount)
+ allDecodedMsgs := make([]*decoder.DecodedMsg, transactionAmount)
wg1 := sync.WaitGroup{}
wg1.Add(transactionAmount)
// Launch goroutines to collect addresses concurrently
- for _, transaction := range transactions {
- go func(transaction TrasnactionsData) {
+ for idx, transaction := range transactions {
+ go func(idx int, transaction TrasnactionsData) {
defer wg1.Done()
decodedMsg := decoder.NewDecodedMsg(transaction.Response.Result.Tx)
if decodedMsg == nil {
- addressCollectionChan <- []*decoder.DecodedMsg{nil}
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "The transaction couldn't be decoded, tx hash: %s",
+ transaction.Response.GetHash(),
+ )
return
}
@@ -323,23 +360,13 @@ func (d *DataProcessor) ProcessMessages(
for _, address := range addresses {
addressesMap[address] = struct{}{}
}
+ allDecodedMsgs[idx] = decodedMsg
mu.Unlock()
- addressCollectionChan <- []*decoder.DecodedMsg{decodedMsg}
- }(transaction)
+ }(idx, transaction)
}
- // Close channel when all address collection goroutines finish
- go func() {
- wg1.Wait()
- close(addressCollectionChan)
- }()
-
- // Collect decoded messages
- allDecodedMsgs := make([]*decoder.DecodedMsg, 0, len(transactions))
- for decodedMsgs := range addressCollectionChan {
- allDecodedMsgs = append(allDecodedMsgs, decodedMsgs[0])
- }
+ wg1.Wait()
// Extract addresses from map[string]struct{} and resolve to IDs
allAddresses := make([]string, 0)
@@ -349,11 +376,14 @@ func (d *DataProcessor) ProcessMessages(
if len(allAddresses) > 0 {
d.addressCache.AddressSolver(allAddresses, d.chainName, false, 3, nil)
- log.Printf("Resolved %d unique addresses for messages from %d to %d", len(allAddresses), fromHeight, toHeight)
+ l.Info().
+ Msgf(
+ "Resolved %d unique addresses for messages from %d to %d",
+ len(allAddresses), fromHeight, toHeight,
+ )
}
- // Phase 2: Concurrent message processing without channels
- // Use a mutex for aggregation instead of channels
+ // Phase 2: Process message groups
aggregatedDbGroups := &decoder.DbMessageGroups{
MsgSend: make([]sqlDataTypes.MsgSend, 0),
MsgCall: make([]sqlDataTypes.MsgCall, 0),
@@ -380,20 +410,42 @@ func (d *DataProcessor) ProcessMessages(
// There might be an error here
// but any kind of retry mechanism will probably not help
// log it for.
- log.Printf("The transaction couldn't be decoded, tx hash: %s", transaction.Response.GetHash())
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "The transaction couldn't be decoded, tx hash: %s",
+ transaction.Response.GetHash(),
+ )
return
}
// Convert directly to database-ready messages with address IDs
txHash, err := base64.StdEncoding.DecodeString(transaction.Response.GetHash())
if err != nil {
- log.Printf("Failed to decode tx hash %s: %v", transaction.Response.GetHash(), err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to decode tx hash %s: %v",
+ transaction.Response.GetHash(),
+ err,
+ )
return
}
- dbMessageGroups, err := decodedMsg.ConvertToDbMessages(d.addressCache, txHash, d.chainName, transaction.Timestamp, decodedMsg.GetSigners())
+ dbMessageGroups, err := decodedMsg.ConvertToDbMessages(
+ d.addressCache, txHash, d.chainName, transaction.Timestamp, decodedMsg.GetSigners(),
+ )
if err != nil {
- log.Printf("Failed to convert messages for tx %s: %v", transaction.Response.GetHash(), err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to convert messages for tx %s: %v",
+ transaction.Response.GetHash(),
+ err,
+ )
return
}
@@ -428,7 +480,7 @@ func (d *DataProcessor) ProcessMessages(
return fmt.Errorf("failed to insert optimized messages: %w", err)
}
- log.Printf("Messages processed concurrently from %d to %d: MsgSend=%d, MsgCall=%d, MsgAddPkg=%d, MsgRun=%d",
+ l.Info().Msgf("Messages processed concurrently from %d to %d: MsgSend=%d, MsgCall=%d, MsgAddPkg=%d, MsgRun=%d",
fromHeight, toHeight,
len(aggregatedDbGroups.MsgSend),
len(aggregatedDbGroups.MsgCall),
@@ -525,13 +577,14 @@ func (d *DataProcessor) ProcessValidatorSignings(
toHeight uint64) {
commitAmount := len(commits)
- validatorChan := make(chan *sqlDataTypes.ValidatorBlockSigning, commitAmount)
+ mu := sync.Mutex{}
+ validatorData := make([]sqlDataTypes.ValidatorBlockSigning, commitAmount)
wg := sync.WaitGroup{}
wg.Add(commitAmount)
// Process blocks concurrently
- for _, commit := range commits {
- go func(commit *rpcClient.CommitResponse) {
+ for idx, commit := range commits {
+ go func(idx int, commit *rpcClient.CommitResponse) {
defer wg.Done()
signedVals := struct {
@@ -552,39 +605,43 @@ func (d *DataProcessor) ProcessValidatorSignings(
height, err := commit.GetHeight()
if err != nil {
- log.Printf("Failed to get commit height: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to get commit height: %v", err,
+ )
return
}
- validatorChan <- &sqlDataTypes.ValidatorBlockSigning{
+ mu.Lock()
+ validatorData[idx] = sqlDataTypes.ValidatorBlockSigning{
BlockHeight: height,
Timestamp: commit.GetTimestamp(),
Proposer: signedVals.Proposer,
SignedVals: signedVals.SignedVals,
ChainName: d.chainName,
}
- }(commit)
+ mu.Unlock()
+ }(idx, commit)
}
- // Close channel when all goroutines finish
- go func() {
- wg.Wait()
- close(validatorChan)
- }()
-
- validatorData := make([]sqlDataTypes.ValidatorBlockSigning, 0, len(commits))
- for validator := range validatorChan {
- validatorData = append(validatorData, *validator)
- }
+ wg.Wait()
timeout := 10*time.Second + (time.Duration(commitAmount) * time.Second / 5)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
err := d.dbPool.InsertValidatorBlockSignings(ctx, validatorData)
cancel()
if err != nil {
- log.Printf("Failed to insert validator commit signings: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "Failed to insert validator commit signings: %v", err,
+ )
}
- log.Printf("Validator commit signings processed from %d to %d", fromHeight, toHeight)
+ l.Info().
+ Msgf("Validator commit signings processed from %d to %d", fromHeight, toHeight)
}
// createAddressTx is a private func that creates a slice of sqlDataTypes.AddressTx from a
diff --git a/indexer/db_init/hypertable.go b/indexer/db_init/hypertable.go
index aeeacca..2877e7b 100644
--- a/indexer/db_init/hypertable.go
+++ b/indexer/db_init/hypertable.go
@@ -3,10 +3,13 @@ package dbinit
import (
"context"
"fmt"
- "log"
"strings"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
+var l = logger.Get()
+
// Hypertable management for TimescaleDB
//
// This package supports both modern (2.19.3+) and legacy TimescaleDB versions:
@@ -42,7 +45,12 @@ func (init *DBInitializer) ConvertToHypertables(tableNames []string) {
sql := fmt.Sprintf("SELECT create_hypertable('%s', 'timestamp', chunk_time_interval => INTERVAL '1 weeks')", tableName)
_, err := init.pool.Exec(context.Background(), sql)
if err != nil {
- log.Fatalf("failed to convert table %s to hypertable: %v", tableName, err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "failed to convert table %s to hypertable: %v", tableName, err,
+ )
}
}
}
@@ -72,7 +80,12 @@ func (init *DBInitializer) AlterCompressionSegments(tables map[string][]string)
`, tableName, columnsString)
_, err := init.pool.Exec(context.Background(), sql)
if err != nil {
- log.Fatalf("failed to alter compression segments for table %s: %v", tableName, err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "failed to alter compression segments for table %s: %v", tableName, err,
+ )
}
}
}
@@ -96,7 +109,12 @@ func (init *DBInitializer) AddCompressionPolicy(tableNames []string) {
`, tableName)
_, err := init.pool.Exec(context.Background(), sql)
if err != nil {
- log.Fatalf("failed to add compression policy for table %s: %v", tableName, err)
+ l.Error().
+ Caller().
+ Stack().
+ Msgf(
+ "failed to add compression policy for table %s: %v", tableName, err,
+ )
}
}
}
diff --git a/indexer/db_init/tables.go b/indexer/db_init/tables.go
index 57d542d..cf3f7a1 100644
--- a/indexer/db_init/tables.go
+++ b/indexer/db_init/tables.go
@@ -3,7 +3,6 @@ package dbinit
import (
"context"
"fmt"
- "log"
"reflect"
"strconv"
"strings"
@@ -464,16 +463,16 @@ func (db *DBInitializer) CreateHypertableFromStruct(structType interface{}, tabl
version, err := db.GetTimescaleDBVersion()
if err != nil {
// If we can't detect version, fall back to legacy method
- log.Printf("Warning: Could not detect TimescaleDB version (%v), using legacy method", err)
+ l.Warn().Err(err).Msg("Warning: Could not detect TimescaleDB version, using legacy method")
return db.createHypertableLegacy(tableInfo, partitionColumn, chunkInterval)
}
// Use modern syntax for 2.19.3+, legacy for older versions
if version.IsModernVersion() {
- log.Printf("Using modern hypertable syntax for TimescaleDB %d.%d.%d", version.Major, version.Minor, version.Patch)
+ l.Info().Msgf("Using modern hypertable syntax for TimescaleDB %d.%d.%d", version.Major, version.Minor, version.Patch)
return db.createHypertableModern(tableInfo, partitionColumn, chunkInterval)
} else {
- log.Printf("Using legacy hypertable syntax for TimescaleDB %d.%d.%d", version.Major, version.Minor, version.Patch)
+ l.Info().Msgf("Using legacy hypertable syntax for TimescaleDB %d.%d.%d", version.Major, version.Minor, version.Patch)
return db.createHypertableLegacy(tableInfo, partitionColumn, chunkInterval)
}
}
@@ -487,7 +486,7 @@ func (db *DBInitializer) createHypertableModern(tableInfo *TableInfo, partitionC
return fmt.Errorf("failed to create hypertable %s using modern syntax: %w", tableInfo.TableName, err)
}
- log.Printf("Successfully created hypertable: %s", tableInfo.TableName)
+ l.Info().Msgf("Successfully created hypertable: %s", tableInfo.TableName)
return nil
}
@@ -517,7 +516,7 @@ func (db *DBInitializer) createHypertableLegacy(tableInfo *TableInfo, partitionC
return fmt.Errorf("failed to convert table %s to hypertable: %w", tableInfo.TableName, err)
}
- log.Printf("Successfully created hypertable (legacy): %s", tableInfo.TableName)
+ l.Info().Msgf("Successfully created hypertable (legacy): %s", tableInfo.TableName)
return nil
}
@@ -549,11 +548,12 @@ func (db *DBInitializer) CreateChainTypeEnum(enumValues []string) error {
// - nil: if the function is successful
// - error: if the function fails
func (db *DBInitializer) CreateUser(userName string) error {
- fmt.Printf("Creating user %s.....\n", userName)
- fmt.Printf("Enter the password for the new user: ")
+ l.Info().Msgf("Creating user %s.....\n", userName)
+ l.Info().Msg("Enter the password for the new user: ")
bytePassword, err := term.ReadPassword(int(syscall.Stdin))
if err != nil {
- log.Fatalf("failed to read password: %v", err)
+ l.Error().Err(err).Msg("failed to read password")
+ return fmt.Errorf("failed to read password: %w", err)
}
password := string(bytePassword)
@@ -562,7 +562,7 @@ func (db *DBInitializer) CreateUser(userName string) error {
// if error is related to user already existing mark it as a warning and continue
if err != nil {
if strings.Contains(err.Error(), "already exists") {
- log.Printf("User %s already exists, skipping creation", userName)
+ l.Warn().Msgf("User %s already exists, skipping creation", userName)
return nil
} else {
return fmt.Errorf("failed to create user %s: %w", userName, err)
@@ -586,11 +586,11 @@ func (db *DBInitializer) AppointPrivileges(
switch privilage {
case "reader":
for _, tableName := range tableNames {
- sql.WriteString(fmt.Sprintf("GRANT SELECT ON TABLE %s TO %s;\n", tableName, userName))
+ fmt.Fprintf(&sql, "GRANT SELECT ON TABLE %s TO %s;\n", tableName, userName)
}
case "writer":
for _, tableName := range tableNames {
- sql.WriteString(fmt.Sprintf("GRANT SELECT, INSERT, UPDATE ON TABLE %s TO %s;\n", tableName, userName))
+ fmt.Fprintf(&sql, "GRANT SELECT, INSERT, UPDATE ON TABLE %s TO %s;\n", tableName, userName)
}
default:
return fmt.Errorf("invalid privilage: %s", privilage)
diff --git a/indexer/events_proto/events.pb.go b/indexer/events_proto/events.pb.go
deleted file mode 100644
index 2950dc0..0000000
--- a/indexer/events_proto/events.pb.go
+++ /dev/null
@@ -1,340 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// protoc-gen-go v1.36.8
-// protoc v6.31.1
-// source: events.proto
-
-package events_proto
-
-import (
- protoreflect "google.golang.org/protobuf/reflect/protoreflect"
- protoimpl "google.golang.org/protobuf/runtime/protoimpl"
- reflect "reflect"
- sync "sync"
- unsafe "unsafe"
-)
-
-const (
- // Verify that this generated code is sufficiently up-to-date.
- _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
- // Verify that runtime/protoimpl is sufficiently up-to-date.
- _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-type TxEvents struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- Events []*Event `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
-}
-
-func (x *TxEvents) Reset() {
- *x = TxEvents{}
- mi := &file_events_proto_msgTypes[0]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
-}
-
-func (x *TxEvents) String() string {
- return protoimpl.X.MessageStringOf(x)
-}
-
-func (*TxEvents) ProtoMessage() {}
-
-func (x *TxEvents) ProtoReflect() protoreflect.Message {
- mi := &file_events_proto_msgTypes[0]
- if x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
-}
-
-// Deprecated: Use TxEvents.ProtoReflect.Descriptor instead.
-func (*TxEvents) Descriptor() ([]byte, []int) {
- return file_events_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *TxEvents) GetEvents() []*Event {
- if x != nil {
- return x.Events
- }
- return nil
-}
-
-type Event struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- AtType string `protobuf:"bytes,1,opt,name=at_type,json=atType,proto3" json:"at_type,omitempty"`
- Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"`
- Attributes []*Attribute `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty"`
- PkgPath *string `protobuf:"bytes,4,opt,name=pkg_path,json=pkgPath,proto3,oneof" json:"pkg_path,omitempty"` // Can be null or string
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
-}
-
-func (x *Event) Reset() {
- *x = Event{}
- mi := &file_events_proto_msgTypes[1]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
-}
-
-func (x *Event) String() string {
- return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Event) ProtoMessage() {}
-
-func (x *Event) ProtoReflect() protoreflect.Message {
- mi := &file_events_proto_msgTypes[1]
- if x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
-}
-
-// Deprecated: Use Event.ProtoReflect.Descriptor instead.
-func (*Event) Descriptor() ([]byte, []int) {
- return file_events_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *Event) GetAtType() string {
- if x != nil {
- return x.AtType
- }
- return ""
-}
-
-func (x *Event) GetType() string {
- if x != nil {
- return x.Type
- }
- return ""
-}
-
-func (x *Event) GetAttributes() []*Attribute {
- if x != nil {
- return x.Attributes
- }
- return nil
-}
-
-func (x *Event) GetPkgPath() string {
- if x != nil && x.PkgPath != nil {
- return *x.PkgPath
- }
- return ""
-}
-
-type Attribute struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
- // Types that are valid to be assigned to Value:
- //
- // *Attribute_StringValue
- // *Attribute_Int64Value
- // *Attribute_BoolValue
- // *Attribute_DoubleValue
- Value isAttribute_Value `protobuf_oneof:"value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
-}
-
-func (x *Attribute) Reset() {
- *x = Attribute{}
- mi := &file_events_proto_msgTypes[2]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
-}
-
-func (x *Attribute) String() string {
- return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Attribute) ProtoMessage() {}
-
-func (x *Attribute) ProtoReflect() protoreflect.Message {
- mi := &file_events_proto_msgTypes[2]
- if x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
-}
-
-// Deprecated: Use Attribute.ProtoReflect.Descriptor instead.
-func (*Attribute) Descriptor() ([]byte, []int) {
- return file_events_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *Attribute) GetKey() string {
- if x != nil {
- return x.Key
- }
- return ""
-}
-
-func (x *Attribute) GetValue() isAttribute_Value {
- if x != nil {
- return x.Value
- }
- return nil
-}
-
-func (x *Attribute) GetStringValue() string {
- if x != nil {
- if x, ok := x.Value.(*Attribute_StringValue); ok {
- return x.StringValue
- }
- }
- return ""
-}
-
-func (x *Attribute) GetInt64Value() int64 {
- if x != nil {
- if x, ok := x.Value.(*Attribute_Int64Value); ok {
- return x.Int64Value
- }
- }
- return 0
-}
-
-func (x *Attribute) GetBoolValue() bool {
- if x != nil {
- if x, ok := x.Value.(*Attribute_BoolValue); ok {
- return x.BoolValue
- }
- }
- return false
-}
-
-func (x *Attribute) GetDoubleValue() float64 {
- if x != nil {
- if x, ok := x.Value.(*Attribute_DoubleValue); ok {
- return x.DoubleValue
- }
- }
- return 0
-}
-
-type isAttribute_Value interface {
- isAttribute_Value()
-}
-
-type Attribute_StringValue struct {
- StringValue string `protobuf:"bytes,2,opt,name=string_value,json=stringValue,proto3,oneof"`
-}
-
-type Attribute_Int64Value struct {
- Int64Value int64 `protobuf:"varint,3,opt,name=int64_value,json=int64Value,proto3,oneof"`
-}
-
-type Attribute_BoolValue struct {
- BoolValue bool `protobuf:"varint,4,opt,name=bool_value,json=boolValue,proto3,oneof"`
-}
-
-type Attribute_DoubleValue struct {
- DoubleValue float64 `protobuf:"fixed64,5,opt,name=double_value,json=doubleValue,proto3,oneof"` // leave this as double for now
-}
-
-func (*Attribute_StringValue) isAttribute_Value() {}
-
-func (*Attribute_Int64Value) isAttribute_Value() {}
-
-func (*Attribute_BoolValue) isAttribute_Value() {}
-
-func (*Attribute_DoubleValue) isAttribute_Value() {}
-
-var File_events_proto protoreflect.FileDescriptor
-
-const file_events_proto_rawDesc = "" +
- "\n" +
- "\fevents.proto\x12\veventsproto\"6\n" +
- "\bTxEvents\x12*\n" +
- "\x06events\x18\x01 \x03(\v2\x12.eventsproto.EventR\x06events\"\x99\x01\n" +
- "\x05Event\x12\x17\n" +
- "\aat_type\x18\x01 \x01(\tR\x06atType\x12\x12\n" +
- "\x04type\x18\x02 \x01(\tR\x04type\x126\n" +
- "\n" +
- "attributes\x18\x03 \x03(\v2\x16.eventsproto.AttributeR\n" +
- "attributes\x12\x1e\n" +
- "\bpkg_path\x18\x04 \x01(\tH\x00R\apkgPath\x88\x01\x01B\v\n" +
- "\t_pkg_path\"\xb4\x01\n" +
- "\tAttribute\x12\x10\n" +
- "\x03key\x18\x01 \x01(\tR\x03key\x12#\n" +
- "\fstring_value\x18\x02 \x01(\tH\x00R\vstringValue\x12!\n" +
- "\vint64_value\x18\x03 \x01(\x03H\x00R\n" +
- "int64Value\x12\x1f\n" +
- "\n" +
- "bool_value\x18\x04 \x01(\bH\x00R\tboolValue\x12#\n" +
- "\fdouble_value\x18\x05 \x01(\x01H\x00R\vdoubleValueB\a\n" +
- "\x05valueB\x19Z\x17../indexer/events_protob\x06proto3"
-
-var (
- file_events_proto_rawDescOnce sync.Once
- file_events_proto_rawDescData []byte
-)
-
-func file_events_proto_rawDescGZIP() []byte {
- file_events_proto_rawDescOnce.Do(func() {
- file_events_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_events_proto_rawDesc), len(file_events_proto_rawDesc)))
- })
- return file_events_proto_rawDescData
-}
-
-var file_events_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
-var file_events_proto_goTypes = []any{
- (*TxEvents)(nil), // 0: eventsproto.TxEvents
- (*Event)(nil), // 1: eventsproto.Event
- (*Attribute)(nil), // 2: eventsproto.Attribute
-}
-var file_events_proto_depIdxs = []int32{
- 1, // 0: eventsproto.TxEvents.events:type_name -> eventsproto.Event
- 2, // 1: eventsproto.Event.attributes:type_name -> eventsproto.Attribute
- 2, // [2:2] is the sub-list for method output_type
- 2, // [2:2] is the sub-list for method input_type
- 2, // [2:2] is the sub-list for extension type_name
- 2, // [2:2] is the sub-list for extension extendee
- 0, // [0:2] is the sub-list for field type_name
-}
-
-func init() { file_events_proto_init() }
-func file_events_proto_init() {
- if File_events_proto != nil {
- return
- }
- file_events_proto_msgTypes[1].OneofWrappers = []any{}
- file_events_proto_msgTypes[2].OneofWrappers = []any{
- (*Attribute_StringValue)(nil),
- (*Attribute_Int64Value)(nil),
- (*Attribute_BoolValue)(nil),
- (*Attribute_DoubleValue)(nil),
- }
- type x struct{}
- out := protoimpl.TypeBuilder{
- File: protoimpl.DescBuilder{
- GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
- RawDescriptor: unsafe.Slice(unsafe.StringData(file_events_proto_rawDesc), len(file_events_proto_rawDesc)),
- NumEnums: 0,
- NumMessages: 3,
- NumExtensions: 0,
- NumServices: 0,
- },
- GoTypes: file_events_proto_goTypes,
- DependencyIndexes: file_events_proto_depIdxs,
- MessageInfos: file_events_proto_msgTypes,
- }.Build()
- File_events_proto = out.File
- file_events_proto_goTypes = nil
- file_events_proto_depIdxs = nil
-}
diff --git a/indexer/indexer.go b/indexer/indexer.go
index 3dff1ca..a2380bd 100644
--- a/indexer/indexer.go
+++ b/indexer/indexer.go
@@ -1,13 +1,43 @@
package main
import (
- "log"
+ "context"
+ "os"
+ "os/signal"
+ "syscall"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/cmd"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
+ "github.com/rs/zerolog"
+ "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
+ "go.opentelemetry.io/otel/log/global"
+ sdklog "go.opentelemetry.io/otel/sdk/log"
)
func main() {
- if err := cmd.RootCmd.Execute(); err != nil {
- log.Fatalf("failed to execute command: %v", err)
+ ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer cancel()
+
+ // Only set up OTel when an endpoint is explicitly configured.
+ // Without this guard the OTLP exporter dials lazily and prints a
+ // "connection refused" error on shutdown even when no collector is running.
+ if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" {
+ if exp, err := otlploghttp.New(ctx); err == nil {
+ provider := sdklog.NewLoggerProvider(
+ sdklog.WithProcessor(sdklog.NewBatchProcessor(exp)),
+ )
+ global.SetLoggerProvider(provider)
+ defer provider.Shutdown(ctx) //nolint:errcheck
+ }
+ }
+
+ logger.Init(logger.Config{
+ Level: zerolog.InfoLevel,
+ ServiceName: "spectra-indexer",
+ Pretty: true,
+ })
+
+ if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
+ logger.Get().Fatal().Err(err).Msg("failed to execute command")
}
}
diff --git a/indexer/main_operator/operator.go b/indexer/main_operator/operator.go
index c737a45..1dee544 100644
--- a/indexer/main_operator/operator.go
+++ b/indexer/main_operator/operator.go
@@ -3,7 +3,6 @@ package mainoperator
import (
"encoding/json"
"fmt"
- "log"
"os"
"path/filepath"
"time"
@@ -17,8 +16,11 @@ import (
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/query"
rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
+var l = logger.Get()
+
// This function is not ready to be used
// this is just a placeholder
// every part of the indexer should be initialized within the main operator
@@ -30,12 +32,12 @@ func InitMainOperator(
// load config
conf, err := config.LoadConfig(configPath)
if err != nil {
- log.Fatalf("failed to load config: %v", err)
+ l.Fatal().Caller().Stack().Err(err).Msg("failed to load config")
}
// load environment
env, err := config.LoadEnvironment(envPath)
if err != nil {
- log.Fatalf("failed to load environment: %v", err)
+ l.Fatal().Caller().Stack().Err(err).Msg("failed to load environment")
}
// get the chain name
@@ -51,60 +53,60 @@ func InitMainOperator(
// Setup signal handling with proper cleanup and state dump functions
signalHandler := contextHook.NewSignalHandler(
func() error {
- log.Printf("Starting main operator cleanup...")
+ l.Info().Msg("Starting main operator cleanup...")
// Cleanup orchestrator first
if err := orch.Cleanup(); err != nil {
- log.Printf("Orchestrator cleanup failed: %v", err)
+ l.Error().Caller().Stack().Err(err).Msg("Orchestrator cleanup failed")
}
// Cleanup major constructors
if err := mc.cleanup(); err != nil {
- log.Printf("Major constructors cleanup failed: %v", err)
+ l.Error().Caller().Stack().Err(err).Msg("Major constructors cleanup failed")
}
- log.Printf("Main operator cleanup completed")
+ l.Info().Msg("Main operator cleanup completed")
return nil
},
func() error {
- log.Printf("Creating emergency state dump...")
+ l.Info().Msg("Creating emergency state dump...")
// Dump orchestrator state
if err := orch.DumpState(); err != nil {
- log.Printf("Orchestrator state dump failed: %v", err)
+ l.Error().Caller().Stack().Err(err).Msg("Orchestrator state dump failed")
}
// Dump major constructors state if needed
if err := mc.dumpState(); err != nil {
- log.Printf("Major constructors state dump failed: %v", err)
+ l.Error().Caller().Stack().Err(err).Msg("Major constructors state dump failed")
}
- log.Printf("Emergency state dump completed")
+ l.Info().Msg("Emergency state dump completed")
return nil
},
)
// Start signal listening
signalHandler.StartListening()
- log.Printf("Signal handler started, listening for termination signals")
+ l.Info().Msg("Signal handler started, listening for termination signals")
// let the orchestrator do it's thing
switch runningFlags.RunningMode {
case "live":
- orch.LiveProcess(signalHandler.Context(), runningFlags.SkipInitialDbCheck)
+ orch.LiveProcess(signalHandler.Context(), runningFlags.SkipInitialDbCheck, runningFlags.CompressEvents)
case "historic":
if runningFlags.FromHeight == 0 || runningFlags.ToHeight == 0 {
- log.Fatalf("from height and to height are required for historic mode")
+ l.Fatal().Caller().Stack().Msg("from height and to height are required for historic mode")
} else if runningFlags.FromHeight > runningFlags.ToHeight {
- log.Fatalf("from height must be less than to height")
+ l.Fatal().Caller().Stack().Msg("from height must be less than to height")
}
// Historic processing doesn't need context cancellation in the same way,
// but we should still respect shutdown signals
go func() {
<-signalHandler.Context().Done()
- log.Printf("Shutdown signal received during historic processing")
+ l.Info().Msg("Shutdown signal received during historic processing")
}()
- orch.HistoricProcess(runningFlags.FromHeight, runningFlags.ToHeight)
+ orch.HistoricProcess(runningFlags.FromHeight, runningFlags.ToHeight, runningFlags.CompressEvents)
default:
- log.Fatalf("invalid running mode, please choose between live and historic")
+ l.Fatal().Caller().Stack().Msg("invalid running mode, please choose between live and historic")
}
}
@@ -122,7 +124,7 @@ func initializeDatabase(conf *config.Config, env *config.Environment) *database.
// check if the config has any null
// if rpc is null throw an error and exit
if conf.RpcUrl == "" {
- log.Fatalf("rpc url is required")
+ l.Fatal().Caller().Stack().Msg("rpc url is required")
}
// if pool max connections is 0 or nil set a default of 100
if conf.PoolMaxConns == 0 {
@@ -200,7 +202,7 @@ func initializeMajorConstructors(
// set it to a default of 1 minute
rpcFlags.TimeWindow = 1 * time.Minute
} else if rpcFlags.TimeWindow <= 0 {
- log.Fatalf("time window must be greater than 0")
+ l.Fatal().Caller().Stack().Msg("time window must be greater than 0")
}
// init all of the major constructors
@@ -213,7 +215,7 @@ func initializeMajorConstructors(
conf.RpcUrl, nil, rpcFlags.RequestsPerWindow, rpcFlags.TimeWindow,
)
if err != nil {
- log.Fatalf("failed to initialize rpc client: %v", err)
+ l.Fatal().Caller().Stack().Err(err).Msg("failed to initialize rpc client")
}
// initialize the validator cache
@@ -242,33 +244,33 @@ func initializeMajorConstructors(
// cleanup performs cleanup operations on all major constructors
func (mc *MajorConstructors) cleanup() error {
- log.Printf("Starting major constructors cleanup...")
+ l.Info().Msg("Starting major constructors cleanup...")
// Close database connection pool
if mc.db != nil {
- log.Printf("Closing database connection pool...")
+ l.Info().Msg("Closing database connection pool...")
mc.db.GetPool().Close()
- log.Printf("Database connection pool closed successfully")
+ l.Info().Msg("Database connection pool closed successfully")
}
// Close RPC client (closes the rate limiter)
if mc.gnoRpcClient != nil {
- log.Printf("Closing RPC client...")
+ l.Info().Msg("Closing RPC client...")
mc.gnoRpcClient.Close()
- log.Printf("RPC client closed successfully")
+ l.Info().Msg("RPC client closed successfully")
}
// Other components (caches, data processor, query operator) don't need explicit cleanup
// as they rely on the database and RPC client connections that we've already closed
- log.Printf("Address caches, data processor, and query operator don't require explicit cleanup")
+ l.Info().Msg("Address caches, data processor, and query operator don't require explicit cleanup")
- log.Printf("Major constructors cleanup completed successfully")
+ l.Info().Msg("Major constructors cleanup completed successfully")
return nil
}
// dumpState creates a state dump of the major constructors
func (mc *MajorConstructors) dumpState() error {
- log.Printf("Creating major constructors state dump...")
+ l.Info().Msg("Creating major constructors state dump...")
// Create basic state information
state := map[string]interface{}{
@@ -312,6 +314,6 @@ func (mc *MajorConstructors) dumpState() error {
return fmt.Errorf("failed to write major constructors state: %w", err)
}
- log.Printf("Major constructors state dump saved to %s", filepath)
+ l.Info().Msgf("Major constructors state dump saved to %s", filepath)
return nil
}
diff --git a/indexer/orchestrator/operator.go b/indexer/orchestrator/operator.go
index a895759..9528990 100644
--- a/indexer/orchestrator/operator.go
+++ b/indexer/orchestrator/operator.go
@@ -6,7 +6,6 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
- "log"
"os"
"path/filepath"
"strings"
@@ -16,6 +15,7 @@ import (
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config"
dataprocessor "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/data_processor"
rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
const (
@@ -23,6 +23,8 @@ const (
Historic = "historic"
)
+var l = logger.Get()
+
func NewOrchestrator(
runningMode string,
config *config.Config,
@@ -50,8 +52,9 @@ func NewOrchestrator(
func (or *Orchestrator) HistoricProcess(
fromHeight uint64,
- toHeight uint64) {
- log.Printf("Starting historic process from %d to %d", fromHeight, toHeight)
+ toHeight uint64,
+ compressEvents bool) {
+ l.Info().Msgf("Starting historic process from %d to %d", fromHeight, toHeight)
startTime := time.Now()
// Track processing state
@@ -59,21 +62,27 @@ func (or *Orchestrator) HistoricProcess(
or.currentProcessingHeight = fromHeight
defer func() {
or.isProcessing = false
- log.Printf("Historic processing completed at height %d", or.currentProcessingHeight)
+ l.Info().Msgf("Historic processing completed at height %d", or.currentProcessingHeight)
}()
for startHeight := fromHeight; startHeight <= toHeight; {
chunkEndHeight := min(startHeight+or.config.MaxBlockChunkSize-1, toHeight)
- log.Printf("Processing chunk from %d to %d", startHeight, chunkEndHeight)
+ l.Info().Msgf("Processing chunk from %d to %d", startHeight, chunkEndHeight)
// Update current processing height
or.currentProcessingHeight = startHeight
// Process the chunk
- err := or.processChunk(startHeight, chunkEndHeight)
+ err := or.processChunk(startHeight, chunkEndHeight, compressEvents)
if err != nil {
- log.Printf("Error processing chunk %d-%d: %v", startHeight, chunkEndHeight, err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf(
+ "Error processing chunk %d-%d", startHeight, chunkEndHeight,
+ )
}
@@ -84,11 +93,11 @@ func (or *Orchestrator) HistoricProcess(
}
totalDuration := time.Since(startTime)
- log.Printf("Historic process completed from %d to %d in %v", fromHeight, toHeight, totalDuration)
+ l.Info().Msgf("Historic process completed from %d to %d in %v", fromHeight, toHeight, totalDuration)
}
-func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool) {
- log.Printf("Starting live block processing")
+func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool, compressEvents bool) {
+ l.Info().Msg("Starting live block processing")
var lastProcessedHeight uint64
var err error
@@ -98,7 +107,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
or.currentProcessingHeight = 0
defer func() {
or.isProcessing = false
- log.Printf("Live processing stopped at height %d", or.currentProcessingHeight)
+ l.Info().Msgf("Live processing stopped at height %d", or.currentProcessingHeight)
}()
// Initial setup - get starting height
@@ -107,22 +116,24 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
defer cancel()
lastProcessedHeight, err = or.db.GetLastBlockHeight(ctx, or.chainName)
if err != nil {
- log.Printf("Failed to get last block height from database: %v", err)
- log.Printf("Either there are no blocks in the database or the database is not properly configured.")
- log.Printf("Use skipInitialDbCheck=true if this is expected to run from the latest chain height without previous data.")
- log.Printf("Starting from height 1")
+ l.Error().
+ Caller().Stack().Err(err).Msgf("Failed to get last block height from database: %v", err)
+ l.Info().Msg("Either there are no blocks in the database or the database is not properly configured.")
+ l.Info().Msg("Use skipInitialDbCheck=true if this is expected to run from the latest chain height without previous data.")
+ l.Info().Msg("Starting from height 1")
lastProcessedHeight = 0
}
- log.Printf("Retrieved last processed height from database: %d", lastProcessedHeight)
+ l.Info().Msgf("Retrieved last processed height from database: %d", lastProcessedHeight)
} else {
// Get latest block height from chain
latestHeight, rpcErr := or.gnoRpcClient.GetLatestBlockHeight()
if rpcErr != nil {
- log.Printf("Failed to get latest block height from chain: %v", rpcErr)
+ l.Error().
+ Caller().Stack().Err(rpcErr).Msgf("Failed to get latest block height from chain: %v", rpcErr)
return
}
lastProcessedHeight = latestHeight
- log.Printf("Starting from latest chain height: %d (skipping database check)", lastProcessedHeight)
+ l.Info().Msgf("Starting from latest chain height: %d (skipping database check)", lastProcessedHeight)
}
or.currentProcessingHeight = lastProcessedHeight
@@ -132,7 +143,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
for {
select {
case <-ctx.Done():
- log.Printf("Live process interrupted by context cancellation")
+ l.Info().Msg("Live process interrupted by context cancellation")
or.saveProcessingState(lastProcessedHeight, "live_interrupted")
return
default:
@@ -141,7 +152,11 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
// Get the latest block height from the chain
latestHeight, rpcErr := or.gnoRpcClient.GetLatestBlockHeight()
if rpcErr != nil {
- log.Printf("Error fetching latest block height: %v", rpcErr)
+ l.Error().
+ Caller().
+ Stack().
+ Err(rpcErr).
+ Msgf("Error fetching latest block height")
time.Sleep(or.config.LivePooling)
continue
}
@@ -150,7 +165,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
// If caught up, wait and continue
if blocksBehind <= 0 {
- log.Printf("Caught up to height %d. Waiting %d seconds...", latestHeight, or.config.LivePooling/time.Second)
+ l.Info().Msgf("Caught up to height %d. Waiting %d seconds...", latestHeight, or.config.LivePooling/time.Second)
time.Sleep(or.config.LivePooling)
continue
}
@@ -161,15 +176,19 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
chunkStart := lastProcessedHeight + 1
chunkEnd := min(chunkStart+currentChunkSize-1, latestHeight)
- log.Printf("Processing live chunk %d-%d (behind by %d blocks)", chunkStart, chunkEnd, blocksBehind)
+ l.Info().Msgf("Processing live chunk %d-%d (behind by %d blocks)", chunkStart, chunkEnd, blocksBehind)
// Update current processing height
or.currentProcessingHeight = chunkStart
// Process this chunk
- err = or.processChunk(chunkStart, chunkEnd)
+ err = or.processChunk(chunkStart, chunkEnd, compressEvents)
if err != nil {
- log.Printf("Error processing live chunk %d-%d: %v", chunkStart, chunkEnd, err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Error processing live chunk %d-%d", chunkStart, chunkEnd)
time.Sleep(or.config.LivePooling)
continue
}
@@ -185,7 +204,7 @@ func (or *Orchestrator) LiveProcess(ctx context.Context, skipInitialDbCheck bool
}
// processChunk processes a single chunk of blocks for live processing
-func (or *Orchestrator) processChunk(chunkStart, chunkEnd uint64) error {
+func (or *Orchestrator) processChunk(chunkStart, chunkEnd uint64, compressEvents bool) error {
chunkStartTime := time.Now()
// Step 1: Get blocks concurrently
@@ -207,22 +226,22 @@ func (or *Orchestrator) processChunk(chunkStart, chunkEnd uint64) error {
wg.Wait()
if len(blocks) == 0 && len(commits) == 0 {
- log.Printf("No valid blocks in live chunk %d-%d", chunkStart, chunkEnd)
+ l.Info().Msgf("No valid blocks in live chunk %d-%d", chunkStart, chunkEnd)
return nil
}
// Step 2: Collect all transactions from all blocks in this chunk
allTransactions := or.collectTransactionsFromBlocks(blocks)
- log.Printf("Collected %d transactions from %d blocks in live chunk", len(allTransactions), len(blocks))
+ l.Info().Msgf("Collected %d transactions from %d blocks in live chunk", len(allTransactions), len(blocks))
// Step 3: Process all data concurrently
- if err := or.processAll(blocks, commits, allTransactions, false, chunkStart, chunkEnd); err != nil {
+ if err := or.processAll(blocks, commits, allTransactions, compressEvents, chunkStart, chunkEnd); err != nil {
return fmt.Errorf("failed to process live chunk %d-%d: %w", chunkStart, chunkEnd, err)
}
chunkDuration := time.Since(chunkStartTime)
- log.Printf("Chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration)
+ l.Info().Msgf("Chunk %d-%d completed in %v", chunkStart, chunkEnd, chunkDuration)
return nil
}
@@ -238,7 +257,7 @@ func (or *Orchestrator) updateProgressMetrics(
// Log progress every 30 seconds or significant milestones
if timeSinceLastProgress >= 30*time.Second || blocksBehind <= 10 {
blocksProcessed := chunkEnd - chunkStart + 1
- log.Printf("Live progress: processed %d blocks (%d-%d), %d blocks behind, last update: %v ago",
+ l.Info().Msgf("Live progress: processed %d blocks (%d-%d), %d blocks behind, last update: %v ago",
blocksProcessed, chunkStart, chunkEnd, blocksBehind, timeSinceLastProgress.Round(time.Second))
*lastProgressTime = now
}
@@ -285,7 +304,11 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR
txHashFinal := base64.StdEncoding.EncodeToString(txHashSha256[:])
blockHeight, err := block.GetHeight()
if err != nil {
- log.Printf("Failed to get block height: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Failed to get block height")
continue
}
allTxHashes = append(allTxHashes, txHashFinal)
@@ -305,7 +328,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR
return make([]dataprocessor.TrasnactionsData, 0)
}
- log.Printf("Fetching %d transactions concurrently", len(allTxHashes))
+ l.Info().Msgf("Fetching %d transactions concurrently", len(allTxHashes))
// Query all transactions concurrently
transactions := or.queryOperator.GetTransactions(allTxHashes)
@@ -326,7 +349,7 @@ func (or *Orchestrator) collectTransactionsFromBlocks(blocks []*rpcClient.BlockR
}
}
- log.Printf("Successfully collected %d valid transactions", len(txData))
+ l.Info().Msgf("Successfully collected %d valid transactions", len(txData))
return txData
}
@@ -365,29 +388,29 @@ func (or *Orchestrator) processAll(
go func() {
defer wg1.Done()
defer close(validatorAddressesDone) // Signal completion
- log.Printf("Phase 1: Starting ProcessValidatorAddresses")
+ l.Info().Msg("Phase 1: Starting ProcessValidatorAddresses")
or.dataProcessor.ProcessValidatorAddresses(blocks, fromHeight, toHeight)
- log.Printf("Phase 1: ProcessValidatorAddresses completed")
+ l.Info().Msg("Phase 1: ProcessValidatorAddresses completed")
}()
// 2. Process transactions (no dependencies)
go func() {
defer wg1.Done()
- log.Printf("Phase 1: Starting ProcessTransactions")
+ l.Info().Msg("Phase 1: Starting ProcessTransactions")
or.dataProcessor.ProcessTransactions(transactions, compressEvents, fromHeight, toHeight)
- log.Printf("Phase 1: ProcessTransactions completed")
+ l.Info().Msg("Phase 1: ProcessTransactions completed")
}()
// 3. Process messages (uses separate address cache)
go func() {
defer wg1.Done()
- log.Printf("Phase 1: Starting ProcessMessages")
+ l.Info().Msg("Phase 1: Starting ProcessMessages")
if err := or.dataProcessor.ProcessMessages(transactions, fromHeight, toHeight); err != nil {
errorsMutex.Lock()
errors = append(errors, fmt.Errorf("ProcessMessages failed: %w", err))
errorsMutex.Unlock()
}
- log.Printf("Phase 1: ProcessMessages completed")
+ l.Info().Msg("Phase 1: ProcessMessages completed")
}()
// Wait for Phase 1 to complete
@@ -408,28 +431,28 @@ func (or *Orchestrator) processAll(
// Wait for validator addresses to be ready (should already be done)
<-validatorAddressesDone
- log.Printf("Phase 2: Validator addresses ready, starting dependent operations")
+ l.Info().Msg("Phase 2: Validator addresses ready, starting dependent operations")
// 4. Process blocks (needs validator address cache for proposer addresses)
go func() {
defer wg2.Done()
- log.Printf("Phase 2: Starting ProcessBlocks")
+ l.Info().Msg("Phase 2: Starting ProcessBlocks")
or.dataProcessor.ProcessBlocks(blocks, fromHeight, toHeight)
- log.Printf("Phase 2: ProcessBlocks completed")
+ l.Info().Msg("Phase 2: ProcessBlocks completed")
}()
// 5. Process validator signings (needs validator address cache)
go func() {
defer wg2.Done()
- log.Printf("Phase 2: Starting ProcessValidatorSignings")
+ l.Info().Msg("Phase 2: Starting ProcessValidatorSignings")
or.dataProcessor.ProcessValidatorSignings(commits, fromHeight, toHeight)
- log.Printf("Phase 2: ProcessValidatorSignings completed")
+ l.Info().Msg("Phase 2: ProcessValidatorSignings completed")
}()
// Wait for Phase 2 to complete
wg2.Wait()
- log.Printf("All processing completed successfully from %d to %d", fromHeight, toHeight)
+ l.Info().Msgf("All processing completed successfully from %d to %d", fromHeight, toHeight)
return nil
}
@@ -459,7 +482,11 @@ func (or *Orchestrator) saveProcessingState(height uint64, reason string) {
// Create state directory if it doesn't exist
stateDir := "state_dumps"
if err := os.MkdirAll(stateDir, 0755); err != nil {
- log.Printf("Failed to create state directory: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Failed to create state directory")
return
}
@@ -471,17 +498,25 @@ func (or *Orchestrator) saveProcessingState(height uint64, reason string) {
// Marshal to JSON
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
- log.Printf("Failed to marshal processing state: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Failed to marshal processing state")
return
}
// Write to file
if err := os.WriteFile(filepath, data, 0644); err != nil {
- log.Printf("Failed to write processing state: %v", err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Failed to write processing state")
return
}
- log.Printf("Processing state saved to %s", filepath)
+ l.Info().Msgf("Processing state saved to %s", filepath)
}
// Cleanup performs cleanup operations for the orchestrator
@@ -489,18 +524,18 @@ func (or *Orchestrator) saveProcessingState(height uint64, reason string) {
// Returns:
// - error: if cleanup fails
func (or *Orchestrator) Cleanup() error {
- log.Printf("Starting orchestrator cleanup...")
+ l.Info().Msg("Starting orchestrator cleanup...")
// Save current state before cleanup
or.saveProcessingState(or.currentProcessingHeight, "cleanup_requested")
- log.Printf("Orchestrator cleanup completed - state saved successfully")
+ l.Info().Msg("Orchestrator cleanup completed - state saved successfully")
return nil
}
// DumpState creates an emergency state dump with current processing information
func (or *Orchestrator) DumpState() error {
- log.Printf("Creating emergency state dump...")
+ l.Info().Msg("Creating emergency state dump...")
// Save processing state
or.saveProcessingState(or.currentProcessingHeight, "emergency_dump")
@@ -543,6 +578,6 @@ func (or *Orchestrator) DumpState() error {
return fmt.Errorf("failed to write diagnostics: %w", err)
}
- log.Printf("Emergency state dump saved to %s", filepath)
+ l.Info().Msgf("Emergency state dump saved to %s", filepath)
return nil
}
diff --git a/indexer/orchestrator/operator_test.go b/indexer/orchestrator/operator_test.go
index 37cc34f..cca5dc1 100644
--- a/indexer/orchestrator/operator_test.go
+++ b/indexer/orchestrator/operator_test.go
@@ -162,7 +162,7 @@ func TestOrchestrator_HistoricProcess_CallsAllProcessors(t *testing.T) {
)
// Test historic processing
- orch.HistoricProcess(1, 5)
+ orch.HistoricProcess(1, 5, false)
// Verify orchestration: all processors should be called
if !mockDataProcessor.ProcessValidatorAddressesCalled {
@@ -210,7 +210,7 @@ func TestOrchestrator_HistoricProcess_SkipsProcessingWhenNoBlocks(t *testing.T)
)
// Test historic processing
- orch.HistoricProcess(1, 5)
+ orch.HistoricProcess(1, 5, false)
// Verify query was attempted
if mockQueryOperator.CallCount == 0 {
@@ -253,7 +253,7 @@ func TestOrchestrator_LiveProcess_RespectsContext(t *testing.T) {
start := time.Now()
// Test live processing - should return quickly due to context cancellation
- orch.LiveProcess(ctx, false)
+ orch.LiveProcess(ctx, false, false)
// Should return quickly (within 1 second)
elapsed := time.Since(start)
@@ -291,7 +291,7 @@ func TestOrchestrator_LiveProcess_SkipDbCheck(t *testing.T) {
defer cancel()
// Test live processing with skip DB check - should not fail even though DB errors
- orch.LiveProcess(ctx, true) // skipInitialDbCheck = true
+ orch.LiveProcess(ctx, true, false) // skipInitialDbCheck = true
// Test passes if we get here without panic/fatal error
}
diff --git a/indexer/query/operator.go b/indexer/query/operator.go
index 41ee6ce..b267027 100644
--- a/indexer/query/operator.go
+++ b/indexer/query/operator.go
@@ -1,14 +1,16 @@
package query
import (
- "log"
"sync"
"time"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/retry"
rc "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
+var l = logger.Get()
+
var (
defaultRetryAmount = 6
defaultPause = 3
@@ -46,7 +48,7 @@ func NewQueryOperator(
}
// A swarm method to get blocks from a to b chain height inclusive
-// This is a fan out method that lauches async workers for each block and wait to get the resaults
+// This is a fan out method that launches async workers for each block and wait to get the results
// The order of the blocks is not guaranteed but it shouldn't matter because at the end of the process
// the indexer should store them all together as one huge slice of blocks, so the order is not important
// the speed is what matters here.
@@ -119,7 +121,11 @@ func (q *QueryOperator) GetFromToBlocks(fromHeight uint64, toHeight uint64) []*r
wg.Done()
},
func(retryErr error) {
- log.Printf("failed to get block %d after retries: %v", height, retryErr)
+ l.Error().
+ Caller().
+ Stack().
+ Err(retryErr).
+ Msgf("failed to get block %d after retries", height)
mu.Lock()
blocks[idx] = nil
mu.Unlock()
@@ -179,7 +185,11 @@ func (q *QueryOperator) GetFromToCommits(fromHeight uint64, toHeight uint64) []*
wg.Done()
},
func(retryErr error) {
- log.Printf("failed to get commit %d after retries: %v", height, retryErr)
+ l.Error().
+ Caller().
+ Stack().
+ Err(retryErr).
+ Msgf("failed to get commit %d after retries", height)
mu.Lock()
commits[idx] = nil
mu.Unlock()
@@ -259,7 +269,11 @@ func (q *QueryOperator) GetTransactions(txs []string) []*rc.TxResponse {
wg.Done()
},
func(retryErr error) {
- log.Printf("failed to get tx %s after retries: %v", tx, retryErr)
+ l.Error().
+ Caller().
+ Stack().
+ Err(retryErr).
+ Msgf("failed to get tx %s after retries", tx)
mu.Lock()
transactions[idx] = nil
mu.Unlock()
diff --git a/indexer/retry/worker.go b/indexer/retry/worker.go
index d5fa5e2..f087b25 100644
--- a/indexer/retry/worker.go
+++ b/indexer/retry/worker.go
@@ -1,10 +1,13 @@
package retry
import (
- "log"
"time"
+
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
)
+var l = logger.Get()
+
// RetryResult holds the result of a retry operation
//
// Sort of kinda like union or enum (depending on your programming language)
@@ -60,7 +63,11 @@ func GenericRetryQuery[T any](
// if the result is not successful, store the error
lastErr = err
- log.Printf("Retry attempt %d failed: %v", i+1, err)
+ l.Error().
+ Caller().
+ Stack().
+ Err(err).
+ Msgf("Retry attempt %d failed", i+1)
// Don't sleep on the last retry attempt
if i < retryAmount-1 {
@@ -112,7 +119,11 @@ func RetryWithContext[T any](
onSuccess(result.Value)
} else {
if result.Error != nil {
- log.Printf("Error: All retry attempts failed: %v", result.Error)
+ l.Error().
+ Caller().
+ Stack().
+ Err(result.Error).
+ Msgf("Error: All retry attempts failed")
}
onFailure(result.Error)
}
diff --git a/indexer/rpc_client/client.go b/indexer/rpc_client/client.go
index 33b8e04..907c2d1 100644
--- a/indexer/rpc_client/client.go
+++ b/indexer/rpc_client/client.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"log"
"net/http"
"strconv"
@@ -94,8 +95,21 @@ func (r *RpcGnoland) performRequest(method string, params map[string]any, result
}
}()
- if err := json.NewDecoder(resp.Body).Decode(result); err != nil {
- return fmt.Errorf("failed to decode response: %w", err)
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("http error %s: %s", resp.Status, string(body))
+ }
+
+ if err := json.Unmarshal(body, result); err != nil {
+ preview := string(body)
+ if len(preview) > 200 {
+ preview = preview[:200] + "..."
+ }
+ return fmt.Errorf("failed to decode response (non-JSON body): %w; body preview: %q", err, preview)
}
return nil
diff --git a/integration/HISTORIC_REPORTS.MD b/integration/HISTORIC_REPORTS.MD
index 2050b4c..1e3f3e9 100644
--- a/integration/HISTORIC_REPORTS.MD
+++ b/integration/HISTORIC_REPORTS.MD
@@ -106,4 +106,19 @@ Most of the vm messages I guess were ok but without looking at the real data and
Regular addresses were stored but the validator addresses were not. I will need to fix this later.
-Overall the test was a success. There were bugs related to the synthetic data generator and the indexer.
\ No newline at end of file
+Overall the test was a success. There were bugs related to the synthetic data generator and the indexer.
+
+## Test run on 2026-02-23
+
+### Test configuration
+
+- **Chain ID**: gnoland
+- **Max height**: 200000
+- **From height**: 1
+- **To height**: 200000
+- **Block production rate**: 1 block per 5 seconds
+- **Regular addresses**: 500
+- **Validators**: 50
+- **Number of transactions**: 528365
+
+The test was improved to run more efficiently. The whole test took around 5 minutes to conclude.
diff --git a/integration/synthetic/init.go b/integration/synthetic/init.go
index cf642dc..12efe72 100644
--- a/integration/synthetic/init.go
+++ b/integration/synthetic/init.go
@@ -3,6 +3,7 @@ package synthetic
import (
"context"
"log"
+ "sync"
addressCache "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/address_cache"
"github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/config"
@@ -31,27 +32,39 @@ func RunSyntheticIntegrationTest(testConfig *SyntheticIntegrationTestConfig) err
log.Printf("Connected to database successfully")
// Initialize address caches (required by data processor)
- validatorCache := addressCache.NewAddressCache(testConfig.ChainID, db, true)
- addrCache := addressCache.NewAddressCache(testConfig.ChainID, db, false)
+ wg := sync.WaitGroup{}
+ wg.Add(2)
+ var validatorCache *addressCache.AddressCache
+ var addrCache *addressCache.AddressCache
+ go func() {
+ defer wg.Done()
+ validatorCache = addressCache.NewAddressCache(testConfig.ChainID, db, true)
+ }()
+ go func() {
+ defer wg.Done()
+ addrCache = addressCache.NewAddressCache(testConfig.ChainID, db, false)
+ }()
+ wg.Wait()
log.Printf("Initialized address caches")
// Initialize data processor
dataProc := dataProcessor.NewDataProcessor(db, addrCache, validatorCache, testConfig.ChainID)
log.Printf("Initialized data processor")
- // Create synthetic query operator (this replaces the real RPC queries!)
+ // Create synthetic query operator
syntheticQueryOp := NewSyntheticQueryOperator(testConfig.ChainID, testConfig.FromHeight, testConfig.MaxHeight)
log.Printf("Created synthetic query operator with max height %d", testConfig.MaxHeight)
// Create a mock database height interface for the orchestrator
mockDbHeight := &MockDatabaseHeight{lastHeight: testConfig.FromHeight - 1}
- // Create a mock gnoland rpc clien
+ // Create a mock gnoland rpc client
mockGnoRpc := &MockGnolandRpcClient{latestHeight: testConfig.MaxHeight}
// Create basic config for orchestrator
orchConfig := &config.Config{
- MaxBlockChunkSize: 50, // Cap it to 50 for now
+ MaxBlockChunkSize: 500,
+ MaxTransactionChunkSize: 1000,
}
// Create orchestrator with synthetic components
@@ -69,7 +82,7 @@ func RunSyntheticIntegrationTest(testConfig *SyntheticIntegrationTestConfig) err
// Run the historic process - this will use synthetic data but process it through
// the real data processor and store it in the real database
- orch.HistoricProcess(testConfig.FromHeight, testConfig.ToHeight)
+ orch.HistoricProcess(testConfig.FromHeight, testConfig.ToHeight, false)
log.Printf("Synthetic integration test completed successfully!")
return nil
diff --git a/integration/synthetic/query_operator.go b/integration/synthetic/query_operator.go
index c083fb7..3135e1d 100644
--- a/integration/synthetic/query_operator.go
+++ b/integration/synthetic/query_operator.go
@@ -4,7 +4,7 @@ import (
"crypto/sha256"
"encoding/base64"
"log"
- "math/rand"
+ "math/rand/v2"
"time"
rpcClient "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/rpc_client"
diff --git a/pkgs/database/queries_address.go b/pkgs/database/queries_address.go
new file mode 100644
index 0000000..dbc17e5
--- /dev/null
+++ b/pkgs/database/queries_address.go
@@ -0,0 +1,317 @@
+package database
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "strings"
+ "time"
+)
+
+var defaultLimit = uint64(10)
+
+// GetAddressTxs gets the transactions for a given address for a certain time period
+//
+// Usage:
+//
+// # Used to get the transactions for a given address for a certain time period
+//
+// Parameters:
+// - address: the address
+// - chainName: the name of the chain
+// - fromTimestamp: the starting timestamp
+// - toTimestamp: the ending timestamp
+//
+// Returns:
+// - []*AddressTx: the transactions
+// - error: if the query fails
+func (t *TimescaleDb) GetAddressTxs(
+ ctx context.Context,
+ address string,
+ chainName string,
+ fromTimestamp *time.Time,
+ toTimestamp *time.Time,
+ limit *uint64,
+ page *uint64,
+ cursor *string,
+) (*[]AddressTx, string, uint64, error) {
+ hasTsRange := fromTimestamp != nil && toTimestamp != nil
+ noTsRange := fromTimestamp == nil && toTimestamp == nil
+
+ var mode string
+ switch {
+ case hasTsRange:
+ mode = "timestamp"
+ case noTsRange && page == nil:
+ mode = "cursor"
+ case noTsRange && cursor == nil:
+ mode = "limit_page"
+ default:
+ return nil, "", 0, fmt.Errorf("invalid query parameters")
+ }
+
+ accountId, err := t.getAccountId(ctx, address, chainName)
+ if err != nil {
+ return nil, "", 0, fmt.Errorf("error getting account id: %w", err)
+ }
+
+ txCount, err := t.getTxsCount(ctx, accountId, chainName)
+ if err != nil {
+ return nil, "", 0, fmt.Errorf("error getting tx count: %w", err)
+ }
+
+ var addressTxs *[]AddressTx
+ var nextCursor string
+
+ switch mode {
+ case "timestamp":
+ addressTxs, err = t.getAddressTxsTimestampQuery(
+ ctx, accountId, chainName, *fromTimestamp, *toTimestamp, limit,
+ )
+ if err != nil {
+ return nil, "", 0, err
+ }
+ case "cursor":
+ addressTxs, nextCursor, err = t.getAddressTxsCursorQuery(
+ ctx, accountId, chainName, cursor, limit, txCount,
+ )
+ if err != nil {
+ return nil, "", 0, err
+ }
+ case "limit_page":
+ addressTxs, err = t.getAddressTxsLimitPageQuery(
+ ctx, accountId, chainName, limit, *page,
+ )
+ if err != nil {
+ return nil, "", 0, err
+ }
+ }
+
+ return addressTxs, nextCursor, txCount, nil
+}
+
+func (t *TimescaleDb) getAddressTxsTimestampQuery(
+ ctx context.Context,
+ accountId int32,
+ chainName string,
+ fromTimestamp time.Time,
+ toTimestamp time.Time,
+ limit *uint64,
+) (*[]AddressTx, error) {
+ if limit == nil {
+ limit = &defaultLimit
+ }
+ var args []any
+
+ query := `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.msg_types
+ FROM address_tx tx
+ WHERE tx.address = $1
+ AND tx.chain_name = $2
+ AND tx.timestamp >= $3
+ AND tx.timestamp <= $4
+ ORDER BY tx.timestamp DESC
+ LIMIT $5
+ `
+ args = append(args, accountId, chainName, fromTimestamp, toTimestamp, *limit)
+
+ addressTxs, err := t.execAccQuery(ctx, query, args)
+ if err != nil {
+ return nil, err
+ }
+ return addressTxs, nil
+}
+
+func (t *TimescaleDb) getAddressTxsCursorQuery(
+ ctx context.Context,
+ accountId int32,
+ chainName string,
+ cursor *string,
+ limit *uint64,
+ txCount uint64,
+) (*[]AddressTx, string, error) {
+ if limit == nil {
+ limit = &defaultLimit
+ }
+ // Fetch limit+1 to detect if there are more rows; only return limit to the caller.
+ fetchLimit := *limit + 1
+ var query string
+ var args []any
+
+ if cursor == nil {
+ query = `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.msg_types
+ FROM address_tx tx
+ WHERE tx.address = $1
+ AND tx.chain_name = $2
+ ORDER BY tx.timestamp DESC, tx.tx_hash DESC
+ LIMIT $3
+ `
+ args = append(args, accountId, chainName, fetchLimit)
+ } else {
+ timestamp, txHash, err := unmarshalCursorParam(*cursor)
+ if err != nil {
+ return nil, "", err
+ }
+ decodedTxHash, err := base64.URLEncoding.Strict().DecodeString(txHash)
+ if err != nil {
+ return nil, "", fmt.Errorf("error decoding tx hash: %w", err)
+ }
+ query = `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.msg_types
+ FROM address_tx tx
+ WHERE tx.address = $1
+ AND tx.chain_name = $2
+ AND (tx.timestamp, tx.tx_hash) < ($3::timestamptz, $4)
+ ORDER BY tx.timestamp DESC, tx.tx_hash DESC
+ LIMIT $5
+ `
+ args = append(args, accountId, chainName, timestamp, decodedTxHash, fetchLimit)
+ }
+
+ addressTxs, err := t.execAccQuery(ctx, query, args)
+ if err != nil {
+ return nil, "", err
+ }
+ // If we got more than limit, there is a next page: return only the first limit and set nextCursor.
+ if len(*addressTxs) > int(*limit) {
+ page := (*addressTxs)[:int(*limit)]
+ lastAddressTx := page[len(page)-1]
+ nextCursor := makeCursorParam(lastAddressTx.Timestamp, lastAddressTx.Hash)
+ return &page, nextCursor, nil
+ }
+ return addressTxs, "", nil
+}
+
+func (t *TimescaleDb) getAddressTxsLimitPageQuery(
+ ctx context.Context,
+ accountId int32,
+ chainName string,
+ limit *uint64,
+ page uint64,
+) (*[]AddressTx, error) {
+ if limit == nil {
+ limit = &defaultLimit
+ }
+
+ var query string
+ var args []any
+
+ offset := page * *limit
+
+ query = `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.msg_types
+ FROM address_tx tx
+ WHERE tx.address = $1
+ AND tx.chain_name = $2
+ ORDER BY tx.timestamp DESC
+ LIMIT $3 OFFSET $4
+ `
+ args = append(args, accountId, chainName, *limit, offset)
+
+ addressTxs, err := t.execAccQuery(ctx, query, args)
+ if err != nil {
+ return nil, err
+ }
+ return addressTxs, nil
+}
+
+func (t *TimescaleDb) getTxsCount(
+ ctx context.Context,
+ accountId int32,
+ chainName string,
+) (uint64, error) {
+ query := `
+ SELECT COUNT(*) FROM address_tx WHERE address = $1 AND chain_name = $2
+ `
+ row := t.pool.QueryRow(ctx, query, accountId, chainName)
+ var count uint64
+ err := row.Scan(&count)
+ if err != nil {
+ return 0, err
+ }
+ return count, nil
+}
+
+func makeCursorParam(
+ timestamp time.Time,
+ txHash string,
+) string {
+ txHashBytes, err := base64.StdEncoding.DecodeString(txHash)
+ if err != nil {
+ // TODO: log error
+ return "error decoding tx hash"
+ }
+ timestamp = timestamp.Round(time.Second)
+ base64Url := base64.URLEncoding.Strict().EncodeToString(txHashBytes)
+ return timestamp.Format(time.RFC3339) + "|" + base64Url
+}
+
+func unmarshalCursorParam(
+ cursor string,
+) (time.Time, string, error) {
+ parts := strings.Split(cursor, "|")
+ if len(parts) != 2 {
+ return time.Time{}, "", fmt.Errorf("invalid cursor")
+ }
+ timestamp, err := time.Parse(time.RFC3339, parts[0])
+ if err != nil {
+ return time.Time{}, "", err
+ }
+ return timestamp, parts[1], nil
+}
+
+func (t *TimescaleDb) execAccQuery(
+ ctx context.Context,
+ query string,
+ args []any,
+) (*[]AddressTx, error) {
+
+ addressTxs := make([]AddressTx, 0)
+ rows, err := t.pool.Query(ctx, query, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var addressTx AddressTx
+ err := rows.Scan(&addressTx.Hash, &addressTx.Timestamp, &addressTx.MsgTypes)
+ if err != nil {
+ return nil, err
+ }
+ addressTxs = append(addressTxs, addressTx)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return &addressTxs, nil
+}
+
+func (t *TimescaleDb) getAccountId(
+ ctx context.Context,
+ address string,
+ chainName string,
+) (int32, error) {
+ query := `
+ SELECT id FROM gno_addresses WHERE address = $1 AND chain_name = $2
+ `
+ row := t.pool.QueryRow(ctx, query, address, chainName)
+ var id int32
+ err := row.Scan(&id)
+ if err != nil {
+ return 0, err
+ }
+ return id, nil
+}
diff --git a/pkgs/database/queries_tx.go b/pkgs/database/queries_tx.go
new file mode 100644
index 0000000..50b0cd4
--- /dev/null
+++ b/pkgs/database/queries_tx.go
@@ -0,0 +1,343 @@
+package database
+
+import (
+ "context"
+ "encoding/base64"
+ "log"
+ "strconv"
+
+ dictloader "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/dict_loader"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/events_proto"
+ "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/logger"
+ "github.com/cosmos/gogoproto/proto"
+ "github.com/klauspost/compress/zstd"
+)
+
+var dictBytes = dictloader.LoadDict()
+var zstdDict = zstd.WithDecoderDicts(dictBytes)
+var zstdReader *zstd.Decoder
+
+var l = logger.Get()
+
+func init() {
+ var err error
+ zstdReader, err = zstd.NewReader(nil, zstdDict)
+ if err != nil {
+ l.Fatal().Caller().Stack().Err(err).Msg("failed to initialize zstd reader")
+ }
+}
+
+// GetTransaction gets the transaction for a given transaction hash
+//
+// Usage:
+//
+// # Used to get the transaction for a given transaction hash
+//
+// Parameters:
+// - txHash: the hash of the transaction
+// - chainName: the name of the chain
+//
+// Returns:
+// - *Transaction: the transaction
+// - error: if the query fails
+func (t *TimescaleDb) GetTransaction(ctx context.Context, txHash string, chainName string) (*Transaction, error) {
+ query := `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.block_height,
+ tx.tx_events,
+ tx.tx_events_compressed,
+ tx.compression_on,
+ tx.gas_used,
+ tx.gas_wanted,
+ tx.fee,
+ tx.msg_types
+ FROM transaction_general tx
+ WHERE tx.tx_hash = decode($1, 'base64')
+ AND tx.chain_name = $2
+ `
+ row := t.pool.QueryRow(ctx, query, txHash, chainName)
+ var transaction FullTxData
+ err := row.Scan(
+ &transaction.TxHash,
+ &transaction.Timestamp,
+ &transaction.BlockHeight,
+ &transaction.TxEvents,
+ &transaction.TxEventsCompressed,
+ &transaction.CompressionOn,
+ &transaction.GasUsed,
+ &transaction.GasWanted,
+ &transaction.Fee,
+ &transaction.MsgTypes,
+ )
+ if err != nil {
+ log.Println("error getting transaction", err)
+ return nil, err
+ }
+ tx, err := transaction.ToTransaction(decodeEvents)
+ if err != nil {
+ return nil, err
+ }
+ return tx, nil
+}
+
+// GetLastXTransactions gets the last x transactions from the database for a given chain name
+//
+// Usage:
+//
+// # Used to get the last x transactions from the database for a given chain name
+//
+// Parameters:
+// - chainName: the name of the chain
+// - x: the number of transactions to get
+func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*Transaction, error) {
+ query := `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.block_height,
+ tx.tx_events,
+ tx.tx_events_compressed,
+ tx.compression_on,
+ tx.gas_used,
+ tx.gas_wanted,
+ tx.fee,
+ tx.msg_types
+ FROM transaction_general tx
+ WHERE tx.chain_name = $1
+ ORDER BY tx.timestamp DESC
+ LIMIT $2
+ `
+ rows, err := t.pool.Query(ctx, query, chainName, x)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ transactions := make([]*Transaction, 0)
+ for rows.Next() {
+ transaction := &FullTxData{}
+ err := rows.Scan(
+ &transaction.TxHash,
+ &transaction.Timestamp,
+ &transaction.BlockHeight,
+ &transaction.TxEvents,
+ &transaction.TxEventsCompressed,
+ &transaction.CompressionOn,
+ &transaction.GasUsed,
+ &transaction.GasWanted,
+ &transaction.Fee,
+ &transaction.MsgTypes)
+ if err != nil {
+ return nil, err
+ }
+ tx, err := transaction.ToTransaction(decodeEvents)
+ if err != nil {
+ return nil, err
+ }
+ transactions = append(transactions, tx)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return transactions, nil
+}
+
+// GetTransactionsByOffset gets the transactions by offset for a given chain name
+//
+// Usage:
+//
+// # Used to get the transactions by offset for a given chain name
+//
+// Parameters:
+// - chainName: the name of the chain
+// - limit: the limit of the transactions to get
+// - offset: the offset of the transactions to get
+//
+// Additional info:
+//
+// This part of the logic won't be officially present in the API. It's current only usage
+// is with training the zstd dictionary. You are welcome to modify this function and add it to the API.
+func (t *TimescaleDb) GetTransactionsByOffset(
+ ctx context.Context,
+ chainName string,
+ limit uint64,
+ offset uint64,
+) ([]*Transaction, error) {
+ query := `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.block_height,
+ tx.tx_events,
+ tx.gas_used,
+ tx.gas_wanted,
+ tx.fee,
+ tx.msg_types
+ FROM transaction_general tx
+ WHERE tx.chain_name = $1
+ ORDER BY tx.timestamp DESC
+ LIMIT $2 OFFSET $3
+ `
+ rows, err := t.pool.Query(ctx, query, chainName, limit, offset)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ transactions := make([]*Transaction, 0)
+ for rows.Next() {
+ transaction := &FullTxData{}
+ err := rows.Scan(&transaction.TxHash, &transaction.Timestamp, &transaction.BlockHeight, &transaction.TxEvents, &transaction.GasUsed, &transaction.GasWanted, &transaction.Fee, &transaction.MsgTypes)
+ if err != nil {
+ return nil, err
+ }
+ tx, err := transaction.ToTransaction(decodeEvents)
+ if err != nil {
+ return nil, err
+ }
+ transactions = append(transactions, tx)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return transactions, nil
+}
+
+func (t *TimescaleDb) GetTransactionsByCursor(
+ ctx context.Context,
+ chainName string,
+ cursor string,
+ limit uint64,
+) ([]*Transaction, error) {
+ var query string
+ // if txHash and timestamp are nil make same query as GetLastXTransactions
+ if cursor == "" {
+ return t.GetLastXTransactions(ctx, chainName, limit)
+ }
+ timestamp, txHash, err := unmarshalCursorParam(cursor)
+ if err != nil {
+ return nil, err
+ }
+
+ decodedTxHash, err := base64.URLEncoding.Strict().DecodeString(txHash)
+ if err != nil {
+ return nil, err
+ }
+ query = `
+ SELECT
+ encode(tx.tx_hash, 'base64') AS tx_hash,
+ tx.timestamp,
+ tx.block_height,
+ tx.tx_events,
+ tx.tx_events_compressed,
+ tx.compression_on,
+ tx.gas_used,
+ tx.gas_wanted,
+ tx.fee,
+ tx.msg_types
+ FROM transaction_general tx
+ WHERE tx.chain_name = $1
+ AND (tx.timestamp, tx.tx_hash) < ($2, $3)
+ ORDER BY tx.timestamp DESC, tx.tx_hash DESC
+ LIMIT $4
+ `
+ args := []any{chainName, timestamp, decodedTxHash, limit}
+ rows, err := t.pool.Query(ctx, query, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ transactions := make([]*Transaction, 0)
+ for rows.Next() {
+ transaction := &FullTxData{}
+ err := rows.Scan(
+ &transaction.TxHash,
+ &transaction.Timestamp,
+ &transaction.BlockHeight,
+ &transaction.TxEvents,
+ &transaction.TxEventsCompressed,
+ &transaction.CompressionOn,
+ &transaction.GasUsed,
+ &transaction.GasWanted,
+ &transaction.Fee,
+ &transaction.MsgTypes,
+ )
+ if err != nil {
+ return nil, err
+ }
+ tx, err := transaction.ToTransaction(decodeEvents)
+ if err != nil {
+ return nil, err
+ }
+ transactions = append(transactions, tx)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return transactions, nil
+}
+
+func decompressEvents(txEvents []byte) ([]byte, error) {
+ decompressed, err := zstdReader.DecodeAll(txEvents, nil)
+ if err != nil {
+ return nil, err
+ }
+ return decompressed, nil
+}
+
+func protoUnmarshal(rawData []byte) (*events_proto.TxEvents, error) {
+ txEvents := &events_proto.TxEvents{}
+ err := proto.Unmarshal(rawData, txEvents)
+ if err != nil {
+ return nil, err
+ }
+ return txEvents, nil
+}
+
+func decodeEvents(txEvents []byte) (*[]Event, error) {
+ if len(txEvents) == 0 {
+ return &[]Event{}, nil
+ }
+ decompressed, err := decompressEvents(txEvents)
+ if err != nil {
+ return nil, err
+ }
+ txEventsProto, err := protoUnmarshal(decompressed)
+ if err != nil {
+ return nil, err
+ }
+ events := make([]Event, 0, len(txEventsProto.Events))
+ for _, event := range txEventsProto.Events {
+ attributes := make([]Attribute, 0, len(event.Attributes))
+ for _, attribute := range event.Attributes {
+ var value string
+ switch v := attribute.Value.(type) {
+ case *events_proto.Attribute_StringValue:
+ value = v.StringValue
+ case *events_proto.Attribute_Int64Value:
+ value = strconv.FormatInt(v.Int64Value, 10)
+ case *events_proto.Attribute_BoolValue:
+ value = strconv.FormatBool(v.BoolValue)
+ case *events_proto.Attribute_DoubleValue:
+ value = strconv.FormatFloat(v.DoubleValue, 'g', -1, 64)
+ default:
+ value = ""
+ }
+ attributes = append(attributes, Attribute{
+ Key: attribute.Key,
+ Value: value,
+ })
+ }
+ pkgPath := ""
+ if event.PkgPath != nil {
+ pkgPath = *event.PkgPath
+ }
+ events = append(events, Event{
+ AtType: event.AtType,
+ Type: event.Type,
+ Attributes: attributes,
+ PkgPath: pkgPath,
+ })
+ }
+ return &events, nil
+}
diff --git a/pkgs/database/query_block.go b/pkgs/database/query_block.go
new file mode 100644
index 0000000..580b552
--- /dev/null
+++ b/pkgs/database/query_block.go
@@ -0,0 +1,152 @@
+package database
+
+import "context"
+
+// GetBlock gets a block from the database for a given height and chain name
+//
+// Usage:
+//
+// # Used to get a block from the database for a given height and chain name
+//
+// Parameters:
+// - height: the height of the block
+// - chainName: the name of the chain
+//
+// Returns:
+// - *BlockData: the block data
+// - error: if the query fails
+func (t *TimescaleDb) GetBlock(ctx context.Context, height uint64, chainName string) (*BlockData, error) {
+ query := `
+ SELECT encode(hash, 'base64'),
+ height,
+ timestamp,
+ chain_id,
+ (SELECT array_agg(upper(encode(tx, 'base64')))
+ FROM unnest(blocks.txs) AS tx
+ ) AS txs
+ FROM blocks
+ WHERE height = $1
+ AND chain_name = $2
+ `
+ row := t.pool.QueryRow(ctx, query, height, chainName)
+ var block BlockData
+ err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
+ if err != nil {
+ return nil, err
+ }
+ return &block, nil
+}
+
+func (t *TimescaleDb) GetLatestBlock(ctx context.Context, chainName string) (*BlockData, error) {
+ query := `
+ SELECT encode(hash, 'base64'),
+ height,
+ timestamp,
+ chain_id,
+ (SELECT array_agg(upper(encode(tx, 'base64')))
+ FROM unnest(blocks.txs) AS tx
+ ) AS txs
+ FROM blocks
+ WHERE chain_name = $1
+ ORDER BY height DESC
+ LIMIT 1
+ `
+ row := t.pool.QueryRow(ctx, query, chainName)
+ var block BlockData
+ err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
+ if err != nil {
+ return nil, err
+ }
+ return &block, nil
+}
+
+// GetLastXBlocks gets the last x blocks from the database for a given chain name
+//
+// Usage:
+//
+// # Used to get the last x blocks from the database for a given chain name
+//
+// Parameters:
+// - chainName: the name of the chain
+// - x: the number of blocks to get
+//
+// Returns:
+// - []*BlockData: the last x blocks
+// - error: if the query fails
+func (t *TimescaleDb) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*BlockData, error) {
+ query := `
+ SELECT encode(hash, 'base64'),
+ height,
+ timestamp,
+ chain_id,
+ (SELECT array_agg(upper(encode(tx, 'base64')))
+ FROM unnest(blocks.txs) AS tx
+ ) AS txs
+ FROM blocks
+ WHERE chain_name = $1
+ ORDER BY height DESC
+ LIMIT $2
+ `
+ rows, err := t.pool.Query(ctx, query, chainName, x)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ blocks := make([]*BlockData, 0)
+ for rows.Next() {
+ block := &BlockData{}
+ err := rows.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
+ if err != nil {
+ return nil, err
+ }
+ blocks = append(blocks, block)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return blocks, nil
+}
+
+// GetFromToBlocks gets a range of blocks from the database for a given height range and chain name
+//
+// Usage:
+//
+// # Used to get a range of blocks from the database for a given height range and chain name
+//
+// Parameters:
+// - fromHeight: the starting height of the block
+// - toHeight: the ending height of the block (inclusive)
+// - chainName: the name of the chain
+//
+// Returns:
+// - []*BlockData: the range of block data
+// - error: if the query fails
+func (t *TimescaleDb) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) {
+ query := `
+ SELECT encode(hash, 'base64'),
+ height,
+ timestamp,
+ chain_id,
+ (SELECT array_agg(upper(encode(tx, 'base64')))
+ FROM unnest(blocks.txs) AS tx
+ ) AS txs
+ FROM blocks
+ WHERE height >= $1 AND height <= $2
+ AND chain_name = $3
+ `
+ rows, err := t.pool.Query(ctx, query, fromHeight, toHeight, chainName)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ blocks := make([]*BlockData, 0)
+ for rows.Next() {
+ block := &BlockData{}
+ err := rows.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
+ if err != nil {
+ return nil, err
+ }
+ blocks = append(blocks, block)
+ }
+ return blocks, nil
+}
diff --git a/pkgs/database/queries_api.go b/pkgs/database/query_msg.go
similarity index 51%
rename from pkgs/database/queries_api.go
rename to pkgs/database/query_msg.go
index 216907a..cea19b9 100644
--- a/pkgs/database/queries_api.go
+++ b/pkgs/database/query_msg.go
@@ -1,159 +1,6 @@
package database
-import (
- "context"
- "log"
- "time"
-)
-
-// GetBlock gets a block from the database for a given height and chain name
-//
-// Usage:
-//
-// # Used to get a block from the database for a given height and chain name
-//
-// Parameters:
-// - height: the height of the block
-// - chainName: the name of the chain
-//
-// Returns:
-// - *BlockData: the block data
-// - error: if the query fails
-func (t *TimescaleDb) GetBlock(ctx context.Context, height uint64, chainName string) (*BlockData, error) {
- query := `
- SELECT encode(hash, 'base64'),
- height,
- timestamp,
- chain_id,
- (SELECT array_agg(upper(encode(tx, 'base64')))
- FROM unnest(blocks.txs) AS tx
- ) AS txs
- FROM blocks
- WHERE height = $1
- AND chain_name = $2
- `
- row := t.pool.QueryRow(ctx, query, height, chainName)
- var block BlockData
- err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
- if err != nil {
- return nil, err
- }
- return &block, nil
-}
-
-func (t *TimescaleDb) GetLatestBlock(ctx context.Context, chainName string) (*BlockData, error) {
- query := `
- SELECT encode(hash, 'base64'),
- height,
- timestamp,
- chain_id,
- (SELECT array_agg(upper(encode(tx, 'base64')))
- FROM unnest(blocks.txs) AS tx
- ) AS txs
- FROM blocks
- WHERE chain_name = $1
- ORDER BY height DESC
- LIMIT 1
- `
- row := t.pool.QueryRow(ctx, query, chainName)
- var block BlockData
- err := row.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
- if err != nil {
- return nil, err
- }
- return &block, nil
-}
-
-// GetLastXBlocks gets the last x blocks from the database for a given chain name
-//
-// Usage:
-//
-// # Used to get the last x blocks from the database for a given chain name
-//
-// Parameters:
-// - chainName: the name of the chain
-// - x: the number of blocks to get
-//
-// Returns:
-// - []*BlockData: the last x blocks
-// - error: if the query fails
-func (t *TimescaleDb) GetLastXBlocks(ctx context.Context, chainName string, x uint64) ([]*BlockData, error) {
- query := `
- SELECT encode(hash, 'base64'),
- height,
- timestamp,
- chain_id,
- (SELECT array_agg(upper(encode(tx, 'base64')))
- FROM unnest(blocks.txs) AS tx
- ) AS txs
- FROM blocks
- WHERE chain_name = $1
- ORDER BY height DESC
- LIMIT $2
- `
- rows, err := t.pool.Query(ctx, query, chainName, x)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- blocks := make([]*BlockData, 0)
- for rows.Next() {
- block := &BlockData{}
- err := rows.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
- if err != nil {
- return nil, err
- }
- blocks = append(blocks, block)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return blocks, nil
-}
-
-// GetFromToBlocks gets a range of blocks from the database for a given height range and chain name
-//
-// Usage:
-//
-// # Used to get a range of blocks from the database for a given height range and chain name
-//
-// Parameters:
-// - fromHeight: the starting height of the block
-// - toHeight: the ending height of the block (inclusive)
-// - chainName: the name of the chain
-//
-// Returns:
-// - []*BlockData: the range of block data
-// - error: if the query fails
-func (t *TimescaleDb) GetFromToBlocks(ctx context.Context, fromHeight uint64, toHeight uint64, chainName string) ([]*BlockData, error) {
- query := `
- SELECT encode(hash, 'base64'),
- height,
- timestamp,
- chain_id,
- (SELECT array_agg(upper(encode(tx, 'base64')))
- FROM unnest(blocks.txs) AS tx
- ) AS txs
- FROM blocks
- WHERE height >= $1 AND height <= $2
- AND chain_name = $3
- `
- rows, err := t.pool.Query(ctx, query, fromHeight, toHeight, chainName)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- blocks := make([]*BlockData, 0)
- for rows.Next() {
- block := &BlockData{}
- err := rows.Scan(&block.Hash, &block.Height, &block.Timestamp, &block.ChainID, &block.Txs)
- if err != nil {
- return nil, err
- }
- blocks = append(blocks, block)
- }
- return blocks, nil
-}
+import "context"
// GetAllBlockSigners gets all of the validators that signed that block + the proposer
//
@@ -469,98 +316,6 @@ func (t *TimescaleDb) GetMsgRun(
return msgRuns, nil
}
-// GetTransaction gets the transaction for a given transaction hash
-//
-// Usage:
-//
-// # Used to get the transaction for a given transaction hash
-//
-// Parameters:
-// - txHash: the hash of the transaction
-// - chainName: the name of the chain
-//
-// Returns:
-// - *Transaction: the transaction
-// - error: if the query fails
-func (t *TimescaleDb) GetTransaction(ctx context.Context, txHash string, chainName string) (*Transaction, error) {
- query := `
- SELECT
- encode(tx.tx_hash, 'base64') AS tx_hash,
- tx.timestamp,
- tx.block_height,
- tx.tx_events,
- tx.gas_used,
- tx.gas_wanted,
- tx.fee,
- tx.msg_types
- FROM transaction_general tx
- WHERE tx.tx_hash = decode($1, 'base64')
- AND tx.chain_name = $2
- `
- row := t.pool.QueryRow(ctx, query, txHash, chainName)
- var transaction Transaction
- err := row.Scan(
- &transaction.TxHash,
- &transaction.Timestamp,
- &transaction.BlockHeight,
- &transaction.TxEvents,
- &transaction.GasUsed,
- &transaction.GasWanted,
- &transaction.Fee,
- &transaction.MsgTypes,
- )
- if err != nil {
- log.Println("error getting transaction", err)
- return nil, err
- }
- return &transaction, nil
-}
-
-// GetLastXTransactions gets the last x transactions from the database for a given chain name
-//
-// Usage:
-//
-// # Used to get the last x transactions from the database for a given chain name
-//
-// Parameters:
-// - chainName: the name of the chain
-// - x: the number of transactions to get
-func (t *TimescaleDb) GetLastXTransactions(ctx context.Context, chainName string, x uint64) ([]*Transaction, error) {
- query := `
- SELECT
- encode(tx.tx_hash, 'base64') AS tx_hash,
- tx.timestamp,
- tx.block_height,
- tx.tx_events,
- tx.gas_used,
- tx.gas_wanted,
- tx.fee,
- tx.msg_types
- FROM transaction_general tx
- WHERE tx.chain_name = $1
- ORDER BY tx.timestamp DESC
- LIMIT $2
- `
- rows, err := t.pool.Query(ctx, query, chainName, x)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- transactions := make([]*Transaction, 0)
- for rows.Next() {
- transaction := &Transaction{}
- err := rows.Scan(&transaction.TxHash, &transaction.Timestamp, &transaction.BlockHeight, &transaction.TxEvents, &transaction.GasUsed, &transaction.GasWanted, &transaction.Fee, &transaction.MsgTypes)
- if err != nil {
- return nil, err
- }
- transactions = append(transactions, transaction)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return transactions, nil
-}
-
// GetMsgTypes gets the message type for a given transaction hash
//
// Usage:
@@ -589,58 +344,3 @@ func (t *TimescaleDb) GetMsgTypes(ctx context.Context, txHash string, chainName
}
return msgTypes, nil
}
-
-// GetAddressTxs gets the transactions for a given address for a certain time period
-//
-// Usage:
-//
-// # Used to get the transactions for a given address for a certain time period
-//
-// Parameters:
-// - address: the address
-// - chainName: the name of the chain
-// - fromTimestamp: the starting timestamp
-// - toTimestamp: the ending timestamp
-//
-// Returns:
-// - []*AddressTx: the transactions
-// - error: if the query fails
-func (t *TimescaleDb) GetAddressTxs(
- ctx context.Context,
- address string,
- chainName string,
- fromTimestamp time.Time,
- toTimestamp time.Time,
-) (*[]AddressTx, error) {
- query := `
- SELECT
- encode(tx.tx_hash, 'base64') AS tx_hash,
- tx.timestamp,
- tx.msg_types
- FROM address_tx tx
- WHERE tx.address = (SELECT id FROM gno_addresses WHERE address = $1 AND chain_name = $2)
- AND tx.chain_name = $2
- AND tx.timestamp >= $3
- AND tx.timestamp <= $4
- `
-
- addressTxs := make([]AddressTx, 0)
-
- rows, err := t.pool.Query(ctx, query, address, chainName, fromTimestamp, toTimestamp)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- for rows.Next() {
- var addressTx AddressTx
- err := rows.Scan(&addressTx.Hash, &addressTx.Timestamp, &addressTx.MsgTypes)
- if err != nil {
- return nil, err
- }
- addressTxs = append(addressTxs, addressTx)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return &addressTxs, nil
-}
diff --git a/pkgs/database/types_api.go b/pkgs/database/types_api.go
index 7824e54..63d9627 100644
--- a/pkgs/database/types_api.go
+++ b/pkgs/database/types_api.go
@@ -1,6 +1,9 @@
package database
-import "time"
+import (
+ "errors"
+ "time"
+)
// BlockData represents the actual block data returned in the response body
type BlockData struct {
@@ -89,6 +92,47 @@ type Transaction struct {
MsgTypes []string `json:"msg_types" doc:"Message types"`
}
+type FullTxData struct {
+ TxHash string
+ Timestamp time.Time
+ BlockHeight uint64
+ TxEvents []Event
+ TxEventsCompressed []byte
+ CompressionOn bool
+ GasUsed uint64
+ GasWanted uint64
+ Fee Amount
+ MsgTypes []string
+}
+
+func (f *FullTxData) ToTransaction(decode func([]byte) (*[]Event, error)) (*Transaction, error) {
+ if decode == nil {
+ return nil, errors.New("decode function is nil")
+ }
+ tx := &Transaction{
+ TxHash: f.TxHash,
+ Timestamp: f.Timestamp,
+ BlockHeight: f.BlockHeight,
+ GasUsed: f.GasUsed,
+ GasWanted: f.GasWanted,
+ Fee: f.Fee,
+ MsgTypes: f.MsgTypes,
+ }
+ if f.CompressionOn {
+ events, err := decode(f.TxEventsCompressed)
+ if err != nil {
+ return nil, err
+ }
+ if events == nil {
+ return nil, errors.New("events are nil")
+ }
+ tx.TxEvents = *events
+ } else {
+ tx.TxEvents = f.TxEvents
+ }
+ return tx, nil
+}
+
type BlockSigners struct {
BlockHeight uint64 `json:"block_height" doc:"Block height"`
Proposer string `json:"proposer" doc:"Proposer (addresses)"`
diff --git a/pkgs/dict_loader/events.zstd.bin b/pkgs/dict_loader/events.zstd.bin
new file mode 100644
index 0000000..3ba8e89
Binary files /dev/null and b/pkgs/dict_loader/events.zstd.bin differ
diff --git a/pkgs/dict_loader/loader.go b/pkgs/dict_loader/loader.go
new file mode 100644
index 0000000..5dcda18
--- /dev/null
+++ b/pkgs/dict_loader/loader.go
@@ -0,0 +1,17 @@
+package dictloader
+
+import _ "embed"
+
+//go:embed events.zstd.bin
+var DictBytes []byte
+
+// LoadDict loads the zstd dictionary from the embedded file
+//
+// Parameters:
+// - none
+//
+// Returns:
+// - []byte: the zstd dictionary
+func LoadDict() []byte {
+ return DictBytes
+}
diff --git a/pkgs/events_proto/events.pb.go b/pkgs/events_proto/events.pb.go
new file mode 100644
index 0000000..562c7b8
--- /dev/null
+++ b/pkgs/events_proto/events.pb.go
@@ -0,0 +1,388 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.34.2
+// protoc (unknown)
+// source: events.proto
+
+package events_proto
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type TxEvents struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Events []*Event `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"`
+}
+
+func (x *TxEvents) Reset() {
+ *x = TxEvents{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *TxEvents) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TxEvents) ProtoMessage() {}
+
+func (x *TxEvents) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use TxEvents.ProtoReflect.Descriptor instead.
+func (*TxEvents) Descriptor() ([]byte, []int) {
+ return file_events_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *TxEvents) GetEvents() []*Event {
+ if x != nil {
+ return x.Events
+ }
+ return nil
+}
+
+type Event struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ AtType string `protobuf:"bytes,1,opt,name=at_type,json=atType,proto3" json:"at_type,omitempty"`
+ Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"`
+ Attributes []*Attribute `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty"`
+ PkgPath *string `protobuf:"bytes,4,opt,name=pkg_path,json=pkgPath,proto3,oneof" json:"pkg_path,omitempty"` // Can be null or string
+}
+
+func (x *Event) Reset() {
+ *x = Event{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Event) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Event) ProtoMessage() {}
+
+func (x *Event) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Event.ProtoReflect.Descriptor instead.
+func (*Event) Descriptor() ([]byte, []int) {
+ return file_events_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Event) GetAtType() string {
+ if x != nil {
+ return x.AtType
+ }
+ return ""
+}
+
+func (x *Event) GetType() string {
+ if x != nil {
+ return x.Type
+ }
+ return ""
+}
+
+func (x *Event) GetAttributes() []*Attribute {
+ if x != nil {
+ return x.Attributes
+ }
+ return nil
+}
+
+func (x *Event) GetPkgPath() string {
+ if x != nil && x.PkgPath != nil {
+ return *x.PkgPath
+ }
+ return ""
+}
+
+type Attribute struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ // Types that are assignable to Value:
+ //
+ // *Attribute_StringValue
+ // *Attribute_Int64Value
+ // *Attribute_BoolValue
+ // *Attribute_DoubleValue
+ Value isAttribute_Value `protobuf_oneof:"value"`
+}
+
+func (x *Attribute) Reset() {
+ *x = Attribute{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Attribute) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Attribute) ProtoMessage() {}
+
+func (x *Attribute) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Attribute.ProtoReflect.Descriptor instead.
+func (*Attribute) Descriptor() ([]byte, []int) {
+ return file_events_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Attribute) GetKey() string {
+ if x != nil {
+ return x.Key
+ }
+ return ""
+}
+
+func (m *Attribute) GetValue() isAttribute_Value {
+ if m != nil {
+ return m.Value
+ }
+ return nil
+}
+
+func (x *Attribute) GetStringValue() string {
+ if x, ok := x.GetValue().(*Attribute_StringValue); ok {
+ return x.StringValue
+ }
+ return ""
+}
+
+func (x *Attribute) GetInt64Value() int64 {
+ if x, ok := x.GetValue().(*Attribute_Int64Value); ok {
+ return x.Int64Value
+ }
+ return 0
+}
+
+func (x *Attribute) GetBoolValue() bool {
+ if x, ok := x.GetValue().(*Attribute_BoolValue); ok {
+ return x.BoolValue
+ }
+ return false
+}
+
+func (x *Attribute) GetDoubleValue() float64 {
+ if x, ok := x.GetValue().(*Attribute_DoubleValue); ok {
+ return x.DoubleValue
+ }
+ return 0
+}
+
+type isAttribute_Value interface {
+ isAttribute_Value()
+}
+
+type Attribute_StringValue struct {
+ StringValue string `protobuf:"bytes,2,opt,name=string_value,json=stringValue,proto3,oneof"`
+}
+
+type Attribute_Int64Value struct {
+ Int64Value int64 `protobuf:"varint,3,opt,name=int64_value,json=int64Value,proto3,oneof"`
+}
+
+type Attribute_BoolValue struct {
+ BoolValue bool `protobuf:"varint,4,opt,name=bool_value,json=boolValue,proto3,oneof"`
+}
+
+type Attribute_DoubleValue struct {
+ DoubleValue float64 `protobuf:"fixed64,5,opt,name=double_value,json=doubleValue,proto3,oneof"` // leave this as double for now
+}
+
+func (*Attribute_StringValue) isAttribute_Value() {}
+
+func (*Attribute_Int64Value) isAttribute_Value() {}
+
+func (*Attribute_BoolValue) isAttribute_Value() {}
+
+func (*Attribute_DoubleValue) isAttribute_Value() {}
+
+var File_events_proto protoreflect.FileDescriptor
+
+var file_events_proto_rawDesc = []byte{
+ 0x0a, 0x0c, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c,
+ 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x37, 0x0a, 0x08,
+ 0x54, 0x78, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e,
+ 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74,
+ 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65,
+ 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x9a, 0x01, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12,
+ 0x17, 0x0a, 0x07, 0x61, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x06, 0x61, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
+ 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x37, 0x0a, 0x0a,
+ 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b,
+ 0x32, 0x17, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+ 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69,
+ 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x08, 0x70, 0x6b, 0x67, 0x5f, 0x70, 0x61, 0x74,
+ 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6b, 0x67, 0x50, 0x61,
+ 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x6b, 0x67, 0x5f, 0x70, 0x61,
+ 0x74, 0x68, 0x22, 0xb4, 0x01, 0x0a, 0x09, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
+ 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
+ 0x65, 0x79, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c,
+ 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69,
+ 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x36, 0x34,
+ 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x0a,
+ 0x69, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f,
+ 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00,
+ 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64,
+ 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28,
+ 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65,
+ 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x16, 0x5a, 0x14, 0x2e, 0x2e, 0x2f,
+ 0x70, 0x6b, 0x67, 0x73, 0x2f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74,
+ 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+ file_events_proto_rawDescOnce sync.Once
+ file_events_proto_rawDescData = file_events_proto_rawDesc
+)
+
+func file_events_proto_rawDescGZIP() []byte {
+ file_events_proto_rawDescOnce.Do(func() {
+ file_events_proto_rawDescData = protoimpl.X.CompressGZIP(file_events_proto_rawDescData)
+ })
+ return file_events_proto_rawDescData
+}
+
+var file_events_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_events_proto_goTypes = []any{
+ (*TxEvents)(nil), // 0: events_proto.TxEvents
+ (*Event)(nil), // 1: events_proto.Event
+ (*Attribute)(nil), // 2: events_proto.Attribute
+}
+var file_events_proto_depIdxs = []int32{
+ 1, // 0: events_proto.TxEvents.events:type_name -> events_proto.Event
+ 2, // 1: events_proto.Event.attributes:type_name -> events_proto.Attribute
+ 2, // [2:2] is the sub-list for method output_type
+ 2, // [2:2] is the sub-list for method input_type
+ 2, // [2:2] is the sub-list for extension type_name
+ 2, // [2:2] is the sub-list for extension extendee
+ 0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_events_proto_init() }
+func file_events_proto_init() {
+ if File_events_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_events_proto_msgTypes[0].Exporter = func(v any, i int) any {
+ switch v := v.(*TxEvents); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_events_proto_msgTypes[1].Exporter = func(v any, i int) any {
+ switch v := v.(*Event); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_events_proto_msgTypes[2].Exporter = func(v any, i int) any {
+ switch v := v.(*Attribute); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ file_events_proto_msgTypes[1].OneofWrappers = []any{}
+ file_events_proto_msgTypes[2].OneofWrappers = []any{
+ (*Attribute_StringValue)(nil),
+ (*Attribute_Int64Value)(nil),
+ (*Attribute_BoolValue)(nil),
+ (*Attribute_DoubleValue)(nil),
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_events_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 3,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_events_proto_goTypes,
+ DependencyIndexes: file_events_proto_depIdxs,
+ MessageInfos: file_events_proto_msgTypes,
+ }.Build()
+ File_events_proto = out.File
+ file_events_proto_rawDesc = nil
+ file_events_proto_goTypes = nil
+ file_events_proto_depIdxs = nil
+}
diff --git a/pkgs/events_proto/events_proto/events.pb.go b/pkgs/events_proto/events_proto/events.pb.go
new file mode 100644
index 0000000..727f95e
--- /dev/null
+++ b/pkgs/events_proto/events_proto/events.pb.go
@@ -0,0 +1,390 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.34.2
+// protoc (unknown)
+// source: events_proto/events.proto
+
+package events_proto
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type TxEvents struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Events []*Event `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"`
+}
+
+func (x *TxEvents) Reset() {
+ *x = TxEvents{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_events_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *TxEvents) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TxEvents) ProtoMessage() {}
+
+func (x *TxEvents) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_events_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use TxEvents.ProtoReflect.Descriptor instead.
+func (*TxEvents) Descriptor() ([]byte, []int) {
+ return file_events_proto_events_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *TxEvents) GetEvents() []*Event {
+ if x != nil {
+ return x.Events
+ }
+ return nil
+}
+
+type Event struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ AtType string `protobuf:"bytes,1,opt,name=at_type,json=atType,proto3" json:"at_type,omitempty"`
+ Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"`
+ Attributes []*Attribute `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty"`
+ PkgPath *string `protobuf:"bytes,4,opt,name=pkg_path,json=pkgPath,proto3,oneof" json:"pkg_path,omitempty"` // Can be null or string
+}
+
+func (x *Event) Reset() {
+ *x = Event{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_events_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Event) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Event) ProtoMessage() {}
+
+func (x *Event) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_events_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Event.ProtoReflect.Descriptor instead.
+func (*Event) Descriptor() ([]byte, []int) {
+ return file_events_proto_events_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Event) GetAtType() string {
+ if x != nil {
+ return x.AtType
+ }
+ return ""
+}
+
+func (x *Event) GetType() string {
+ if x != nil {
+ return x.Type
+ }
+ return ""
+}
+
+func (x *Event) GetAttributes() []*Attribute {
+ if x != nil {
+ return x.Attributes
+ }
+ return nil
+}
+
+func (x *Event) GetPkgPath() string {
+ if x != nil && x.PkgPath != nil {
+ return *x.PkgPath
+ }
+ return ""
+}
+
+type Attribute struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ // Types that are assignable to Value:
+ //
+ // *Attribute_StringValue
+ // *Attribute_Int64Value
+ // *Attribute_BoolValue
+ // *Attribute_DoubleValue
+ Value isAttribute_Value `protobuf_oneof:"value"`
+}
+
+func (x *Attribute) Reset() {
+ *x = Attribute{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_events_proto_events_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Attribute) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Attribute) ProtoMessage() {}
+
+func (x *Attribute) ProtoReflect() protoreflect.Message {
+ mi := &file_events_proto_events_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Attribute.ProtoReflect.Descriptor instead.
+func (*Attribute) Descriptor() ([]byte, []int) {
+ return file_events_proto_events_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Attribute) GetKey() string {
+ if x != nil {
+ return x.Key
+ }
+ return ""
+}
+
+func (m *Attribute) GetValue() isAttribute_Value {
+ if m != nil {
+ return m.Value
+ }
+ return nil
+}
+
+func (x *Attribute) GetStringValue() string {
+ if x, ok := x.GetValue().(*Attribute_StringValue); ok {
+ return x.StringValue
+ }
+ return ""
+}
+
+func (x *Attribute) GetInt64Value() int64 {
+ if x, ok := x.GetValue().(*Attribute_Int64Value); ok {
+ return x.Int64Value
+ }
+ return 0
+}
+
+func (x *Attribute) GetBoolValue() bool {
+ if x, ok := x.GetValue().(*Attribute_BoolValue); ok {
+ return x.BoolValue
+ }
+ return false
+}
+
+func (x *Attribute) GetDoubleValue() float64 {
+ if x, ok := x.GetValue().(*Attribute_DoubleValue); ok {
+ return x.DoubleValue
+ }
+ return 0
+}
+
+type isAttribute_Value interface {
+ isAttribute_Value()
+}
+
+type Attribute_StringValue struct {
+ StringValue string `protobuf:"bytes,2,opt,name=string_value,json=stringValue,proto3,oneof"`
+}
+
+type Attribute_Int64Value struct {
+ Int64Value int64 `protobuf:"varint,3,opt,name=int64_value,json=int64Value,proto3,oneof"`
+}
+
+type Attribute_BoolValue struct {
+ BoolValue bool `protobuf:"varint,4,opt,name=bool_value,json=boolValue,proto3,oneof"`
+}
+
+type Attribute_DoubleValue struct {
+ DoubleValue float64 `protobuf:"fixed64,5,opt,name=double_value,json=doubleValue,proto3,oneof"` // leave this as double for now
+}
+
+func (*Attribute_StringValue) isAttribute_Value() {}
+
+func (*Attribute_Int64Value) isAttribute_Value() {}
+
+func (*Attribute_BoolValue) isAttribute_Value() {}
+
+func (*Attribute_DoubleValue) isAttribute_Value() {}
+
+var File_events_proto_events_proto protoreflect.FileDescriptor
+
+var file_events_proto_events_proto_rawDesc = []byte{
+ 0x0a, 0x19, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65,
+ 0x76, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x65, 0x76, 0x65,
+ 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x31, 0x22, 0x3a, 0x0a, 0x08,
+ 0x54, 0x78, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2e, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e,
+ 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74,
+ 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74,
+ 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x9d, 0x01, 0x0a, 0x05, 0x45, 0x76, 0x65,
+ 0x6e, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
+ 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74,
+ 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
+ 0x3a, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20,
+ 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f,
+ 0x74, 0x6f, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x52,
+ 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x08, 0x70,
+ 0x6b, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52,
+ 0x07, 0x70, 0x6b, 0x67, 0x50, 0x61, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f,
+ 0x70, 0x6b, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x22, 0xb4, 0x01, 0x0a, 0x09, 0x41, 0x74, 0x74,
+ 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
+ 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69,
+ 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00,
+ 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a,
+ 0x0b, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01,
+ 0x28, 0x03, 0x48, 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65,
+ 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04,
+ 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75,
+ 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75,
+ 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c,
+ 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42,
+ 0x19, 0x5a, 0x17, 0x2e, 0x2e, 0x2f, 0x2e, 0x2e, 0x2f, 0x70, 0x6b, 0x67, 0x73, 0x2f, 0x65, 0x76,
+ 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+ 0x6f, 0x33,
+}
+
+var (
+ file_events_proto_events_proto_rawDescOnce sync.Once
+ file_events_proto_events_proto_rawDescData = file_events_proto_events_proto_rawDesc
+)
+
+func file_events_proto_events_proto_rawDescGZIP() []byte {
+ file_events_proto_events_proto_rawDescOnce.Do(func() {
+ file_events_proto_events_proto_rawDescData = protoimpl.X.CompressGZIP(file_events_proto_events_proto_rawDescData)
+ })
+ return file_events_proto_events_proto_rawDescData
+}
+
+var file_events_proto_events_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_events_proto_events_proto_goTypes = []any{
+ (*TxEvents)(nil), // 0: events_proto.v1.TxEvents
+ (*Event)(nil), // 1: events_proto.v1.Event
+ (*Attribute)(nil), // 2: events_proto.v1.Attribute
+}
+var file_events_proto_events_proto_depIdxs = []int32{
+ 1, // 0: events_proto.v1.TxEvents.events:type_name -> events_proto.v1.Event
+ 2, // 1: events_proto.v1.Event.attributes:type_name -> events_proto.v1.Attribute
+ 2, // [2:2] is the sub-list for method output_type
+ 2, // [2:2] is the sub-list for method input_type
+ 2, // [2:2] is the sub-list for extension type_name
+ 2, // [2:2] is the sub-list for extension extendee
+ 0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_events_proto_events_proto_init() }
+func file_events_proto_events_proto_init() {
+ if File_events_proto_events_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_events_proto_events_proto_msgTypes[0].Exporter = func(v any, i int) any {
+ switch v := v.(*TxEvents); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_events_proto_events_proto_msgTypes[1].Exporter = func(v any, i int) any {
+ switch v := v.(*Event); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_events_proto_events_proto_msgTypes[2].Exporter = func(v any, i int) any {
+ switch v := v.(*Attribute); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ file_events_proto_events_proto_msgTypes[1].OneofWrappers = []any{}
+ file_events_proto_events_proto_msgTypes[2].OneofWrappers = []any{
+ (*Attribute_StringValue)(nil),
+ (*Attribute_Int64Value)(nil),
+ (*Attribute_BoolValue)(nil),
+ (*Attribute_DoubleValue)(nil),
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_events_proto_events_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 3,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_events_proto_events_proto_goTypes,
+ DependencyIndexes: file_events_proto_events_proto_depIdxs,
+ MessageInfos: file_events_proto_events_proto_msgTypes,
+ }.Build()
+ File_events_proto_events_proto = out.File
+ file_events_proto_events_proto_rawDesc = nil
+ file_events_proto_events_proto_goTypes = nil
+ file_events_proto_events_proto_depIdxs = nil
+}
diff --git a/pkgs/events_proto/parse.go b/pkgs/events_proto/parse.go
new file mode 100644
index 0000000..7b66f27
--- /dev/null
+++ b/pkgs/events_proto/parse.go
@@ -0,0 +1,41 @@
+package events_proto
+
+import "strconv"
+
+// NewAttributeFromString builds an Attribute from key and a string value.
+// The value is parsed at runtime: we try bool, then int64, then float64;
+// if any parse fails (e.g. overflow or invalid), we keep the value as string.
+// So "true"/"false" become bool, "42" becomes int64, "3.14" becomes double,
+// and anything else (including too-large numbers) stays as string_value.
+func NewAttributeFromString(key, value string) *Attribute {
+ return &Attribute{
+ Key: key,
+ Value: parseAttributeValue(value),
+ }
+}
+
+// parseAttributeValue tries to interpret s as bool, int64, or float64, in that order.
+// On failure (including overflow), it returns a string_value so the value is never lost.
+func parseAttributeValue(s string) isAttribute_Value {
+ if b, ok := parseBool(s); ok {
+ return &Attribute_BoolValue{BoolValue: b}
+ }
+ if i, err := strconv.ParseInt(s, 10, 64); err == nil {
+ return &Attribute_Int64Value{Int64Value: i}
+ }
+ if f, err := strconv.ParseFloat(s, 64); err == nil {
+ return &Attribute_DoubleValue{DoubleValue: f}
+ }
+ return &Attribute_StringValue{StringValue: s}
+}
+
+func parseBool(s string) (bool, bool) {
+ switch s {
+ case "true":
+ return true, true
+ case "false":
+ return false, true
+ default:
+ return false, false // return false and false if the value is not a boolean
+ }
+}
diff --git a/pkgs/generator/general.go b/pkgs/generator/general.go
index 623a567..2409f8a 100644
--- a/pkgs/generator/general.go
+++ b/pkgs/generator/general.go
@@ -8,7 +8,7 @@ import (
"strings"
"time"
- eventsProto "github.com/Cogwheel-Validator/spectra-gnoland-indexer/indexer/events_proto"
+ eventsProto "github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/events_proto"
"google.golang.org/protobuf/proto"
"github.com/gnolang/gno/gno.land/pkg/sdk/vm"
@@ -27,7 +27,6 @@ import (
//
// The code does contain some of the implementation meant for training the zstandard and protobuf experiments.
// For now this will serve for the integration purposes.
-// TODO: rework this later to be somewhat more accurate and to have a seperated training and integration code.
type TxEvents struct {
Events []eventsProto.Event `json:"Events"`
@@ -46,9 +45,8 @@ func NewDataGenerator(size int) *DataGenerator {
cryptoGen := NewCryptoGenerator(seed)
return &DataGenerator{
- rand: rand.New(rand.NewSource(seed)),
- cryptoGen: cryptoGen,
- // declare ammount of keys by size integer
+ rand: rand.New(rand.NewSource(seed)),
+ cryptoGen: cryptoGen,
keyPairPool: cryptoGen.GenerateKeyPairPool(size),
}
}
@@ -243,7 +241,7 @@ var eventTemplates = map[string]func(*DataGenerator) eventsProto.Event{
// Generate synthetic transactions for integration tests
func (g *DataGenerator) GenerateTransaction() (TxEvents, std.Tx) {
// some transactions do not have any events like bank send, other vm msgs might have events
- // but untill some real events are available use the event templates
+ // but until some real events are available use the event templates
// declare the transaction type first
// bank send should be maybe 40% of the time?
// vm. MsgCall and MsgRun should be 50% (25/25) of the time?
@@ -414,33 +412,33 @@ func (g *DataGenerator) GenerateArgs() string {
// mostly I have seen some longer ones for validator registration
// so I think we will just generate a radnom lenght depending on the generator
// since it is hard to know how much each transaction will have we will just generate a random length
- var lenght int
+ var length int
var randomNum float32
switch randomNum = g.rand.Float32(); {
// 15% chance of having short arguments
case randomNum <= 0.15:
- lenght = 12
+ length = 12
// 15% chance of having medium arguments
case randomNum > 0.15 && randomNum <= 0.3:
- lenght = 24
+ length = 24
// 15% chance of having long arguments
case randomNum > 0.3 && randomNum <= 0.45:
- lenght = 36
+ length = 36
// 30% chance of having very long arguments
case randomNum > 0.45 && randomNum <= 0.75:
- lenght = 48
+ length = 48
// 15% chance of having very long arguments
case randomNum > 0.75 && randomNum <= 0.9:
- lenght = 60
+ length = 60
// 9% chance of having very long arguments
case randomNum > 0.9 && randomNum <= 0.99:
- lenght = 72
+ length = 72
// 0.9% chance of having very long arguments
case randomNum > 0.99 && randomNum <= 0.999:
- lenght = 84
+ length = 84
// 0.1% chance of having very long arguments
case randomNum > 0.999 && randomNum <= 1.0:
- lenght = 3000
+ length = 3000
}
loremIpsum := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
@@ -448,10 +446,10 @@ func (g *DataGenerator) GenerateArgs() string {
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum.`
- if lenght > len(loremIpsum) {
+ if length > len(loremIpsum) {
loremIpsum = strings.Repeat(loremIpsum, 10)
}
- return loremIpsum[:lenght]
+ return loremIpsum[:length]
}
func (g *DataGenerator) GeneratePackageName() string {
diff --git a/pkgs/logger/logger.go b/pkgs/logger/logger.go
new file mode 100644
index 0000000..32f0f48
--- /dev/null
+++ b/pkgs/logger/logger.go
@@ -0,0 +1,33 @@
+package logger
+
+import (
+ "io"
+ "os"
+ "time"
+
+ "github.com/rs/zerolog"
+ "go.opentelemetry.io/contrib/bridges/otelzerolog"
+ "go.opentelemetry.io/otel/log/global"
+)
+
+var log zerolog.Logger
+
+type Config struct {
+ Level zerolog.Level
+ ServiceName string
+ Pretty bool
+}
+
+func Init(cfg Config) {
+ var w io.Writer = os.Stderr
+ if cfg.Pretty {
+ w = zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}
+ }
+
+ base := zerolog.New(w).Level(cfg.Level).With().Timestamp().Logger()
+
+ hook := otelzerolog.NewHook(cfg.ServiceName, otelzerolog.WithLoggerProvider(global.GetLoggerProvider()))
+ log = base.Hook(hook)
+}
+
+func Get() *zerolog.Logger { return &log }
diff --git a/pkgs/sql_data_types/table.go b/pkgs/sql_data_types/table.go
index 0b8c21a..c32052a 100644
--- a/pkgs/sql_data_types/table.go
+++ b/pkgs/sql_data_types/table.go
@@ -94,7 +94,7 @@ func (gv GnoValidatorAddress) GetTableInfo() (*dbinit.TableInfo, error) {
// - Timestamp (time.Time)
// - Chain ID (string)
// - Proposer address (int32)
-// - Txs ([]string)
+// - Txs ([][]byte)
// - Chain Name (string)
//
// PRIMARY KEY (height, timestamp, chain_name)
@@ -132,21 +132,15 @@ func (b Blocks) TableColumns() []string {
// - Timestamp (time.Time)
// - Proposer (int32)
// - Signed validators (int32 all of the validators that signed the block)
-// - Chain ID (string)
-// - Missed validators (int32 all of the validators that missed the block)
+// - Chain Name (string)
//
-// PRIMARY KEY (block_height, timestamp, chain_id)
+// PRIMARY KEY (block_height, timestamp, chain_name)
type ValidatorBlockSigning struct {
BlockHeight uint64 `db:"block_height" dbtype:"bigint" nullable:"false" primary:"true"`
Timestamp time.Time `db:"timestamp" dbtype:"timestamptz" nullable:"false" primary:"true"`
Proposer int32 `db:"proposer" dbtype:"integer" nullable:"false" primary:"false"`
SignedVals []int32 `db:"signed_vals" dbtype:"integer[]" nullable:"false" primary:"false"`
ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"` // use type enum chain_name from postgres
- // MissedVals []int32 `db:"missed_vals" dbtype:"integer" nullable:"false" primary:"false"`
- // can't confirm who is in the active set without making a smart contract query to the DAO smart contract
- // which could cost a lot of gas if the program checks it constantly, also in "historical" mode when the program
- // is running it can't check previous active set without having some over engeneered system to track the votes from the DAO
- // so only store the signed validators because that is the only thing we can gather and confirm they did sign
}
// TableName returns the name of the table for the ValidatorBlockSigning struct
@@ -221,11 +215,6 @@ func (at AddressTx) TableColumns() []string {
// - Fee (Fee)
//
// PRIMARY KEY (tx_hash, chain_name, timestamp)
-//
-// INFO about this type!
-// This project is open source hence for the sake of wider addoption this table storer both compressed and native format
-// But what kind of data will be stored should be managed by the config.
-// It is not recommended to use both modes at the same time.
type TransactionGeneral struct {
TxHash []byte `db:"tx_hash" dbtype:"bytea" nullable:"false" primary:"true"`
ChainName string `db:"chain_name" dbtype:"chain_name" nullable:"false" primary:"true"`
diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml
new file mode 100644
index 0000000..b93e7a7
--- /dev/null
+++ b/proto/buf.gen.yaml
@@ -0,0 +1,5 @@
+version: v2
+plugins:
+ - local: protoc-gen-go
+ out: ../pkgs/events_proto
+ opt: paths=source_relative
\ No newline at end of file
diff --git a/proto/buf.yaml b/proto/buf.yaml
new file mode 100644
index 0000000..0a714b4
--- /dev/null
+++ b/proto/buf.yaml
@@ -0,0 +1,7 @@
+version: v2
+lint:
+ use:
+ - STANDARD
+breaking:
+ use:
+ - FILE
diff --git a/proto/events.proto b/proto/events_proto/events.proto
similarity index 86%
rename from proto/events.proto
rename to proto/events_proto/events.proto
index 470516e..45c8a2e 100644
--- a/proto/events.proto
+++ b/proto/events_proto/events.proto
@@ -1,7 +1,7 @@
syntax = "proto3";
-package eventsproto;
+package events_proto.v1;
-option go_package = "../indexer/events_proto";
+option go_package = "../../pkgs/events_proto";
message TxEvents {
repeated Event events = 1;
diff --git a/training-config.example.yml b/training-config.example.yml
new file mode 100644
index 0000000..4518218
--- /dev/null
+++ b/training-config.example.yml
@@ -0,0 +1,17 @@
+# Example config file for the training
+# Please note that the user and password are just an example and you should use your own values
+# However the training process will only ever be done prior to the indexer being deployed and only be
+# done once, or if you want to re-train the dictionary. Therefore it is not a big deal since you would
+# only do this in development or testing.
+user: postgres
+password: 12345678
+host: localhost
+port: 5432
+dbname: gnoland
+sslmode: disable
+pool_max_conns: 50
+pool_min_conns: 10
+pool_max_conn_lifetime: 5m
+pool_max_conn_idle_time: 5m
+pool_health_check_period: 1m
+pool_max_conn_lifetime_jitter: 1m
\ No newline at end of file