0.2.0

.env.example

@@ -1,7 +1,24 @@
 # Environment example use your own values
+# just do not use these values unless you are testing or developing
 DB_HOST=127.0.0.1
 DB_PORT=5432
-DB_USER=postgres
+DB_USER=writer
 DB_SSLMODE=disable
 DB_PASSWORD=12345678
-DB_NAME=gnoland
+DB_NAME=gnoland
+
+# Envs for the API
+# If you plan to use the API than use this values
+API_DB_HOST=127.0.0.1
+API_DB_PORT=5432
+API_DB_USER=reader
+API_DB_SSLMODE=disable
+API_DB_PASSWORD=12345678
+API_DB_NAME=gnoland
+
+API_DB_POOL_MAX_CONNS=50
+API_DB_POOL_MIN_CONNS=10
+API_DB_POOL_MAX_CONN_LIFETIME=10s
+API_DB_POOL_MAX_CONN_IDLE_TIME=5m
+API_DB_POOL_HEALTH_CHECK_PERIOD=1m
+API_DB_POOL_MAX_CONN_LIFETIME_JITTER=1m

.github/workflows/release.yml

@@ -40,8 +40,11 @@ jobs:
         run: |
           mkdir -p build
 
+          # Get git commit hash
+          GIT_COMMIT=$(git rev-parse --short HEAD)
+
           # Linux AMD64
-          GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Version=${{ steps.version.outputs.version }}" -o build/indexer indexer/main.go
+          GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Commit=${GIT_COMMIT} -X main.Version=${{ steps.version.outputs.version }}" -o build/indexer-linux-amd64 indexer/main.go
 
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v1

.gitignore

@@ -37,7 +37,12 @@ go.work.sum
 config.yml
 test_config.yml
 .env
+config-api.yml
 
 # State dumps and diagnostics directories
 state_dumps/
 diagnostics/
+
+# Cert files
+server.crt
+server.key

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.0] - 2025-10-04
+
+Added the REST API with some basic routes that will come in handy. Small bug fixes and changes.
+
+### Added
+
+- Rest API with 5 basic routes that will come in handy. [90bef0e](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/90bef0ec5a0bff468d4a1d6771b82706029f4ea9),[f2a39d6](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/f2a39d65c5622e0a5953d1f6113ab5eea1996cad),[d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e)
+- Some basic documentation for the API and modified the existing docs. [b2c20d2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/b2c20d222e4f52b74b333974a7930df3cddad29e)
+- Database queries for the API ( althoguh they can be used for any other app or service if needed ) [d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e), [f2a39d6](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/f2a39d65c5622e0a5953d1f6113ab5eea1996cad)
+
+### Fixed
+
+- Some table fixes, the msg_types would insert the empty string because when the make slice function was called by the accident it added the empty string instead of just allocating the size of the slice. [d875be2](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/d875be2c42eb5fd71a02b4b29d1496c4b7c3de1e)
+- The validator signing has the column name addresses for the signed validators. While this is not a bug it wasn't intended. [90bef0e](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/commit/90bef0ec5a0bff468d4a1d6771b82706029f4ea9)
+
 ## [0.1.1] - 2025-10-02
 
 Mostly some fixes. The live should work now there was a bug with the RPC client when making a request to the last block height recorded. The retry worker could sometimes send the data to the closed channel. Now the query operator calls the wg.Done functions directly. It should work now but there might be some other bugs.

Dockerfile

@@ -4,9 +4,10 @@ WORKDIR /app
 
 COPY . .
 
-RUN go build -o indexer indexer/main.go
-
-RUN chmod +x indexer
+ARG VERSION=unknown
+RUN GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
+    go build -ldflags="-X main.Commit=${GIT_COMMIT} -X main.Version=${VERSION}" -o indexer indexer/main.go && \
+    chmod +x indexer
 
 FROM debian:trixie-slim
 

Makefile

@@ -1,21 +1,30 @@
-.PHONY: build install clean build-experimental install-experimental
+.PHONY: build install clean build-experimental install-experimental build-api
+
+# Get git information
+GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+GIT_TAG := $(shell git describe --tags --exact-match 2>/dev/null || echo "")
+VERSION := $(if $(GIT_TAG),$(GIT_TAG),dev-$(GIT_COMMIT))
 
 build:
 	mkdir -p build
-	go build -o build/indexer indexer/main.go
+	go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer indexer/main.go
 
 install:
-	cd indexer && go install ./...
+	cd indexer && go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)"
+
+build-api:
+	mkdir -p build
+	go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/api api/main.go
 
 clean:
 	rm -rf build
 
 # experimental build with greentea garbage collection
 # use at your own risk
 build-experimental:
-	GOEXPERIMENT=greenteagc go build -o build/indexer-tea indexer/main.go
+	GOEXPERIMENT=greenteagc go build -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)" -o build/indexer-tea indexer/main.go
 
 # experimental install with greentea garbage collection
 # use at your own risk
 install-experimental:
-	cd indexer && GOEXPERIMENT=greenteagc go install ./...
+	cd indexer && GOEXPERIMENT=greenteagc go install ./... -ldflags="-X main.Commit=$(GIT_COMMIT) -X main.Version=$(VERSION)"

README.md

@@ -2,26 +2,27 @@
 <div align="center">
 <img src="./images/spectra-gnoland-indexer.svg" alt="Spectra Gnoland Indexer" width="250" style="padding: 25px;">
 </div>
+<p align="center" style="font-style: italic;">This project is still in development. Future releases might have some breaking changes.</p>
 
 The Spectra Gnoland Indexer(SGI) is a tool that records the data from the Gnoland blockchain and stores the data 
 in the timeseries database, Timescale DB. This indexer will be a part of the Spectra explorer and will be used to 
 store the data for the explorer. This program can be used for any kind of data/analytics program or app.
 
 The biggest problem when dealing with the blockchain data is how to store the data in a way that is easy to query 
 and analyze. Some projects rely on the direct access to the data from the blockchain nodes and the problem in this 
-case is that the nodes are meant for multiple purposes. They usually provide at least one, sometimes even more,
-endpoints for querying the data, then this node need to be able to work with other nodes and to acquire the block 
-data from other nodes via P2P network. Most nodes can be used as a miners/validators that need to sign the blocks, 
-and maintain the blockchain state. While they are the first point of contact for data and contact with the chain
-they are not most efficient when it comes to making some queries easy and fast. Plus the nodes are made to be fast 
-and efficient when it comes to preformance of the nodes, so they are not most efficient when it comes to storing 
-the data.
+case is that the nodes are meant for multiple purposes. 
+
+They usually provide at least one, sometimes even more, endpoints for querying the data, then this node need to be 
+able to work with other nodes and to acquire the block data from other nodes via P2P network. Most nodes can be 
+used as a miners/validators that need to sign the blocks. Even if these nodes are only to be used as a RPC nodes
+they can still provide the data but depending on the underlying SDK, tech stack and other factors it might be 
+not the best choice. And finally the nodes are not the best when it comes to storing the data and scalability.
 
 ## Table of content
 - [Table of content](#table-of-content)
 - [The solution](#the-solution)
 - [Why TimescaleDB? And can it work on other SQL databases?](#why-timescaledb-and-can-it-work-on-other-sql-databases)
-- [The indexer and how it works](#the-indexer-and-how-it-works)
+- [How does the indexer work?](#how-does-the-indexer-work)
 - [What is indexed and what is not](#what-is-indexed-and-what-is-not)
   - [Blocks:](#blocks)
   - [Validator signings:](#validator-signings)
@@ -53,6 +54,12 @@ TimescaleDB (Tiger Data is their commercial version) is a extension of the Postg
 store the time series data in a way that is easy to query and analyze. It also has a lot of features that can 
 extend the capabilities of the PostgreSQL and make it more powerful.
 
+The TimescaleDB has also feature of it's own that is not present in the Postgres. The user could extend the indexer
+by adding the data aggregation features, automatic jobs scheduling, hyperfunctions and more.
+
+This database extension also has a data compression feature that can be used to reduce the storage space and 
+segment the data into smaller chunks by using time based intervals making the queries faster.
+
 Technically speaking the indexer can work on Postgres database, however you would need to create the tables and 
 types manually. This might be added later in the future if there is a demand for it but for now the focus is on the 
 TimescaleDB.
@@ -65,21 +72,7 @@ There are some other SQL databases that could work in theory. For them to work t
 If all of the above are met then the indexer can work on them. It might work on CockroachDB for example but this is 
 out of the scope of this project. Maybe in the future it might be an interesting idea to support it.
 
-There were some other contenders for the database choice. They were: InfluxDB and QuestDB. 
-
-At the time when the cosmos indexer was made the InfluxDB had a opensource version 2 but at the time it was lacking
-some of the features that are now present in the version 3. Maybe it could have been used for the indexer but at 
-the time it seemed to be not the best choice.
-
-QuestDB was a interesting choice and it is probably even better choice than TimescaleDB. But it does have some 
-limitations but nothing that would make it impossible to use it. However with the TimescaleDB it is pretty much an 
-extention of Postgres and it is more mature and has more features and it can be extended with any other extensions 
-that are available for the Postgres.
-
-The TimescaleDB has also feature of it's own that is not present in the Postgres. The user could extend the indexer
-by adding the data aggregation features, automatic jobs scheduling, hyperfunctions and more.
-
-## The indexer and how it works
+## How does the indexer work?
 
 The indexer has 2 main modes of operation:
 - Live mode
@@ -136,6 +129,7 @@ Stored data:
 - Validator block signing height
 - Validator block signing timestamp
 - Validator block signing signed validators
+- Proposer address
 
 Not stored data:
 - Missed validators
@@ -149,11 +143,12 @@ VM message Add Package and Call where in theory one could extract even the body
 
 ### 🦾 Pros:
 
-- The indexer process the data using goroutines and channels, which can provide a faster processing of the data. Some early test on the real data showed about 2vCPU can index 10K blocks in about 2m30s while the 4vCPU can index 10K blocks in about 1m30s.
+- The indexer process the data using goroutines and channels, which can provide a faster processing of the data.
+- Fast data processing. [see benchmarks](./docs/benchmarks.md)
 - The program has 2 modes that can be used for the indexing of the data. This can be useful for the testing, partial indexing of the chain or gradual indexing of the chain.
 - The data is stored ready to be used for any kind of analytics and visualization with any programming language.
 - No need to deal with Amino encoding and decoding for the messagess as the indexer decodes the messages and stores them in the database.
-- It comes with a REST API to get you started quickly.(To be added in the future)
+- It comes with a REST API to get you started quickly.
 - It relies on a SQL database, which can provide a easier experience for any user that is familiar with the SQL.
 - It uses TimescaleDB, a PostgresSQL extenstion that can be extended with any other extensions, plus the TimescaleDB has a lot of features that are not present in the Postgres.
 
@@ -165,4 +160,4 @@ VM message Add Package and Call where in theory one could extract even the body
 
 ## In depth documentation
 
-For more detailed documentation, please refer to the [docs](https://github.com/Cogwheel-Validator/spectra-gnoland-indexer/blob/main/docs/README.md) directory.
+For more detailed documentation, please refer to the [docs](./docs/README.md) directory.

api/config/loader.go

@@ -0,0 +1,70 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/caarlos0/env/v6"
+	"github.com/joho/godotenv"
+	"go.yaml.in/yaml/v4"
+)
+
+func LoadConfig(path string) (*ApiConfig, error) {
+	yamlFile, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var config ApiConfig
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		return nil, err
+	}
+	// check if any of the fields are empty
+	if config.Host == "" {
+		config.Host = "localhost"
+	}
+	if config.Port == 0 {
+		config.Port = 8080
+	}
+	// any cors method should be auto filled by the cors middleware
+	// TODO: maybe add more options later
+	return &config, nil
+}
+
+func LoadEnvironment(path string) (*ApiEnv, error) {
+	possibleEnvFiles := []string{
+		".env", // accept only .env for now
+	}
+	// check if any of the files exist within the current path
+	existingFiles := []string{}
+	for _, envFile := range possibleEnvFiles {
+		formPath := filepath.Join(path, envFile)
+		if _, err := os.Stat(formPath); err == nil {
+			existingFiles = append(existingFiles, formPath)
+		}
+	}
+	// if there are multiple files, decide which has highest priority
+	// 1. production , 2. development, 3. local, 4. default
+	// only use the regular .env for now return to this laster
+	if len(existingFiles) == 0 {
+		fmt.Println("No environment file found. Searching for os environment variables.")
+	} else if len(existingFiles) == 1 {
+		absPath, err := filepath.Abs(existingFiles[0])
+		if err != nil {
+			return nil, err
+		}
+		err = godotenv.Load(absPath)
+		if err != nil {
+			return nil, fmt.Errorf("error loading .env file: %w", err)
+		}
+		fmt.Printf("Loaded environment variables from %s\n", absPath)
+	}
+
+	environment := ApiEnv{}
+	if err := env.Parse(&environment); err != nil {
+		return nil, fmt.Errorf("failed to parse environment variables: %w", err)
+	}
+
+	return &environment, nil
+}

api/config/types.go

@@ -0,0 +1,30 @@
+package config
+
+import "time"
+
+type ApiConfig struct {
+	// Basic connection info
+	Host string `yaml:"host"`
+	Port int    `yaml:"port"`
+	// CORS config
+	CorsAllowedOrigins []string `yaml:"cors_allowed_origins"`
+	CorsAllowedMethods []string `yaml:"cors_allowed_methods"`
+	CorsAllowedHeaders []string `yaml:"cors_allowed_headers"`
+	CorsMaxAge         int      `yaml:"cors_max_age"`
+	ChainName          string   `yaml:"chain_name"`
+}
+
+type ApiEnv struct {
+	ApiDbHost                      string        `env:"API_DB_HOST" envDefault:"localhost"`
+	ApiDbPort                      int           `env:"API_DB_PORT" envDefault:"5432"`
+	ApiDbUser                      string        `env:"API_DB_USER" envDefault:"postgres"`
+	ApiDbPassword                  string        `env:"API_DB_PASSWORD" envDefault:"12345678"`
+	ApiDbName                      string        `env:"API_DB_NAME" envDefault:"gnoland"`
+	ApiDbSslmode                   string        `env:"API_DB_SSLMODE" envDefault:"disable"`
+	ApiDbPoolMaxConns              int           `env:"API_DB_POOL_MAX_CONNS" envDefault:"50"`
+	ApiDbPoolMinConns              int           `env:"API_DB_POOL_MIN_CONNS" envDefault:"10"`
+	ApiDbPoolMaxConnLifetime       time.Duration `env:"API_DB_POOL_MAX_CONN_LIFETIME" envDefault:"10m"`
+	ApiDbPoolMaxConnIdleTime       time.Duration `env:"API_DB_POOL_MAX_CONN_IDLE_TIME" envDefault:"5m"`
+	ApiDbPoolHealthCheckPeriod     time.Duration `env:"API_DB_POOL_HEALTH_CHECK_PERIOD" envDefault:"1m"`
+	ApiDbPoolMaxConnLifetimeJitter time.Duration `env:"API_DB_POOL_MAX_CONN_LIFETIME_JITTER" envDefault:"1m"`
+}

api/handlers/address.go

@@ -0,0 +1,31 @@
+package handlers
+
+import (
+	"context"
+
+	humatypes "github.com/Cogwheel-Validator/spectra-gnoland-indexer/api/huma-types"
+	"github.com/Cogwheel-Validator/spectra-gnoland-indexer/pkgs/database"
+	"github.com/danielgtaylor/huma/v2"
+)
+
+type AddressHandler struct {
+	db        *database.TimescaleDb
+	chainName string
+}
+
+func NewAddressHandler(db *database.TimescaleDb, chainName string) *AddressHandler {
+	return &AddressHandler{db: db, chainName: chainName}
+}
+
+func (h *AddressHandler) GetAddressTxs(
+	ctx context.Context,
+	input *humatypes.AddressGetInput,
+) (*humatypes.AddressGetOutput, error) {
+	address, err := h.db.GetAddressTxs(input.Address, h.chainName, input.FromTimestamp, input.ToTimestamp)
+	if err != nil {
+		return nil, huma.Error404NotFound("Address not found", err)
+	}
+	return &humatypes.AddressGetOutput{
+		Body: *address,
+	}, nil
+}