feat(otel): add support for open telemetry traces and instrumentation (#46)

Author: cloud-j-luna

cmd/main.go

@@ -23,10 +23,13 @@ import (
 	"golang.org/x/net/http2/h2c"
 	"golang.org/x/sync/errgroup"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
 	"github.com/akash-network/rpc-proxy/internal/block"
 	"github.com/akash-network/rpc-proxy/internal/config"
 	"github.com/akash-network/rpc-proxy/internal/halt"
 	"github.com/akash-network/rpc-proxy/internal/metrics"
+	proxyotel "github.com/akash-network/rpc-proxy/internal/otel"
 	"github.com/akash-network/rpc-proxy/internal/proxy"
 	"github.com/akash-network/rpc-proxy/internal/seed"
 	"github.com/spf13/cobra"
@@ -121,6 +124,14 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 	rootCmd.PersistentFlags().String("metrics.path", "/metrics", "Path to expose metrics on")
 	rootCmd.PersistentFlags().String("metrics.service-name", "", "Service name for metrics labeling (defaults to HOSTNAME env var)")
 
+	// OpenTelemetry configuration
+	rootCmd.PersistentFlags().Bool("otel.enabled", false, "Enable OpenTelemetry tracing")
+	rootCmd.PersistentFlags().String("otel.exporter-type", "grpc", "OTLP exporter type (grpc or http)")
+	rootCmd.PersistentFlags().String("otel.endpoint", "", "OTLP collector endpoint")
+	rootCmd.PersistentFlags().Bool("otel.insecure", false, "Use insecure connection to OTLP collector")
+	rootCmd.PersistentFlags().String("otel.service-name", "", "Service name for tracing (defaults to metrics service name or HOSTNAME)")
+	rootCmd.PersistentFlags().Float64("otel.sample-rate", 0.1, "Trace sampling rate (0.0 to 1.0, e.g. 0.1 = 10%)")
+
 	// Configuration file support
 	rootCmd.PersistentFlags().StringP("config", "c", "", "config file (default is $HOME/.akash-proxy/config.yaml)")
 
@@ -130,6 +141,13 @@ func NewRootCmd(v *viper.Viper) *cobra.Command {
 func runProxy(cfg config.Config) error {
 	log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
 
+	// Initialize OpenTelemetry tracing
+	otelShutdown, err := proxyotel.Init(context.Background(), cfg.OTEL, cfg.Metrics.ServiceName)
+	if err != nil {
+		return fmt.Errorf("initializing OpenTelemetry: %w", err)
+	}
+	defer otelShutdown(context.Background())
+
 	var metricsServer *http.Server
 	if cfg.Metrics.Enabled {
 		// Determine service name from flag or HOSTNAME environment variable
@@ -174,7 +192,6 @@ func runProxy(cfg config.Config) error {
 	blockTime := 6 * time.Second
 
 	var haltDetector *halt.Detector
-	var err error
 	if cfg.Halt.Enabled {
 		haltDetector, err = newHaltDetector(cfg.Halt, log)
 		if err != nil {
@@ -362,7 +379,7 @@ func prepareRestAndRPCServer(log *slog.Logger, cfg config.Config, rpcProxyHandle
 
 	srv := &http.Server{
 		Addr:         cfg.Server.Listen,
-		Handler:      cors.WithCorsMiddleware(corsHeaders, m),
+		Handler:      otelhttp.NewHandler(cors.WithCorsMiddleware(corsHeaders, m), "akash-proxy"),
 		TLSConfig:    am.TLSConfig(),
 		ReadTimeout:  cfg.Server.Timeouts.Read,
 		IdleTimeout:  cfg.Server.Timeouts.Idle,
@@ -384,7 +401,7 @@ func prepareGRPCServer(log *slog.Logger, cfg config.Config, p *proxy.GRPCProxy)
 	}
 
 	mux := http.NewServeMux()
-	mux.Handle("/", cors.WithCorsMiddleware(corsHeaders, p))
+	mux.Handle("/", otelhttp.NewHandler(cors.WithCorsMiddleware(corsHeaders, p), "akash-proxy-grpc"))
 
 	// Start HTTP/2.0 server.
 	grpcServer := &http.Server{

deploy/docker-compose.yml

@@ -1,20 +1,35 @@
-version: '3.8'
-
 services:
   prometheus:
     image: prom/prometheus:latest
-    container_name: prometheus
     ports:
       - "9090:9090"
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-remote-write-receiver'
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  tempo:
+    image: grafana/tempo:latest
+    ports:
+      - "3200:3200"   # Tempo HTTP API
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP
+    volumes:
+      - ./tempo/tempo.yml:/etc/tempo/tempo.yml
+    command:
+      - '-config.file=/etc/tempo/tempo.yml'
+    depends_on:
+      - prometheus
+    networks:
+      - monitoring
     restart: unless-stopped
 
   grafana:
     image: grafana/grafana:latest
-    container_name: grafana
     ports:
       - "3000:3000"
     volumes:
@@ -23,4 +38,13 @@ services:
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
-    restart: unless-stopped 
+    depends_on:
+      - prometheus
+      - tempo
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+networks:
+  monitoring:
+    driver: bridge

deploy/grafana/provisioning/datasources/prometheus.yml

@@ -2,8 +2,9 @@ apiVersion: 1
 
 datasources:
   - name: Prometheus
+    uid: prometheus
     type: prometheus
     access: proxy
     url: http://prometheus:9090
     isDefault: true
-    editable: false 
+    editable: false

deploy/grafana/provisioning/datasources/tempo.yml

@@ -0,0 +1,18 @@
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+    editable: false
+    jsonData:
+      nodeGraph:
+        enabled: true
+      serviceMap:
+        datasourceUid: prometheus
+      tracesToLogsV2:
+        datasourceUid: ""
+      tracesToMetrics:
+        datasourceUid: prometheus

deploy/prometheus/prometheus.yml

@@ -2,9 +2,15 @@ global:
   scrape_interval: 15s
   evaluation_interval: 15s
 
+remote_write: []
+
 scrape_configs:
   - job_name: 'rpc-proxy'
     static_configs:
-      - targets: ['host.docker.internal:4000'] 
+      - targets: ['host.docker.internal:4000']
         labels:
           service: 'rpc-proxy'
+
+  - job_name: 'tempo'
+    static_configs:
+      - targets: ['tempo:3200']

deploy/tempo/tempo.yml

@@ -0,0 +1,41 @@
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:4317"
+        http:
+          endpoint: "0.0.0.0:4318"
+
+metrics_generator:
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+  processor:
+    service_graphs:
+      dimensions:
+        - service.name
+    span_metrics:
+      dimensions:
+        - service.name
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/traces
+    wal:
+      path: /var/tempo/wal
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors:
+        - service-graphs
+        - span-metrics

go.mod

@@ -1,6 +1,6 @@
 module github.com/akash-network/rpc-proxy
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/cometbft/cometbft v0.38.12
@@ -9,11 +9,17 @@ require (
 	github.com/prometheus/client_model v0.6.1
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.10.0
-	golang.org/x/crypto v0.32.0
-	golang.org/x/net v0.34.0
-	golang.org/x/sync v0.10.0
-	google.golang.org/grpc v1.70.0
+	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0
+	go.opentelemetry.io/otel v1.43.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0
+	go.opentelemetry.io/otel/sdk v1.43.0
+	go.opentelemetry.io/otel/trace v1.43.0
+	golang.org/x/crypto v0.49.0
+	golang.org/x/net v0.52.0
+	golang.org/x/sync v0.20.0
+	google.golang.org/grpc v1.80.0
 )
 
 require (
@@ -35,6 +41,7 @@ require (
 	github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cockroachdb/errors v1.11.3 // indirect
 	github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
@@ -68,6 +75,8 @@ require (
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -77,11 +86,13 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/flatbuffers v24.12.23+incompatible // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -116,7 +127,7 @@ require (
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -134,15 +145,19 @@ require (
 	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.etcd.io/bbolt v1.4.0-alpha.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/term v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 // indirect
-	google.golang.org/protobuf v1.36.5 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect