feat(telemetry): Adding instrumentation around http client for tracing

README.md

@@ -81,6 +81,7 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga
     - [OIDC](#oidc)
   - [TLS Encryption](#tls-encryption)
   - [Metrics](#metrics)
+  - [Traces](#traces)
   - [Connectivity](#connectivity)
   - [Remote instances (EXPERIMENTAL)](#remote-instances-experimental)
 - [Deployment](#deployment)
@@ -1756,6 +1757,17 @@ endpoint on the same port your application is configured to run on (`web.port`).
 See [examples/docker-compose-grafana-prometheus](.examples/docker-compose-grafana-prometheus) for further documentation as well as an example.
 
 
+### Traces
+To export traces around http.client requests you can set up standard Open Telemetry exporter and resource environment variables to configure your trace provider. For more details on viable configuration options for the [exporter](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md) and [resource](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/sdk.md) variables, please reference the Open Telemetry documentation. An example configuration is listed below.
+
+```bash
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
+OTEL_SERVICE_NAME=gatus
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=local,service.version=1.0.0
+```
+
 ### Connectivity
 | Parameter                       | Description                                | Default       |
 |:--------------------------------|:-------------------------------------------|:--------------|

client/config.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/TwiN/logr"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 	"google.golang.org/api/idtoken"
@@ -212,29 +213,18 @@ func (c *Config) getHTTPClient() *http.Client {
 		tlsConfig = configureTLS(tlsConfig, *c.TLS)
 	}
 	if c.httpClient == nil {
-		c.httpClient = &http.Client{
-			Timeout: c.Timeout,
-			Transport: &http.Transport{
-				MaxIdleConns:        100,
-				MaxIdleConnsPerHost: 20,
-				Proxy:               http.ProxyFromEnvironment,
-				TLSClientConfig:     tlsConfig,
-			},
-			CheckRedirect: func(req *http.Request, via []*http.Request) error {
-				if c.IgnoreRedirect {
-					// Don't follow redirects
-					return http.ErrUseLastResponse
-				}
-				// Follow redirects
-				return nil
-			},
+		baseTransport := &http.Transport{
+			MaxIdleConns:        100,
+			MaxIdleConnsPerHost: 20,
+			Proxy:               http.ProxyFromEnvironment,
+			TLSClientConfig:     tlsConfig,
 		}
 		if c.ProxyURL != "" {
 			proxyURL, err := url.Parse(c.ProxyURL)
 			if err != nil {
 				logr.Errorf("[client.getHTTPClient] THIS SHOULD NOT HAPPEN. Silently ignoring custom proxy due to error: %s", err.Error())
 			} else {
-				c.httpClient.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyURL)
+				baseTransport.Proxy = http.ProxyURL(proxyURL)
 			}
 		}
 		if c.HasCustomDNSResolver() {
@@ -253,11 +243,22 @@ func (c *Config) getHTTPClient() *http.Client {
 						},
 					},
 				}
-				c.httpClient.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+				baseTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
 					return dialer.DialContext(ctx, network, addr)
 				}
 			}
 		}
+		wrappedTransport := otelhttp.NewTransport(baseTransport)
+		c.httpClient = &http.Client{
+			Timeout:   c.Timeout,
+			Transport: wrappedTransport,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				if c.IgnoreRedirect {
+					return http.ErrUseLastResponse
+				}
+				return nil
+			},
+		}
 		if c.HasOAuth2Config() && c.HasIAPConfig() {
 			logr.Errorf("[client.getHTTPClient] Error: Both Identity-Aware-Proxy and Oauth2 configuration are present.")
 		} else if c.HasOAuth2Config() {

config/endpoint/endpoint.go

@@ -2,6 +2,7 @@ package endpoint
 
 import (
 	"bytes"
+	"context"
 	"crypto/x509"
 	"encoding/json"
 	"errors"
@@ -13,11 +14,16 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+
 	"github.com/TwiN/gatus/v5/alerting/alert"
 	"github.com/TwiN/gatus/v5/client"
 	"github.com/TwiN/gatus/v5/config/endpoint/dns"
 	sshconfig "github.com/TwiN/gatus/v5/config/endpoint/ssh"
 	"github.com/TwiN/gatus/v5/config/endpoint/ui"
+	"go.opentelemetry.io/otel"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -251,6 +257,43 @@ func (e *Endpoint) Close() {
 
 // EvaluateHealth sends a request to the endpoint's URL and evaluates the conditions of the endpoint.
 func (e *Endpoint) EvaluateHealth() *Result {
+	tp := otel.GetTracerProvider()
+	tracer := tp.Tracer("gatus")
+	ctx, span := tracer.Start(context.Background(), "HealthCheck")
+	defer span.End()
+	templateBase := "gatus.io"
+	serviceAttr := templateBase + ".service"
+	serviceEnvAttr := templateBase + ".env"
+	visibilityAttr := templateBase + ".visibility"
+	urlAttr := templateBase + ".url"
+
+	// Set OpenTelemetry attributes
+	span.SetAttributes(attribute.String(urlAttr, e.URL))
+	teamAndService := strings.Split(e.Group, ".")
+	if len(teamAndService) != 2 {
+		semconv.ServiceNamespace("none")
+	} else {
+		team, service := teamAndService[0], teamAndService[1]
+		span.SetAttributes(
+			semconv.ServiceNamespace(team),
+			attribute.String(serviceAttr, service),
+		)
+	}
+	envAndVisibility := strings.Split(e.Name, "-")
+	if len(envAndVisibility) != 2 {
+		span.SetAttributes(
+			attribute.String(serviceEnvAttr, e.Name),
+			attribute.String(visibilityAttr, "public"),
+			attribute.String("user_agent.synthetic.type", "bot"),
+		)
+	}
+	env, visibility := envAndVisibility[0], envAndVisibility[1]
+	span.SetAttributes(
+		attribute.String(serviceEnvAttr, env),
+		attribute.String(visibilityAttr, visibility),
+		attribute.String("user_agent.synthetic.type", "bot"),
+	)
+
 	result := &Result{Success: true, Errors: []string{}}
 	// Parse or extract hostname from URL
 	if e.DNSConfig != nil {
@@ -276,7 +319,7 @@ func (e *Endpoint) EvaluateHealth() *Result {
 	}
 	// Call the endpoint (if there's no errors)
 	if len(result.Errors) == 0 {
-		e.call(result)
+		e.call(ctx, result)
 	} else {
 		result.Success = false
 	}
@@ -287,6 +330,9 @@ func (e *Endpoint) EvaluateHealth() *Result {
 			result.Success = false
 		}
 	}
+	if !result.Success {
+		span.SetStatus(codes.Error, "failure")
+	}
 	result.Timestamp = time.Now()
 	// Clean up parameters that we don't need to keep in the results
 	if e.UIConfig.HideURL {
@@ -315,7 +361,7 @@ func (e *Endpoint) getIP(result *Result) {
 	}
 }
 
-func (e *Endpoint) call(result *Result) {
+func (e *Endpoint) call(ctx context.Context, result *Result) {
 	var request *http.Request
 	var response *http.Response
 	var err error
@@ -388,6 +434,8 @@ func (e *Endpoint) call(result *Result) {
 		}
 		result.Duration = time.Since(startTime)
 	} else {
+
+		request = request.WithContext(ctx)
 		response, err = client.GetHTTPClient(e.ClientConfig).Do(request)
 		result.Duration = time.Since(startTime)
 		if err != nil {

go.mod

@@ -22,21 +22,25 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/valyala/fasthttp v1.58.0
 	github.com/wcharczuk/go-chart/v2 v2.1.2
-	golang.org/x/crypto v0.31.0
-	golang.org/x/net v0.33.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0
+	go.opentelemetry.io/otel v1.34.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0
+	go.opentelemetry.io/otel/sdk v1.34.0
+	go.opentelemetry.io/otel/trace v1.34.0
+	golang.org/x/crypto v0.32.0
+	golang.org/x/net v0.34.0
 	golang.org/x/oauth2 v0.25.0
-	google.golang.org/api v0.214.0
+	google.golang.org/api v0.153.0
 	gopkg.in/mail.v2 v2.3.1
 	gopkg.in/yaml.v3 v3.0.1
 	modernc.org/sqlite v1.34.4
 )
 
 require (
-	cloud.google.com/go/auth v0.13.0 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
 	cloud.google.com/go/compute/metadata v0.6.0 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -46,15 +50,16 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -66,22 +71,23 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
-	go.opentelemetry.io/otel v1.29.0 // indirect
-	go.opentelemetry.io/otel/metric v1.29.0 // indirect
-	go.opentelemetry.io/otel/trace v1.29.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect
+	go.opentelemetry.io/otel/metric v1.34.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/grpc v1.67.1 // indirect
-	google.golang.org/protobuf v1.35.2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/grpc v1.69.4 // indirect
+	google.golang.org/protobuf v1.36.3 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	modernc.org/gc/v3 v3.0.0-20240304020402-f0dba7c97c2b // indirect
 	modernc.org/libc v1.55.3 // indirect