Merge pull request #9 from williamchanrico/fix_pkgs

Re-use http.Transport Scrape() calls

Author: williamchanrico

collector/task/darkstat/darkstat.go

@@ -16,8 +16,10 @@ package darkstat
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
+	"net/http"
 	"planet-exporter/collector/task/inventory"
 	"planet-exporter/pkg/network"
 	"planet-exporter/pkg/prometheus"
@@ -31,8 +33,9 @@ import (
 
 // task that queries darkstat metrics and aggregates them into usable planet metrics
 type task struct {
-	enabled      bool
-	darkstatAddr string
+	enabled          bool
+	darkstatAddr     string
+	prometheusClient *prometheus.Client
 
 	hosts []Metric
 	mu    sync.Mutex
@@ -44,10 +47,24 @@ var (
 )
 
 func init() {
+	httpTransport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
 	singleton = task{
-		enabled: false,
-		hosts:   []Metric{},
-		mu:      sync.Mutex{},
+		enabled:          false,
+		hosts:            []Metric{},
+		mu:               sync.Mutex{},
+		prometheusClient: prometheus.New(httpTransport),
 	}
 }
 
@@ -91,6 +108,9 @@ func Collect(ctx context.Context) error {
 
 	startTime := time.Now()
 
+	ctxCollect, ctxCollectCancel := context.WithCancel(ctx)
+	defer ctxCollectCancel()
+
 	inventoryHosts := inventory.Get()
 
 	localAddr, err := network.DefaultLocalAddr()
@@ -109,7 +129,7 @@ func Collect(ctx context.Context) error {
 
 	// Scrape darkstat prometheus endpoint for host_bytes_total
 	var darkstatHostBytesTotal *prom2json.Family
-	darkstatScrape, err := prometheus.Scrape(singleton.darkstatAddr)
+	darkstatScrape, err := singleton.prometheusClient.Scrape(ctxCollect, singleton.darkstatAddr)
 	if err != nil {
 		return err
 	}

collector/task/ebpf/tcptop.go

@@ -18,8 +18,10 @@ package ebpf
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
+	"net/http"
 	"planet-exporter/collector/task/inventory"
 	"planet-exporter/pkg/network"
 	"planet-exporter/pkg/prometheus"
@@ -33,8 +35,9 @@ import (
 
 // task that queries ebpf metrics and aggregates them into usable planet metrics.
 type task struct {
-	enabled  bool
-	ebpfAddr string
+	enabled          bool
+	ebpfAddr         string
+	prometheusClient *prometheus.Client
 
 	hosts []Metric
 	mu    sync.Mutex
@@ -53,10 +56,24 @@ const (
 )
 
 func init() {
+	httpTransport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
 	singleton = task{
-		enabled: false,
-		hosts:   []Metric{},
-		mu:      sync.Mutex{},
+		enabled:          false,
+		hosts:            []Metric{},
+		mu:               sync.Mutex{},
+		prometheusClient: prometheus.New(httpTransport),
 	}
 }
 
@@ -100,8 +117,11 @@ func Collect(ctx context.Context) error {
 
 	startTime := time.Now()
 
+	ctxCollect, ctxCollectCancel := context.WithCancel(ctx)
+	defer ctxCollectCancel()
+
 	// Scrape ebpf prometheus endpoint for send_bytes_metric and recv_bytes_metric.
-	ebpfScrape, err := prometheus.Scrape(singleton.ebpfAddr)
+	ebpfScrape, err := singleton.prometheusClient.Scrape(ctxCollect, singleton.ebpfAddr)
 	if err != nil {
 		return err
 	}

pkg/prometheus/prometheus.go

@@ -15,8 +15,11 @@
 package prometheus
 
 import (
+	"context"
 	"crypto/tls"
+	"net"
 	"net/http"
+	"time"
 
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/prom2json"
@@ -25,17 +28,44 @@ import (
 // TODO: Complete package
 // e.g. abstract prom2json data structures and maybe share http client
 
-// Scrape metrics from a prometheus HTTP endpoint
-func Scrape(url string) ([]*prom2json.Family, error) {
+// Client for Prometheus endpoints
+type Client struct {
+	httpTransport *http.Transport
+}
+
+// New Prometheus client used to consume Prometheus metrics endpoints
+func New(httpTransport *http.Transport) *Client {
+	if httpTransport == nil {
+		// Use sane defaults from http.DefaultTransport
+		httpTransport = &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+			ExpectContinueTimeout: 1 * time.Second,
+		}
+	}
+
+	return &Client{
+		httpTransport: httpTransport,
+	}
+}
+
+// Scrape metrics from a Prometheus HTTP endpoint
+func (c *Client) Scrape(ctx context.Context, url string) ([]*prom2json.Family, error) {
 	var err error
 
 	mfChan := make(chan *dto.MetricFamily, 1024)
 
-	transport, err := makeTransport()
 	if err != nil {
 		return nil, err
 	}
-	err = prom2json.FetchMetricFamilies(url, mfChan, transport)
+	err = prom2json.FetchMetricFamilies(url, mfChan, c.httpTransport)
 	if err != nil {
 		return nil, err
 	}
@@ -47,10 +77,3 @@ func Scrape(url string) ([]*prom2json.Family, error) {
 
 	return result, nil
 }
-
-func makeTransport() (*http.Transport, error) {
-	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	return transport, nil
-}