Add http pprof profiling endpoint

Signed-off-by: Tim Vaillancourt <[email protected]>

Author: timvaillancourt

control.go

@@ -18,12 +18,12 @@ import (
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
 type Control struct {
-	f          *Interface
-	l          *logrus.Logger
-	cancel     context.CancelFunc
-	sshStart   func()
-	statsStart func()
-	dnsStart   func()
+	f         *Interface
+	l         *logrus.Logger
+	cancel    context.CancelFunc
+	sshStart  func()
+	httpStart func()
+	dnsStart  func()
 }
 
 type ControlHostInfo struct {
@@ -48,8 +48,8 @@ func (c *Control) Start() {
 	if c.sshStart != nil {
 		go c.sshStart()
 	}
-	if c.statsStart != nil {
-		go c.statsStart()
+	if c.httpStart != nil {
+		go c.httpStart()
 	}
 	if c.dnsStart != nil {
 		go c.dnsStart()

examples/config.yml

@@ -244,6 +244,9 @@ logging:
   # As an example, to log as RFC3339 with millisecond precision, set to:
   #timestamp_format: "2006-01-02T15:04:05.000Z07:00"
 
+#http:
+  #listen: 127.0.0.1:8080
+
 #stats:
   #type: graphite
   #prefix: nebula
@@ -252,7 +255,6 @@ logging:
   #interval: 10s
 
   #type: prometheus
-  #listen: 127.0.0.1:8080
   #path: /metrics
   #namespace: prometheusns
   #subsystem: nebula

http.go

@@ -0,0 +1,34 @@
+package nebula
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
+)
+
+func startHttp(l *logrus.Logger, c *config.C, statsHandler statsHandlerFunc, listen string) (f func(), err error) {
+	if listen == "" {
+		return nil, nil
+	}
+
+	var statsPath string
+	if statsHandler != nil {
+		statsPath = c.GetString("stats.path", "")
+		if statsPath == "" {
+			return nil, fmt.Errorf("stats.path should not be empty")
+		}
+	}
+
+	f = func() {
+		l.Infof("Go pprof handler listening on %s at /debug/pprof", listen)
+		if statsHandler != nil {
+			http.Handle(statsPath, statsHandler(listen, statsPath))
+		}
+		l.Fatal(http.ListenAndServe(listen, nil))
+	}
+
+	return f, err
+}

main.go

@@ -324,14 +324,25 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		go lightHouse.LhUpdateWorker(ctx, ifce)
 	}
 
+	httpListen := c.GetString("http.listen", "")
+	if httpListen == "" {
+		if httpListen = c.GetString("stats.listen", ""); httpListen != "" {
+			l.Warn("http.listen is undef, falling back to stats.listen. stats.listen will be deprecated in a future release.")
+		}
+	}
+
 	// TODO - stats third-party modules start uncancellable goroutines. Update those libs to accept
 	// a context so that they can exit when the context is Done.
-	statsStart, err := startStats(l, c, buildVersion, configTest)
-
+	statsHTTPHandler, err := startStats(l, c, httpListen, buildVersion, configTest)
 	if err != nil {
 		return nil, util.NewContextualError("Failed to start stats emitter", nil, err)
 	}
 
+	httpStart, err := startHttp(l, c, statsHTTPHandler, httpListen)
+	if err != nil {
+		return nil, util.NewContextualError("Failed to start http server", nil, err)
+	}
+
 	if configTest {
 		return nil, nil
 	}
@@ -348,5 +359,5 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		dnsStart = dnsMain(l, hostMap, c)
 	}
 
-	return &Control{ifce, l, cancel, sshStart, statsStart, dnsStart}, nil
+	return &Control{ifce, l, cancel, sshStart, httpStart, dnsStart}, nil
 }

stats.go

@@ -3,7 +3,6 @@ package nebula
 import (
 	"errors"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"runtime"
@@ -19,10 +18,12 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
+type statsHandlerFunc func(listen, path string) http.Handler
+
 // startStats initializes stats from config. On success, if any further work
-// is needed to serve stats, it returns a func to handle that work. If no
+// is needed to serve stats, it returns an http.Handler for that work. If no
 // work is needed, it'll return nil. On failure, it returns nil, error.
-func startStats(l *logrus.Logger, c *config.C, buildVersion string, configTest bool) (func(), error) {
+func startStats(l *logrus.Logger, c *config.C, listen, buildVersion string, configTest bool) (f statsHandlerFunc, err error) {
 	mType := c.GetString("stats.type", "")
 	if mType == "" || mType == "none" {
 		return nil, nil
@@ -33,16 +34,14 @@ func startStats(l *logrus.Logger, c *config.C, buildVersion string, configTest b
 		return nil, fmt.Errorf("stats.interval was an invalid duration: %s", c.GetString("stats.interval", ""))
 	}
 
-	var startFn func()
 	switch mType {
 	case "graphite":
 		err := startGraphiteStats(l, interval, c, configTest)
 		if err != nil {
 			return nil, err
 		}
 	case "prometheus":
-		var err error
-		startFn, err = startPrometheusStats(l, interval, c, buildVersion, configTest)
+		f, err = startPrometheusStats(l, interval, c, listen, buildVersion, configTest)
 		if err != nil {
 			return nil, err
 		}
@@ -56,7 +55,7 @@ func startStats(l *logrus.Logger, c *config.C, buildVersion string, configTest b
 	go metrics.CaptureDebugGCStats(metrics.DefaultRegistry, interval)
 	go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, interval)
 
-	return startFn, nil
+	return f, nil
 }
 
 func startGraphiteStats(l *logrus.Logger, i time.Duration, c *config.C, configTest bool) error {
@@ -79,13 +78,12 @@ func startGraphiteStats(l *logrus.Logger, i time.Duration, c *config.C, configTe
 	return nil
 }
 
-func startPrometheusStats(l *logrus.Logger, i time.Duration, c *config.C, buildVersion string, configTest bool) (func(), error) {
+func startPrometheusStats(l *logrus.Logger, i time.Duration, c *config.C, listen, buildVersion string, configTest bool) (statsHandlerFunc, error) {
 	namespace := c.GetString("stats.namespace", "")
 	subsystem := c.GetString("stats.subsystem", "")
 
-	listen := c.GetString("stats.listen", "")
 	if listen == "" {
-		return nil, fmt.Errorf("stats.listen should not be empty")
+		return nil, fmt.Errorf("http.listen or stats.listen must be defined to use promtheus stats")
 	}
 
 	path := c.GetString("stats.path", "")
@@ -114,14 +112,13 @@ func startPrometheusStats(l *logrus.Logger, i time.Duration, c *config.C, buildV
 	pr.MustRegister(g)
 	g.Set(1)
 
-	var startFn func()
+	var f statsHandlerFunc
 	if !configTest {
-		startFn = func() {
+		f = func(listen, path string) http.Handler {
 			l.Infof("Prometheus stats listening on %s at %s", listen, path)
-			http.Handle(path, promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: l}))
-			log.Fatal(http.ListenAndServe(listen, nil))
+			return promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: l})
 		}
 	}
 
-	return startFn, nil
+	return f, nil
 }