Parse request / response size histograms (via JSON stats)

This implements parsing of the DNS request / response traffic size
histograms, for the JSON statistics channel.

Refs: #64

Signed-off-by: Daniel Swarbrick <[email protected]>

Author: dswarbrick

Diff for: bind/bind.go

@@ -26,24 +26,38 @@ type Client interface {
 const (
 	// QryRTT is the common prefix of query round-trip histogram counters.
 	QryRTT = "QryRTT"
+
+	// trafficBucketSize is the size of one traffic histogram bucket, defined as
+	// DNS_SIZEHISTO_QUANTUM in BIND source code.
+	TrafficBucketSize = 16
+
+	// trafficInMaxSize is the maximum size inbound request reported by BIND, referred to by
+	// DNS_SIZEHISTO_MAXIN in BIND source code.
+	TrafficInMaxSize = 288
+
+	// trafficOutMaxSize is the maximum size outbound response reported by BIND, referred to by
+	// DNS_SIZEHISTO_MAXOUT in BIND source code.
+	TrafficOutMaxSize = 4096
 )
 
 // StatisticGroup describes a sub-group of BIND statistics.
 type StatisticGroup string
 
 // Available statistic groups.
 const (
-	ServerStats StatisticGroup = "server"
-	ViewStats   StatisticGroup = "view"
-	TaskStats   StatisticGroup = "tasks"
+	ServerStats  StatisticGroup = "server"
+	TaskStats    StatisticGroup = "tasks"
+	TrafficStats StatisticGroup = "traffic"
+	ViewStats    StatisticGroup = "view"
 )
 
 // Statistics is a generic representation of BIND statistics.
 type Statistics struct {
-	Server      Server
-	Views       []View
-	ZoneViews   []ZoneView
-	TaskManager TaskManager
+	Server            Server
+	Views             []View
+	ZoneViews         []ZoneView
+	TaskManager       TaskManager
+	TrafficHistograms TrafficHistograms
 }
 
 // Server represents BIND server statistics.
@@ -111,3 +125,16 @@ type ThreadModel struct {
 	DefaultQuantum uint64 `xml:"default-quantum"`
 	TasksRunning   uint64 `xml:"tasks-running"`
 }
+
+// TrafficHistograms contains slices representing sent / received traffic, with each slice element
+// corresponding to a `TrafficBucketSize` range. The last slice element represents the +Inf bucket.
+type TrafficHistograms struct {
+	ReceivedUDPv4 []uint64
+	SentUDPv4     []uint64
+	ReceivedTCPv4 []uint64
+	SentTCPv4     []uint64
+	ReceivedUDPv6 []uint64
+	SentUDPv6     []uint64
+	ReceivedTCPv6 []uint64
+	SentTCPv6     []uint64
+}

Diff for: bind/json/json.go

@@ -20,6 +20,7 @@ import (
 	"net/url"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/prometheus-community/bind_exporter/bind"
@@ -30,6 +31,8 @@ const (
 	ServerPath = "/json/v1/server"
 	// TasksPath is the HTTP path of the JSON v1 tasks resource.
 	TasksPath = "/json/v1/tasks"
+	// TrafficPath is the HTTP path of the JSON v1 traffic resource.
+	TrafficPath = "/json/v1/traffic"
 	// ZonesPath is the HTTP path of the JSON v1 zones resource.
 	ZonesPath = "/json/v1/zones"
 )
@@ -71,6 +74,19 @@ type TaskStatistics struct {
 	} `json:"taskmgr"`
 }
 
+type TrafficStatistics struct {
+	Traffic struct {
+		ReceivedUDPv4 map[string]uint64 `json:"dns-udp-requests-sizes-received-ipv4"`
+		SentUDPv4     map[string]uint64 `json:"dns-udp-responses-sizes-sent-ipv4"`
+		ReceivedTCPv4 map[string]uint64 `json:"dns-tcp-requests-sizes-sent-ipv4"`
+		SentTCPv4     map[string]uint64 `json:"dns-tcp-responses-sizes-sent-ipv4"`
+		ReceivedUDPv6 map[string]uint64 `json:"dns-udp-requests-sizes-received-ipv6"`
+		SentUDPv6     map[string]uint64 `json:"dns-udp-responses-sizes-sent-ipv6"`
+		ReceivedTCPv6 map[string]uint64 `json:"dns-tcp-requests-sizes-sent-ipv6"`
+		SentTCPv6     map[string]uint64 `json:"dns-tcp-responses-sizes-sent-ipv6"`
+	} `json:"traffic"`
+}
+
 // Client implements bind.Client and can be used to query a BIND JSON v1 API.
 type Client struct {
 	url  string
@@ -191,5 +207,76 @@ func (c *Client) Stats(groups ...bind.StatisticGroup) (bind.Statistics, error) {
 		s.TaskManager.ThreadModel.WorkerThreads = taskstats.TaskMgr.WorkerThreads
 	}
 
+	if m[bind.TrafficStats] {
+		var trafficStats TrafficStatistics
+		if err := c.Get(TrafficPath, &trafficStats); err != nil {
+			return s, err
+		}
+
+		var err error
+
+		// Make IPv4 traffic histograms.
+		if s.TrafficHistograms.ReceivedUDPv4, err = parseTrafficHist(trafficStats.Traffic.ReceivedUDPv4, bind.TrafficInMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.SentUDPv4, err = parseTrafficHist(trafficStats.Traffic.SentUDPv4, bind.TrafficOutMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.ReceivedTCPv4, err = parseTrafficHist(trafficStats.Traffic.ReceivedTCPv4, bind.TrafficInMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.SentTCPv4, err = parseTrafficHist(trafficStats.Traffic.SentTCPv4, bind.TrafficOutMaxSize); err != nil {
+			return s, err
+		}
+
+		// Make IPv6 traffic histograms.
+		if s.TrafficHistograms.ReceivedUDPv6, err = parseTrafficHist(trafficStats.Traffic.ReceivedUDPv6, bind.TrafficInMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.SentUDPv6, err = parseTrafficHist(trafficStats.Traffic.SentUDPv6, bind.TrafficOutMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.ReceivedTCPv6, err = parseTrafficHist(trafficStats.Traffic.ReceivedTCPv6, bind.TrafficInMaxSize); err != nil {
+			return s, err
+		}
+		if s.TrafficHistograms.SentTCPv6, err = parseTrafficHist(trafficStats.Traffic.SentTCPv6, bind.TrafficOutMaxSize); err != nil {
+			return s, err
+		}
+	}
+
 	return s, nil
 }
+
+func parseTrafficHist(traffic map[string]uint64, maxBucket uint) ([]uint64, error) {
+	trafficHist := make([]uint64, maxBucket/bind.TrafficBucketSize)
+
+	for k, v := range traffic {
+		// Keys are in the format "lowerBound-upperBound". We are only interested in the upper
+		// bound.
+		parts := strings.Split(k, "-")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("malformed traffic bucket range: %q", k)
+		}
+
+		upperBound, err := strconv.ParseUint(parts[1], 10, 16)
+		if err != nil {
+			return nil, fmt.Errorf("cannot convert bucket upper bound to uint: %w", err)
+		}
+
+		if (upperBound+1)%bind.TrafficBucketSize != 0 {
+			return nil, fmt.Errorf("upper bucket bound is not a multiple of %d minus one: %d",
+				bind.TrafficBucketSize, upperBound)
+		}
+
+		if upperBound < uint64(maxBucket) {
+			// idx is offset, since there is no 0-16 bucket reported by BIND.
+			idx := (upperBound+1)/bind.TrafficBucketSize - 2
+			trafficHist[idx] += v
+		} else {
+			// Final slice element aggregates packet sizes from maxBucket to +Inf.
+			trafficHist[len(trafficHist)-1] += v
+		}
+	}
+
+	return trafficHist, nil
+}

Diff for: bind_exporter.go

@@ -218,6 +218,16 @@ var (
 		"Zone serial number.",
 		[]string{"view", "zone_name"}, nil,
 	)
+	trafficReceived = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "traffic", "received_size"),
+		"Received traffic packet sizes.",
+		[]string{"type"}, nil,
+	)
+	trafficSent = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, "traffic", "sent_size"),
+		"Received traffic packet sizes.",
+		[]string{"transport"}, nil,
+	)
 )
 
 type collectorConstructor func(log.Logger, *bind.Statistics) prometheus.Collector
@@ -387,6 +397,87 @@ func (c *taskCollector) Collect(ch chan<- prometheus.Metric) {
 	)
 }
 
+type trafficCollector struct {
+	logger log.Logger
+	stats  *bind.Statistics
+}
+
+// newTrafficCollector implements collectorConstructor.
+func newTrafficCollector(logger log.Logger, s *bind.Statistics) prometheus.Collector {
+	return &trafficCollector{logger: logger, stats: s}
+}
+
+// Describe implements prometheus.Collector.
+func (c *trafficCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- trafficReceived
+	ch <- trafficSent
+}
+
+// Collect implements prometheus.Collector.
+func (c *trafficCollector) Collect(ch chan<- prometheus.Metric) {
+	// IPv4 traffic histograms.
+	buckets, count := c.makeHistogram(c.stats.TrafficHistograms.ReceivedUDPv4)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficReceived, count, math.NaN(), buckets, "udpv4",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.SentUDPv4)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficSent, count, math.NaN(), buckets, "udpv4",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.ReceivedTCPv4)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficReceived, count, math.NaN(), buckets, "tcpv4",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.SentTCPv4)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficSent, count, math.NaN(), buckets, "tcpv4",
+	)
+
+	// IPv6 traffic histograms.
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.ReceivedUDPv6)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficReceived, count, math.NaN(), buckets, "udpv6",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.SentUDPv6)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficSent, count, math.NaN(), buckets, "udpv6",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.ReceivedTCPv6)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficReceived, count, math.NaN(), buckets, "tcpv6",
+	)
+	buckets, count = c.makeHistogram(c.stats.TrafficHistograms.SentTCPv6)
+	ch <- prometheus.MustNewConstHistogram(
+		trafficSent, count, math.NaN(), buckets, "tcpv6",
+	)
+}
+
+// makeHistogram translates the non-aggregated bucket slice into an aggregated map, suitable for
+// use by prometheus.MustNewConstHistogram().
+func (c *trafficCollector) makeHistogram(rawBuckets []uint64) (map[float64]uint64, uint64) {
+	var (
+		buckets = map[float64]uint64{}
+		count   uint64
+	)
+
+	for i, v := range rawBuckets {
+		if v > 0 {
+			var idx float64
+
+			if i == len(rawBuckets)-1 {
+				idx = math.Inf(1)
+			} else {
+				idx = float64((i+2)*bind.TrafficBucketSize) - 1
+			}
+
+			count += v
+			buckets[idx] = count
+		}
+	}
+
+	return buckets, count
+}
+
 // Exporter collects Binds stats from the given server and exports them using
 // the prometheus metrics package.
 type Exporter struct {
@@ -411,10 +502,12 @@ func NewExporter(logger log.Logger, version, url string, timeout time.Duration,
 		switch g {
 		case bind.ServerStats:
 			cs = append(cs, newServerCollector)
-		case bind.ViewStats:
-			cs = append(cs, newViewCollector)
 		case bind.TaskStats:
 			cs = append(cs, newTaskCollector)
+		case bind.TrafficStats:
+			cs = append(cs, newTrafficCollector)
+		case bind.ViewStats:
+			cs = append(cs, newViewCollector)
 		}
 	}
 
@@ -503,10 +596,12 @@ func (s *statisticGroups) Set(value string) error {
 		switch dt {
 		case string(bind.ServerStats):
 			sg = bind.ServerStats
-		case string(bind.ViewStats):
-			sg = bind.ViewStats
 		case string(bind.TaskStats):
 			sg = bind.TaskStats
+		case string(bind.TrafficStats):
+			sg = bind.TrafficStats
+		case string(bind.ViewStats):
+			sg = bind.ViewStats
 		default:
 			return fmt.Errorf("unknown stats group %q", dt)
 		}
@@ -544,7 +639,8 @@ func main() {
 	toolkitFlags := webflag.AddFlags(kingpin.CommandLine, ":9119")
 
 	kingpin.Flag("bind.stats-groups",
-		"Comma-separated list of statistics to collect",
+		"Comma-separated list of statistics to collect. "+
+			"One or more of: [server, tasks, traffic, view]",
 	).Default((&statisticGroups{
 		bind.ServerStats, bind.ViewStats,
 	}).String()).SetValue(&groups)