go-kafka/producer_instrumenting.go at master · ricardo-ch/go-kafka · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
package kafka

import (
	"context"
	"sync"
	"time"

	"github.com/IBM/sarama"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	producerRecordSendCounter     *prometheus.CounterVec
	producerDeadletterSendCounter *prometheus.CounterVec
	producerRecordSendLatency     *prometheus.HistogramVec
	producerRecordErrorCounter    *prometheus.CounterVec

	producerMetricsLabel = []string{"kafka_topic"}

	producerSendOnce       sync.Once
	producerDeadletterOnce sync.Once
	producerLatencyOnce    sync.Once
	producerErrorOnce      sync.Once
)

// ProducerMetricsService is a service that provides metrics for the producer.
type ProducerMetricsService struct {
	recordSendCounter           *prometheus.CounterVec
	deadletterRecordSendCounter *prometheus.CounterVec
	recordSendLatency           *prometheus.HistogramVec
	errorCounter                *prometheus.CounterVec
}

func getProducerRecordSendCounter() *prometheus.CounterVec {
	producerSendOnce.Do(func() {
		producerRecordSendCounter = prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "kafka",
				Subsystem: "producer",
				Name:      "record_send_total",
				Help:      "Number of records sent",
			}, producerMetricsLabel)
		prometheus.MustRegister(producerRecordSendCounter)
	})

	return producerRecordSendCounter
}

func getProducerDeadletterSendCounter() *prometheus.CounterVec {
	producerDeadletterOnce.Do(func() {
		producerDeadletterSendCounter = prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "kafka",
				Subsystem: "producer",
				Name:      "dead_letter_created_total",
				Help:      "Number of dead letters created",
			}, producerMetricsLabel)
		prometheus.MustRegister(producerDeadletterSendCounter)
	})

	return producerDeadletterSendCounter
}

func getProducerRecordSendLatency() *prometheus.HistogramVec {
	producerLatencyOnce.Do(func() {
		producerRecordSendLatency = prometheus.NewHistogramVec(
			prometheus.HistogramOpts{
				Namespace: "kafka",
				Subsystem: "producer",
				Name:      "record_send_latency_seconds",
				Help:      "Latency of records sent in seconds",
			}, producerMetricsLabel)
		prometheus.MustRegister(producerRecordSendLatency)
	})

	return producerRecordSendLatency
}

func getProducerRecordErrorCounter() *prometheus.CounterVec {
	producerErrorOnce.Do(func() {
		producerRecordErrorCounter = prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "kafka",
				Subsystem: "producer",
				Name:      "record_error_total",
				Help:      "Number of record send errors",
			}, producerMetricsLabel)
		prometheus.MustRegister(producerRecordErrorCounter)
	})

	return producerRecordErrorCounter
}

func NewProducerMetricsService() *ProducerMetricsService {
	return &ProducerMetricsService{
		recordSendCounter: getProducerRecordSendCounter(),
		recordSendLatency: getProducerRecordSendLatency(),
		errorCounter:      getProducerRecordErrorCounter(),
	}
}

func NewDeadletterProducerMetricsService() *ProducerMetricsService {
	return &ProducerMetricsService{
		deadletterRecordSendCounter: getProducerDeadletterSendCounter(),
		recordSendLatency:           getProducerRecordSendLatency(),
		errorCounter:                getProducerRecordErrorCounter(),
	}
}

// Instrumentation is a middleware that provides metrics for the producer.
func (p *ProducerMetricsService) Instrumentation(next producerHandler) producerHandler {
	return func(ctx context.Context, producer *producer, msg *sarama.ProducerMessage) (err error) {
		defer func(begin time.Time) {
			p.recordSendLatency.WithLabelValues(msg.Topic).Observe(time.Since(begin).Seconds())
		}(time.Now())

		err = next(ctx, producer, msg)

		if err != nil {
			p.errorCounter.WithLabelValues(msg.Topic).Inc()
		} else {
			p.recordSendCounter.WithLabelValues(msg.Topic).Inc()
		}

		return
	}
}

// DeadletterInstrumentation is a middleware that provides metrics for a deadletter producer.
func (p *ProducerMetricsService) DeadletterInstrumentation(next producerHandler) producerHandler {
	return func(ctx context.Context, producer *producer, msg *sarama.ProducerMessage) (err error) {
		defer func(begin time.Time) {
			p.recordSendLatency.WithLabelValues(msg.Topic).Observe(time.Since(begin).Seconds())
		}(time.Now())

		err = next(ctx, producer, msg)

		if err != nil {
			p.errorCounter.WithLabelValues(msg.Topic).Inc()
		} else {
			p.deadletterRecordSendCounter.WithLabelValues(msg.Topic).Inc()
		}

		return
	}
}