Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,7 @@ func newDefaultMetricsConfig() *MetricsConfig {
type PrometheusConfig struct {
Port int `mapstructure:"port"`
Enabled bool `mapstructure:"enabled"`
Objectives map[string]float64 `mapstructure:"objectives"`
}

// newDefaultPrometheusConfig provides default configuration for PrometheusReporter
Expand Down
39 changes: 39 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,45 @@ func TestNewConfig(t *testing.T) {
}
}

// The built-in summary objectives are intentionally NOT seeded with a viper
// default. If they were, viper would deep-merge the default quantiles back into
// any partial override, so dropping a percentile via config would silently fail.
// Unset must therefore resolve to nil, which the reporter turns into the
// historical default.
func TestPrometheusObjectivesDefaultIsUnset(t *testing.T) {
t.Parallel()

cfg := NewDefaultPitayaConfig()
assert.Nil(t, cfg.Metrics.Prometheus.Objectives)
}

func TestPrometheusObjectivesConfigResolution(t *testing.T) {
t.Parallel()

t.Run("unset-resolves-to-nil", func(t *testing.T) {
cfg := NewPitayaConfig(NewConfig())
assert.Nil(t, cfg.Metrics.Prometheus.Objectives)
})

t.Run("partial-override-drops-unlisted-percentiles", func(t *testing.T) {
v := viper.New()
v.Set("pitaya.metrics.prometheus.objectives", map[string]interface{}{
"0.95": 0.005,
"0.99": 0.001,
})
cfg := NewPitayaConfig(NewConfig(v))
assert.Equal(t, map[string]float64{"0.95": 0.005, "0.99": 0.001}, cfg.Metrics.Prometheus.Objectives)
})

t.Run("explicit-empty-map-stays-empty", func(t *testing.T) {
v := viper.New()
v.Set("pitaya.metrics.prometheus.objectives", map[string]interface{}{})
cfg := NewPitayaConfig(NewConfig(v))
assert.NotNil(t, cfg.Metrics.Prometheus.Objectives)
assert.Empty(t, cfg.Metrics.Prometheus.Objectives)
})
}

func TestGetDuration(t *testing.T) {
t.Parallel()

Expand Down
4 changes: 4 additions & 0 deletions docs/configuration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,10 @@ Metrics Reporting
- 9090
- int
- Port to expose prometheus metrics
* - pitaya.metrics.prometheus.objectives
- unset (equivalent to {"0.7": 0.02, "0.95": 0.005, "0.99": 0.001})
- map[string]float64
- Quantiles (keyed by the quantile string) and their allowed error for the built-in ``handler_response_time_ns`` and ``handler_handler_delay_ns`` summaries. When unset, the historical ``{0.7, 0.95, 0.99}`` objectives are used. Set a subset (e.g. only ``0.95``/``0.99``) to drop the percentiles you omit, or an empty map to emit only ``_sum``/``_count`` and no quantile series, reducing series cardinality. Note that client-side summary quantiles are per-pod and cannot be aggregated across pods.
* - pitaya.metrics.constTags
- map[string]string{}
- map[string]string
Expand Down
89 changes: 70 additions & 19 deletions metrics/prometheus_reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package metrics

import (
"fmt"
"strconv"

"github.com/topfreegames/pitaya/v2/logger"

Expand Down Expand Up @@ -53,6 +54,52 @@ type PrometheusReporter struct {

var _ Reporter = (*PrometheusReporter)(nil)

// defaultSummaryObjectives returns the historical hard-coded objectives used by
// the built-in handler summaries. Kept as a constructor so callers get an
// independent copy they can pass to Prometheus.
func defaultSummaryObjectives() map[float64]float64 {
return map[float64]float64{0.7: 0.02, 0.95: 0.005, 0.99: 0.001}
}

// buildObjectives converts the YAML/env-friendly quantile→error config map into
// the map[float64]float64 Prometheus expects. A nil map falls back to the
// historical default; an explicit (non-nil) empty map yields no quantile
// series, leaving only _sum and _count.
func buildObjectives(configured map[string]float64) (map[float64]float64, error) {
if configured == nil {
return defaultSummaryObjectives(), nil
}

objectives := make(map[float64]float64, len(configured))
for quantile, allowedError := range configured {
q, err := strconv.ParseFloat(quantile, 64)
if err != nil {
return nil, fmt.Errorf("invalid prometheus summary objective quantile %q: %w", quantile, err)
}
objectives[q] = allowedError
}
return objectives, nil
}

func newSummaryVec(
subsystem, name, help string,
objectives map[float64]float64,
constLabels map[string]string,
labelKeys []string,
) *prometheus.SummaryVec {
return prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "pitaya",
Subsystem: subsystem,
Name: name,
Help: help,
Objectives: objectives,
ConstLabels: constLabels,
},
labelKeys,
)
}

func (p *PrometheusReporter) registerCustomMetrics(
constLabels map[string]string,
additionalLabelsKeys []string,
Expand Down Expand Up @@ -115,6 +162,7 @@ func (p *PrometheusReporter) registerCustomMetrics(

func (p *PrometheusReporter) registerMetrics(
constLabels, additionalLabels map[string]string,
objectives map[float64]float64,
spec *models.CustomMetricsSpec,
) {

Expand All @@ -130,15 +178,12 @@ func (p *PrometheusReporter) registerMetrics(
p.registerCustomMetrics(constLabels, additionalLabelsKeys, spec)

// HandlerResponseTimeMs summary
p.summaryReportersMap[ResponseTime] = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "pitaya",
Subsystem: "handler",
Name: ResponseTime,
Help: "the time to process a msg in nanoseconds",
Objectives: map[float64]float64{0.7: 0.02, 0.95: 0.005, 0.99: 0.001},
ConstLabels: constLabels,
},
p.summaryReportersMap[ResponseTime] = newSummaryVec(
"handler",
ResponseTime,
"the time to process a msg in nanoseconds",
objectives,
constLabels,
append([]string{"route", "status", "type", "code"}, additionalLabelsKeys...),
)

Expand All @@ -155,15 +200,12 @@ func (p *PrometheusReporter) registerMetrics(
)

// ProcessDelay summary
p.summaryReportersMap[ProcessDelay] = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "pitaya",
Subsystem: "handler",
Name: ProcessDelay,
Help: "the delay to start processing a msg in nanoseconds",
Objectives: map[float64]float64{0.7: 0.02, 0.95: 0.005, 0.99: 0.001},
ConstLabels: constLabels,
},
p.summaryReportersMap[ProcessDelay] = newSummaryVec(
"handler",
ProcessDelay,
"the delay to start processing a msg in nanoseconds",
objectives,
constLabels,
append([]string{"route", "type"}, additionalLabelsKeys...),
)

Expand Down Expand Up @@ -353,6 +395,15 @@ func getPrometheusReporter(
config config.MetricsConfig,
metricsSpecs *models.CustomMetricsSpec,
) (*PrometheusReporter, error) {
var configuredObjectives map[string]float64
if config.Prometheus != nil {
configuredObjectives = config.Prometheus.Objectives
}
objectives, err := buildObjectives(configuredObjectives)
if err != nil {
return nil, err
}

once.Do(func() {
prometheusReporter = &PrometheusReporter{
serverType: serverType,
Expand All @@ -362,7 +413,7 @@ func getPrometheusReporter(
summaryReportersMap: make(map[string]*prometheus.SummaryVec),
gaugeReportersMap: make(map[string]*prometheus.GaugeVec),
}
prometheusReporter.registerMetrics(config.ConstLabels, config.AdditionalLabels, metricsSpecs)
prometheusReporter.registerMetrics(config.ConstLabels, config.AdditionalLabels, objectives, metricsSpecs)
http.Handle("/metrics", promhttp.Handler())
go (func() {
err := http.ListenAndServe(fmt.Sprintf(":%d", config.Prometheus.Port), nil)
Expand Down
127 changes: 127 additions & 0 deletions metrics/prometheus_reporter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
// Copyright (c) TFG Co. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package metrics

import (
"sort"
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestBuildObjectives(t *testing.T) {
t.Parallel()

t.Run("nil-config-falls-back-to-default", func(t *testing.T) {
objectives, err := buildObjectives(nil)
require.NoError(t, err)
assert.Equal(t, map[float64]float64{0.7: 0.02, 0.95: 0.005, 0.99: 0.001}, objectives)
})

t.Run("custom-objectives-are-used-verbatim", func(t *testing.T) {
objectives, err := buildObjectives(map[string]float64{"0.95": 0.005, "0.99": 0.001})
require.NoError(t, err)
assert.Equal(t, map[float64]float64{0.95: 0.005, 0.99: 0.001}, objectives)
})

t.Run("explicit-empty-map-yields-no-quantiles", func(t *testing.T) {
objectives, err := buildObjectives(map[string]float64{})
require.NoError(t, err)
assert.Empty(t, objectives)
assert.NotNil(t, objectives)
})

t.Run("invalid-quantile-key-errors", func(t *testing.T) {
_, err := buildObjectives(map[string]float64{"not-a-float": 0.01})
assert.Error(t, err)
})
}

// quantilesFromSummary registers a summary built exactly as registerMetrics
// builds the built-in ones, observes a value, scrapes it, and returns the sorted
// list of quantiles emitted plus whether _sum/_count were present.
func quantilesFromSummary(t *testing.T, objectives map[float64]float64) (quantiles []float64, hasSum, hasCount bool) {
t.Helper()

registry := prometheus.NewRegistry()
summary := newSummaryVec(
"handler",
ResponseTime,
"the time to process a msg in nanoseconds",
objectives,
map[string]string{},
[]string{"route"},
)
require.NoError(t, registry.Register(summary))

summary.With(map[string]string{"route": "test.route"}).Observe(42)

families, err := registry.Gather()
require.NoError(t, err)
require.Len(t, families, 1)

for _, m := range families[0].GetMetric() {
s := m.GetSummary()
for _, q := range s.GetQuantile() {
quantiles = append(quantiles, q.GetQuantile())
}
hasSum = s.SampleSum != nil
hasCount = s.SampleCount != nil
}
sort.Float64s(quantiles)
return quantiles, hasSum, hasCount
}

func TestBuiltinSummaryObjectivesScrape(t *testing.T) {
t.Parallel()

t.Run("default-emits-standard-quantiles", func(t *testing.T) {
objectives, err := buildObjectives(nil)
require.NoError(t, err)

quantiles, hasSum, hasCount := quantilesFromSummary(t, objectives)
assert.Equal(t, []float64{0.7, 0.95, 0.99}, quantiles)
assert.True(t, hasSum)
assert.True(t, hasCount)
})

t.Run("custom-objectives-drop-unconfigured-quantiles", func(t *testing.T) {
objectives, err := buildObjectives(map[string]float64{"0.95": 0.005, "0.99": 0.001})
require.NoError(t, err)

quantiles, hasSum, hasCount := quantilesFromSummary(t, objectives)
assert.Equal(t, []float64{0.95, 0.99}, quantiles)
assert.True(t, hasSum)
assert.True(t, hasCount)
})

t.Run("empty-objectives-emit-only-sum-and-count", func(t *testing.T) {
objectives, err := buildObjectives(map[string]float64{})
require.NoError(t, err)

quantiles, hasSum, hasCount := quantilesFromSummary(t, objectives)
assert.Empty(t, quantiles)
assert.True(t, hasSum)
assert.True(t, hasCount)
})
}
Loading