Add support for configurable histogram percentiles

Author: sean-heller

aggregator.py

@@ -223,6 +223,7 @@ def flush(self, timestamp, interval):
 
 DEFAULT_HISTOGRAM_AGGREGATES = ['max', 'median', 'avg', 'count']
 DEFAULT_HISTOGRAM_PERCENTILES = [0.95]
+DEFAULT_HISTOGRAM_NAME_FORMAT = '%s.%spercentile'
 
 class Histogram(Metric):
     """ A metric to track the distribution of a set of values. """
@@ -238,6 +239,9 @@ def __init__(self, formatter, name, tags, hostname, device_name, extra_config=No
         self.percentiles = extra_config['percentiles'] if\
             extra_config is not None and extra_config.get('percentiles') is not None\
             else DEFAULT_HISTOGRAM_PERCENTILES
+        self.name_format = extra_config['name_format'] if\
+            extra_config is not None and extra_config.get('name_format') is not None\
+            else DEFAULT_HISTOGRAM_NAME_FORMAT
         self.tags = tags
         self.hostname = hostname
         self.device_name = device_name
@@ -287,7 +291,11 @@ def flush(self, ts, interval):
 
         for p in self.percentiles:
             val = self.samples[int(round(p * length - 1))]
-            name = '%s.%spercentile' % (self.name, int(p * 100))
+            try:
+                name = self.name_format % (self.name, int(p * 100))
+            except TypeError:
+                log.warn("Invalid histogram name format %s, defaulting to '%s'" % (self.name_format, DEFAULT_HISTOGRAM_NAME_FORMAT))
+                name = DEFAULT_HISTOGRAM_NAME_FORMAT % (self.name, int(p * 100))
             metrics.append(self.formatter(
                 hostname=self.hostname,
                 tags=self.tags,
@@ -401,6 +409,7 @@ class Aggregator(object):
     def __init__(self, hostname, interval=1.0, expiry_seconds=300,
             formatter=None, recent_point_threshold=None,
             histogram_aggregates=None, histogram_percentiles=None,
+            histogram_name_format=None,
             utf8_decoding=False):
         self.events = []
         self.service_checks = []
@@ -421,7 +430,8 @@ def __init__(self, hostname, interval=1.0, expiry_seconds=300,
         self.metric_config = {
             Histogram: {
                 'aggregates': histogram_aggregates,
-                'percentiles': histogram_percentiles
+                'percentiles': histogram_percentiles,
+                'name_format': histogram_name_format
             }
         }
 
@@ -723,6 +733,7 @@ class MetricsBucketAggregator(Aggregator):
     def __init__(self, hostname, interval=1.0, expiry_seconds=300,
             formatter=None, recent_point_threshold=None,
             histogram_aggregates=None, histogram_percentiles=None,
+            histogram_name_format=None,
             utf8_decoding=False):
         super(MetricsBucketAggregator, self).__init__(
             hostname,
@@ -732,6 +743,7 @@ def __init__(self, hostname, interval=1.0, expiry_seconds=300,
             recent_point_threshold,
             histogram_aggregates,
             histogram_percentiles,
+            histogram_name_format,
             utf8_decoding
         )
         self.metric_by_bucket = {}
@@ -863,6 +875,7 @@ class MetricsAggregator(Aggregator):
     def __init__(self, hostname, interval=1.0, expiry_seconds=300,
             formatter=None, recent_point_threshold=None,
             histogram_aggregates=None, histogram_percentiles=None,
+            histogram_name_format=None,
             utf8_decoding=False):
         super(MetricsAggregator, self).__init__(
             hostname,
@@ -872,6 +885,7 @@ def __init__(self, hostname, interval=1.0, expiry_seconds=300,
             recent_point_threshold,
             histogram_aggregates,
             histogram_percentiles,
+            histogram_name_format,
             utf8_decoding
         )
         self.metrics = {}

config.py

@@ -317,7 +317,6 @@ def get_histogram_percentiles(configstr=None):
 
     return result
 
-
 def clean_dd_url(url):
     url = url.strip()
     if not url.startswith('http'):
@@ -477,6 +476,9 @@ def get_config(parse_args=True, cfg_path=None, options=None):
         if config.has_option('Main', 'histogram_percentiles'):
             agentConfig['histogram_percentiles'] = get_histogram_percentiles(config.get('Main', 'histogram_percentiles'))
 
+        if config.has_option('Main', 'histogram_name_format'):
+            agentConfig['histogram_name_format'] = config.get('Main', 'histogram_name_format')
+
         # Disable Watchdog (optionally)
         if config.has_option('Main', 'watchdog'):
             if config.get('Main', 'watchdog').lower() in ('no', 'false'):

dogstatsd.py

@@ -503,6 +503,7 @@ def init(config_path=None, use_watchdog=False, use_forwarder=False, args=None):
         formatter=get_formatter(c),
         histogram_aggregates=c.get('histogram_aggregates'),
         histogram_percentiles=c.get('histogram_percentiles'),
+        histogram_name_format=c.get('histogram_name_format'),
         utf8_decoding=c['utf8_decoding']
     )
 

tests/core/test_histogram.py

@@ -57,6 +57,74 @@ def test_custom_single_percentile(self):
 
         self.assertEquals(value_by_type['40percentile'], 7, value_by_type)
 
+    def test_custom_single_percentile_withCustomName(self):
+        configstr = '0.40'
+        configname = '%s.p%s'
+
+        stats = MetricsAggregator(
+            'myhost',
+            histogram_percentiles=get_histogram_percentiles(configstr),
+            histogram_name_format=configname
+        )
+
+        self.assertEquals(
+            stats.metric_config[Histogram]['percentiles'],
+            [0.40],
+            stats.metric_config[Histogram]
+        )
+        self.assertEquals(
+            stats.metric_config[Histogram]['name_format'],
+            '%s.p%s',
+            stats.metric_config[Histogram]
+        )
+
+        for i in xrange(20):
+            stats.submit_packets('myhistogram:{0}|h'.format(i))
+
+        metrics = stats.flush()
+
+        self.assertEquals(len(metrics), 5, metrics)
+
+        value_by_type = {}
+        for k in metrics:
+            value_by_type[k['metric'][len('myhistogram')+1:]] = k['points'][0][1]
+
+        self.assertEquals(value_by_type['p40'], 7, value_by_type)
+
+    def test_custom_single_percentile_withInvalidCustomName(self):
+        configstr = '0.40'
+        configname = '%s.p%s.%s'
+
+        stats = MetricsAggregator(
+            'myhost',
+            histogram_percentiles=get_histogram_percentiles(configstr),
+            histogram_name_format=configname
+        )
+
+        self.assertEquals(
+            stats.metric_config[Histogram]['percentiles'],
+            [0.40],
+            stats.metric_config[Histogram]
+        )
+        self.assertEquals(
+            stats.metric_config[Histogram]['name_format'],
+            '%s.p%s.%s',
+            stats.metric_config[Histogram]
+        )
+
+        for i in xrange(20):
+            stats.submit_packets('myhistogram:{0}|h'.format(i))
+
+        metrics = stats.flush()
+
+        self.assertEquals(len(metrics), 5, metrics)
+
+        value_by_type = {}
+        for k in metrics:
+            value_by_type[k['metric'][len('myhistogram')+1:]] = k['points'][0][1]
+
+        self.assertEquals(value_by_type['40percentile'], 7, value_by_type)
+
     def test_custom_multiple_percentile(self):
         configstr = '0.4, 0.65, 0.999'
         stats = MetricsAggregator(