- Add the ability to pass custom collectors and type collectors to the web instrumenter

splittingred · splittingred · commit ac59efe5196e · 2019-12-18T10:40:31.000-06:00
- Add base collector and type collector classes for ease of development of custom integrations
- Change railtie to after initialization to allow for customization
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@ Changelog for the bc-prometheus-ruby gem.
 
 h3. Pending Release
 
+- Add the ability to pass custom collectors and type collectors to the web instrumenter
+- Add base collector and type collector classes for ease of development of custom integrations
+- Change railtie to after initialization to allow for customization
+
 h3. 0.1.5
 
 - Fix issue where puma collector was not being registered on the server
diff --git a/lib/bigcommerce/prometheus.rb b/lib/bigcommerce/prometheus.rb
@@ -30,7 +30,9 @@
 require_relative 'prometheus/server'
 require_relative 'prometheus/client'
 
+require_relative 'prometheus/collectors/base'
 require_relative 'prometheus/collectors/resque'
+require_relative 'prometheus/type_collectors/base'
 require_relative 'prometheus/type_collectors/resque'
 
 require_relative 'prometheus/instrumentors/web'
diff --git a/lib/bigcommerce/prometheus/collectors/base.rb b/lib/bigcommerce/prometheus/collectors/base.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2019-present, BigCommerce Pty. Ltd. All rights reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+module Bigcommerce
+  module Prometheus
+    module Collectors
+      ##
+      # Base class for collectors
+      #
+      class Base
+        ##
+        # Start the collector
+        #
+        def self.start(*args, &block)
+          process_collector = new(*args, &block)
+
+          stop if @thread
+
+          @thread = Thread.new do
+            Kernel.loop do
+              process_collector.run
+            end
+          end
+        end
+
+        ##
+        # Stop the collector
+        #
+        def self.stop
+          t = @thread
+          return unless t
+
+          t.kill
+          @thread = nil
+        end
+
+        ##
+        # @param [PrometheusExporter::Client] client
+        # @param [String] type
+        # @param [Integer] frequency
+        # @param [Hash] options
+        #
+        def initialize(client: nil, type: nil, frequency: nil, options: nil)
+          @client = client || PrometheusExporter::Client.default
+          @type = type || self.class.to_s.downcase.gsub('::', '_').gsub('collector', '')
+          @frequency = frequency || 15
+          @options = options || {}
+          @logger = Bigcommerce::Prometheus.logger
+        end
+
+        ##
+        # Run the collector and send stats
+        #
+        def run
+          metrics = { type: @type }
+          metrics = collect(metrics)
+          @logger.debug "[bigcommerce-prometheus] Pushing #{@type} metrics to type collector: #{metrics.inspect}"
+          @client.send_json(metrics)
+        rescue StandardError => e
+          @logger.error("Prometheus Exporter Failed To Collect #{@type} Stats #{e}")
+        ensure
+          sleep @frequency
+        end
+
+        ##
+        # Collect metrics. This should be overridden in derivative collectors
+        #
+        # @param [Hash] metrics
+        # @return [Hash]
+        #
+        def collect(metrics = {})
+          metrics
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bigcommerce/prometheus/collectors/resque.rb b/lib/bigcommerce/prometheus/collectors/resque.rb
@@ -22,54 +22,21 @@ module Collectors
       # Collect metrics to push to the server type collector
       #
       class Resque
-        include Bigcommerce::Prometheus::Loggable
-
-        ##
-        # @param [Bigcommerce::Prometheus::Client] client
-        # @param [Integer] frequency
-        #
-        def initialize(client:, frequency: nil)
-          @client = client || Bigcommerce::Prometheus.client
-          @frequency = frequency || 15
-        end
-
         ##
-        # Start the collector
+        # @param [Hash] metrics
+        # @return [Hash]
         #
-        def self.start(client: nil, frequency: nil)
-          collector = new(client: client, frequency: frequency)
-          Thread.new do
-            loop do
-              collector.run
-            end
-          end
-        end
-
-        def run
-          metric = collect
-          logger.debug "[bigcommerce-prometheus] Pushing resque metrics to type collector: #{metric.inspect}"
-          @client.send_json metric
-        rescue StandardError => e
-          logger.error "[bigcommerce-prometheus] Failed to collect resque prometheus stats: #{e.message}"
-        ensure
-          sleep @frequency
-        end
-
-        private
-
-        def collect
+        def collect(metrics = {})
           info = ::Resque.info
 
-          metric = {}
-          metric[:type] = 'resque'
-          metric[:environment] = info[:environment].to_s
-          metric[:workers_total] = info[:workers].to_i
-          metric[:jobs_failed_total] = info[:failed].to_i
-          metric[:jobs_pending_total] = info[:pending].to_i
-          metric[:jobs_processed_total] = info[:processed].to_i
-          metric[:queues_total] = info[:queues].to_i
-          metric[:queues] = queue_sizes
-          metric
+          metrics[:environment] = info[:environment].to_s
+          metrics[:workers_total] = info[:workers].to_i
+          metrics[:jobs_failed_total] = info[:failed].to_i
+          metrics[:jobs_pending_total] = info[:pending].to_i
+          metrics[:jobs_processed_total] = info[:processed].to_i
+          metrics[:queues_total] = info[:queues].to_i
+          metrics[:queues] = queue_sizes
+          metrics
         end
 
         def queue_sizes
diff --git a/lib/bigcommerce/prometheus/configuration.rb b/lib/bigcommerce/prometheus/configuration.rb
@@ -34,7 +34,9 @@ module Configuration
         server_host: '0.0.0.0',
         server_port: PrometheusExporter::DEFAULT_PORT,
         server_timeout: PrometheusExporter::DEFAULT_TIMEOUT,
-        server_prefix: PrometheusExporter::DEFAULT_PREFIX
+        server_prefix: PrometheusExporter::DEFAULT_PREFIX,
+        web_collectors: [],
+        web_type_collectors: []
       }.freeze
 
       attr_accessor *VALID_CONFIG_KEYS.keys
@@ -89,6 +91,7 @@ def reset
 
         self.puma_process_label = ENV.fetch('PROMETHEUS_PUMA_PROCESS_LABEL', 'web').to_s
         self.puma_collection_frequency = ENV.fetch('PROMETHEUS_PUMA_COLLECTION_FREQUENCY', 30).to_i
+        self.web_type_collectors = []
       end
 
       ##
diff --git a/lib/bigcommerce/prometheus/instrumentors/web.rb b/lib/bigcommerce/prometheus/instrumentors/web.rb
@@ -31,6 +31,8 @@ def initialize(app:)
           @server_timeout = Bigcommerce::Prometheus.server_timeout
           @server_prefix = Bigcommerce::Prometheus.server_prefix
           @process_name = Bigcommerce::Prometheus.process_name
+          @collectors = Bigcommerce::Prometheus.web_collectors || []
+          @type_collectors = Bigcommerce::Prometheus.web_type_collectors || []
         end
 
         ##
@@ -65,6 +67,9 @@ def setup_before_fork
             server.add_type_collector(PrometheusExporter::Server::ActiveRecordCollector.new)
             server.add_type_collector(PrometheusExporter::Server::WebCollector.new)
             server.add_type_collector(PrometheusExporter::Server::PumaCollector.new)
+            @type_collectors.each do |tc|
+              server.add_type_collector(tc)
+            end
             server.start
           end
         end
@@ -73,6 +78,7 @@ def setup_after_fork
           @app.config.after_fork_callbacks = [] unless @app.config.after_fork_callbacks
           @app.config.after_fork_callbacks << lambda do
             ::Bigcommerce::Prometheus::Integrations::Puma.start
+            @collectors.each(&:start)
           end
         end
 
diff --git a/lib/bigcommerce/prometheus/integrations/railtie.rb b/lib/bigcommerce/prometheus/integrations/railtie.rb
@@ -22,7 +22,7 @@ module Integrations
       # Railtie for automatic configuration of Rails environments
       #
       class Railtie < ::Rails::Railtie
-        initializer 'zzz_bigcommerce.prometheus.configure_rails_initialization' do |app|
+        config.after_initialize do |app|
           Bigcommerce::Prometheus::Instrumentors::Web.new(app: app).start
         end
       end
diff --git a/lib/bigcommerce/prometheus/type_collectors/base.rb b/lib/bigcommerce/prometheus/type_collectors/base.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2019-present, BigCommerce Pty. Ltd. All rights reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+module Bigcommerce
+  module Prometheus
+    module TypeCollectors
+      ##
+      # Base type collector class
+      #
+      class Base < PrometheusExporter::Server::TypeCollector
+        attr_reader :type
+
+        ##
+        # @param [String] type
+        # @param [Hash] default_labels
+        #
+        def initialize(type: nil, default_labels: {})
+          @type = type || self.class.to_s.downcase.gsub('::', '_').gsub('typecollector', '')
+          @default_labels = default_labels || {}
+          @metrics = build_metrics
+        end
+
+        ##
+        # @return [Array]
+        #
+        def metrics
+          return [] unless @metrics.any?
+
+          @metrics.values
+        end
+
+        ##
+        # @param [Symbol] key
+        #
+        def metric(key)
+          @metrics.fetch(key.to_sym, nil)
+        end
+
+        ##
+        # Collect metrics from input data
+        #
+        # @param [Hash] data
+        #
+        def collect(data)
+          custom_labels = data.fetch('custom_labels', nil)
+          labels = custom_labels.nil? ? @default_labels : @default_labels.merge(custom_labels)
+          collect_metrics(data: data, labels: labels)
+        end
+
+        private
+
+        ##
+        # Collect metrics. Implementations of translating metrics to observed calls should happen here.
+        #
+        def collect_metrics(*)
+          raise NotImplementedError, 'Must implement collect_metrics'
+        end
+
+        ##
+        # Build and return all observable metrics. This should be a hash of symbol keys that map to
+        # PrometheusExporter::Metric::Base objects.
+        #
+        # @return [Hash]
+        #
+        def build_metrics
+          {}
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bigcommerce/prometheus/type_collectors/resque.rb b/lib/bigcommerce/prometheus/type_collectors/resque.rb
@@ -21,59 +21,33 @@ module TypeCollectors
       ##
       # Collect resque data from collectors and parse them into metrics
       #
-      class Resque < PrometheusExporter::Server::TypeCollector
+      class Resque < Bigcommerce::Prometheus::TypeCollectors::Base
         ##
         # Initialize the collector
         #
-        def initialize
-          @workers_total = PrometheusExporter::Metric::Gauge.new('resque_workers_total', 'Number of active workers')
-          @jobs_failed_total = PrometheusExporter::Metric::Gauge.new('jobs_failed_total', 'Number of failed jobs')
-          @jobs_pending_total = PrometheusExporter::Metric::Gauge.new('jobs_pending_total', 'Number of pending jobs')
-          @jobs_processed_total = PrometheusExporter::Metric::Gauge.new('jobs_processed_total', 'Number of processed jobs')
-          @queues_total = PrometheusExporter::Metric::Gauge.new('queues_total', 'Number of total queues')
-          @queue_sizes = PrometheusExporter::Metric::Gauge.new('queue_sizes', 'Size of each queue')
-        end
-
-        ##
-        # @return [String]
-        #
-        def type
-          'resque'
-        end
-
-        ##
-        # @return [Array]
-        #
-        def metrics
-          return [] unless @workers_total
-
-          [
-            @workers_total,
-            @jobs_failed_total,
-            @jobs_pending_total,
-            @jobs_processed_total,
-            @queues_total,
-            @queue_sizes
-          ]
+        def build_metrics
+          {
+            workers_total: PrometheusExporter::Metric::Gauge.new('resque_workers_total', 'Number of active workers'),
+            jobs_failed_total: PrometheusExporter::Metric::Gauge.new('jobs_failed_total', 'Number of failed jobs'),
+            jobs_pending_total: PrometheusExporter::Metric::Gauge.new('jobs_pending_total', 'Number of pending jobs'),
+            jobs_processed_total: PrometheusExporter::Metric::Gauge.new('jobs_processed_total', 'Number of processed jobs'),
+            queues_total: PrometheusExporter::Metric::Gauge.new('queues_total', 'Number of total queues'),
+            queue_sizes: PrometheusExporter::Metric::Gauge.new('queue_sizes', 'Size of each queue')
+          }
         end
 
         ##
         # Collect resque metrics from input data
         #
-        def collect(obj)
-          default_labels = { environment: obj['environment'] }
-
-          custom_labels = obj['custom_labels']
-          labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
-
-          @workers_total.observe(obj['workers_total'], labels)
-          @jobs_failed_total.observe(obj['jobs_failed_total'], labels)
-          @jobs_pending_total.observe(obj['jobs_pending_total'], labels)
-          @jobs_processed_total.observe(obj['jobs_processed_total'], labels)
-          @queues_total.observe(obj['queues_total'], labels)
-
-          obj['queues'].each do |name, size|
-            @queue_sizes.observe(size, labels.merge(queue: name))
+        def collect_metrics(data:, labels: {})
+          metric(:workers_total).observe(obj['workers_total'], labels)
+          metric(:jobs_failed_total).observe(obj['jobs_failed_total'], labels)
+          metric(:jobs_pending_total).observe(obj['jobs_pending_total'], labels)
+          metric(:jobs_processed_total).observe(obj['jobs_processed_total'], labels)
+          metric(:queues_total).observe(obj['queues_total'], labels)
+
+          data['queues'].each do |name, size|
+            metric(:queue_sizes).observe(size, labels.merge(queue: name))
           end
         end
       end
diff --git a/lib/bigcommerce/prometheus/version.rb b/lib/bigcommerce/prometheus/version.rb
diff --git a/spec/demo/server b/spec/demo/server
diff --git a/spec/demo/support.rb b/spec/demo/support.rb

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ module Integrations`
`22`	`22`	`# Railtie for automatic configuration of Rails environments`
`23`	`23`	`#`
`24`	`24`	`class Railtie < ::Rails::Railtie`
`25`		`- initializer 'zzz_bigcommerce.prometheus.configure_rails_initialization' do \|app\|`
	`25`	`+ config.after_initialize do \|app\|`
`26`	`26`	`Bigcommerce::Prometheus::Instrumentors::Web.new(app: app).start`
`27`	`27`	`end`
`28`	`28`	`end`