Add prometheus reporting for replicas scaled (#1)

* Add prometheus reporting for replicas scaled
* Bump rake version to address a CVE

Author: hammady

.gitignore

@@ -6,7 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
-Gemfile.lock
 .env
 scaltainer.yml
 scaltainer.yml.state

Gemfile.lock

@@ -0,0 +1,104 @@
+PATH
+  remote: .
+  specs:
+    scaltainer (0.3.0)
+      docker-api
+      dotenv
+      excon (>= 0.47.0)
+      kubeclient
+      prometheus-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coderay (1.1.1)
+    coveralls (0.8.21)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.14.1)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.4)
+      tins (~> 1.6)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    docker-api (1.34.2)
+      excon (>= 0.47.0)
+      multi_json
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    dotenv (2.7.5)
+    excon (0.73.0)
+    ffi (1.12.2)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    http (4.4.1)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    http-parser (1.2.1)
+      ffi-compiler (>= 1.0, < 2.0)
+    json (2.1.0)
+    kubeclient (4.6.0)
+      http (>= 3.0, < 5.0)
+      recursive-open-struct (~> 1.0, >= 1.0.4)
+      rest-client (~> 2.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0425)
+    multi_json (1.14.1)
+    netrc (0.11.0)
+    prometheus-client (2.0.0)
+    public_suffix (4.0.4)
+    rake (13.0.1)
+    recursive-open-struct (1.1.1)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    simplecov (0.14.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    term-ansicolor (1.6.0)
+      tins (~> 1.0)
+    thor (0.19.4)
+    tins (1.13.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.15)
+  coderay (~> 1.1)
+  coveralls (~> 0.8)
+  rake (>= 12.3.3)
+  rspec (~> 3.5)
+  scaltainer!
+
+BUNDLED WITH
+   1.17.3

README.md

@@ -55,6 +55,21 @@ specify the wait time between repetitions using the `-w` parameter in seconds:
 
 This will repeatedly call scaltainer every 60 seconds, sleeping in-between.
 
+If you would like to monitor the changes in scaling out and in. You can install
+Prometheus and add a configuration parameter pointing to its Push Gateway:
+
+    scaltainer -g prometheus-pushgateway.monitoring.svc.cluster.local:9091
+
+Where `prometheus-pushgateway.monitoring.svc.cluster.local:9091` is the address
+of the push gateway. For Kubernetes environments the above denotes the gateway service
+name (`prometheus-pushgateway`), where it is installed in the namespace called
+`monitoring`. Scaltainer will report the following metrics to Prometheus:
+
+- `rayyan_controller_replicas`: number of replicas scaled (or untouched thereof).
+This is labeled by the namespace and controller name, both matching the scaltainer
+configuration file.
+- `rayyan_scaltainer_ticks`: iterations scaltainer has performed (if `-w` is used)
+
 ## Configuration
 
 ### Environment variables
@@ -121,7 +136,7 @@ The configuration file (determined by `-f FILE` command line parameter) should b
 
     # to get worker metrics
     endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info
-    # optional docker swarm stack name or kubernetes namespace
+    # optional docker swarm stack name or kubernetes namespace (useful if having push gateway)
     namespace: mynamespace
     # list of web services to monitor
     web_services:

exe/scaltainer

@@ -3,8 +3,8 @@
 require 'scaltainer'
 
 begin
-  configfile, statefile, logger, wait, orchestrator = Scaltainer::Command.parse ARGV
-  Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator
+  configfile, statefile, logger, wait, orchestrator, pushgateway = Scaltainer::Command.parse ARGV
+  Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator, pushgateway
 rescue => e
   $stderr.puts e.message
   $stderr.puts e.backtrace

lib/scaltainer/command.rb

@@ -4,7 +4,7 @@
 module Scaltainer
   class Command
     def self.parse(args)
-      configfile, statefile, wait, orchestrator = 'scaltainer.yml', nil, 0, :swarm
+      configfile, statefile, wait, orchestrator, pushgateway = 'scaltainer.yml', nil, 0, :swarm, nil
       OptionParser.new do |opts|
         opts.banner = "Usage: scaltainer [options]"
         opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file|
@@ -19,6 +19,9 @@ def self.parse(args)
         opts.on("-o", "--orchestrator swarm:kubernetes", [:swarm, :kubernetes], "Specify orchestrator type (default: swarm)") do |o|
           orchestrator = o
         end
+        opts.on("-g", "--prometheus-push-gateway ADDRESS", "Specify prometheus push gateway address in the form of host:port") do |gw|
+          pushgateway = gw
+        end
         opts.on("-v", "--version", "Show version and exit") do
           puts Scaltainer::VERSION
           exit 0
@@ -55,7 +58,7 @@ def self.parse(args)
       logger = Logger.new(STDOUT)
       logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1
 
-      return configfile, statefile, logger, wait, orchestrator
+      return configfile, statefile, logger, wait, orchestrator, pushgateway
     end
 
     private

lib/scaltainer/runner.rb

@@ -1,8 +1,10 @@
 require "yaml"
+require 'prometheus/client'
+require 'prometheus/client/push'
 
 module Scaltainer
   class Runner
-    def initialize(configfile, statefile, logger, wait, orchestrator)
+    def initialize(configfile, statefile, logger, wait, orchestrator, pushgateway)
       @orchestrator = orchestrator
       @logger = logger
       @default_service_config = {
@@ -19,18 +21,20 @@ def initialize(configfile, statefile, logger, wait, orchestrator)
       endpoint = config["endpoint"]
       service_type_web = ServiceTypeWeb.new(endpoint)
       service_type_worker = ServiceTypeWorker.new(endpoint)
+      register_pushgateway(pushgateway) if pushgateway
+      namespace = config["namespace"] || config["stack_name"]
       loop do
-        run config, state, service_type_web, service_type_worker
+        run config, state, service_type_web, service_type_worker, namespace
         save_state statefile, state
+        sync_pushgateway(namespace, state) if pushgateway
         sleep wait
         break if wait == 0
       end
     end
 
     private
 
-    def run(config, state, service_type_web, service_type_worker)
-      namespace = config["namespace"] || config["stack_name"]
+    def run(config, state, service_type_web, service_type_worker, namespace)
       iterate_services config["web_services"], namespace, service_type_web, state
       iterate_services config["worker_services"], namespace, service_type_worker, state
     end
@@ -82,9 +86,11 @@ def process_service(service_name, config, state, namespace, type, metrics)
       adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config)
       @logger.debug "Desired number of replicas for #{service.type} #{service.name} is adjusted to #{adjusted_replicas}"
       replica_diff = adjusted_replicas - current_replicas
+      state["replicas"] = current_replicas
       type.yield_to_scale(replica_diff, config, state, metric,
         service.name, @logger) do
           scale_out service, current_replicas, adjusted_replicas
+          state["replicas"] = adjusted_replicas
         end
     end
 
@@ -113,5 +119,27 @@ def scale_out(service, current_replicas, desired_replicas)
       end
     end
 
+    def register_pushgateway(pushgateway)
+      @registry = Prometheus::Client.registry
+      @replicas_gauge = @registry.gauge(:rayyan_controller_replicas, docstring: 'Rayyan replicas', labels: [:controller, :namespace])
+      @ticks_counter = @registry.counter(:rayyan_scaltainer_ticks, docstring: 'Rayyan Scaltainer ticks', labels: [:namespace])
+
+      @pushgateway = Prometheus::Client::Push.new("scaltainer", "scaltainer", "http://#{pushgateway}")
+    end
+
+    def sync_pushgateway(namespace, state)
+      @logger.debug("Now syncing state #{state} in namespace #{namespace}")
+      state.each do |service, state|
+        @replicas_gauge.set(state["replicas"], labels: {namespace: namespace, controller: service}) if state["replicas"]
+      end
+      @ticks_counter.increment(labels: {namespace: namespace})
+      begin
+        @pushgateway.add(@registry)
+      rescue => e
+        @logger.warn "[#{e.class}] Error pushing metrics to the configured Prometheus Push Gateway: #{e.message}"
+      end
+      @logger.info "Pushed metrics successfully to the configured Prometheus Push Gateway"
+    end
+
   end # class
 end # module

lib/scaltainer/version.rb

@@ -1,3 +1,3 @@
 module Scaltainer
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

scaltainer.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.15"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency 'rspec', '~> 3.5'
   spec.add_development_dependency 'coderay', '~> 1.1'
   spec.add_development_dependency 'coveralls', '~> 0.8'
@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "docker-api"
   spec.add_runtime_dependency "kubeclient"
   spec.add_runtime_dependency "dotenv"
+  spec.add_runtime_dependency "prometheus-client"
 end