Ajvb/v0.1.8 (#24)

* reputation cache improvements

* Up default cache ttl to 60 seconds
* Up default cache size to 5000 items
* Add metric for cache hits
* Only set reputation in cache when it's new

* bump version

* Set HTTP keepalive header

* Added iprepd.cache_hit to README

* add separate error cache ttl

Author: ajvb

README.md

@@ -81,12 +81,13 @@ violations for your environment.
 --  Optional parameters:
 --    url - The base URL to iprepd (defaults to "http://localhost:8080/")
 --    timeout - The timeout for making requests to iprepd in milliseconds (defaults to 10)
---    cache_ttl - The iprepd response cache ttl in seconds (defaults to 30)
---    cache_buffer_count - Max number of entries allowed in the cache. (defaults to 200)
+--    cache_ttl - The iprepd response cache ttl in seconds (defaults to 60)
+--    cache_buffer_count - Max number of entries allowed in the cache. (defaults to 5000)
 --    cache_errors - Enables (1) or disables (0) caching errors. Caching errors is a good
 --                   idea in production, as it can reduce the average additional latency
 --                   caused by this module if anything goes wrong with the underlying
 --                   infrastructure. (defaults to disabled)
+--    cache_errors_ttl - The iprepd response cache ttl for error responses (not 200 or 404) in seconds (defaults to 10)
 --    statsd_host - Host of statsd collector. Setting this will enable statsd metrics collection
 --    statsd_port - Port of statsd collector. (defaults to 8125)
 --    statsd_max_buffer_count - Max number of metrics in buffer before metrics should be submitted
@@ -107,6 +108,7 @@ client = require("resty.iprepd").new({
   cache_ttl = 30,
   cache_buffer_count = 1000,
   cache_errors = 1,
+  cache_errors_ttl = 10,
   statsd_host = "127.0.0.1",
   statsd_port = 8125,
   statsd_max_buffer_count = 100,
@@ -127,6 +129,7 @@ client = require("resty.iprepd").new({
 | iprepd.status.rejected | count | The request was blocked (won’t be sent if `blocking_mode` is disabled). |
 | iprepd.status.accepted | count | The request was accepted. The reputation can still be below the threshold if `blocking_mode` is disabled.
 | iprepd.get_reputation | count | Request to iprepd |
+| iprepd.cache_hit | count | Got reputation from internal cache |
 | iprepd.err.timeout | count | Request to iprepd timed out |
 | iprepd.err.500 | count | Got a 500 response from iprepd |
 | iprepd.err.401 | count | Got a 401 response from iprepd, usually means the API key in use is invalid or being sent incorrectly by nginx. |

dist.ini

@@ -1,7 +1,7 @@
 name = iprepd-nginx
 abstract = iprepd openresty module
 author = AJ Bahnken (ajvb)
-version = 0.1.7
+version = 0.1.8
 is_original = yes
 license = mozilla2
 lib_dir = lib

lib/resty/iprepd.lua

@@ -19,7 +19,7 @@ function _M.new(options)
     iprepd_url = iprepd_url:sub(1, -2)
   end
 
-  local cache_buffer_count = options.cache_buffer_count or 200
+  local cache_buffer_count = options.cache_buffer_count or 5000
 
   local iprepd_threshold = options.threshold or fatal_error('Need to pass in a threshold')
   local iprepd_api_key = options.api_key or fatal_error('Need to pass in an api_key')
@@ -44,12 +44,14 @@ function _M.new(options)
     url = iprepd_url,
     timeout = options.timeout or 10,
     threshold = iprepd_threshold,
-    api_key_hdr = {
+    iprepd_hdrs = {
       ['Authorization'] = string.format('APIKey %s', iprepd_api_key),
+      ['Connection'] = 'keep-alive',
     },
     cache = cache,
-    cache_ttl = options.cache_ttl or 30,
+    cache_ttl = options.cache_ttl or 60,
     cache_errors = options.cache_errors or 0,
+    cache_errors_ttl = options.cache_errors_ttl or 10,
     statsd = statsd_client,
     statsd_host = options.statsd_host,
     statsd_port = options.statsd_port or 8125,
@@ -112,7 +114,7 @@ function _M.get_reputation(self, ip)
     httpc:set_timeout(self.timeout)
     local resp, err = httpc:request_uri(string.format("%s/%s", self.url, ip), {
       method  = "GET",
-      headers = self.api_key_hdr,
+      headers = self.iprepd_hdrs,
     })
     self.statsd.incr("iprepd.get_reputation")
     if err then
@@ -147,12 +149,16 @@ function _M.get_reputation(self, ip)
       if self.cache_errors == 1 then
         reputation = 100
         self:debug_log(string.format("cache_errors is enabled, setting reputation of %s to 100 within the cache", ip))
+        self.cache:set(ip, reputation, self.cache_errors_ttl)
+        return reputation
       end
     end
-  end
 
-  if reputation and reputation >= 0 and reputation <= 100 then
-    self.cache:set(ip, reputation, self.cache_ttl)
+    if reputation and reputation >= 0 and reputation <= 100 then
+      self.cache:set(ip, reputation, self.cache_ttl)
+    end
+  else
+    self.statsd.incr("iprepd.cache_hit")
   end
 
   return reputation