Merge pull request #17 from mozilla-services/ajvb/0.1.5

Version 0.1.5

Author: ajvb

README.md

@@ -33,6 +33,9 @@ init_by_lua_block {
     statsd_host = os.getenv("STATSD_HOST") or nil,
     statsd_port = tonumber(os.getenv("STATSD_PORT")) or 8125,
     statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100,
+    statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5,
+    dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0,
+    whitelist = {},
   })
 }
 
@@ -101,8 +104,9 @@ violations for your environment.
 --
 --  Optional parameters:
 --    url - The base URL to iprepd (defaults to "http://localhost:8080/")
---    cache_ttl - The iprepd response cache ttl in seconds (defaults to 30)
 --    timeout - The timeout for making requests to iprepd in milliseconds (defaults to 10)
+--    cache_ttl - The iprepd response cache ttl in seconds (defaults to 30)
+--    cache_buffer_count - Max number of entries allowed in the cache. (defaults to 200)
 --    cache_errors - Enables (1) or disables (0) caching errors. Caching errors is a good
 --                   idea in production, as it can reduce the average additional latency
 --                   caused by this module if anything goes wrong with the underlying
@@ -111,17 +115,24 @@ violations for your environment.
 --    statsd_port - Port of statsd collector. (defaults to 8125)
 --    statsd_max_buffer_count - Max number of metrics in buffer before metrics should be submitted
 --                              to statsd (defaults to 100)
+--    statsd_flush_timer - Interval for attempting to flush the stats in seconds. (defaults to 5)
+--    dont_block - Enables (1) or disables (0) not blocking within nginx by returning a 403. (defaults to disabled)
+--    whitelist - List of whitelisted IP's and IP CIDR's. (defaults to empty)
 --
 client = require("resty.iprepd").new({
-  url = "http://127.0.0.1:8080",
   api_key = os.getenv("IPREPD_API_KEY"),
   threshold = 50,
-  cache_ttl = 30,
+  url = "http://127.0.0.1:8080",
   timeout = 10,
+  cache_ttl = 30,
+  cache_buffer_count = 1000,
   cache_errors = 1,
   statsd_host = "127.0.0.1",
   statsd_port = 8125,
   statsd_max_buffer_count = 100,
+  statsd_flush_timer = 10,
+  dont_block = 0,
+  whitelist = {"127.0.0.1", "10.10.10.0/24", "192.168.0.0/16"}
 })
 ```
 
@@ -137,7 +148,7 @@ $ make run_dev
 
 Then you will be able to hit this proxy with: `curl http://localhost:80`
 
-### Environment Variables
+### Environment Variables for Dev
 
 #### Note:
 
@@ -153,10 +164,12 @@ IPREPD_REPUTATION_THRESHOLD=50  # iprepd reputation threshold, block all IP's wi
 #
 # optional
 #
-IPREPD_TIMEOUT=10  # iprepd client timeout in milliseconds (default is 10ms)
-IPREPD_CACHE_TTL=60 # iprepd response cache ttl in seconds (default is 30s)
-IPREPD_CACHE_ERRORS=1 # enables caching iprepd non-200 responses (1 enables, 0 disables, default is 0)
-STATSD_HOST=127.0.0.1 # statsd host, setting this will also enable statsd metrics collection.
-STATSD_PORT=8125 # statsd port (default is 8125)
-STATSD_MAX_BUFFER_COUNT=200 # statsd max number of buffer items before submitting (default is 100)
+IPREPD_TIMEOUT=10
+IPREPD_CACHE_TTL=30
+IPREPD_CACHE_ERRORS=0
+STATSD_HOST=127.0.0.1
+STATSD_PORT=8125
+STATSD_MAX_BUFFER_COUNT=200
+STATSD_FLUSH_TIMER=2
+DONT_BLOCK=0
 ```

dist.ini

@@ -1,11 +1,11 @@
 name = iprepd-nginx
 abstract = iprepd openresty module
 author = AJ Bahnken (ajvb)
-version = 0.1.4
+version = 0.1.5
 is_original = yes
 license = mozilla2
 lib_dir = lib
 doc_dir = lib
 repo_link = https://github.com/mozilla-services/iprepd-nginx
 main_module = lib/resty/iprepd.lua
-requires = openresty/lua-resty-lrucache, pintsized/lua-resty-http
+requires = openresty/lua-resty-lrucache, pintsized/lua-resty-http, hamishforbes/lua-resty-iputils

etc/conf.d/server.conf

@@ -9,6 +9,9 @@ init_by_lua_block {
     statsd_host = os.getenv("STATSD_HOST") or nil,
     statsd_port = tonumber(os.getenv("STATSD_PORT")) or 8125,
     statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100,
+    statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5,
+    dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0,
+    whitelist = {},
   })
 }
 

lib/resty/iprepd.lua

@@ -1,5 +1,6 @@
 local cjson = require('cjson')
 local http = require('resty.http')
+local iputils = require('resty.iputils')
 local lrucache = require('resty.lrucache')
 local statsd = require('resty.statsd')
 
@@ -18,13 +19,12 @@ function _M.new(options)
     iprepd_url = iprepd_url:sub(1, -2)
   end
 
-  local cache_ttl = options.cache_ttl or 30
+  local cache_buffer_count = options.cache_buffer_count or 200
 
   local iprepd_threshold = options.threshold or fatal_error('Need to pass in a threshold')
   local iprepd_api_key = options.api_key or fatal_error('Need to pass in an api_key')
 
-  -- TODO: Make configurable?
-  local cache, err = lrucache.new(200)
+  local cache, err = lrucache.new(cache_buffer_count)
   if not cache then
     fatal_error('failed to create the cache: ' .. (err or 'unknown'))
   end
@@ -34,33 +34,82 @@ function _M.new(options)
     statsd_client = statsd
   end
 
+  local whitelist = nil
+  local whitelist_list = options.whitelist or nil
+  if whitelist_list then
+    whitelist = iputils.parse_cidrs(whitelist_list)
+  end
+
   local self = {
     url = iprepd_url,
+    timeout = options.timeout or 10,
     threshold = iprepd_threshold,
     api_key_hdr = {
       ['Authorization'] = 'APIKey ' .. iprepd_api_key,
     },
-    cache_ttl = cache_ttl,
-    timeout = options.timeout or 10,
     cache = cache,
+    cache_ttl = options.cache_ttl or 30,
     cache_errors = options.cache_errors or 0,
     statsd = statsd_client,
     statsd_host = options.statsd_host,
     statsd_port = options.statsd_port or 8125,
     statsd_max_buffer_count =  options.statsd_max_buffer_count or 100,
+    statsd_flush_timer = options.statsd_flush_timer or 5,
+    dont_block = options.dont_block or 0,
+    whitelist = whitelist,
   }
 
   return setmetatable(self, mt)
 end
 
 function _M.check(self, ip)
-  local httpc = http.new()
-  -- set timeout in ms
-  httpc:set_timeout(self.timeout)
+  ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false')
+  ngx.req.set_header('X-Foxsec-Block', 'false')
+  if self.whitelist then
+    if iputils.ip_in_cidrs(ip, self.whitelist) then
+      return
+    end
+  end
 
-  -- Get reputation for ip
+  local reputation = self:get_reputation(ip)
+  if reputation then
+    ngx.req.set_header('X-Foxsec-IP-Reputation', tostring(reputation))
+    if reputation <= self.threshold then
+      ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'true')
+      ngx.req.set_header('X-Foxsec-Block', 'true')
+      if self.statsd then
+        self.statsd.incr("iprepd.status.below_threshold")
+      end
+
+      if self.dont_block == 1 then
+        ngx.log(ngx.ERR, ip .. ' is below threshold with a reputation of ' .. reputation)
+      else
+        ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation)
+        if self.statsd then
+          self.statsd.incr("iprepd.status.rejected")
+        end
+        ngx.exit(ngx.HTTP_FORBIDDEN)
+      end
+    else
+      if self.statsd then
+        self.statsd.incr("iprepd.status.accepted")
+      end
+    end
+
+    return
+  end
+
+  if self.statsd then
+    self.statsd.incr("iprepd.status.accepted")
+  end
+end
+
+function _M.get_reputation(self, ip)
   local reputation = self.cache:get(ip)
+
   if not reputation then
+    local httpc = http.new()
+    httpc:set_timeout(self.timeout)
     local resp, err = httpc:request_uri(self.url .. '/' .. ip, {
       method  = "GET",
       headers = self.api_key_hdr,
@@ -70,7 +119,7 @@ function _M.check(self, ip)
         self.statsd.incr("iprepd.err.timeout")
       end
       ngx.log(ngx.ERR, 'Error with request to iprepd: ' .. err)
-      return
+      return nil
     end
 
     -- If the IP was found
@@ -85,26 +134,17 @@ function _M.check(self, ip)
       self.cache:set(ip, 100, self.cache_ttl)
     else
       ngx.log(ngx.ERR, 'iprepd responded with a ' .. resp.status .. ' http status code')
+      if self.statsd then
+        self.statsd.incr("iprepd.err." .. resp.status)
+      end
       if self.cache_errors == 1 then
         ngx.log(ngx.ERR, 'cache_errors is enabled, setting reputation of ' .. ip .. ' to 100 within the cache')
         self.cache:set(ip, 100, self.cache_ttl)
       end
     end
   end
 
-  -- check reputation against threshold
-  if reputation and reputation <= self.threshold then
-    -- return 403 and log rejections
-    ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation)
-    if self.statsd then
-      self.statsd.incr("iprepd.status.rejected")
-    end
-    ngx.exit(ngx.HTTP_FORBIDDEN)
-  else
-    if self.statsd then
-      self.statsd.incr("iprepd.status.accepted")
-    end
-  end
+  return reputation
 end
 
 function _M.flush_stats(self)
@@ -120,7 +160,7 @@ function _M.async_flush_stats(premature, self)
 end
 
 function _M.config_flush_timer(self)
-  ngx.timer.every(5, self.async_flush_stats, self)
+  ngx.timer.every(self.statsd_flush_timer, self.async_flush_stats, self)
 end
 
 return _M