Merge pull request #384 from neslinesli93/feature/remove_keys_from_realtime_stats

Replace KEYS with a cursored call to SCAN for realtime stats

Author: akira

lib/exq/redis/job_stat.ex

@@ -118,11 +118,10 @@ defmodule Exq.Redis.JobStat do
   end
 
   def realtime_stats(redis, namespace) do
-    {:ok, [failure_keys, success_keys]} =
-      Connection.qp(redis, [
-        ["KEYS", JobQueue.full_key(namespace, "stat:failed_rt:*")],
-        ["KEYS", JobQueue.full_key(namespace, "stat:processed_rt:*")]
-      ])
+    failure_keys = realtime_stats_scanner(redis, JobQueue.full_key(namespace, "stat:failed_rt:*"))
+
+    success_keys =
+      realtime_stats_scanner(redis, JobQueue.full_key(namespace, "stat:processed_rt:*"))
 
     formatter = realtime_stats_formatter(redis, namespace)
     failures = formatter.(failure_keys, "stat:failed_rt:")
@@ -131,6 +130,24 @@ defmodule Exq.Redis.JobStat do
     {:ok, failures, successes}
   end
 
+  defp realtime_stats_scanner(redis, namespace) do
+    {:ok, [[cursor, result]]} =
+      Connection.qp(redis, [["SCAN", 0, "MATCH", namespace, "COUNT", 1_000]])
+
+    realtime_stats_scan_keys(redis, namespace, cursor, result)
+  end
+
+  defp realtime_stats_scan_keys(_redis, _namespace, "0", accumulator) do
+    accumulator
+  end
+
+  defp realtime_stats_scan_keys(redis, namespace, cursor, accumulator) do
+    {:ok, [[new_cursor, result]]} =
+      Connection.qp(redis, [["SCAN", cursor, "MATCH", namespace, "COUNT", 1_000]])
+
+    realtime_stats_scan_keys(redis, namespace, new_cursor, accumulator ++ result)
+  end
+
   defp realtime_stats_formatter(redis, namespace) do
     fn keys, ns ->
       if Enum.empty?(keys) do