Add rate limiter to helm agent fetches

Pretty sure these can get overwhelmed at large scale, and a simple rate limit should improve the load shedding here.

Author: michaeljguarino

lib/console/deployments/git/agent.ex

@@ -18,7 +18,7 @@ defmodule Console.Deployments.Git.Agent do
 
   @poll :timer.seconds(120)
   @timeout :timer.seconds(10)
-  @limit 50
+  @limit 100
   @limit_interval :timer.seconds(1)
 
   defmodule State, do: defstruct [:git, :cache, :last_pull, :url]
@@ -269,12 +269,15 @@ defmodule Console.Deployments.Git.Agent do
 
   defp touch(pid, line), do: send(pid, {:touch, line})
 
-  defp rate_limited(key, fun) when is_function(fun, 0) do
+  @doc false
+  def rate_limited(key, fun) when is_function(fun, 0) do
     :erlang.term_to_binary(key)
     |> Hammer.check_rate(@limit_interval, @limit)
     |> case do
       {:allow, _} -> fun.()
-      {:deny, _} -> {:error, :rate_limited}
+      {:deny, _} ->
+        Logger.warn "rate limiting git/helm agent fetch"
+        {:error, :rate_limited}
     end
   end
 

lib/console/deployments/helm/agent.ex

@@ -1,5 +1,6 @@
 defmodule Console.Deployments.Helm.Agent do
   use GenServer, restart: :temporary
+  import Console.Deployments.Git.Agent, only: [rate_limited: 2]
   alias Console.Repo
   alias Console.Deployments.Git
   alias Console.Deployments.Helm.{AgentCache, Discovery, Supervisor}
@@ -21,7 +22,8 @@ defmodule Console.Deployments.Helm.Agent do
          _ <- touch(pid, line) do
       {:ok, opener(pid, f), d, i}
     else
-      _ -> GenServer.call(pid, {:fetch, chart, vsn}, @timeout)
+      _ ->
+        rate_limited({:helm, pid}, fn -> GenServer.call(pid, {:fetch, chart, vsn}, @timeout) end)
     end
   end
 
@@ -32,7 +34,8 @@ defmodule Console.Deployments.Helm.Agent do
          _ <- touch(pid, line) do
       {:ok, d}
     else
-      _ -> GenServer.call(pid, {:digest, chart, vsn}, @timeout)
+      _ ->
+        rate_limited({:helm, pid}, fn -> GenServer.call(pid, {:digest, chart, vsn}, @timeout) end)
     end
   end