Cache dashboard queue previews

Author: mjc

lib/reencodarr/dashboard/state.ex

@@ -352,25 +352,14 @@ defmodule Reencodarr.Dashboard.State do
   defp fetch_queue_items(current_items) do
     timeout = queue_query_timeout()
 
+    preview_items =
+      fetch_queue_preview_items(timeout)
+      |> Enum.group_by(& &1.queue_type)
+
     %{
-      analyzer:
-        fetch_queue_preview(
-          "analyzer",
-          current_items.analyzer,
-          fn -> VideoQueries.videos_needing_analysis_preview(5, timeout: timeout) end
-        ),
-      crf_searcher:
-        fetch_queue_preview(
-          "crf_searcher",
-          current_items.crf_searcher,
-          fn -> VideoQueries.videos_for_crf_search_preview(5, timeout: timeout) end
-        ),
-      encoder:
-        fetch_queue_preview(
-          "encoder",
-          current_items.encoder,
-          fn -> VideoQueries.videos_ready_for_encoding_preview(5, timeout: timeout) end
-        )
+      analyzer: queue_preview_rows(preview_items, :analyzer, current_items.analyzer),
+      crf_searcher: queue_preview_rows(preview_items, :crf_searcher, current_items.crf_searcher),
+      encoder: queue_preview_rows(preview_items, :encoder, current_items.encoder)
     }
   end
 
@@ -402,18 +391,10 @@ defmodule Reencodarr.Dashboard.State do
     %{
       analyzer: stats.needs_analysis || current_queue_counts.analyzer || 0,
       crf_searcher: stats.analyzed || current_queue_counts.crf_searcher || 0,
-      encoder: refresh_encoder_queue_count(current_queue_counts.encoder)
+      encoder: stats.encoding_queue_count || current_queue_counts.encoder || 0
     }
   end
 
-  defp refresh_encoder_queue_count(previous_count) do
-    VideoQueries.encoding_queue_count(timeout: queue_query_timeout()) || previous_count || 0
-  rescue
-    error ->
-      Logger.warning("Dashboard.State encoder queue count refresh failed: #{inspect(error)}")
-      previous_count || 0
-  end
-
   defp queue_refresh_enabled? do
     Application.get_env(:reencodarr, :dashboard_queue_refresh_enabled, true)
   end
@@ -427,19 +408,67 @@ defmodule Reencodarr.Dashboard.State do
     |> Map.put(:percent, percent)
   end
 
-  defp fetch_queue_preview(label, fallback, fun) do
-    fun.()
+  defp queue_preview_rows(preview_items_by_type, queue_type, fallback) do
+    preview_items_by_type
+    |> Map.get(queue_type, [])
+    |> Enum.sort_by(&queue_preview_sort_key(queue_type, &1))
+    |> Enum.map(&queue_preview_item/1)
+    |> case do
+      [] -> fallback
+      rows -> rows
+    end
+  end
+
+  defp queue_preview_sort_key(:analyzer, row) do
+    {
+      -normalize_queue_int(row.priority),
+      -normalize_queue_int(row.size),
+      -timestamp(row.inserted_at),
+      -timestamp(row.updated_at)
+    }
+  end
+
+  defp queue_preview_sort_key(:crf_searcher, row) do
+    {
+      -normalize_queue_int(row.priority),
+      -normalize_queue_int(row.bitrate),
+      -normalize_queue_int(row.size),
+      timestamp(row.updated_at)
+    }
+  end
+
+  defp queue_preview_sort_key(:encoder, row) do
+    {
+      -normalize_queue_int(row.priority),
+      -normalize_queue_int(row.savings),
+      -timestamp(row.updated_at)
+    }
+  end
+
+  defp normalize_queue_int(nil), do: 0
+  defp normalize_queue_int(value) when is_integer(value), do: value
+
+  defp timestamp(nil), do: 0
+  defp timestamp(%DateTime{} = dt), do: DateTime.to_unix(dt, :microsecond)
+
+  defp queue_preview_item(row), do: %{id: row.video_id, path: row.path}
+
+  defp fetch_queue_preview_items(timeout) do
+    VideoQueries.dashboard_queue_preview_items(timeout: timeout)
   rescue
     error in [DBConnection.ConnectionError, Exqlite.Error] ->
       Logger.debug(
-        "Dashboard.State #{label} queue preview failed, keeping previous items: #{inspect(error)}"
+        "Dashboard.State queue preview cache read failed, falling back to previous items: #{inspect(error)}"
       )
 
-      fallback
+      []
   catch
     :exit, {:timeout, _} ->
-      Logger.debug("Dashboard.State #{label} queue preview timed out, keeping previous items")
-      fallback
+      Logger.debug(
+        "Dashboard.State queue preview cache read timed out, falling back to previous items"
+      )
+
+      []
   end
 
   defp stats_query_timeout do

lib/reencodarr/media.ex

@@ -1828,6 +1828,7 @@ defmodule Reencodarr.Media do
       avg_duration_minutes: 0.0,
       most_recent_video_update: nil,
       most_recent_inserted_video: nil,
+      encoding_queue_count: 0,
       total_savings_gb: 0.0
     }
   end

lib/reencodarr/media/dashboard_queue_cache.ex

@@ -0,0 +1,17 @@
+defmodule Reencodarr.Media.DashboardQueueCache do
+  use Ecto.Schema
+
+  @primary_key {:id, :id, autogenerate: true}
+
+  schema "dashboard_queue_cache" do
+    field :queue_type, Ecto.Enum, values: [:analyzer, :crf_searcher, :encoder]
+    field :video_id, :integer
+    field :path, :string
+    field :priority, :integer
+    field :bitrate, :integer
+    field :size, :integer
+    field :savings, :integer
+    field :inserted_at, :utc_datetime
+    field :updated_at, :utc_datetime
+  end
+end

lib/reencodarr/media/dashboard_stats_cache.ex

@@ -18,6 +18,7 @@ defmodule Reencodarr.Media.DashboardStatsCache do
     field :most_recent_inserted_video, :utc_datetime
     field :total_vmafs, :integer
     field :chosen_vmafs, :integer
+    field :encoding_queue_count, :integer
     field :encoded_savings_bytes, :integer
     field :predicted_savings_bytes, :integer
   end

lib/reencodarr/media/shared_queries.ex

@@ -107,6 +107,7 @@ defmodule Reencodarr.Media.SharedQueries do
           ),
         most_recent_video_update: c.most_recent_video_update,
         most_recent_inserted_video: c.most_recent_inserted_video,
+        encoding_queue_count: c.encoding_queue_count,
         total_savings_gb: 0.0
       }
   end

lib/reencodarr/media/video_queries.ex

@@ -7,7 +7,7 @@ defmodule Reencodarr.Media.VideoQueries do
   """
 
   import Ecto.Query
-  alias Reencodarr.{Media.Video, Media.Vmaf, Repo}
+  alias Reencodarr.{Media.DashboardQueueCache, Media.Video, Media.Vmaf, Repo}
 
   @doc """
   Gets videos ready for CRF search (state: analyzed).
@@ -76,22 +76,8 @@ defmodule Reencodarr.Media.VideoQueries do
   Gets a lightweight preview of videos needing analysis for dashboard display.
   """
   @spec videos_needing_analysis_preview(integer(), keyword()) :: [map()]
-  def videos_needing_analysis_preview(limit \\ 10, opts \\ []) do
-    Repo.all(
-      from(v in Video,
-        where: v.state == :needs_analysis,
-        order_by: [
-          desc: v.priority,
-          desc: v.size,
-          desc: v.inserted_at,
-          desc: v.updated_at
-        ],
-        limit: ^limit,
-        select: %{id: v.id, path: v.path}
-      ),
-      opts
-    )
-  end
+  def videos_needing_analysis_preview(limit \\ 10, opts \\ []),
+    do: cached_dashboard_queue_preview(:analyzer, limit, opts)
 
   @doc """
   Atomically claims videos for analysis by transitioning them from
@@ -146,17 +132,8 @@ defmodule Reencodarr.Media.VideoQueries do
   Gets a lightweight preview of videos ready for CRF search for dashboard display.
   """
   @spec videos_for_crf_search_preview(integer(), keyword()) :: [map()]
-  def videos_for_crf_search_preview(limit \\ 10, opts \\ []) do
-    Repo.all(
-      from(v in Video,
-        where: v.state == :analyzed,
-        order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at],
-        limit: ^limit,
-        select: %{id: v.id, path: v.path}
-      ),
-      opts
-    )
-  end
+  def videos_for_crf_search_preview(limit \\ 10, opts \\ []),
+    do: cached_dashboard_queue_preview(:crf_searcher, limit, opts)
 
   @doc """
   Gets videos ready for encoding with complex alternation logic between services and libraries.
@@ -184,18 +161,72 @@ defmodule Reencodarr.Media.VideoQueries do
   Gets a lightweight preview of videos ready for encoding for dashboard display.
   """
   @spec videos_ready_for_encoding_preview(integer(), keyword()) :: [map()]
-  def videos_ready_for_encoding_preview(limit, opts \\ []) do
+  def videos_ready_for_encoding_preview(limit, opts \\ []),
+    do: cached_dashboard_queue_preview(:encoder, limit, opts)
+
+  @doc """
+  Returns all cached dashboard queue preview items.
+
+  The dashboard uses this single cache-backed query so queue previews no longer
+  hit the live `videos` table during refresh.
+  """
+  @spec dashboard_queue_preview_items(keyword()) :: [DashboardQueueCache.t()]
+  def dashboard_queue_preview_items(opts \\ []) do
     Repo.all(
-      from(v in Video,
-        where: v.state == :crf_searched and not is_nil(v.chosen_vmaf_id),
-        order_by: [desc: v.priority, desc: v.updated_at],
-        limit: ^limit,
-        select: %{id: v.id, path: v.path}
+      from(q in DashboardQueueCache,
+        where: q.queue_type in [:analyzer, :crf_searcher, :encoder],
+        select: q
       ),
       opts
     )
   end
 
+  defp queue_preview_items(queue_type, opts) do
+    dashboard_queue_preview_items(opts)
+    |> Enum.filter(&(&1.queue_type == queue_type))
+    |> Enum.sort_by(&queue_preview_sort_key(queue_type, &1))
+  end
+
+  defp cached_dashboard_queue_preview(queue_type, limit, opts) do
+    queue_preview_items(queue_type, opts)
+    |> Enum.take(limit)
+    |> Enum.map(&queue_preview_item/1)
+  end
+
+  defp queue_preview_sort_key(:analyzer, row) do
+    {
+      -normalize_int(row.priority),
+      -normalize_int(row.size),
+      -timestamp(row.inserted_at),
+      -timestamp(row.updated_at)
+    }
+  end
+
+  defp queue_preview_sort_key(:crf_searcher, row) do
+    {
+      -normalize_int(row.priority),
+      -normalize_int(row.bitrate),
+      -normalize_int(row.size),
+      timestamp(row.updated_at)
+    }
+  end
+
+  defp queue_preview_sort_key(:encoder, row) do
+    {
+      -normalize_int(row.priority),
+      -normalize_int(row.savings),
+      -timestamp(row.updated_at)
+    }
+  end
+
+  defp normalize_int(nil), do: 0
+  defp normalize_int(value) when is_integer(value), do: value
+
+  defp timestamp(nil), do: 0
+  defp timestamp(%DateTime{} = dt), do: DateTime.to_unix(dt, :microsecond)
+
+  defp queue_preview_item(row), do: %{id: row.video_id, path: row.path}
+
   @doc """
   Counts total videos ready for encoding.
   """