Added far faster random devlogs

Author: jam06452

assets/js/app.js

@@ -563,6 +563,7 @@ function initHomePage() {
 function initProjectsPage() {
   let currentFilter       = "random"
   let allProjects         = []
+  const seenProjectIds    = new Set()
   let displayedCount      = 0
   const itemsPerLoad      = 24
   const defaultFetchLimit = 120
@@ -612,6 +613,11 @@ function initProjectsPage() {
 
   async function fetchProjects(filter, limit) {
     const params = new URLSearchParams({filter, limit: String(limit)})
+
+    if (filter === "random" && seenProjectIds.size > 0) {
+      params.set("exclude_ids", Array.from(seenProjectIds).join(","))
+    }
+
     const response = await fetch(`/api/random_projects?${params.toString()}`)
 
     if (!response.ok) {
@@ -650,6 +656,8 @@ function initProjectsPage() {
     hideEmpty()
 
     batch.forEach((p, idx) => {
+      seenProjectIds.add(String(p.id))
+
       const card = document.createElement("div")
       card.className = "project-card"
       card.onclick = () => viewProject(p.id)

lib/ftpdb/DB.ex

@@ -1,6 +1,10 @@
 defmodule Ftpdb.DB do
   require Logger
 
+  @random_project_batch_size 100
+  @random_project_cache_ttl :timer.minutes(30)
+  @default_project_banner_url "https://flavortown.hackclub.com/assets/default-banner-3d4e1b67.png"
+
   defp project_duration_fields(duration_seconds) do
     duration_seconds = duration_seconds || 0
 
@@ -258,23 +262,61 @@ defmodule Ftpdb.DB do
     end)
   end
 
-  defp get_max_project_id do
-    {:ok, response} =
-      Supabase.PostgREST.from(client(), "projects")
-      |> Supabase.PostgREST.select(["id"])
-      |> Supabase.PostgREST.order("id", desc: true)
-      |> Supabase.PostgREST.limit(1)
-      |> Map.put(:method, :get)
-      |> Supabase.PostgREST.execute()
+  def random_projects(limit \\ 10, excluded_project_ids \\ [])
 
-    case response.body do
-      [%{"id" => max_id}] -> max_id
-      _ -> 0
+  def random_projects(limit, excluded_project_ids)
+      when is_integer(limit) and limit > 0 and is_list(excluded_project_ids) do
+    excluded_project_ids =
+      excluded_project_ids
+      |> Enum.map(&to_string/1)
+      |> MapSet.new()
+
+    limit
+    |> build_random_project_pool(0, [], excluded_project_ids)
+    |> Enum.take_random(limit)
+  end
+
+  def random_projects(_limit, _excluded_project_ids), do: []
+
+  defp build_random_project_pool(limit, _batch_index, acc, _excluded_project_ids)
+       when length(acc) >= limit,
+       do: acc
+
+  defp build_random_project_pool(limit, batch_index, acc, excluded_project_ids) do
+    batch =
+      Cachex.fetch!(
+        :random_project_cache,
+        random_project_batch_key(batch_index),
+        fn _key -> fetch_recent_project_batch(batch_index) end,
+        expiration: @random_project_cache_ttl
+      )
+
+    case batch do
+      [] ->
+        acc
+
+      projects ->
+        filtered_projects =
+          Enum.reject(projects, fn project ->
+            MapSet.member?(excluded_project_ids, to_string(project.id))
+          end)
+
+        build_random_project_pool(
+          limit,
+          batch_index + 1,
+          acc ++ filtered_projects,
+          excluded_project_ids
+        )
     end
   end
 
-  defp fetch_and_format_project_for_random(id) do
-    result =
+  defp random_project_batch_key(batch_index), do: "recent_projects_batch:#{batch_index}"
+
+  defp fetch_recent_project_batch(batch_index) do
+    start_index = batch_index * @random_project_batch_size
+    end_index = start_index + @random_project_batch_size - 1
+
+    {:ok, response} =
       Supabase.PostgREST.from(client(), "projects")
       |> Supabase.PostgREST.select([
         "id",
@@ -284,84 +326,84 @@ defmodule Ftpdb.DB do
         "stat_hot_score",
         "stat_total_likes"
       ])
-      |> Supabase.PostgREST.eq("id", id)
+      |> Supabase.PostgREST.order("id", desc: true)
+      |> Supabase.PostgREST.range(start_index, end_index)
       |> Map.put(:method, :get)
       |> Supabase.PostgREST.execute()
 
-    case result do
-      {:ok, %{body: [item]}} when not is_nil(item) ->
-        banner_url = item["banner_url"]
-
-        if is_nil(banner_url) or
-             banner_url == "https://flavortown.hackclub.com/assets/default-banner-3d4e1b67.png" do
-          nil
-        else
-          user_id = get_user_id(item["id"])
-
-          {user_avatar, user_display_name} =
-            if user_id do
-              case get_user_info(user_id) do
-                [u] -> {u.avatar_url, u.display_name || "Unknown User"}
-                _ -> {nil, "Unknown User"}
-              end
-            else
-              {nil, "Unknown User"}
-            end
-
-          duration = item["stat_total_duration_seconds"] || 0
-
-          %{
-            id: to_string(item["id"]),
-            title: item["title"],
-            banner_url: banner_url,
-            display_name: user_display_name,
-            avatar_url: user_avatar,
-            stat_hot_score: item["stat_hot_score"] || 0,
-            stat_total_likes: item["stat_total_likes"] || 0,
-            total_duration_seconds: duration,
-            total_hours: div(duration, 3600)
-          }
-        end
-
-      _ ->
-        nil
-    end
+    response.body
+    |> List.wrap()
+    |> Task.async_stream(&format_recent_project_for_random/1,
+      max_concurrency: System.schedulers_online() * 2,
+      ordered: false,
+      timeout: :infinity
+    )
+    |> Enum.reduce([], fn
+      {:ok, nil}, acc ->
+        acc
+
+      {:ok, project}, acc ->
+        [project | acc]
+
+      {:exit, reason}, acc ->
+        Logger.warning("Failed to build random project batch #{batch_index}: #{inspect(reason)}")
+        acc
+    end)
+    |> Enum.reverse()
   end
 
-  def random_projects(limit \\ 10) do
-    max_id = get_max_project_id()
+  defp format_recent_project_for_random(item) do
+    banner_url = item["banner_url"]
 
-    if max_id == 0 do
-      []
+    if is_nil(banner_url) or banner_url == @default_project_banner_url do
+      nil
     else
-      random_ids =
-        1..(limit * 10)
-        |> Enum.map(fn _ -> :rand.uniform(max_id) end)
-        |> Enum.uniq()
-
-      random_ids
-      |> Enum.reduce_while([], fn id, acc ->
-        if length(acc) >= limit do
-          {:halt, acc}
-        else
-          project =
-            Cachex.fetch!(
-              :random_project_cache,
-              to_string(id),
-              fn _key -> fetch_and_format_project_for_random(id) end,
-              expiration: :timer.minutes(30)
-            )
+      user_id = cached_user_id(item["id"])
 
-          if project do
-            {:cont, [project | acc]}
-          else
-            {:cont, acc}
+      {user_avatar, user_display_name} =
+        if user_id do
+          case cached_user_info(user_id) do
+            [user_info] -> {user_info.avatar_url, user_info.display_name || "Unknown User"}
+            _ -> {nil, "Unknown User"}
           end
+        else
+          {nil, "Unknown User"}
         end
-      end)
+
+      duration = item["stat_total_duration_seconds"] || 0
+
+      %{
+        id: to_string(item["id"]),
+        title: item["title"],
+        banner_url: banner_url,
+        display_name: user_display_name,
+        avatar_url: user_avatar,
+        stat_hot_score: item["stat_hot_score"] || 0,
+        stat_total_likes: item["stat_total_likes"] || 0,
+        total_duration_seconds: duration,
+        total_hours: div(duration, 3600)
+      }
     end
   end
 
+  defp cached_user_id(project_id) do
+    Cachex.fetch!(
+      :user_id_cache,
+      to_string(project_id),
+      fn _key -> get_user_id(project_id) end,
+      expiration: @random_project_cache_ttl
+    )
+  end
+
+  defp cached_user_info(user_id) do
+    Cachex.fetch!(
+      :user_cache,
+      to_string(user_id),
+      fn _key -> get_user_info(user_id) end,
+      expiration: @random_project_cache_ttl
+    )
+  end
+
   def search_projects(query) when is_binary(query) do
     cleaned_query = String.trim(query)
 

lib/ftpdb_web/controllers/api_controller.ex

@@ -97,6 +97,7 @@ defmodule FtpdbWeb.ApiController do
 
   def random_projects(conn, %{"filter" => filter} = params) do
     limit = String.to_integer(Map.get(params, "limit", "10"))
+    excluded_project_ids = parse_excluded_project_ids(params)
 
     case filter do
       "hottest" ->
@@ -109,13 +110,20 @@ defmodule FtpdbWeb.ApiController do
         json(conn, Ftpdb.DB.most_active_projects(limit))
 
       _ ->
-        json(conn, Ftpdb.DB.random_projects(limit))
+        json(conn, Ftpdb.DB.random_projects(limit, excluded_project_ids))
     end
   end
 
   def random_projects(conn, params) do
     limit = String.to_integer(Map.get(params, "limit", "10"))
-    json(conn, Ftpdb.DB.random_projects(limit))
+    excluded_project_ids = parse_excluded_project_ids(params)
+    json(conn, Ftpdb.DB.random_projects(limit, excluded_project_ids))
+  end
+
+  defp parse_excluded_project_ids(params) do
+    params
+    |> Map.get("exclude_ids", "")
+    |> String.split(",", trim: true)
   end
 
   def random_devlogs(conn, _params) do