perf: push all catalog filtering into sqlite, remove JS full-table load

The subscriptions editor was loading 20k+ all_courses rows into a JS
array via chunked setTimeouts, then filtering in-memory getters that
re-ran on every Alpine reactive tick. This froze the browser for 10+
seconds and caused stale state bugs when toggling terms.

Now searchAllCourses / countAllCourses / getCoursesByIds /
getAllCoursesDepts all query sqlite directly with WHERE clauses, so
the JS heap never holds the full catalog. The middle column is capped
at 200 rows with a "refine your search" hint. Cache arrays for the
subscribed and single-run columns are explicitly refreshed when IDs
change, decoupling them from the filter state.

Author: LeafCreeper

frontend/index.html

@@ -481,7 +481,7 @@ <h4 class="font-medium text-gray-800 text-sm mt-0.5" x-text="r.sub_title || 'Unt
               <span class="float-right ml-2">▾</span>
             </button>
             <div x-show="subsDeptOpen" class="absolute z-10 mt-1 bg-white border rounded-lg shadow-lg max-h-64 overflow-y-auto min-w-[240px]">
-              <input x-model="deptSearchQuery" @input="rebuildSubsFiltered()" type="text"
+              <input x-model="deptSearchQuery" @input="onDeptSearchInput()" type="text"
                      placeholder="搜索院系…"
                      class="w-full border-b px-3 py-2 text-sm sticky top-0 bg-white focus:outline-none">
               <template x-for="d in subsDeptFiltered" :key="d">
@@ -555,7 +555,10 @@ <h3 class="text-sm font-semibold text-gray-700">已订阅</h3>
           <div class="flex-[2] min-w-0 flex flex-col">
             <div class="flex items-center justify-between px-2 py-1.5">
               <h3 class="text-sm font-semibold text-gray-700">搜索结果</h3>
-              <span class="text-xs text-gray-400" x-text="subsFiltered.length + '门'"></span>
+              <span class="text-xs text-gray-400"
+                    x-text="subsFilteredTotal > subsFiltered.length
+                            ? (subsFiltered.length + '/' + subsFilteredTotal + '门')
+                            : (subsFiltered.length + '门')"></span>
             </div>
             <div class="bg-white border border-gray-200 rounded-lg overflow-y-auto max-h-[55vh] divide-y divide-gray-100 flex-1">
               <template x-for="c in subsFiltered" :key="c.course_id + '@' + c.term">
@@ -578,6 +581,11 @@ <h3 class="text-sm font-semibold text-gray-700">搜索结果</h3>
                 </label>
               </template>
               <p x-show="subsFiltered.length === 0" class="text-center text-gray-400 text-xs py-8">没有匹配的课程</p>
+              <p x-show="subsFilteredTotal > subsFiltered.length"
+                 class="text-center text-gray-400 text-xs py-3 bg-gray-50 border-t border-gray-100">
+                仅显示前 <span x-text="subsFiltered.length"></span> 条（共
+                <span x-text="subsFilteredTotal"></span> 条），请细化搜索条件。
+              </p>
             </div>
           </div>
 

frontend/js/app.js

@@ -252,14 +252,19 @@ document.addEventListener("alpine:init", () => {
     /* Subscriptions editor state — three-column layout:
        left (subscribed) | middle (catalog search) | right (single-run).
        The left column persists to GitHub Secret on demand; the right
-       column is session-only and cleared after triggering a workflow. */
-    allCourses: [], allCoursesTerms: [],
+       column is session-only and cleared after triggering a workflow.
+       Columns are driven by SQL queries against the in-memory shard DB
+       so we never hold the full 20k-row catalog in JS. */
+    allCoursesTerms: [],
     subsTerms: [], subsDepts: [], deptSearchQuery: "",
     subsSearchTitle: "", subsSearchTeacher: "",
     subsTermOpen: false, subsDeptOpen: false,
     subscribedIds: [], singleRunIds: [],
     subsSelLeft: [], subsSelMiddle: [], subsSelRight: [],
-    subsFiltered: [],
+    subsFiltered: [], subsFilteredTotal: 0,
+    subsLimit: 200,
+    _subsSubscribedCache: [], _subsSingleRunCache: [], _subsDeptCache: [],
+    _subsFilterTimer: null, _deptFilterTimer: null,
     subsSaving: false, subsError: "",
     singleRunTriggering: false,
     /* Per-browser pinned-courses set, lazily synced to localStorage. */
@@ -613,13 +618,9 @@ document.addEventListener("alpine:init", () => {
 
     // ── Subscriptions editor (three-column) ──────────────────────────
     openSubscriptions() {
-      // Debug: log entry
-      console.log("openSubscriptions called, view=", this.view);
-      // Enter the page FIRST — before any DB work.
       try {
         this._go("subscriptions");
-        console.log("_go returned, view now=", this.view);
-      } catch(e) {
+      } catch (e) {
         console.error("_go failed:", e);
         return;
       }
@@ -632,13 +633,17 @@ document.addEventListener("alpine:init", () => {
       this.subsSearchTeacher = "";
       this.subsTermOpen = false;
       this.subsDeptOpen = false;
-      this.allCourses = [];
       this.subsFiltered = [];
+      this.subsFilteredTotal = 0;
       this.singleRunIds = [];
       this.subsSelLeft = [];
       this.subsSelMiddle = [];
       this.subsSelRight = [];
+      this._subsSubscribedCache = [];
+      this._subsSingleRunCache = [];
+      this._subsDeptCache = [];
       this.subsError = "";
+
       // Load subscription (localStorage → meta table fallback)
       this.subscribedIds = [];
       try {
@@ -661,45 +666,22 @@ document.addEventListener("alpine:init", () => {
           } catch {}
         }
       }
-      // Background-load the catalog after the page has rendered
-      var self = this;
-      setTimeout(function () { self._loadCoursesForTerms(); }, 200);
+      this._refreshSubscribedCache();
+      this._refreshSingleRunCache();
+      this._refreshDeptCache();
+      this.rebuildSubsFiltered();
     },
-    // ── Async batch-load courses to avoid freezing the UI ───────────
-    _loadCoursesForTerms() {
-      var self = this;
-      var terms = this.subsTerms.length
-        ? this.subsTerms
-        : this.allCoursesTerms;
-
-      var tIdx = 0;
-      var CHUNK = 200;
-      var skipCount = 0;
-
-      function nextTerm() {
-        if (tIdx >= terms.length) return;
-        var rows = ICS.db.getAllCourses(terms[tIdx]);
-        tIdx++;
-        feed(0, rows);
-      }
-
-      function feed(offset, rows) {
-        var sub = rows.slice(offset, offset + CHUNK);
-        self.allCourses = self.allCourses.concat(sub);
-        // Only re-filter every 3 chunks (~600 rows) to reduce DOM churn
-        skipCount++;
-        if (skipCount % 3 === 0) {
-          self.rebuildSubsFiltered();
-        }
-        if (offset + CHUNK >= rows.length) {
-          self.rebuildSubsFiltered();  // Final flush
-          setTimeout(nextTerm, 80);
-        } else {
-          setTimeout(function () { feed(offset + CHUNK, rows); }, 80);
-        }
-      }
-
-      setTimeout(nextTerm, 300);  // Wait 300ms after page render
+    // ── Cache refreshers (called explicitly when underlying IDs change) ─
+    _refreshSubscribedCache() {
+      this._subsSubscribedCache = ICS.db.getCoursesByIds(this.subscribedIds);
+    },
+    _refreshSingleRunCache() {
+      this._subsSingleRunCache = ICS.db.getCoursesByIds(this.singleRunIds);
+    },
+    _refreshDeptCache() {
+      this._subsDeptCache = ICS.db.getAllCoursesDepts(
+        this.subsTerms, this.deptSearchQuery,
+      );
     },
     // ── Term badge color (cyclic palette for the search results) ─────
     _TERM_COLORS: [
@@ -716,15 +698,9 @@ document.addEventListener("alpine:init", () => {
       for (var i = 0; i < term.length; i++) idx = (idx * 31 + term.charCodeAt(i)) | 0;
       return this._TERM_COLORS[Math.abs(idx) % this._TERM_COLORS.length];
     },
-    // ── Column data ─────────────────────────────────────────────────
-    get subscribedCourses() {
-      var ids = new Set(this.subscribedIds.map(String));
-      return this.allCourses.filter(function (c) { return ids.has(String(c.course_id)); });
-    },
-    get singleRunCourses() {
-      var ids = new Set(this.singleRunIds.map(String));
-      return this.allCourses.filter(function (c) { return ids.has(String(c.course_id)); });
-    },
+    // ── Column data (plain arrays kept in sync by cache refreshers) ─
+    get subscribedCourses() { return this._subsSubscribedCache; },
+    get singleRunCourses() { return this._subsSingleRunCache; },
     // ── Dropdown labels ─────────────────────────────────────────────
     get subsTermLabel() {
       if (!this.subsTerms.length) return '全部学期';
@@ -734,39 +710,46 @@ document.addEventListener("alpine:init", () => {
       if (!this.subsDepts.length) return '全部院系';
       return this.subsDepts.length + '个院系';
     },
-    get subsDeptFiltered() {
-      var q = (this.deptSearchQuery || '').toLowerCase();
-      var deptSet = new Set();
-      for (var i = 0; i < this.allCourses.length; i++) {
-        var d = this.allCourses[i].dept;
-        if (d && (!q || d.toLowerCase().indexOf(q) !== -1)) deptSet.add(d);
-      }
-      return Array.from(deptSet).sort();
+    get subsDeptFiltered() { return this._subsDeptCache; },
+    onDeptSearchInput() {
+      // Debounced — typing in the dept search box requeries distinct depts
+      // from sqlite filtered by term + substring.
+      var self = this;
+      clearTimeout(this._deptFilterTimer);
+      this._deptFilterTimer = setTimeout(function () {
+        self._refreshDeptCache();
+      }, 150);
     },
     // ── Toggle multi-select ─────────────────────────────────────────
     toggleSubsTerm(term, checked) {
       var s = new Set(this.subsTerms);
       if (checked) s.add(term); else s.delete(term);
       this.subsTerms = Array.from(s);
-      this._loadCoursesForTerms();
+      // Term change invalidates both the dept dropdown (term-narrowed)
+      // and the middle column (different rows match).
+      this._refreshDeptCache();
+      this.rebuildSubsFiltered();
     },
     toggleSubsDept(dept, checked) {
       var s = new Set(this.subsDepts);
       if (checked) s.add(dept); else s.delete(dept);
       this.subsDepts = Array.from(s);
       this.rebuildSubsFiltered();
     },
-    // ── Filter middle column ─────────────────────────────────────────
+    // ── Filter middle column (debounced SQL query) ───────────────────
     rebuildSubsFiltered() {
-      var deptSet = new Set(this.subsDepts.map(function (d) { return d.toLowerCase(); }));
-      var titleQ = (this.subsSearchTitle || "").trim().toLowerCase();
-      var teacherQ = (this.subsSearchTeacher || "").trim().toLowerCase();
-      this.subsFiltered = this.allCourses.filter(function (c) {
-        if (deptSet.size && (!c.dept || !deptSet.has(c.dept.toLowerCase()))) return false;
-        if (titleQ && (!c.title || c.title.toLowerCase().indexOf(titleQ) === -1)) return false;
-        if (teacherQ && (!c.teacher || c.teacher.toLowerCase().indexOf(teacherQ) === -1)) return false;
-        return true;
-      });
+      var self = this;
+      clearTimeout(this._subsFilterTimer);
+      this._subsFilterTimer = setTimeout(function () {
+        var filters = {
+          terms: self.subsTerms,
+          depts: self.subsDepts,
+          title: self.subsSearchTitle,
+          teacher: self.subsSearchTeacher,
+        };
+        self.subsFiltered = ICS.db.searchAllCourses(filters, self.subsLimit);
+        self.subsFilteredTotal = ICS.db.countAllCourses(filters);
+      }, 150);
     },
     // ── Per-column multi-select ──────────────────────────────────────
     _toggleSel(arr, id, checked) {
@@ -791,27 +774,31 @@ document.addEventListener("alpine:init", () => {
       for (var i = 0; i < selected.length; i++) target.add(selected[i]);
       this.subscribedIds = Array.from(target);
       this.subsSelMiddle = [];
+      this._refreshSubscribedCache();
     },
     moveFromSubscribed() {
       var target = new Set(this.subscribedIds.map(String));
       var selected = this.subsSelLeft;
       for (var i = 0; i < selected.length; i++) target.delete(selected[i]);
       this.subscribedIds = Array.from(target);
       this.subsSelLeft = [];
+      this._refreshSubscribedCache();
     },
     moveToSingleRun() {
       var target = new Set(this.singleRunIds.map(String));
       var selected = this.subsSelMiddle;
       for (var i = 0; i < selected.length; i++) target.add(selected[i]);
       this.singleRunIds = Array.from(target);
       this.subsSelMiddle = [];
+      this._refreshSingleRunCache();
     },
     moveFromSingleRun() {
       var target = new Set(this.singleRunIds.map(String));
       var selected = this.subsSelRight;
       for (var i = 0; i < selected.length; i++) target.delete(selected[i]);
       this.singleRunIds = Array.from(target);
       this.subsSelRight = [];
+      this._refreshSingleRunCache();
     },
     // ── Save left column to GitHub Secret ────────────────────────────
     async saveSubscriptions() {

frontend/js/db.js

@@ -209,6 +209,109 @@ function _getAllCoursesTerms() {
   ).map((r) => r.term);
 }
 
+function _buildCatalogWhere(filters) {
+  // Shared WHERE/params builder for paged search + count + dept distinct.
+  // Filters: { terms: string[], depts: string[], title: string, teacher: string }
+  var clauses = [];
+  var params = [];
+  if (filters.terms && filters.terms.length) {
+    clauses.push("term IN (" + filters.terms.map(function () { return "?"; }).join(",") + ")");
+    for (var i = 0; i < filters.terms.length; i++) params.push(filters.terms[i]);
+  }
+  if (filters.depts && filters.depts.length) {
+    clauses.push("dept IN (" + filters.depts.map(function () { return "?"; }).join(",") + ")");
+    for (var j = 0; j < filters.depts.length; j++) params.push(filters.depts[j]);
+  }
+  if (filters.title && filters.title.trim()) {
+    clauses.push("title LIKE ?");
+    params.push("%" + filters.title.trim() + "%");
+  }
+  if (filters.teacher && filters.teacher.trim()) {
+    clauses.push("teacher LIKE ?");
+    params.push("%" + filters.teacher.trim() + "%");
+  }
+  return {
+    where: clauses.length ? "WHERE " + clauses.join(" AND ") : "",
+    params: params,
+  };
+}
+
+function _searchAllCourses(filters, limit) {
+  // Paged catalog search — used by the subscriptions editor middle column.
+  // Pushes all filtering into sqlite so the JS heap never holds the full
+  // 20k-row catalog.
+  var w = _buildCatalogWhere(filters || {});
+  var sql = "SELECT course_id, term, title, teacher, dept FROM all_courses "
+          + w.where + " ORDER BY term DESC, title LIMIT ?";
+  var p = w.params.slice();
+  p.push(limit || 200);
+  return _queryAll(sql, p);
+}
+
+function _countAllCourses(filters) {
+  var w = _buildCatalogWhere(filters || {});
+  var rows = _queryAll("SELECT COUNT(*) AS n FROM all_courses " + w.where, w.params);
+  return rows.length ? rows[0].n : 0;
+}
+
+function _getCoursesByIds(ids) {
+  // Look up the catalog rows for an arbitrary set of course_ids.  Used by
+  // the left ("已订阅") and right ("单次运行") columns so they can render
+  // without holding the full catalog in JS.  Deduplicates by course_id;
+  // a course offered in multiple terms is shown with its most recent term.
+  if (!ids || !ids.length) return [];
+  var placeholders = ids.map(function () { return "?"; }).join(",");
+  // Pull every term row that matches, then collapse to one per course_id
+  // (preferring the most recent term) in JS — keeps the SQL simple.
+  var rows = _queryAll(
+    "SELECT course_id, term, title, teacher, dept FROM all_courses "
+    + "WHERE course_id IN (" + placeholders + ") ORDER BY term DESC",
+    ids.map(String),
+  );
+  var seen = {};
+  var out = [];
+  for (var i = 0; i < rows.length; i++) {
+    var cid = String(rows[i].course_id);
+    if (seen[cid]) continue;
+    seen[cid] = true;
+    out.push(rows[i]);
+  }
+  // Synthesize empty placeholders for IDs the catalog doesn't know about,
+  // so the count shown to the user matches what's actually in their list.
+  var foundIds = {};
+  for (var k = 0; k < out.length; k++) foundIds[String(out[k].course_id)] = true;
+  for (var m = 0; m < ids.length; m++) {
+    var idStr = String(ids[m]);
+    if (!foundIds[idStr]) {
+      out.push({ course_id: idStr, term: "", title: "", teacher: "", dept: "" });
+      foundIds[idStr] = true;
+    }
+  }
+  return out;
+}
+
+function _getAllCoursesDepts(termFilter, search) {
+  // Distinct dept list, optionally narrowed to a set of terms and a
+  // case-insensitive substring search.  Lets the dept dropdown stay
+  // responsive without iterating the JS catalog.
+  var clauses = ["dept IS NOT NULL", "dept != ''"];
+  var params = [];
+  if (termFilter && termFilter.length) {
+    clauses.push("term IN (" + termFilter.map(function () { return "?"; }).join(",") + ")");
+    for (var i = 0; i < termFilter.length; i++) params.push(termFilter[i]);
+  }
+  if (search && search.trim()) {
+    clauses.push("LOWER(dept) LIKE ?");
+    params.push("%" + search.trim().toLowerCase() + "%");
+  }
+  var rows = _queryAll(
+    "SELECT DISTINCT dept FROM all_courses WHERE "
+    + clauses.join(" AND ") + " ORDER BY dept",
+    params,
+  );
+  return rows.map(function (r) { return r.dept; });
+}
+
 function _getSubscribedCourseIds() {
   // The ``courses`` table holds courses we've actually run.  This is our
   // best signal of "currently subscribed" without reading the
@@ -233,6 +336,10 @@ window.ICS.db = {
   searchSummaries: _searchSummaries,
   getAllCourses: _getAllCourses,
   getAllCoursesTerms: _getAllCoursesTerms,
+  searchAllCourses: _searchAllCourses,
+  countAllCourses: _countAllCourses,
+  getCoursesByIds: _getCoursesByIds,
+  getAllCoursesDepts: _getAllCoursesDepts,
   getSubscribedCourseIds: _getSubscribedCourseIds,
   getMeta: _getMeta,
 };