Address review: project curves column only when displayed (LMFDB#6975)

The curves column is default-off but was added to the fixed db_cols
projection, so every abvar/Fq search (and every download) fetched and
deserialized the full curve arrays that the user never sees - up to
~2.2MB on the worst realistic browse page (g=2, q=167).

Drop "curves" from the static db_cols and fetch it only when the column
is actually shown: the search sets __projection__ to include curves when
curves_requested(info) is true, and AbvarFq_download overrides a new
Downloader.get_projection hook (a behavior-preserving extraction of the
existing projection logic) to add curves only when it is in the download.

Verified with the flask test client: default searches/downloads no longer
project curves (worst page drops from 33 to 32 columns, saving 2.2MB),
while showcol=curves still displays and downloads the equations; all 20
abvar tests plus number-field and ECNF download tests pass; pyflakes clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: roed314

lmfdb/abvar/fq/download.py

@@ -29,5 +29,14 @@ def download_curves(self, label, lang='text'):
                           lang=lang,
                           title='Curves in abelian variety isogeny class %s,' % (label))
 
+    def get_projection(self, info, table, columns):
+        proj = super().get_projection(info, table, columns)
+        # The curves arrays are large, so only fetch them when the Curves column is
+        # actually included in the download (see LMFDB#6975).
+        if ("curves" in table.search_cols and "curves" not in proj
+                and any(col.name == "curves" and col.default(info) for col in columns.columns)):
+            proj = proj + ["curves"]
+        return proj
+
     def postprocess(self, rec, info, query):
         return AbvarFq_isoclass(rec)

lmfdb/abvar/fq/main.py

@@ -714,7 +714,15 @@ def extended_code(c):
     ProcessedCol("number_fields", "av.fq.number_field", "Number fields", lambda nfs: ", ".join(nf_display_knowl(nf, field_pretty(nf)) for nf in nfs), default=False),
     SearchCol("galois_groups_pretty", "nf.galois_group", "Galois groups", download_col="galois_groups", default=False),
     SearchCol("decomposition_display_search", "av.decomposition", "Isogeny factors", download_col="decompositionraw")],
-    db_cols=["label", "g", "q", "poly", "p_rank", "p_rank_deficit", "is_simple", "is_geometrically_simple", "simple_distinct", "simple_multiplicities", "is_primitive", "primitive_models", "curve_count", "curve_counts", "curves", "abvar_count", "abvar_counts", "jacobian_count", "hyp_count", "number_fields", "galois_groups", "slopes", "newton_elevation", "twist_count", "max_twist_degree", "geometric_extension_degree", "angle_rank", "angle_corank", "is_supersingular", "has_principal_polarization", "has_jacobian", "is_cyclic", "noncyclic_primes"])
+    db_cols=["label", "g", "q", "poly", "p_rank", "p_rank_deficit", "is_simple", "is_geometrically_simple", "simple_distinct", "simple_multiplicities", "is_primitive", "primitive_models", "curve_count", "curve_counts", "abvar_count", "abvar_counts", "jacobian_count", "hyp_count", "number_fields", "galois_groups", "slopes", "newton_elevation", "twist_count", "max_twist_degree", "geometric_extension_degree", "angle_rank", "angle_corank", "is_supersingular", "has_principal_polarization", "has_jacobian", "is_cyclic", "noncyclic_primes"])
+
+# The curves column holds potentially long lists of equations (up to ~6000 per row, and
+# 2+ MB per page of search results), so - unlike the other columns - it is left out of the
+# fixed db_cols projection above and only fetched from the database when the column is
+# actually displayed or included in a download.  See LMFDB#6975.
+def curves_requested(info):
+    """Whether the (heavy, default-off) curves column is included in the current display/download."""
+    return any(col.name == "curves" and col.default(info) for col in abvar_columns.columns)
 
 def abvar_postprocess(res, info, query):
     gals = set()
@@ -742,6 +750,9 @@ def abvar_postprocess(res, info, query):
 )
 def abelian_variety_search(info, query):
     common_parse(info, query)
+    if curves_requested(info):
+        # Fetch the (heavy) curves arrays only when the Curves column is displayed (LMFDB#6975).
+        query["__projection__"] = abvar_columns.db_cols + ["curves"]
 
 @count_wrap(
     template="abvarfq-count-results.html",

lmfdb/utils/downloader.py

@@ -605,6 +605,20 @@ def modify_query(self, info, query):
         """
         pass
 
+    def get_projection(self, info, table, columns):
+        """
+        This is a hook for downloaders to determine the database projection based on info.
+
+        By default it fetches the columns in ``columns.db_cols`` that are present in the
+        search table.  It is overridden when a heavy column should only be fetched when it
+        is actually included in the download (see abelian varieties over Fq, LMFDB#6975).
+        """
+        # It's fairly common to add virtual columns in postprocessing that are then used in MultiProcessedCols.
+        # These virtual columns are often only used in display code and won't be present in the database, so we just strip them out
+        if isinstance(columns.db_cols, list):
+            return [col for col in columns.db_cols if col in table.search_cols]
+        return columns.db_cols # some tables use 1 for project-to-all
+
     def get_sort(self, info, query):
         """
         This determines the sort order requested from the database.
@@ -705,12 +719,7 @@ def __call__(self, info):
 
         # Determine which columns will be fetched from the database
         columns = info["columns"]
-        # It's fairly common to add virtual columns in postprocessing that are then used in MultiProcessedCols.
-        # These virtual columns are often only used in display code and won't be present in the database, so we just strip them out
-        if isinstance(columns.db_cols, list):
-            proj = [col for col in columns.db_cols if col in table.search_cols]
-        else:
-            proj = columns.db_cols # some tables use 1 for project-to-all
+        proj = self.get_projection(info, table, columns)
 
         # Extract the query and modify it
         try: