Address review: pin the Conway generator convention and test the projection (LMFDB#6975)

The download description for the Curves column said only that "a" denotes
a generator of the base field over its prime field, which does not pin
down the stored equations: a finite field has many generators, and a
different choice changes the coefficients.  Give the same convention the
isogeny class homepage does, namely that a is the image of x in
GF(p)[x]/(C(x)) for C the degree r Conway polynomial used by the LMFDB,
and stop conflating the two missing-data states: null means the curves
have not been computed, while an empty list means the computation was
carried out and there are no such curves (spot checks confirm this
reading: whole (g, q) blocks are either uncomputed or fully computed, and
in the computed blocks the empty lists fall exactly on the classes
recorded as containing no Jacobian).

Also cover the conditional projection with tests.  test_curves_projection
pins both directions - curves is absent from db_cols, curves_requested is
false by default and true with showcol=curves, the search sets
__projection__ only when the column is shown, AbvarFq_download.get_projection
adds curves only then, and the generic Downloader.get_projection still
just restricts db_cols to the table's columns.  test_curves_not_computed
replaces the weak "label appears on the page" check with assertions on
the rendered Curves cells and on the downloaded values, and
test_curves_search_column now checks the raw equations plus the Conway
polynomial wording in every non-CSV language, gp included since that is
what the issue asked for.

Verified with sage -python: 22 abvar/fq tests pass (20 before, two new),
number-field and local-field download tests pass, pyflakes and ruff clean,
and sage/gp downloads inspected by hand for a computed row, a NULL row
(None/null) and an empty list row ([]).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: roed314

lmfdb/abvar/fq/main.py

@@ -707,7 +707,12 @@ def extended_code(c):
     MathCol("jacobian_count", "av.jacobian_count", "Jacobians", default=False),
     MathCol("hyp_count", "av.hyperelliptic_count", "Hyperelliptic Jacobians", default=False),
     ProcessedCol("curves", "ag.jacobian", "Curves", curves_display_search, default=False,
-                 download_desc="Equations of curves whose Jacobians are in the isogeny class.  For a base field GF(p^r) with r > 1, the variable a denotes a generator of the base field over its prime field.  The list may be empty or missing if no such curves are known."),
+                 download_desc="Equations of curves whose Jacobians are in the isogeny class.\n"
+                 "For a base field GF(p^r) with r > 1, the variable a is the image of x in GF(p)[x]/(C(x)),\n"
+                 "where C is the degree r Conway polynomial over GF(p) used by the LMFDB;\n"
+                 "equivalently, a is a root of that Conway polynomial.\n"
+                 "A null value means that the curves in this isogeny class have not been computed;\n"
+                 "an empty list means that the computation was carried out and there are no such curves."),
     MathCol("twist_count", "av.twist", "Num. twists", default=False),
     MathCol("max_twist_degree", "av.twist", "Max. twist degree", default=False),
     MathCol("geometric_extension_degree", "av.endomorphism_field", "End. degree", default=False),

lmfdb/abvar/fq/test_av.py

@@ -169,12 +169,91 @@ def test_curves_search_column(self):
         page = self.tc.get("/Variety/Abelian/Fq/?q=4&g=1&showcol=curves").get_data(as_text=True)
         assert "$y^2+y=x^3+a$" in page
         assert "$y^2+a y=x^3$ and 1 more" in page
-        # Search downloads contain the full lists
-        data = self.tc.get("/Variety/Abelian/Fq/?q=4&g=1&showcol=curves&Submit=sage&download=1&query=%7B%27q%27%3A+4%2C+%27g%27%3A+1%7D").get_data(as_text=True)
-        assert '["y^2+a*y=x^3", "y^2+(a+1)*y=x^3"]' in data
-        # Classes for which curves have not been computed display an empty cell
+        # Search downloads contain the full lists of raw equations, over a non-prime
+        # base field as well, and say which generator a of that field is meant.
+        # The gp download matters for LMFDB#6975, which asked for these equations in PARI/GP.
+        query = "query=%7B%27q%27%3A+4%2C+%27g%27%3A+1%7D"
+        for lang in ["sage", "gp", "magma", "oscar", "gap", "text"]:
+            data = self.tc.get("/Variety/Abelian/Fq/?q=4&g=1&showcol=curves&download=1&Submit=%s&%s" % (lang, query)).get_data(as_text=True)
+            assert '["y^2+a*y=x^3", "y^2+(a+1)*y=x^3"]' in data
+            assert "Conway polynomial" in data
+        # csv files hold the equations but cannot carry the explanatory comments
+        data = self.tc.get("/Variety/Abelian/Fq/?q=4&g=1&showcol=curves&download=1&Submit=csv&" + query).get_data(as_text=True)
+        assert "y^2+a*y=x^3" in data
+        assert "Conway polynomial" not in data
+
+    def test_curves_projection(self):
+        r"""
+        Check that the (heavy) curves arrays are fetched from the database only when the
+        Curves column is actually displayed or downloaded (LMFDB#6975).
+        """
+        from lmfdb.utils import Downloader
+        from lmfdb.abvar.fq.download import AbvarFq_download
+        from lmfdb.abvar.fq.main import abelian_variety_search, abvar_columns, curves_requested
+
+        # curves is deliberately absent from the fixed projection used by every search
+        assert "curves" not in abvar_columns.db_cols
+        assert not curves_requested({})
+        assert curves_requested({"showcol": "curves"})
+
+        # the search adds it to the projection only when the column is shown
+        with self.app.test_request_context("/Variety/Abelian/Fq/?q=4&g=1"):
+            query = {}
+            abelian_variety_search.f({"q": "4", "g": "1"}, query)
+            assert "__projection__" not in query
+            query = {}
+            abelian_variety_search.f({"q": "4", "g": "1", "showcol": "curves"}, query)
+            assert "curves" in query["__projection__"]
+
+        # and so do downloads of search results
+        downloader = AbvarFq_download()
+        default_projection = downloader.get_projection({}, self.db.av_fq_isog, abvar_columns)
+        curves_projection = downloader.get_projection({"showcol": "curves"}, self.db.av_fq_isog, abvar_columns)
+        assert "curves" not in default_projection
+        assert "curves" in curves_projection
+
+        # the generic hook is unchanged: it only restricts db_cols to the table's columns
+        generic = Downloader().get_projection({"showcol": "curves"}, self.db.av_fq_isog, abvar_columns)
+        assert generic == default_projection
+        assert generic == [col for col in abvar_columns.db_cols if col in self.db.av_fq_isog.search_cols]
+
+    def test_curves_not_computed(self):
+        r"""
+        Isogeny classes whose curves have not been computed (NULL in the database) must be
+        distinguishable in downloads from those searched without finding a curve (an empty list).
+        """
+        import re
+        from lmfdb.utils.downloader import GPLanguage, SageLanguage
+        from lmfdb.abvar.fq.isog_class import AbvarFq_isoclass
+        from lmfdb.abvar.fq.main import abvar_columns
+
+        curves_col = [col for col in abvar_columns.columns if col.name == "curves"][0]
+        # the curves of 2.729.aee_gmg have not been computed (psycodict omits the NULL column)
+        not_computed = AbvarFq_isoclass(self.db.av_fq_isog.lookup("2.729.aee_gmg", ["label", "curves"]))
+        assert not_computed.curves is None
+        # 2.3.ag_p was searched, and contains no curve
+        no_curve = AbvarFq_isoclass(self.db.av_fq_isog.lookup("2.3.ag_p", ["label", "curves"]))
+        assert no_curve.curves == []
+        # both display as an empty cell
+        assert curves_col.display(not_computed) == ""
+        assert curves_col.display(no_curve) == ""
+        # but downloads give the language's null value for the uncomputed one
+        assert SageLanguage().to_lang(curves_col.download(not_computed)) == "None"
+        assert GPLanguage().to_lang(curves_col.download(not_computed)) == "null"
+        assert SageLanguage().to_lang(curves_col.download(no_curve)) == "[]"
+        assert GPLanguage().to_lang(curves_col.download(no_curve)) == "[]"
+
+        # the cells on the search page are empty, rather than showing None
         page = self.tc.get("/Variety/Abelian/Fq/?q=729&g=2&showcol=curves").get_data(as_text=True)
         assert "2.729.aee_gmg" in page
+        cells = re.findall(r'<td class="col-curves"[^>]*>(.*?)</td>', page, re.S)
+        assert cells and not any(cell.strip() for cell in cells)
+
+        # and downloading such a row gives null (here we hide every other column)
+        hidecol = ".".join(col.name for col in abvar_columns.columns if col.default({}))
+        url = "/Variety/Abelian/Fq/?q=729&g=2&showcol=curves&hidecol=%s&download=1&download_row_count=2&Submit=%s&query=%%7B%%27q%%27%%3A+729%%2C+%%27g%%27%%3A+2%%7D"
+        assert "[None]" in self.tc.get(url % (hidecol, "sage")).get_data(as_text=True)
+        assert "[null]" in self.tc.get(url % (hidecol, "gp")).get_data(as_text=True)
 
     def test_cyclic_group_of_points_display(self):
         r"""