Keep the whole number field record in the character search batch (LMFDB#6008)

The batched postprocessing step kept only the label of each matched number
field, so formatfield built a WebNumberField holding just that label and
field_pretty reloaded the record from the database whenever the pretty name
needed more than the label: degree 3 wants disc_abs/disc_sign/coeffs, generic
degree 4 and degree 2^k want subfields/subfield_mults.  Displaying the kernel
field 4.0.2197.1 (the order 4 characters of modulus 13) therefore still made
5 record level lookups, so a cold results page did database work proportional
to the number of distinct kernel fields on it.

Keep the whole nf_fields record in character_postprocess (one query either
way) and hand it to formatfield as the virtual column kernel_field_data.
Every column field_pretty consults lives in nf_fields, so no nf_fields_extra
batch is needed.  Case 5b of field_pretty also looked up the unique quadratic
subfield of an imprimitive quartic by coefficients; read the squarefree
radicand off the stored subfield polynomial instead, the way cases 5a and 7
already do, which leaves no lucky/lookup call in the render path.

Verified: 5 record lookups -> 0 when displaying that kernel field, and 0 over
a 50 row page of order <= 12 characters; the new test_field_columns_no_lookups
patches nf_fields.lookup, nf_fields.lucky and nf_fields_extra.lookup to raise
and fails on the previous commit; rendered results tables and downloads are
byte identical before and after on 9 searches and 4 downloads; old and new
case 5b radicands agree on the 1450 quartics with a unique quadratic subfield
among the 4000 smallest discriminant quartics; cold render of
?modulus=1-500&order=4&showcol=first drops from 11.2s to 0.9s, order=8 from
7.9s to 0.9s and order=3 from 4.5s to 0.9s against devmirror.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: roed314

lmfdb/characters/main.py

@@ -271,16 +271,15 @@ def display_galois_orbit(modulus, first, last, degree):
             disp = r"$, \cdots ,$".join(disp)
             return f'<p style="margin-top: 0px;margin-bottom:0px;">\n{disp}\n</p>'
 
-def display_kernel_field(modulus, first, order, kernel_poly=None, kernel_label=None):
-    # kernel_poly and kernel_label are precomputed in character_postprocess;
-    # if they are absent we fall back to computing them here, at the cost of
-    # a database query for each call
+def display_kernel_field(modulus, first, order, kernel_poly=None, kernel_field_data=None):
+    # kernel_poly and kernel_field_data are precomputed in
+    # character_postprocess; if they are absent we fall back to computing them
+    # here, at the cost of several database queries for each call
     if order > 12:
         return "not computed"
     if kernel_poly is None:
         kernel_poly = [ZZ(x) for x in ConreyCharacter(modulus,first).kernel_field_poly()]
-    data = {"label": kernel_label} if kernel_label else None
-    return formatfield(kernel_poly, data=data)
+    return formatfield(kernel_poly, data=kernel_field_data)
 
 def display_value_field(order, value_field_label=None):
     # value_field_label is precomputed in character_postprocess ("N/A" when the
@@ -306,8 +305,8 @@ def character_postprocess(res, info, query):
     Each of these columns identifies a number field by its defining polynomial,
     which naively requires two database queries per row.  Instead we compute all
     of the defining polynomials here (the pari computations involved are cheap)
-    and then find all of the matching number field labels in a single database
-    query.  See https://github.com/LMFDB/lmfdb/issues/6008.
+    and then find all of the matching number fields in a single database query.
+    See https://github.com/LMFDB/lmfdb/issues/6008.
     """
     R = PolynomialRing(ZZ, "x")
     cyclo_coeffs = {}  # order2 -> coeffs of the polredabs'ed cyclotomic polynomial
@@ -327,20 +326,26 @@ def character_postprocess(res, info, query):
             rec["kernel_poly"] = [int(ZZ(x)) for x in kernel_poly]
     all_coeffs = {tuple(c) for c in cyclo_coeffs.values() if c is not None}
     all_coeffs.update(tuple(rec["kernel_poly"]) for rec in res if "kernel_poly" in rec)
-    labels = {}
+    nf_data = {}
     if all_coeffs:
         # We use $or rather than $in since psycodict does not apply the
-        # typecast needed to compare integer arrays with the numeric[] column
+        # typecast needed to compare integer arrays with the numeric[] column.
+        # We keep the whole record rather than just the label since
+        # field_pretty needs more than the label (coeffs, disc_abs, disc_sign,
+        # subfields and subfield_mults, all of which live in nf_fields) and
+        # reloads the field from the database when they are missing.
         nf_query = {"$or": [{"coeffs": list(c)} for c in sorted(all_coeffs)]}
-        labels = {tuple(f["coeffs"]): f["label"]
-                  for f in db.nf_fields.search(nf_query, ["coeffs", "label"])}
+        nf_data = {tuple(f["coeffs"]): f for f in db.nf_fields.search(nf_query)}
     for rec in res:
         order = rec["order"]
         order2 = order if order % 4 != 2 else order // 2
         cyclo = cyclo_coeffs[order2]
-        rec["value_field_label"] = "N/A" if cyclo is None else labels.get(tuple(cyclo), "N/A")
+        value_field = None if cyclo is None else nf_data.get(tuple(cyclo))
+        rec["value_field_label"] = "N/A" if value_field is None else value_field["label"]
         if "kernel_poly" in rec:
-            rec["kernel_label"] = labels.get(tuple(rec["kernel_poly"]), "N/A")
+            # formatfield reads the label "N/A" as "not in the database", so a
+            # missing field is recorded without another query
+            rec["kernel_field_data"] = nf_data.get(tuple(rec["kernel_poly"]), {"label": "N/A"})
     return res
 
 character_columns = SearchColumns([
@@ -351,7 +356,7 @@ def character_postprocess(res, info, query):
     MathCol("conductor", "character.dirichlet.conductor", "Conductor"),
     MathCol("order", "character.dirichlet.order", "Order"),
     MultiProcessedCol("first", "character.dirichlet.field_cut_out", "Kernel field",
-                      ["modulus", "first", "order", "kernel_poly", "kernel_label"],
+                      ["modulus", "first", "order", "kernel_poly", "kernel_field_data"],
                       display_kernel_field, align="center", default=False,
                       apply_download=lambda modulus, first, order, *args: [modulus, first, order]),
     MultiProcessedCol("order", "character.dirichlet.value_field", "Value field",

lmfdb/characters/test_characters.py

@@ -68,10 +68,62 @@ def test_field_columns(self):
         assert 'label=2.0.3.1' in data
         # kernel field knowl for Q(zeta_5)
         assert 'label=4.0.125.1' in data
+        # The order 4 characters of modulus 13 cut out 4.0.2197.1, which is
+        # not the value field of any character, so this pins down the kernel
+        # field column (and is a quartic field whose pretty name needs more
+        # than the label)
+        W = self.tc.get('/Character/Dirichlet/?modulus=13&order=4&search_type=List&showcol=first')
+        data = W.get_data(as_text=True)
+        assert 'label=4.0.2197.1' in data
+        assert r'\(\Q(\sqrt{-26 -6 \sqrt{13}})\)' in data
+        # kernel fields that are not in the database
+        W = self.tc.get('/Character/Dirichlet/?modulus=5002&order=12&search_type=List&showcol=first')
+        assert 'knowl="nf.field.missing"' in W.get_data(as_text=True)
+        # value fields that are not in the database
+        W = self.tc.get('/Character/Dirichlet/?order=47&search_type=List')
+        assert r'$\Q(\zeta_{47})$' in W.get_data(as_text=True)
         # kernel fields are not computed for orders larger than 12
         W = self.tc.get('/Character/Dirichlet/?order=13-100&search_type=List')
         assert 'not computed' in W.get_data(as_text=True)
 
+    def test_field_columns_no_lookups(self):
+        # Once character_postprocess has run, displaying the kernel field must
+        # not go back to the database: it is the per-field record lookups that
+        # issue #6008 is about.
+        from unittest.mock import patch
+        from lmfdb import db
+        from lmfdb.characters.main import character_postprocess, display_kernel_field
+        from lmfdb.number_fields.web_number_field import field_pretty
+
+        res = list(db.char_dirichlet.search({'modulus': 13, 'order': 4}))
+        res = character_postprocess(res, {}, {})
+        assert [rec['kernel_field_data']['label'] for rec in res] == ['4.0.2197.1']
+
+        def no_lookup(*args, **kwargs):
+            raise AssertionError("number field record lookup while displaying a kernel field")
+
+        # field_pretty caches by label, so a lookup made by an earlier test
+        # would otherwise hide one made here
+        field_pretty.clear_cache()
+        with patch.object(db.nf_fields, 'lookup', no_lookup), \
+             patch.object(db.nf_fields, 'lucky', no_lookup), \
+             patch.object(db.nf_fields_extra, 'lookup', no_lookup):
+            displayed = [display_kernel_field(rec['modulus'], rec['first'], rec['order'],
+                                              rec['kernel_poly'], rec['kernel_field_data'])
+                         for rec in res]
+        assert 'label=4.0.2197.1' in displayed[0]
+        assert r'\(\Q(\sqrt{-26 -6 \sqrt{13}})\)' in displayed[0]
+
+    def test_field_columns_download(self):
+        # The virtual columns added by character_postprocess must not leak into
+        # downloads: the kernel field column downloads as [modulus, first, order]
+        # and the value field column as the raw order
+        url = ('/Character/Dirichlet/?query=%7B%27order%27%3A+4%2C+%27modulus%27%3A+13%7D'
+               '&Submit=text&download=1&search_type=List&showcol=first')
+        data = self.tc.get(url).get_data(as_text=True)
+        assert '[Orbit label, Conrey labels, Modulus, Conductor, Order, Kernel field,' in data
+        assert '"13.d"\t[13, 5, 8, 2]\t13\t13\t4\t[13, 5, 4]\t' in data
+
 class DirichletTableTest(LmfdbTest):
 
     def test_table(self):

lmfdb/number_fields/web_number_field.py

@@ -232,13 +232,6 @@ def field_pretty(label, wnf=None):
     if d == '1':  # Q
         return r'\(\Q\)'
 
-    # Converts LMFDB label for quadratic field K to the D in K = Q(sqrt(D))
-    def _quad_label_to_D(quad_label):
-        parts = str(quad_label).split('.')
-        z = integer_squarefree_part(ZZ(parts[2]))  # Get squarefree part
-        z *= (-1) ** (1 + int(parts[1]) // 2)      # Get correct sign
-        return ZZ(z)
-
     # Converts D to the latexed form of sqrt(D)
     def _sqrt_symbol(z):
         return 'i' if z == -1 else r'\sqrt{%d}' % z
@@ -289,35 +282,35 @@ def _wnf_with(*keys):
 
         # Case 5b: Imprimitive quartic fields of type Q(\sqrt(A + B*\sqrt(D)))
         if len(subs) == 1:
-            quad_sub = wnf.from_coeffs(string2list(str(subs[0][0])))
-            if not quad_sub._data is None:
-                # Get unique quadratic subfield Q(sqrt(D))
-                quad_label = str(quad_sub.get_label())
-                D = _quad_label_to_D(quad_label)
-                Ksub = QuadraticField(D, 'sqrtD')
-                sqrtD = Ksub.gen(0)
-
-                # Factorise defining polynomial for K over Q(sqrt(D))
-                Rsub = PolynomialRing(Ksub, 'x')
-                relative_poly = Rsub(wnf.poly()).factor()[0][0]
-
-                # Can extract the first quadratic factor
-                rel_coeffs = relative_poly.coefficients(sparse=False)
-                alpha = rel_coeffs[1]**2 - 4*rel_coeffs[0]*rel_coeffs[2]
-                A, B = sqrtD.coordinates_in_terms_of_powers()(alpha)
-
-                # Divide out common square factors
-                g = gcd(A,B)
-                g //= g.squarefree_part()
-                A, B = ZZ(A//g), ZZ(B//g)
-
-                # Return final pretty latex
-                if A == 0:
-                    # Case: Pure quartic field
-                    return r'\(\Q(\sqrt[4]{%d})\)' % (D*B**2)
-                else:
-                    B_str = "+" if B == 1 else "-" if B == -1 else f"{B:+d}"
-                    return r'\(\Q(\sqrt{%d %s %s})\)' % (A, B_str, _sqrt_symbol(D))
+            # Get unique quadratic subfield Q(sqrt(D)) from its defining
+            # polynomial, as in case 5a; looking the subfield up in the
+            # database instead would cost a query on every call
+            qs = subs[0][0].split(',')
+            D = integer_squarefree_part(ZZ(qs[1])**2 - 4*ZZ(qs[0])*ZZ(qs[2]))
+            Ksub = QuadraticField(D, 'sqrtD')
+            sqrtD = Ksub.gen(0)
+
+            # Factorise defining polynomial for K over Q(sqrt(D))
+            Rsub = PolynomialRing(Ksub, 'x')
+            relative_poly = Rsub(wnf.poly()).factor()[0][0]
+
+            # Can extract the first quadratic factor
+            rel_coeffs = relative_poly.coefficients(sparse=False)
+            alpha = rel_coeffs[1]**2 - 4*rel_coeffs[0]*rel_coeffs[2]
+            A, B = sqrtD.coordinates_in_terms_of_powers()(alpha)
+
+            # Divide out common square factors
+            g = gcd(A,B)
+            g //= g.squarefree_part()
+            A, B = ZZ(A//g), ZZ(B//g)
+
+            # Return final pretty latex
+            if A == 0:
+                # Case: Pure quartic field
+                return r'\(\Q(\sqrt[4]{%d})\)' % (D*B**2)
+            else:
+                B_str = "+" if B == 1 else "-" if B == -1 else f"{B:+d}"
+                return r'\(\Q(\sqrt{%d %s %s})\)' % (A, B_str, _sqrt_symbol(D))
 
     # Case 6: Pure cubic fields Q(\sqrt[3]{N})
     if d == '3':