Load the ECNF base field polynomials in a single query

Review follow-up: ECNFDownloader looked up the defining polynomial of each
curve's base field one label at a time (memoized with lru_cache).  There
are only 894 base fields occurring in ec_nfcurves, so fetch all of them
from nf_fields in one search the first time a download happens and keep
them in a dictionary on the downloader.  postprocess now just indexes into
that dictionary, and generating a download issues no further queries.

Also extend the unknown-rank regression test to all six download formats,
asserting on the rank column rather than on a whole serialized row, add a
test that the polynomials are loaded in bulk and reused, and unit test the
Oscar rational serialization in lmfdb/tests/test_utils.py.

Downloading the 712 curves of conductor norm 1 in sage format against
devmirror takes 2.25s including building the dictionary and 0.98s after.
lmfdb/ecnf/test_ecnf.py (14 passed) and lmfdb/tests/test_utils.py (20
passed) pass; pyflakes and ruff clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: roed314

lmfdb/ecnf/main.py

@@ -3,16 +3,15 @@
 
 import ast
 import re
-from functools import lru_cache
 from urllib.parse import quote, unquote
 
 from flask import render_template, request, url_for, redirect, make_response, abort
-from sage.all import factor, is_prime, QQ, ZZ, PolynomialRing
+from sage.all import factor, is_prime, lazy_attribute, QQ, ZZ, PolynomialRing
 
 from lmfdb import db
 from psycodict.encoding import Json
 from lmfdb.utils import (
-    to_dict, flash_error, display_knowl, Downloader,
+    to_dict, flash_error, display_knowl, Downloader, coeff_to_poly,
     parse_ints, parse_ints_to_list_flash, parse_noop, nf_string_to_label, parse_element_of,
     parse_nf_string, parse_nf_jinv, parse_bracketed_posints, parse_floats, parse_primes,
     SearchArray, TextBox, SelectBox, CountBox, SubsetBox, TextBoxWithSelect,
@@ -449,17 +448,22 @@ class ECNFDownloader(Downloader):
     title = "Elliptic curves over number fields"
     short_name = "curves"
 
-    @staticmethod
-    @lru_cache(maxsize=128)
-    def field_poly(field_label):
-        # Look up the defining polynomial of a base field; cached since download
-        # results are sorted by field, so the same field shows up in consecutive rows
-        from lmfdb.utils import coeff_to_poly
-        return coeff_to_poly(db.nf_fields.lookup(field_label, projection='coeffs'))
+    @lazy_attribute
+    def field_polys(self):
+        # The defining polynomials of the base fields, keyed by field label.  There
+        # are under a thousand base fields, so we fetch them all in one query the
+        # first time a download happens rather than looking them up row by row.
+        labels = db.ec_nfcurves.distinct('field_label')
+        polys = {rec['label']: coeff_to_poly(rec['coeffs'])
+                 for rec in db.nf_fields.search({'label': {'$in': labels}}, projection=['label', 'coeffs'])}
+        missing = [label for label in labels if label not in polys]
+        if missing:
+            raise ValueError("Missing defining polynomial for base field %s" % missing[0])
+        return polys
 
     def postprocess(self, row, info, query):
         # Look up the defining polynomial coefficients for the base field of the curve
-        row["field_coeffs"] = self.field_poly(row['field_label'])
+        row["field_coeffs"] = self.field_polys[row['field_label']]
 
         # Convert Weierstrass coefficients from string to a list of list of rationals
         row['ainvs'] = [[QQ(aj) for aj in ai.split(",")] for ai in row['ainvs'].split(";")]

lmfdb/ecnf/test_ecnf.py

@@ -96,11 +96,38 @@ def test_search_download(self):
         # For curves whose rank is not known, the rank bounds should be
         # downloaded rather than a LaTeX string such as "0 \le r \le 1"
         base = '/EllipticCurve/?download=1&query=%7B%27field_label%27%3A+%272.0.868.1%27%2C+%27conductor_norm%27%3A+2%7D&Submit='
-        for lang in ['sage', 'oscar']:
+        for lang in ['sage', 'gp', 'magma', 'text', 'csv', 'oscar']:
             L = self.tc.get(base + lang)
             data = L.get_data(as_text=True)
-            assert '["2.1-b1", "2.1-b", "2.0.868.1", [217, 0, 1], 2, [0, 1], []' in data
-            assert '"0 \\\\le r \\\\le 1"' not in data
+            # The columns preceding the Sato-Tate group are the two labels, the base
+            # field with its defining polynomial, the conductor norm, the rank and the
+            # torsion, so [0, 1] can only be the bounds on the rank of this curve
+            row = [line for line in data.split('\n') if '2.1-b1' in line][0]
+            assert '[0, 1]' in row.split('1.2.A.1.1a')[0]
+            assert r'\le r' not in row
+
+    def test_download_field_polys(self):
+        r"""
+        Check that the base field defining polynomials used when downloading
+        search results are loaded in a single query (issue #7004)
+        """
+        from unittest.mock import patch
+        from lmfdb.ecnf.main import ECNFDownloader
+        downloader = ECNFDownloader()
+        rows = [{'field_label': label, 'ainvs': '0,0,0;0,0,0;0,0,0;0,0,0;0,0,0'}
+                for label in ['3.3.1849.1', '2.0.868.1', '3.3.1849.1']]
+        with patch.object(self.db.nf_fields, 'search', wraps=self.db.nf_fields.search) as search, \
+             patch.object(self.db.nf_fields, 'lookup', wraps=self.db.nf_fields.lookup) as lookup:
+            polys = [downloader.postprocess(row, {}, {})['field_coeffs'] for row in rows]
+            # All of the defining polynomials are fetched by the first row, in bulk
+            assert search.call_count == 1
+            assert lookup.call_count == 0
+            # and the dictionary they were saved in is reused afterwards
+            assert downloader.field_polys is downloader.field_polys
+            assert search.call_count == 1
+        assert polys[0] is polys[2]
+        for row, poly in zip(rows, polys):
+            assert poly.degree() == int(row['field_label'].split('.')[0])
 
     def test_search(self):
         r"""

lmfdb/tests/test_utils.py

@@ -8,7 +8,7 @@
 
 import unittest
 
-from sage.all import var
+from sage.all import var, QQ
 
 from lmfdb.utils import (
     an_list,
@@ -38,6 +38,8 @@
     infinity,
 )
 
+from lmfdb.utils.downloader import OscarLanguage, SageLanguage
+
 class UtilsTest(unittest.TestCase):
     """
     An example of unit tests that are not based on the website itself.
@@ -104,6 +106,15 @@ def test_splitcoeff(self):
         self.assertEqual(splitcoeff("  0  -1.2  \n  3.14  1 "),
                          [[0.0, -1.2], [3.14, 1.0]])
 
+    def test_rational_to_lang(self):
+        r"""
+        Checking utility: DownloadLanguage.rational_to_lang
+        """
+        # In Julia, -3/2 is floating point division, so Oscar needs -3//2
+        self.assertEqual(SageLanguage().to_lang(QQ(-3) / 2), "-3/2")
+        self.assertEqual(OscarLanguage().to_lang(QQ(-3) / 2), "-3//2")
+        self.assertEqual(OscarLanguage().to_lang(QQ(3)), "3")
+
     ################################################################################
     #  display and formatting utilities
     ################################################################################