Merge pull request #7181 from roed-math/ci-test-statement-timeout

Stop CI jobs hanging for hours on a slow database query

Author: edgarcosta

.github/workflows/python-package.yml

@@ -28,6 +28,12 @@ jobs:
   test:
     runs-on: ubuntu-latest
     needs: matrix_prep
+    # A healthy job finishes well inside half an hour; the slowest green ones
+    # observed are around 25 minutes, and that includes building the conda
+    # environment on a cache miss.  GitHub's default is six hours, which is long
+    # enough for a job stuck on the database to hold a runner all morning -- one
+    # recently ran for five and a half.  Fail such a job in an hour instead.
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}

lmfdb/characters/test_characters.py

@@ -35,18 +35,29 @@ def test_modbrowse(self):
         assert '46.d' in W.get_data(as_text=True)
 
     def test_search(self):
-        W = self.tc.get('/Character/Dirichlet/?conductor=15&order=4')
-        assert r'15.e' in W.get_data(as_text=True)
-        W = self.tc.get('/Character/Dirichlet/?conductor=25-50&order=5-7')
-        assert r'25.d' in W.get_data(as_text=True)
-        W = self.tc.get('/Character/Dirichlet/?conductor=25-50&order=5-7&primitive=Yes')
-        assert r'25.d' in W.get_data(as_text=True)
-        W = self.tc.get('/Character/Dirichlet/?conductor=25-50&order=5-7&primitive=No')
-        assert r'50.d' in W.get_data(as_text=True)
-        W = self.tc.get('/Character/Dirichlet/?conductor=25-50&order=5-7&primitive=No&parity=Odd')
-        assert r'56.n' in W.get_data(as_text=True)
-        W = self.tc.get('/Character/Dirichlet/?conductor=25-50&order=5-7&primitive=No&parity=Even')
-        assert r'50.d' in W.get_data(as_text=True)
+        # Each filter is checked in both directions, so a parameter that stops
+        # being parsed widens the search and trips a negative assertion.
+        # Conductor 16, order 4 has a primitive even orbit (16.e), a primitive
+        # odd one (16.f), and imprimitive counterparts at modulus 32.  Labels
+        # carry their tags, since a bare '16.e' also matches '416.e'.
+        page = self.tc.get('/Character/Dirichlet/?conductor=15&order=4').get_data(as_text=True)
+        assert '>15.e<' in page
+        page = self.tc.get('/Character/Dirichlet/?conductor=16&order=4&is_primitive=no').get_data(as_text=True)
+        assert '>32.e<' in page and '>32.f<' in page
+        assert '>16.e<' not in page and '>16.f<' not in page
+        page = self.tc.get('/Character/Dirichlet/?conductor=16&order=4&parity=even').get_data(as_text=True)
+        assert '>16.e<' in page and '>32.e<' in page
+        assert '>16.f<' not in page and '>32.f<' not in page
+        page = self.tc.get('/Character/Dirichlet/?conductor=16&order=4&parity=odd').get_data(as_text=True)
+        assert '>16.f<' in page and '>32.f<' in page
+        assert '>16.e<' not in page and '>32.e<' not in page
+        page = self.tc.get('/Character/Dirichlet/?conductor=16&order=4&is_primitive=no&parity=odd').get_data(as_text=True)
+        assert '>32.f<' in page
+        assert '>16.e<' not in page and '>16.f<' not in page and '>32.e<' not in page
+        # A conductor range matches no index prefix, so this one reads every
+        # orbit of conductor 25-50 before it can sort; allow the timeout page.
+        self.check_args_with_timeout(
+            '/Character/Dirichlet/?conductor=25-50&order=5-7', '>25.d<')
 
     def test_condsearch(self):
         W = self.tc.get('/Character/Dirichlet/?conductor=111')

tox.ini

@@ -13,6 +13,26 @@ passenv =
     SAGE_LOCAL
     HOME
 
+# Bound how long a single database query may run.  The suite talks to a shared
+# server (devmirror by default) that has no statement_timeout of its own, and
+# psycodict only hands one to connections made as the "webserver" role, so a
+# pathological query otherwise runs unbounded: a Dirichlet character search that
+# takes seconds on an idle server has held a CI job for over five hours.
+#
+# libpq reads PGOPTIONS when it opens a connection, including the ones psycodict
+# opens when it resets a broken one, so this needs no cooperation from the test
+# harness and cannot be lost on a reconnect.  A query stopped this way raises
+# the same QueryCanceledError the search pages already handle, so a slow search
+# renders the usual timeout page instead of hanging.
+#
+# This is a hang catcher, not a performance budget.  120s is far above anything
+# the suite issues against a healthy server and far below the hangs it exists to
+# stop; the job-level timeout-minutes in .github/workflows/python-package.yml
+# backstops the case where many queries are slow at once.  Export PGOPTIONS to
+# override, e.g. PGOPTIONS='-c statement_timeout=0' for the old behavior.
+setenv =
+    PGOPTIONS = {env:PGOPTIONS:-c statement_timeout=120s}
+
 commands =
     parallel --group 'echo "Running test on {}"; sage -python -m pytest -vv --durations=0 {}' ::: {posargs}