diff --git a/lmfdb/groups/abstract/stats.py b/lmfdb/groups/abstract/stats.py index 7dc0c3b5aa..6d5c7c7d0d 100644 --- a/lmfdb/groups/abstract/stats.py +++ b/lmfdb/groups/abstract/stats.py @@ -34,7 +34,9 @@ def nilp_formatter(nilp): return str(nilp) def nilp_qformatter(nilp): - if nilp == "not": + # A group that is not nilpotent is stored with nilpotency class -1, which the + # search page does not accept; it is searched for with the nilpotent box. + if nilp in (-1, "not"): return "nilpotent=no" return f"nilpotency_class={nilp}" diff --git a/lmfdb/local_fields/main.py b/lmfdb/local_fields/main.py index 862a9e0a48..31f380e4fd 100644 --- a/lmfdb/local_fields/main.py +++ b/lmfdb/local_fields/main.py @@ -22,9 +22,11 @@ EmbeddedSearchArray, integer_options, redirect_no_cache, raw_typeset) from lmfdb.utils.place_code import CodeSnippet +from psycodict.utils import SearchParsingError, range_formatter +from lmfdb.utils.display_stats import NO_SEARCH_QUERY from lmfdb.utils.interesting import interesting_knowls from lmfdb.utils.search_columns import SearchColumns, LinkCol, MathCol, ProcessedCol, MultiProcessedCol, RationalListCol, PolynomialCol, eval_rational_list -from lmfdb.utils.search_parsing import search_parser +from lmfdb.utils.search_parsing import QQ_DEC_RE, QQ_RE, search_parser from lmfdb.api import datapage from lmfdb.logger import logger from lmfdb.local_fields import local_fields_page @@ -435,10 +437,10 @@ def galcolresponse(n,t,cache): return group_pretty_and_nTj(n, t, cache=cache) def formatbracketcol(blist): + if blist is None or blist == '': + return 'not computed' if blist == []: return r'$[\ ]$' - if blist == '': - return 'not computed' return f'${blist}$' def intcol(j): @@ -1883,7 +1885,9 @@ def ramdisp(p): 'proportioner': proportioners.per_row_total} def discdisp(p): + # c has default buckets for dynamic stats, but here we display each value separately return {'cols': ['n', 'c'], + 'buckets': {}, 'constraint': {'p': p, 'n': {'$lte': 23}}, 'top_title':[('degree', 'lf.degree'), ('and', None), @@ -1905,25 +1909,231 @@ def galdisp(p, n): def galcache(): return knowl_cache(db.lf_fields.distinct("galois_label")) def galformatter(gal): + if gal is None: + return "not computed" n, t = galdata(gal) return '' + group_pretty_and_nTj(n, t, True, cache=galcache()).replace("(as", '
(as') + "" + +def galsortkey(gal): + # galdata cannot handle None, which arises for fields where the Galois group is not computed + if gal is None: + return [-1, -1] + return galdata(gal) + +def galquery(gal): + if gal is None: + # There is no way to search for fields where the Galois group is not computed + return NO_SEARCH_QUERY + return "gal=%s" % galunformatter(gal) + +CONTENT_RE = re.compile(r"\[([0-9/, ]*)\](?:_\{(\d+)\})?(?:\^\{(\d+)\})?") + +def content_sort_key(s): + # A sort key for slope contents: strings such as '[4/3, 4/3, 2]' or '[2, 2]_{2}^{3}', + # as stored in the slopes, visible and hidden columns. None (not computed) sorts + # first, and unparseable values (such as user-entered buckets) sort last by raw string. + if s is None: + return (0, [], 0, 0) + m = CONTENT_RE.fullmatch(s) + if m is None: + return (2, s) + body, t, u = m.groups() + slopes = [QQ(x.strip()) for x in body.split(",")] if body.strip() else [] + return (1, slopes, int(t or 0), int(u or 0)) + +def array_sort_key(v): + # Lists cannot be compared with the -infinity used for None by the default sort key + if v is None: + return (0, []) + return (1, v) + +# top_slope is stored as a fixed-width decimal approximation, which makes the +# database sort text in numerical order, followed by the exact rational; see ratproc. +TOPSLOPE_PREFIX_LEN = 12 +TOPSLOPE_PREFIX_RE = re.compile(r"\d+\.\d+") + +def topslope_encoder(endpoint): + """ + Encode a top slope, as it is written in the topslope search box, into the + form stored in the database. Used for the endpoints of statistics buckets, + which must be compared against the stored values. + """ + if not QQ_DEC_RE.match(endpoint): + raise SearchParsingError("%s is not a non-negative rational number, such as 4/3 or 2.5." % endpoint) + return ratproc(endpoint) + +def topslope_decoder(ts): + """ + The exact rational underlying a stored top slope, or None if the input is not + a stored top slope (a bucket typed by a user, for example, is already exact). + """ + if not isinstance(ts, str) or len(ts) <= TOPSLOPE_PREFIX_LEN: + return None + prefix, rest = ts[:TOPSLOPE_PREFIX_LEN], ts[TOPSLOPE_PREFIX_LEN:] + if TOPSLOPE_PREFIX_RE.fullmatch(prefix) and QQ_RE.match(rest): + return rest + return None + +def topslope_endpoints(ts): + """ + The endpoints of a top slope value or range, as exact rationals: a pair + (lower, upper), either of which may be None if the range is unbounded on that + side, and which are equal for a single value. + + The input is either a value or a range as stored in the database, or a bucket + as typed into the buckets box on the dynamic statistics page. + """ + if isinstance(ts, dict): + lower = ts.get("$gte", ts.get("$gt")) + upper = ts.get("$lte", ts.get("$lt")) + elif isinstance(ts, str) and topslope_decoder(ts) is None and "-" in ts[1:]: + # a range typed into the buckets box, such as '1-2' or '2-' + lower, _, upper = ts.partition("-") + upper = upper or None + else: + lower = upper = ts + return tuple(topslope_decoder(x) or x for x in (lower, upper)) + +def topslope_formatter(ts): + def show(x): + try: + return "$%s$" % latex(QQ(x)) + except (TypeError, ValueError): + return str(x) + lower, upper = topslope_endpoints(ts) + if lower is None and upper is None: + return "not computed" + elif lower == upper: + return show(lower) + elif upper is None: + return "%s-" % show(lower) + elif lower is None: + # top slopes are always non-negative + return "$0$-%s" % show(upper) + return "%s-%s" % (show(lower), show(upper)) + +def topslope_query(ts): + lower, upper = topslope_endpoints(ts) + if lower is None and upper is None: + return NO_SEARCH_QUERY + elif lower == upper: + return "topslope=%s" % lower + elif upper is None: + return "topslope=%s-" % lower + elif lower is None: + return "topslope=0-%s" % upper + return "topslope=%s-%s" % (lower, upper) + +def content_query(shortname, quantifier): + # For columns searched via parse_newton_polygon; the quantifier makes the search + # match the exact value being counted, rather than the default containment search + def inner(val): + if val is None or str(val) in ("", "[]"): + # There is no way to search for an empty or uncomputed list of slopes + return NO_SEARCH_QUERY + return "%s=%s&%s=exactly" % (shortname, val, quantifier) + return inner + +def bracket_query(shortname): + # For columns searched via parse_bracketed_posints, which matches exactly. + # The empty list is searchable, unlike a value that is not computed. + def inner(val): + if val is None: + return NO_SEARCH_QUERY + return "%s=%s" % (shortname, val) + return inner + +def nullable_int_query(shortname): + def inner(val): + if val is None: + # There is no way to search for fields where this is not computed + return NO_SEARCH_QUERY + return "%s=%s" % (shortname, range_formatter(val)) + return inner + class LFStats(StatsDisplay): table = db.lf_fields baseurl_func = ".index" short_display = {'galois_label': 'Galois group', 'n': 'degree', + 'p': 'residue characteristic', 'e': 'ramification index', + 'f': 'residue field degree', 'c': 'discriminant exponent', - 'hidden': 'hidden slopes'} - sort_keys = {'galois_label': galdata} + 'u': 'Galois unramified degree', + 't': 'Galois tame degree', + 'aut': 'automorphisms', + 'top_slope': 'top Artin slope', + 'slopes': 'Galois Artin slopes', + 'visible': 'visible Artin slopes', + 'hidden': 'hidden slopes', + 'ind_of_insep': 'indices of inseparability', + 'associated_inertia': 'associated inertia', + 'jump_set': 'jump set'} + top_titles = {'aut': 'number of automorphisms', + 'e': 'ramification indices', + 'ind_of_insep': 'indices of inseparability', + 'associated_inertia': 'associated inertia'} + knowls = {'galois_label': 'nf.galois_group', + 'n': 'lf.degree', + 'p': 'lf.residue_field', + 'e': 'lf.ramification_index', + 'f': 'lf.residue_field_degree', + 'c': 'lf.discriminant_exponent', + 'u': 'lf.unramified_degree', + 't': 'lf.tame_degree', + 'aut': 'lf.automorphism_group', + 'top_slope': 'lf.top_slope', + 'slopes': 'lf.hidden_slopes', + 'visible': 'lf.slopes', + 'hidden': 'lf.slopes', + 'ind_of_insep': 'lf.indices_of_inseparability', + 'associated_inertia': 'lf.associated_inertia', + 'jump_set': 'lf.jump_set'} + sort_keys = {'galois_label': galsortkey, + 'slopes': content_sort_key, + 'visible': content_sort_key, + 'hidden': content_sort_key, + 'ind_of_insep': array_sort_key, + 'associated_inertia': array_sort_key, + 'jump_set': array_sort_key} formatters = { 'galois_label': galformatter, + 'slopes': latex_content, + 'visible': latex_content, 'hidden': latex_content, + 'top_slope': topslope_formatter, + 'ind_of_insep': formatbracketcol, + 'associated_inertia': formatbracketcol, + # a field with no jump set has an empty one; distinguishing that from a + # field where it is not computed keeps the two from sharing a row + 'jump_set': formatbracketcol, } query_formatters = { - 'galois_label': (lambda gal: r'gal=%s' % (galunformatter(gal))), - 'hidden': (lambda hid: r'hidden=%s' % (content_unformatter(hid))), + 'galois_label': galquery, + 'slopes': content_query('slopes', 'slopes_quantifier'), + 'visible': content_query('visible', 'visible_quantifier'), + 'hidden': (lambda hid: r'hidden=%s' % content_unformatter(hid) if hid else NO_SEARCH_QUERY), + 'top_slope': topslope_query, + 'u': nullable_int_query('u'), + 't': nullable_int_query('t'), + 'ind_of_insep': content_query('ind_of_insep', 'insep_quantifier'), + 'associated_inertia': bracket_query('associated_inertia'), + 'jump_set': bracket_query('jump_set'), } + # The public parameters of the search page, where they differ from the column name + url_params = {'galois_label': ['gal'], + 'top_slope': ['topslope'], + 'slopes': ['slopes', 'slopes_quantifier'], + 'visible': ['visible', 'visible_quantifier'], + 'ind_of_insep': ['ind_of_insep', 'insep_quantifier']} + # top_slope is stored encoded, so bucket endpoints must be encoded before they + # are compared against it + bucket_encoders = {'top_slope': topslope_encoder} + # The last bucket is left open above, so that fields are not omitted from the + # table if the database grows beyond the current maximum (p < 200 and c <= 79) + buckets = {'p': ['2', '3', '5', '7', '11-19', '23-97', '101-'], + 'c': ['0', '1', '2', '3', '4', '5-8', '9-16', '17-32', '33-']} stat_list = [ ramdisp(2), @@ -1965,26 +2175,30 @@ def dynamic_parse(info, query): common_parse(info, query) dynamic_parent_page = "padic-refine-search.html" - dynamic_cols = ["galois_label", "slopes"] + dynamic_cols = ["p", "n", "e", "f", "c", "galois_label", "aut", "u", "t", + "top_slope", "slopes", "visible", "hidden", + "ind_of_insep", "associated_inertia", "jump_set"] @property def short_summary(self): - return 'The database currently contains %s %s, %s absolute %s, and %s relative families. Here are some further statistics.' % ( + return 'The database currently contains %s %s, %s absolute %s, and %s relative families. Here are some further statistics, or you can create your own.' % ( comma(self.numfields), display_knowl("lf.padic_field", r"$p$-adic fields"), comma(self.num_abs_families), display_knowl("lf.family_polynomial", "families"), comma(self.num_rel_families), url_for(".statistics"), + url_for(".dynamic_statistics"), ) @property def summary(self): - return r'The database currently contains %s %s, including all with $p < 200$ and %s $n < 24$. It also contains all %s absolute %s with $p < 200$ and degree $n < 48$, as well as all %s relative families with $p < 200$, base degree $n_0 < 16$ and absolute degree $n_{\mathrm{absolute}} < 48$.' % ( + return r'The database currently contains %s %s, including all with $p < 200$ and %s $n < 24$. It also contains all %s absolute %s with $p < 200$ and degree $n < 48$, as well as all %s relative families with $p < 200$, base degree $n_0 < 16$ and absolute degree $n_{\mathrm{absolute}} < 48$. In addition to the statistics below, you can also create your own.' % ( comma(self.numfields), display_knowl("lf.padic_field", r"$p$-adic fields"), display_knowl("lf.degree", "degree"), comma(self.num_abs_families), display_knowl("lf.family_polynomial", "families"), comma(self.num_rel_families), + url_for(".dynamic_statistics"), ) diff --git a/lmfdb/local_fields/test_localfields.py b/lmfdb/local_fields/test_localfields.py index ebc9225bcf..d25b354bdd 100644 --- a/lmfdb/local_fields/test_localfields.py +++ b/lmfdb/local_fields/test_localfields.py @@ -1,6 +1,49 @@ from lmfdb.tests import LmfdbTest +class StubTable(): + """ + Stands in for a table and its statistics backend, with a controlled set of + counts, so that tests of the statistics framework do not depend on which + statistics happen to be cached in the database. + + INPUT: + + - ``counts`` -- the counts, as a dictionary from tuples of stored values, or + as a list of (tuple of stored values, count) pairs when those values are + unhashable (a bucketed column is stored as a range) + - ``records`` -- the number of records satisfying the constraint, which + exceeds the total when the column is not computed for some of them. + Defaults to the number counted. + """ + def __init__(self, counts, records=None): + self.counts = list(counts.items() if isinstance(counts, dict) else counts) + self.records = records + # the same object serves as the table, its statistics and its counts + self.stats = self.table = self + + def _get_values_counts(self, cols, constraint, split_list, formatter, + query_formatter, base_url, buckets=None): + from psycodict.utils import KeyedDefaultDict + headers = [[] for _ in cols] + data = KeyedDefaultDict(lambda key: {'count': 0, 'query': '', 'proportion': ''}) + for values, cnt in self.counts: + for val, header in zip(values, headers): + header.append(val) + key = tuple(formatter[col](val) for col, val in zip(cols, values)) + data[key if len(cols) > 1 else key[0]] = {'count': cnt, 'query': '', 'proportion': ''} + return (headers, data) if len(cols) > 1 else (headers[0], data) + + def _get_total_avg(self, cols, constraint, avg, split_list): + # as in psycodict, records where the column is null are left out + return sum(cnt for values, cnt in self.counts if values[0] is not None), False + + def count(self, query): + if self.records is not None: + return self.records + return sum(cnt for _values, cnt in self.counts) + + class LocalFieldTest(LmfdbTest): # All tests should pass @@ -23,6 +66,411 @@ def test_search_top_slope(self): L = self.tc.get('/padicField/?p=2&topslope=7/2') assert '2.1.4.9a1.1' in L.get_data(as_text=True) # number of matches + def test_stats_pages(self): + # The browse page and statistics page link to dynamic statistics + L = self.tc.get('/padicField/') + assert 'dynamic_stats' in L.get_data(as_text=True) + L = self.tc.get('/padicField/stats') + dat = L.get_data(as_text=True) + assert 'create your own' in dat and 'dynamic_stats' in dat + + def test_dynamic_stats(self): + # A combination whose statistics are precomputed: degree x ramification index for p=2 + # (there are 6 totally ramified quadratic extensions of Q_2) + L = self.tc.get('/padicField/dynamic_stats?p=2&col1=n&totals1=yes&col2=e&proportions=rows&search_type=DynStats') + dat = L.get_data(as_text=True) + assert 'n=2&e=2' in dat and '>6<' in dat + # Galois groups for p=2, n=4 (also precomputed); 12 of the 59 quartic + # 2-adic fields are cyclic + L = self.tc.get('/padicField/dynamic_stats?p=2&n=4&col1=galois_label&proportions=none&search_type=DynStats') + dat = L.get_data(as_text=True) + assert 'C_4' in dat and '>12<' in dat + # All column options render (values for most combinations are computed and + # cached on demand, so against a read-only database the tables may be empty, + # but the pages should not error) + for col in ['p', 'n', 'e', 'f', 'c', 'galois_label', 'aut', 'u', 't', 'top_slope', + 'slopes', 'visible', 'hidden', 'ind_of_insep', 'associated_inertia', 'jump_set']: + L = self.tc.get('/padicField/dynamic_stats?col1=%s&proportions=recurse&search_type=DynStats' % col) + assert L.status_code == 200 + other = 'p' if col == 'n' else 'n' + L = self.tc.get('/padicField/dynamic_stats?col1=%s&col2=%s&totals1=yes&totals2=yes&proportions=rows&search_type=DynStats' % (col, other)) + assert L.status_code == 200 + + def test_dynamic_stats_null_query_formatters(self): + # Not-computed / empty values have no search representation, so their + # query_formatters return the NO_SEARCH_QUERY sentinel and the drill-down link + # is suppressed rather than pointing at an unfiltered search (LMFDB#6542). + from lmfdb.local_fields.main import ( + LFStats, NO_SEARCH_QUERY, galquery, bracket_query, content_query, + nullable_int_query, formatbracketcol, topslope_query) + # Null / empty -> sentinel; genuine values -> a real search fragment + assert galquery(None) == NO_SEARCH_QUERY + assert galquery('1T1') == 'gal=1T1' + assert nullable_int_query('u')(None) == NO_SEARCH_QUERY + assert nullable_int_query('u')(2) == 'u=2' + assert bracket_query('associated_inertia')(None) == NO_SEARCH_QUERY + assert bracket_query('associated_inertia')([1, 2]) == 'associated_inertia=[1, 2]' + # an empty jump set is searchable, unlike one that is not computed + assert bracket_query('jump_set')([]) == 'jump_set=[]' + cq = content_query('slopes', 'slopes_quantifier') + assert cq(None) == NO_SEARCH_QUERY and cq([]) == NO_SEARCH_QUERY and cq('[]') == NO_SEARCH_QUERY + assert cq('[2, 2]') == 'slopes=[2, 2]&slopes_quantifier=exactly' + assert LFStats.query_formatters['hidden'](None) == NO_SEARCH_QUERY + assert LFStats.query_formatters['hidden']('') == NO_SEARCH_QUERY + assert topslope_query(None) == NO_SEARCH_QUERY + # None array columns render as "not computed", never a literal $None$ (P3) + assert formatbracketcol(None) == 'not computed' + assert formatbracketcol('') == 'not computed' + assert formatbracketcol([]) == r'$[\ ]$' + assert formatbracketcol([1, 2]) == '$[1, 2]$' + # a not-computed jump set and an empty one are distinct values, so they must + # not share a row of the table under one ambiguous label + assert LFStats.formatters['jump_set'](None) != LFStats.formatters['jump_set']([]) + # and the empty one is a search that works, so its count stays linked + from lmfdb import db + assert self._url_count('/padicField/?jump_set=[]') == db.lf_fields.count({'jump_set': []}) + # display_data blanks a sentinel-bearing drill-down while keeping real ones + base = '/padicField/?' + data = {'counts': [ + {'value': 'not computed', 'count': 5, 'query': base + NO_SEARCH_QUERY}, + {'value': 'C_4', 'count': 7, 'query': base + 'gal=4T1'}]} + LFStats._suppress_unsearchable(data) + assert data['counts'][0]['query'] == '' + assert data['counts'][1]['query'] == base + 'gal=4T1' + grid = {'grid': [('C_4', [ + {'count': 1, 'query': base + 'gal=4T1&' + NO_SEARCH_QUERY}, + {'count': 2, 'query': base + 'gal=4T1&e=2'}])]} + LFStats._suppress_unsearchable(grid) + assert grid['grid'][0][1][0]['query'] == '' + assert grid['grid'][0][1][1]['query'] == base + 'gal=4T1&e=2' + + def test_dynamic_stats_null_bucket(self): + # A not-computed Galois group (3996 fields) cannot be expressed as a search, so + # its statistics bucket is shown without a drill-down link -- clicking it must + # not open an unfiltered search returning every field (LMFDB#6542 review). + import re + empty_link = re.compile(r"href='[^']*[?&][A-Za-z_]+=(?:&|')") + for col, param in [('galois_label', 'gal'), ('slopes', 'slopes'), ('hidden', 'hidden')]: + L = self.tc.get('/padicField/dynamic_stats?col1=%s&proportions=none&search_type=DynStats' % col) + assert L.status_code == 200 + dat = L.get_data(as_text=True) + assert 'not computed' in dat # the null bucket is displayed... + assert "?%s='" % param not in dat # ...with no empty-parameter link + assert empty_link.search(dat) is None # no drill-down has an empty value + assert '\x00' not in dat # the sentinel never leaks to output + # Non-null buckets still link to a correctly-filtered search (12 cyclic quartics) + L = self.tc.get('/padicField/dynamic_stats?p=2&n=4&col1=galois_label&proportions=none&search_type=DynStats') + dat = L.get_data(as_text=True) + assert 'gal=4T1' in dat and empty_link.search(dat) is None + + ############################################################################ + # Drill-down links on the dynamic statistics page must describe exactly the + # records that were counted. The helpers below read the table off the page + # and count what each link actually selects, by running its parameters + # through the same parser the search page uses. + ############################################################################ + + @staticmethod + def _stat_rows(dat): + """The rows of a 2d statistics table, as (header, [(count, url), ...]).""" + import re + dat = dat.replace('&', '&') + rows = [] + for header, body in re.findall(r'(.*?)(.*?)', dat, re.S): + cells = [(int(cnt) if cnt else 0, url) for url, cnt in + re.findall(r"(?:)?(\d*)", body)] + rows.append((re.sub('<[^>]*>', '', header).strip(), cells)) + return rows + + @staticmethod + def _stat_counts(dat): + """ + The counts of a 1d statistics table, as (label, count, url). + + Such a table is transposed: a row of values, then the row of counts + underneath it, for each block of ten values. + """ + import re + dat = dat.replace('&', '&') + counts, labels = [], None + for row in re.findall(r']*>(.*?)', dat, re.S): + head = re.match(r'\s*]*>(.*?)', row, re.S) + if head is None: + continue + title = re.sub('<[^>]*>', '', head.group(1)).strip() + cells = row[head.end():] + if title == 'count': + for label, (url, cnt) in zip(labels or [], re.findall( + r"(?:)?(\d+)(?:)?", cells)): + counts.append((label, int(cnt), url)) + labels = None + elif title != 'proportion' and labels is None: + labels = [re.sub('<[^>]*>', '', cell).strip() + for cell in re.findall(r'(.*?)', cells, re.S)] + return counts + + def _url_count(self, url): + """How many fields the search page returns for a drill-down url.""" + from urllib.parse import urlparse, parse_qsl + from lmfdb import db + from lmfdb.local_fields.main import common_parse + from lmfdb.utils import to_dict + parsed = urlparse(url) + assert parsed.path == '/padicField/', url + info = to_dict(dict(parse_qsl(parsed.query, keep_blank_values=True))) + query = {} + with self.app.test_request_context(): + common_parse(info, query) + # the search page flashes an error and sets 'err' for input it rejects + assert 'err' not in info, "search page rejects %s" % url + return db.lf_fields.count(query) + + def test_dynamic_stats_2d_links(self): + # In a sparse grid, the cells with no fields must constrain the search the + # same way the nonempty ones do, so that intersecting the cells of a row + # leaves the row's own constraint in the total's link (LMFDB#6542 review). + L = self.tc.get('/padicField/dynamic_stats?p=2&col1=galois_label&col2=n&totals1=yes&proportions=none&search_type=DynStats') + assert L.status_code == 200 + rows = self._stat_rows(L.get_data(as_text=True)) + assert rows, "no statistics available for Galois group by degree" + sparse = 0 + for header, cells in rows: + counts, total = cells[:-1], cells[-1] + if not any(cnt for cnt, _ in counts) or all(cnt for cnt, _ in counts): + continue # not a sparse row + sparse += 1 + label = header.split('as ')[1].rstrip(')') # eg '$C_4$ (as 4T1)' -> '4T1' + # the cells with fields in them constrain both the group and the degree + for cnt, url in counts: + if cnt: + assert 'gal=%s' % label in url and 'n=' in url, (header, url) + assert '$' not in url and '<' not in url, url + # the total is over the whole row, empty cells included, so its link + # keeps the group but not any one degree + assert 'gal=%s' % label in total[1] and 'n=' not in total[1], (header, total) + assert total[0] == sum(cnt for cnt, _ in counts) + if sparse <= 3: + assert self._url_count(total[1]) == total[0], total + assert sparse >= 10, "expected a sparse grid, found %s sparse rows" % sparse + + def test_dynamic_stats_2d_zero_cell_urls(self): + # The same invariant, checked directly on display_data so that it does not + # depend on which statistics happen to be cached in the database. The + # formatter for slopes produces TeX, which is not valid search input, so an + # empty cell built from the displayed value rather than the stored one is + # visibly wrong (and its row's total would lose the constraint entirely). + from lmfdb.local_fields.main import LFStats + from lmfdb.utils import totaler + + # slopes [2] occurs only in degree 2, and is not computed for some fields + table = StubTable({('[2]', 2): 5, ('[2, 2]', 4): 7, ('[2, 2]', 8): 3, (None, 2): 11}) + with self.app.test_request_context('/padicField/'): + data = LFStats().display_data( + cols=['slopes', 'n'], table=table, proportioner=False, + totaler=totaler(row_counts=True, col_counts=False)) + grid = dict(data['grid']) + assert data['col_headers'] == ['2', '4', '8', 'Total'] + # the empty cells of the [2] row carry the same slopes constraint as its + # nonempty cell, built from the stored value rather than the displayed one + row = grid['$[2]$'] + assert [D['count'] for D in row] == [5, 0, 0, 5] + for D, n in zip(row, [2, 4, 8]): + assert D['query'] == '/padicField/?slopes=[2]&slopes_quantifier=exactly&n=%s' % n + assert row[-1]['query'] == '/padicField/?slopes=[2]&slopes_quantifier=exactly' + # a row spanning two degrees keeps only the row constraint in its total + assert [D['count'] for D in grid['$[2, 2]$']] == [0, 7, 3, 10] + assert grid['$[2, 2]$'][-1]['query'] == '/padicField/?slopes=[2, 2]&slopes_quantifier=exactly' + # slopes that are not computed cannot be searched for, so neither the cells + # of that row nor its total are linked + assert [D['count'] for D in grid['not computed']] == [11, 0, 0, 11] + assert all(D['query'] == '' for D in grid['not computed']) + + def test_dynamic_stats_1d_totals(self): + # The Total of a one-dimensional table counts the fields where the column + # is computed, which is not something the search page can ask for, so it is + # only linked when that is every field matching the constraint. Of the 3784 + # fields with p=7 and n=21, only 1324 have a computed Galois group. + from lmfdb import db + L = self.tc.get('/padicField/dynamic_stats?p=7&n=21&col1=galois_label' + '&totals1=yes&proportions=none&search_type=DynStats') + assert L.status_code == 200 + counts = self._stat_counts(L.get_data(as_text=True)) + assert counts, "no statistics available for Galois groups with p=7, n=21" + total = [c for c in counts if c[0] == 'Total'] + assert len(total) == 1, counts + _label, count, url = total[0] + assert count == db.lf_fields.count({'p': 7, 'n': 21, 'galois_label': {'$exists': True}}) + assert count < db.lf_fields.count({'p': 7, 'n': 21}) + assert url == '', url # a link here would return the other 2460 too + # where the column is computed for every field, the total is the constraint + # itself, and is linked + L = self.tc.get('/padicField/dynamic_stats?p=2&n=8&col1=galois_label' + '&totals1=yes&proportions=none&search_type=DynStats') + counts = self._stat_counts(L.get_data(as_text=True)) + total = [c for c in counts if c[0] == 'Total'][0] + assert total[2] and 'galois_label' not in total[2], total + assert self._url_count(total[2]) == total[1] == db.lf_fields.count({'p': 2, 'n': 8}) + # a total over buckets covers only the buckets displayed, so it is not + # linked, while the buckets themselves still are + L = self.tc.get('/padicField/dynamic_stats?col1=top_slope&buckets1=0-1,1-2' + '&totals1=yes&proportions=none&search_type=DynStats') + counts = self._stat_counts(L.get_data(as_text=True)) + assert [label for label, _cnt, _url in counts] == ['$0$-$1$', '$1$-$2$', 'Total'] + assert counts[-1][2] == '', counts + assert [url for _label, _cnt, url in counts[:2]] == [ + '/padicField/?topslope=0-1', '/padicField/?topslope=1-2'] + + def test_dynamic_stats_1d_totals_policy(self): + # The same rule, on a controlled set of counts. A column that is not + # computed for every field, a total over part of the column, and a total + # over the entries of lists rather than over fields all go unlinked. + from lmfdb.local_fields.main import LFStats + stats = LFStats() + with self.app.test_request_context('/padicField/'): + def total_of(table, **kwds): + counts = stats.display_data(table=table, totaler={'avg': False}, + proportioner=False, **kwds)['counts'] + assert counts[-1]['value'] == 'Total' + return counts[-1]['count'], counts[-1]['query'] + # every field has a Galois group here, so the total is the whole search + assert total_of(StubTable({('4T1',): 5, ('4T3',): 11}), cols=['galois_label'], + link_constraint='p=2&n=4') == (16, '/padicField/?p=2&n=4') + # here it is not computed for 11 of the 27, which no search expresses + assert total_of(StubTable({('4T1',): 5, ('4T3',): 11, (None,): 11}, records=27), + cols=['galois_label'], link_constraint='p=2&n=4') == (16, '') + # a total over buckets covers only the buckets shown + assert total_of(StubTable([(({'$gte': 0, '$lte': 1},), 7), + (({'$gte': 2, '$lte': 3},), 9)]), + cols=['c'], buckets={'c': ['0-1', '2-3']}) == (16, '') + # a constraint on the column being displayed is left out of the urls, + # since each count constrains that column itself, so the total would + # describe more fields than it counted + assert total_of(StubTable({(2,): 5, (4,): 11}), cols=['n'], + constraint={'p': 2, 'n': {'$lte': 4}}, + link_constraint='p=2') == (16, '') + # and a split-list total counts entries of lists rather than fields + assert LFStats._total_url('/padicField/?', [], None, ['cm_discs'], {}, 16, {}, True) == '' + + def test_dynamic_stats_public_urls(self): + # A constraint entered on the dynamic statistics page has to survive + # clicking a count. The parsed query uses internal columns (slopes_tmp and + # friends) that the search page does not accept, so the links are built from + # the search boxes instead, and must reproduce the same query (LMFDB#6542 + # review). + from urllib.parse import urlparse, parse_qsl + from lmfdb.local_fields.main import LFStats, common_parse + from lmfdb.utils import to_dict + constraints = [ + 'slopes=[2, 2]&slopes_quantifier=exactly', + 'slopes=[2]&slopes_quantifier=include', + 'slopes=[2]&slopes_quantifier=exclude', + 'slopes=[2, 2]&slopes_quantifier=subset', + 'visible=[2]&visible_quantifier=exactly', + 'visible=[2]&visible_quantifier=include', + 'ind_of_insep=[1, 0]&insep_quantifier=exactly', + 'ind_of_insep=[1, 0]&insep_quantifier=subset', + 'topslope=1-2&p=2', + 'jump_set=[1]&associated_inertia=[1, 1]', + ] + stats = LFStats() + for constraint in constraints: + info = to_dict(dict(parse_qsl(constraint))) + info.update({'col1': 'n', 'totals1': 'yes', 'search_type': 'DynStats'}) + query = {} + with self.app.test_request_context(): + stats.dynamic_parse(info, query) + link = stats.dynamic_link_constraint(info, ['n']) + # the parameters of the link, parsed as the search page parses them + reparsed_info = to_dict(dict(parse_qsl(link))) + reparsed = {} + common_parse(reparsed_info, reparsed) + assert 'err' not in reparsed_info, (constraint, link) + assert '_tmp' not in link, (constraint, link) + assert '=None' not in link, (constraint, link) + assert reparsed == query, (constraint, link, reparsed, query) + # the quantifier is part of what the user asked for, so it is kept + for key, val in parse_qsl(constraint): + if key.endswith('quantifier'): + assert '%s=%s' % (key, val) in link, (constraint, link) + # and every drill-down link is built from those fragments. Statistics + # for a constrained column are computed on demand, so the page itself + # has nothing to show against a read-only database; the counts are + # supplied here so that the check does not depend on the cache. + with self.app.test_request_context('/padicField/'): + counts = stats.display_data(cols=['n'], table=StubTable({(2,): 5, (4,): 7}), + proportioner=False, link_constraint=link)['counts'] + assert [D['count'] for D in counts] == [5, 7] + for D, n in zip(counts, [2, 4]): + assert D['query'] == '/padicField/?%s&n=%s' % (link, n), (constraint, D['query']) + # the pages themselves render, and never expose an internal column + for constraint in constraints: + url = ('/padicField/dynamic_stats?%s&col1=n&totals1=yes&proportions=none' + '&search_type=DynStats' % constraint) + L = self.tc.get(url) + assert L.status_code == 200, constraint + dat = L.get_data(as_text=True) + assert 'is not a valid input' not in dat, constraint + assert '_tmp' not in dat and '=None' not in dat, constraint + for _label, cnt, link in self._stat_counts(dat): + if link: + assert urlparse(link).path == '/padicField/', link + if cnt: + assert self._url_count(link) == cnt, (constraint, link, cnt) + + def test_dynamic_stats_topslope_buckets(self): + # top_slope is stored as a fixed-width decimal prefix followed by the exact + # rational, so that the database sorts it numerically. Bucket endpoints + # must be encoded the same way before being compared, and decoded again for + # display and for links (LMFDB#6542 review). + import re + from lmfdb import db + from lmfdb.local_fields.main import ratproc + url = ('/padicField/dynamic_stats?col1=top_slope&buckets1=0-1,1-2,2-' + '&totals1=yes&proportions=none&search_type=DynStats') + L = self.tc.get(url) + assert L.status_code == 200 + dat = L.get_data(as_text=True) + # the buckets are labeled and linked with exact rationals, not with the + # encoding, and an endpoint is never dropped (which produced 'topslope=-') + assert '$0$-$1$' in dat and '$1$-$2$' in dat and '$2$-' in dat + assert '00.0000000000' not in dat and 'topslope=-' not in dat + links = {h.replace('&', '&') for h in re.findall(r"href='(/padicField/\?[^']*)'", dat)} + assert {'/padicField/?topslope=0-1', '/padicField/?topslope=1-2', + '/padicField/?topslope=2-'} <= links, sorted(links) + # each bucket's link selects exactly the fields the bucket counts: the + # comparison the statistics backend makes, on the encoded endpoints + for bucket, query in [('0-1', {'$gte': ratproc('0'), '$lte': ratproc('1')}), + ('1-2', {'$gte': ratproc('1'), '$lte': ratproc('2')}), + ('2-', {'$gte': ratproc('2')})]: + assert self._url_count('/padicField/?topslope=%s' % bucket) == db.lf_fields.count({'top_slope': query}) + # the counts shown agree too, wherever the statistics have been computed + for label, cnt, link in self._stat_counts(dat): + if cnt and link: + assert self._url_count(link) == cnt, (label, link, cnt) + # buckets compose with the two-dimensional grid and its totals + L = self.tc.get('/padicField/dynamic_stats?col1=top_slope&buckets1=0-1,1-2,2-' + '&col2=n&totals1=yes&proportions=none&search_type=DynStats') + assert L.status_code == 200 + for header, cells in self._stat_rows(L.get_data(as_text=True)): + for cnt, link in cells: + if link: + assert 'topslope=' in link and '00.00' not in link, (header, link) + # an endpoint that is not a rational number is rejected with the usual + # message, rather than silently counting nothing + L = self.tc.get('/padicField/dynamic_stats?col1=top_slope&buckets1=0-junk&search_type=DynStats') + assert L.status_code == 200 + assert 'not a non-negative rational number' in L.get_data(as_text=True) + + def test_dynamic_stats_bucket_coverage(self): + # Every field has to land in some bucket, so the last bucket of each column + # with defaults is unbounded above + from lmfdb import db + from lmfdb.local_fields.main import LFStats + for col in LFStats.buckets: + buckets = LFStats.buckets[col] + assert buckets[-1].endswith('-'), (col, buckets) + assert db.lf_fields.count({col: {'$lt': int(buckets[0])}}) == 0, col + def test_field_page(self): L = self.tc.get('/padicField/11.6.4.2', follow_redirects=True) assert '11.2.3.4a1.1' in L.get_data(as_text=True) diff --git a/lmfdb/templates/stat_1d.html b/lmfdb/templates/stat_1d.html index 135b90861d..78cdf6e3c1 100644 --- a/lmfdb/templates/stat_1d.html +++ b/lmfdb/templates/stat_1d.html @@ -19,7 +19,11 @@ count {% for c in r %} + {% if c.query %} {{c.count}} + {% else %} + {{c.count}} + {% endif %} {% endfor %} diff --git a/lmfdb/utils/display_stats.py b/lmfdb/utils/display_stats.py index beca6ab94d..ddcaffe664 100644 --- a/lmfdb/utils/display_stats.py +++ b/lmfdb/utils/display_stats.py @@ -1,11 +1,89 @@ from collections import defaultdict +from urllib.parse import quote from flask import url_for from sage.all import UniqueRepresentation, lazy_attribute, infinity -from .utilities import format_percentage +from .utilities import format_percentage, flash_error from .web_display import display_knowl -from psycodict.utils import KeyedDefaultDict, range_formatter +from psycodict.utils import KeyedDefaultDict, SearchParsingError, range_formatter + +# Included in a drill-down url by a query formatter when the value being counted +# cannot be expressed as a search: a not-computed (NULL) value for example. +# ``display_data`` shows the count of such a cell without a link, rather than +# linking to a search that returns the wrong records (an empty url parameter is +# ignored by the search parsers, so it would return everything). A NUL byte +# cannot occur in a url, which makes the marker unambiguous, and it is a url +# fragment in its own right, so it survives the intersection that ``totaler`` +# uses to build the link for a row or column total. +NO_SEARCH_QUERY = "\x00" + +# Characters left alone when a search box value is copied into a drill-down url. +# Brackets, commas and slashes are common in LMFDB search input and are safe in a +# query string; everything else that is special (notably '&', '=', '#', '%', '+' +# and spaces) is escaped. +URL_SAFE = "[](),/:" + +def _hashable(val): + """ + A hashable canonical form of a value as stored in the database, used to match + up the counts returned by the statistics backend with the rows and columns of + the table being displayed. Distinct values always get distinct keys, unlike + the strings produced by the formatters. + """ + if isinstance(val, dict): + return ("$dict",) + tuple((key, _hashable(val[key])) for key in sorted(val)) + elif isinstance(val, (list, tuple)): + return ("$list",) + tuple(_hashable(x) for x in val) + return val + +class StatHeader(): + """ + One row or column of a statistics table. + + Separating the three roles of a value keeps drill-down links correct even + when displaying a value loses information: ``label`` is shown to the user, + ``keys`` are used to look up counts, and ``fragment`` constrains a search to + this value and is built from the value as stored in the database. + + INPUT: + + - ``label`` -- the string displayed as the header + - ``value`` -- a value taken on by the column, as stored in the database + (or the bucket string, for a bucketed column) + - ``key`` -- the key under which the statistics backend stores counts for ``value`` + - ``fragment`` -- url fragment(s) constraining a search to ``value`` + """ + def __init__(self, label, value, key, fragment): + self.label = label + self.value = value + self.keys = [key] + self.fragment = fragment + + def add(self, key, fragment): + """ + Include another value in this header. + + Two values that display identically have to share a row, since the user + cannot tell them apart; their counts are added. A link is only shown if + they also constrain the search in the same way, since a search for the + union of two values usually cannot be expressed. + """ + self.keys.append(key) + if fragment != self.fragment: + self.fragment = NO_SEARCH_QUERY + + def count(self, data, other=None): + """ + The number of rows with this value (and ``other``'s value, in the 2d case), + given the ``data`` returned by the statistics backend. + """ + if other is None: + keys = self.keys + else: + keys = [(key, okey) for key in self.keys for okey in other.keys] + # data is a KeyedDefaultDict, so we must avoid looking up absent keys + return sum(data[key]["count"] for key in keys if key in data) class formatters(): @classmethod @@ -157,6 +235,7 @@ def per_grid_recurse(cls, attr): attr = dict(attr) attr['base_url'] = '' # urls aren't used below attr['constraint'] = {} + attr['link_constraint'] = None attr['proportioner'] = False attr['totaler'] = False @@ -190,6 +269,7 @@ def recurse_1d(cls, attr): attr = dict(attr) attr['base_url'] = '' attr['constraint'] = None + attr['link_constraint'] = None attr['proportioner'] = False attr['totaler'] = False @@ -231,18 +311,24 @@ def common_link(cls, link_list): """ Takes a nonempty list of links to search pages and returns the link with search options the intersection of the search options. The initial part of the link must be the same for all. + + The options are kept in the order they appear in the first link, so that + the same table always produces the same urls. """ def _split(link): H, T = link.split('?') - T = set(T.split('&')) - return H, T - head, tails = _split(link_list[0]) + return H, [frag for frag in T.split('&') if frag] + head, tail = _split(link_list[0]) + common = set(tail) for link in link_list[1:]: H, T = _split(link) if H != head: raise ValueError("Cannot vary main url") - tails.intersection_update(T) - return head + '?' + '&'.join(tails) + common.intersection_update(T) + seen = set() + fragments = [frag for frag in tail + if frag in common and not (frag in seen or seen.add(frag))] + return head + '?' + '&'.join(fragments) def __init__(self, row_counts=True, row_proportions=True, col_counts=True, col_proportions=True, corner_count=None, corner_proportion=None, include_links=True, row_total_label='Total', col_total_label='Total'): if corner_count and not (row_counts and col_counts): @@ -260,7 +346,8 @@ def __init__(self, row_counts=True, row_proportions=True, col_counts=True, col_p self.col_total_label = col_total_label def __call__(self, grid, row_headers, col_headers, stats): - if not grid: + if not grid or not grid[0]: + # No cells to total, which happens when no statistics are available return row_counts = self.row_counts row_proportions = self.row_proportions @@ -285,6 +372,10 @@ def __call__(self, grid, row_headers, col_headers, stats): for i, row in enumerate(grid): total = sum(D['count'] for D in row) query = self.common_link([D['query'] for D in row]) if include_links else None + if query is not None and query[-1] == '?': + # No search options are common to the whole row, so a link + # would return every record rather than the ones counted + query = None if recursive_prop: overall = sum(D['count'] for D in stats._total_grid[i]) if corner_count: @@ -382,8 +473,19 @@ class StatsDisplay(UniqueRepresentation): values or ranges like '2-10'. - ``formatters`` -- callables as values. Input a database value or bucket, output the text to display in the header. - - ``query_formatters`` -- callables as values. Input a database value or output of formatter, - output the text to insert into the url, such as 'level=2-10'. + - ``query_formatters`` -- callables as values. Input a database value or bucket, + output the text to insert into the url, such as 'level=2-10'. The input is + never the output of a formatter, so a formatter is free to produce TeX or + html that could not be parsed as search input. Return ``NO_SEARCH_QUERY`` + for a value that cannot be searched for, such as a value that is not + computed; its count is then displayed without a link. + - ``bucket_encoders`` -- callables as values. Input an endpoint of a bucket, as + typed into the search box for that column, output the corresponding value + to compare against the database. Needed for a column whose database + representation does not sort in the same way as the values shown to users. + - ``url_params`` -- lists of strings as values. The parameters of the search + page that constrain this column, if they are not just the column name. + Used to avoid constraining a column that is being displayed. - ``sort_keys`` -- callables as values. Custom sorting for this column (as in ``sorted``) - ``reverses`` -- boolean values. Whether to reverse the order of the header (as in ``sorted``) - ``split_lists`` -- boolean values. Whether to count entries from lists individually. @@ -413,6 +515,18 @@ def _buckets(self): A.update(getattr(self, 'buckets', {})) return A + @property + def _bucket_encoders(self): + A = defaultdict(lambda: None) + A.update(getattr(self, 'bucket_encoders', {})) + return A + + @property + def _url_params(self): + A = KeyedDefaultDict(lambda col: [col]) + A.update(getattr(self, 'url_params', {})) + return A + @property def _dynamic_cols(self): return [('none', 'None')] + [(col, self._short_display[col]) for col in self.dynamic_cols] @@ -469,9 +583,160 @@ def _split_lists(self): def stats(self): return self + def _bucket_endpoints(self, bucket): + """ + Split a bucket into its endpoints, using the same syntax as the statistics + backend: '2' is a single value, '2-10' a closed range and '10-' a range + unbounded above. + + OUTPUT: + + A pair (endpoints, rebuild), where ``rebuild`` reassembles a bucket from a + list of endpoints of the same length. + """ + if bucket[-1] == '-': + return [bucket[:-1]], (lambda L: L[0] + '-') + elif '-' not in bucket[1:]: + return [bucket], (lambda L: L[0]) + elif bucket[0] == '-': + # a negative lower endpoint, as in '-10-5' + L = bucket[1:].split('-') + L[0] = '-' + L[0] + else: + L = bucket.split('-') + if len(L) != 2: + raise SearchParsingError("%s is not a single value or a range such as 2-10." % bucket) + return L, (lambda L: '%s-%s' % tuple(L)) + + def _encode_buckets(self, buckets): + """ + Rewrite bucket strings into the form used to compare against the database. + + This is the identity for a column with no entry in ``bucket_encoders``, + which covers every column whose database representation sorts in the same + way as the values shown to users. + """ + encoded = {} + for col, bucket_list in buckets.items(): + encoder = self._bucket_encoders[col] + if encoder is None: + encoded[col] = bucket_list + continue + encoded[col] = elist = [] + for bucket in bucket_list: + endpoints, rebuild = self._bucket_endpoints(bucket) + endpoints = [encoder(endpoint) for endpoint in endpoints] + if any('-' in endpoint for endpoint in endpoints): + raise ValueError("Bucket encoder for %s produced a '-'" % col) + elist.append(rebuild(endpoints)) + return encoded + + def _bucket_key(self, bucket): + """ + The key under which the statistics backend stores the counts for a bucket. + + It normalizes a range with equal endpoints, since the backend stores such + a bucket as a single value. + """ + endpoints, _ = self._bucket_endpoints(bucket) + if len(endpoints) == 2 and endpoints[0] == endpoints[1]: + return endpoints[0] + return bucket + + def _make_headers(self, col, values, buckets): + """ + Assemble the headers for the rows or columns of a statistics table. + + INPUT: + + - ``col`` -- the column being displayed + - ``values`` -- the values it takes on, as stored in the database + (ignored when the column is bucketed) + - ``buckets`` -- the list of buckets for this column, or None if it is not bucketed + + OUTPUT: + + A list of ``StatHeader`` objects, in the order they should be displayed. + """ + formatter = self._formatters[col] + query_formatter = self._query_formatters[col] + if buckets is None: + # Distinct values can display the same way, so deduplicate on the values + # themselves, keeping one of each to build the header and its links from + distinct = {} + for val in values: + distinct.setdefault(_hashable(val), val) + pairs = sorted(distinct.items(), key=lambda kv: self._sort_keys[col](kv[1]), + reverse=self._reverses[col]) + else: + encoded = self._encode_buckets({col: buckets})[col] + pairs = [(self._bucket_key(bucket), public) + for public, bucket in zip(buckets, encoded)] + headers = [] + by_label = {} + for key, val in pairs: + label = formatter(val) + fragment = query_formatter(val) + if label in by_label: + by_label[label].add(key, fragment) + else: + by_label[label] = header = StatHeader(label, val, key, fragment) + headers.append(header) + return headers + + @staticmethod + def _total_url(base_url, extras, table, cols, constraint, total, buckets, split_list): + """ + The url for the total of a one-dimensional table, or the empty string when + the records it counts cannot be described by a search. + + The only aggregate a search page can express here is the constraint itself, + which is the right one exactly when every record satisfying the constraint + is included in the total. That fails in four ways: + + - a total over buckets covers only the buckets displayed, which need not + exhaust the column, and a union of buckets is not a search anyway; + - a total over a column whose lists are split counts the entries of those + lists rather than records; + - the statistics backend leaves out records where the column is null, so a + column that is not computed for every record is totaled over only some + of them; + - a constraint on the column being displayed is left out of the urls, since + each cell constrains that column itself, so the total would claim more + records than it counted. + + The column name on its own used to be appended as a marker, but an empty + parameter is ignored by the search parsers, so such a link silently + returned every record rather than the ones counted (and the parameter is + often not one the search page accepts, as with galois_label and gal). + """ + constraint = constraint or {} + if buckets or split_list or any(col in constraint for col in cols): + return '' + if total != table.table.count(constraint): + # Some records have no value in this column, and no search selects + # exactly the ones that do + return '' + return base_url + '&'.join(extras) + + @staticmethod + def _suppress_unsearchable(data): + """ + Blank the url of any cell whose value cannot be searched for, so that its + count is displayed without a link. + """ + def fix(cells): + for cell in cells: + if cell.get('query') and NO_SEARCH_QUERY in cell['query']: + cell['query'] = '' + fix(data.get('counts', [])) + for _row_header, row in data.get('grid', []): + fix(row) + return data + def display_data(self, cols, table=None, constraint=None, avg=None, buckets=None, totaler=None, proportioner=None, - baseurl_func=None, url_extras=None, **kwds): + baseurl_func=None, url_extras=None, link_constraint=None, **kwds): """ Returns statistics data in a common format that is used by page templates. @@ -492,6 +757,10 @@ def display_data(self, cols, table=None, constraint=None, avg=None, - ``baseurl_func`` -- a base url, to which url_for is applied and then col=value tags are appended. Defaults to the url for ``self.baseurl_func``. - ``url_extras`` -- Text to add to the url after the '?'. + - ``link_constraint`` -- url fragments expressing ``constraint`` in terms of + the search page's parameters, used in place of ``constraint`` when + building urls. Needed when the query used for counting is not in terms + of the columns accepted by the search page. - ``kwds`` -- used to discard unused extraneous arguments. OUTPUT: @@ -523,90 +792,93 @@ def display_data(self, cols, table=None, constraint=None, avg=None, raise ValueError("buckets should be a dictionary with columns as keys") else: buckets = {col: buckets[col] for col in cols if col in buckets} - formatter = self._formatters - query_formatter = self._query_formatters - sort_key = self._sort_keys - reverse = self._reverses + if any(col not in cols for col in buckets): + raise ValueError("Bucket keys must be a subset of columns") + buckets = {col: [bucket for bucket in bucket_list if bucket] + for col, bucket_list in buckets.items()} + buckets = {col: bucket_list for col, bucket_list in buckets.items() if bucket_list} if baseurl_func is None: baseurl_func = self.baseurl_func base_url = url_for(baseurl_func) + '?' - if url_extras: - base_url += url_extras - if constraint: - base_url += "".join("%s&" % query_formatter[col](val) for col, val in constraint.items() if col not in cols) + # Url fragments shared by every cell of the table. Each cell's url is + # assembled from these together with the fragments for its row and column, + # so that a totaler can recover the constraint on a row or column by + # intersecting the urls of its cells. + extras = [frag for frag in (url_extras or '').split('&') if frag] + if link_constraint is not None: + extras.extend(frag for frag in link_constraint.split('&') if frag) + elif constraint: + extras.extend(self._query_formatters[col](val) + for col, val in constraint.items() if col not in cols) + + def cell_url(*headers): + return base_url + '&'.join(extras + [header.fragment for header in headers]) + if table is None: table = self.table self._tmp_table = table = table.stats + # The backend indexes its counts by the output of ``formatter``, so we give + # it functions that are injective on database values rather than the ones + # used for display, which may not be. Urls are built here instead, from the + # values themselves, so we do not ask it to build any. + keyer = {col: (range_formatter if col in buckets else _hashable) for col in cols} + no_urls = KeyedDefaultDict(lambda col: (lambda val: '')) + db_buckets = self._encode_buckets(buckets) if len(cols) == 1: avg = totaler.get('avg', False) if totaler else False show_total = bool(totaler) col = cols[0] split_list = self._split_lists[col] - headers, counts = table._get_values_counts(cols, constraint, split_list=split_list, formatter=formatter, query_formatter=query_formatter, base_url=base_url, buckets=buckets) - old_headers = headers ## Preserve the original headers for later lookups - if not buckets: - if show_total or proportioner is None: + if buckets and (split_list or avg): + raise ValueError("Unsupported option") + values, data = table._get_values_counts(cols, constraint, split_list=split_list, formatter=keyer, query_formatter=no_urls, base_url='', buckets=db_buckets) + headers = self._make_headers(col, values, buckets.get(col)) + counts = [{'count': header.count(data), + 'query': cell_url(header), + 'proportion': " 0.00%", # overridden below for nonzero counts + 'value': header.label} + for header in headers] + if 'addl_row_title' in kwds: + addl_row_title = kwds['addl_row_title'] + for D, header in zip(counts, headers): + D['value2'] = self._formatters[addl_row_title](header.value) + if show_total or proportioner is None: + if buckets: + total = sum(D['count'] for D in counts) + else: total, avg = table._get_total_avg(cols, constraint, avg, split_list) - headers = [formatter[col](val) for val in sorted(headers, key=sort_key[col], reverse=reverse[col])] - elif cols == list(buckets): - if split_list or avg or sort_key[col] is not default_sort_key: - raise ValueError("Unsupported option") - headers = [formatter[col](bucket) for bucket in buckets[col]] - if show_total or proportioner is None: - total = sum(counts[bucket]['count'] for bucket in headers) - else: - raise ValueError("Bucket keys must be subset of columns") - counts = [counts[val] for val in headers] - - for D, val, old_val in zip(counts, headers, old_headers): - D['value'] = val - if 'addl_row_title' in kwds.keys(): - addl_row_title = kwds['addl_row_title'] - D['value2'] = formatter[addl_row_title](old_val) - - if proportioner is None or show_total: self._overall = total if proportioner is None or isinstance(proportioner, dict): proportioner = proportioners.ratio_1d(proportioner) if proportioner: - proportioner(counts, headers, self) + proportioner(counts, [header.label for header in headers], self) else: for D in counts: D['proportion'] = '' if show_total: total = {'count': total, - 'query':"{0}{1}".format(base_url, cols[0]), + 'query': self._total_url(base_url, extras, table, cols, constraint, + total, buckets, split_list), 'proportion':_format_percentage(total, self._overall, show_zero=True)} if avg is False: # Want to show avg even if 0 total['value'] = 'Total' else: total['value'] = r'\(\mathrm{avg}\ %.2f\)' % avg counts.append(total) - return {'counts': counts} + return self._suppress_unsearchable({'counts': counts}) elif len(cols) == 2: if avg: raise ValueError("unsupported option") - non_buckets = [col for col in cols if col not in buckets] - if len(buckets) + len(non_buckets) != 2: - raise ValueError("Bucket keys must be a subset of columns") - headers, grid = table._get_values_counts(cols, constraint, split_list=False, formatter=formatter, query_formatter=query_formatter, base_url=base_url, buckets=buckets) - for i, col in enumerate(cols): - if col in buckets: - headers[i] = [formatter[col](bucket) for bucket in buckets[col]] - else: - try: - dup_free = set(headers[i]) - except TypeError: - # The headers may not all be hashable - dup_free = [] - for h in headers[i]: - if h not in dup_free: - dup_free.append(h) - headers[i] = [formatter[col](val) for val in - sorted(dup_free, key=sort_key[col], reverse=reverse[col])] - row_headers, col_headers = headers - grid = [[grid[(rw,cl)] for cl in col_headers] for rw in row_headers] + values, data = table._get_values_counts(cols, constraint, split_list=False, formatter=keyer, query_formatter=no_urls, base_url='', buckets=db_buckets) + rows = self._make_headers(cols[0], values[0], buckets.get(cols[0])) + columns = self._make_headers(cols[1], values[1], buckets.get(cols[1])) + grid = [[{'count': row.count(data, column), + 'query': cell_url(row, column), + 'proportion': ''} + for column in columns] for row in rows] + row_headers = [row.label for row in rows] + col_headers = [column.label for column in columns] # _total_grid is used for recursive proportions; such proportioners # will set it for use in a totaler. Otherwise, we set it to None # here to signal that unrecursive totaling should be used. @@ -615,8 +887,8 @@ def display_data(self, cols, table=None, constraint=None, avg=None, proportioner(grid, row_headers, col_headers, self) if totaler: totaler(grid, row_headers, col_headers, self) - return {'grid': list(zip(row_headers, grid)), - 'col_headers': col_headers} + return self._suppress_unsearchable({'grid': list(zip(row_headers, grid)), + 'col_headers': col_headers}) elif not cols: return {} else: @@ -688,6 +960,7 @@ def setup(self, attributes=None, delete=False): buckets = attr.get('buckets', {col: self._buckets[col] for col in cols if self._buckets[col]}) if isinstance(buckets, list) and len(cols) == 1: buckets = {cols[0]: buckets} + buckets = self._encode_buckets(buckets) constraint = attr.get("constraint") table = attr.get("table", self.table) split_list = all(self._split_lists[col] for col in cols) @@ -736,21 +1009,53 @@ def _dyn_attribute_parse(self, info, attributes): if prop == 'none': attributes['proportioner'] = False + # Parameters of the dynamic statistics page itself, rather than of the search + # that determines which records are counted. + dynamic_params = ['col1', 'col2', 'buckets1', 'buckets2', 'totals1', 'totals2', + 'proportions', 'search_type', 'search_array', 'count', 'start', + 'hst', 'err', 'd', 'stats'] + + def dynamic_link_constraint(self, info, cols): + """ + The url fragments constraining the drill-down links on the dynamic + statistics page to the records being counted. + + We echo back the search boxes that the user filled in, rather than the + query that ``dynamic_parse`` produced from them, since the columns of that + query are often not parameters that the search page accepts: a search box + may be renamed, combined with a quantifier, or encoded before being + compared with the database. The search boxes reproduce the same search by + construction, being the input that produced the query in the first place. + """ + skip = set(self.dynamic_params) + for col in cols: + skip.update(self._url_params[col]) + return "&".join("%s=%s" % (key, quote(val, safe=URL_SAFE)) + for key, val in info.items() + if key not in skip and isinstance(val, str) and val) + def dynamic_setup(self, info): if not info: - attr = {'cols':[], 'buckets':{}} + info["d"] = self.prep({'cols':[], 'buckets':{}}) else: - constraint = {} try: # parse the constraint + constraint = {} self.dynamic_parse(info, constraint) attr = {'constraint': constraint} # add in the columns and proportioner+totaller strategies self._dyn_attribute_parse(info, attr) - except Exception: - # Should provide nice error message - raise - info["d"] = self.prep(attr) + # the same constraint, in terms of the search page's parameters + attr['link_constraint'] = self.dynamic_link_constraint(info, attr['cols']) + info["d"] = self.prep(attr) + except (ValueError, AttributeError, TypeError) as err: + # These are the errors raised for invalid search input. The search + # parsers flash their own message, and set info['err'] when they have. + if "err" not in info: + flash_error("%s", str(err)) + info["err"] = "" + # Show the search form and the error message, without any table + info["d"] = self.prep({'cols':[], 'buckets':{}}) info["stats"] = self info["get_bucket"] = (lambda i: info.get("buckets%s" % i, "")) info["get_col"] = (lambda i: info.get("col%s" % i, "none"))