Improve /api/<table> schema display and surface table-description knowls

lmfdb/api/api.py

@@ -33,6 +33,21 @@ def pretty_document(rec, sep=", ", id=True):
     return "{" + sep.join("'%s': %s" % attr for attr in attrs) + "}"
 
 
+def schema_example(value, typ, max_len=100):
+    """
+    String representation of one value from a random row, truncated
+    for display in the Example column of the schema table.
+    """
+    if value is None:
+        return ""
+    if typ == "bytea" or isinstance(value, buffer):
+        return "[binary data]"
+    value = str(quote_string(value))
+    if len(value) > max_len:
+        value = value[:max_len] + "..."
+    return value
+
+
 def hidden_collection(c):
     """
     hide some collections from the main page (still available via direct requests)
@@ -351,14 +366,26 @@ def apierror(msg, flash_extras=[], code=404, table=True):
         title = "Database - " + location
         bc = [("Database", url_for(".index")), (table,)]
         query_unquote = unquote(data["query"])
-        description = coll.description()
-        if description:
-            title += " (%s)" % description
+        # The table description (the tables.<name> knowl) is displayed by
+        # collection.html rather than being appended to the title
+        try:
+            # projection=3 selects every search and extra column (plus id),
+            # the same full mask the datapage query below uses, so the Example
+            # column can be populated for every row shown in the schema table.
+            example = coll.random(projection=3)
+        except Exception:
+            example = None
+        if example is None:
+            examples = {}
+        else:
+            examples = {table: {col: schema_example(example.get(col), typ)
+                                for col, typ in coll.col_type.items()}}
         search_schema = [(col, coll.col_type[col])
                          for col in sorted(coll.search_cols)]
         return render_template("collection.html",
                                title=title,
                                search_schema={table: search_schema},
+                               examples=examples,
                                single_object=single_object,
                                query_unquote=query_unquote,
                                url_args=url_args,

lmfdb/api/templates/apidata.html

@@ -4,14 +4,6 @@
 
 <style>
   .api-entries > li { margin-bottom: 14px; }
-  #schema-table {
-    width: 95%;
-  }
-  div.schema-holder {
-    height: 300px;
-    width: 97%;
-    overflow-y: scroll;
-  }
 </style>
 
 <div>

lmfdb/api/templates/apischema.html

@@ -1,15 +1,22 @@
   <a id="{{table}}-schema-shower" onclick="return show_schema('{{table}}');" href="#">Show schema</a>
   <a id="{{table}}-schema-hider" onclick="return hide_schema('{{table}}');" href="#" style="display: none;">Hide schema</a>
-  <div class="{{table}}-schema-holder" style="display: none;">
-    <table id="schema-table" class="ntdata">
+  <div class="schema-holder {{table}}-schema-holder" style="display: none;">
+    {% set example = examples.get(table) if examples is defined else none %}
+    <table class="ntdata schema-table">
       <tr>
         <th>Column</th>
         <th>Type</th>
+        {% if example is not none %}
+          <th>Example</th>
+        {% endif %}
       </tr>
       {% for col, typ in search_schema[table] %}
         <tr>
           <td>{{KNOWL('columns.'+table+'.'+col, col)}}</td>
           <td>{{typ}}</td>
+          {% if example is not none %}
+            <td class="schema-example">{{example.get(col, '')}}</td>
+          {% endif %}
         </tr>
       {% endfor %}
     </table>

lmfdb/api/templates/collection.html

@@ -4,14 +4,6 @@
 
 <style>
   .api-entries > li { margin-bottom: 14px; }
-  #schema-table {
-    width: 95%;
-  }
-  div.schema-holder {
-    height: 300px;
-    width: 97%;
-    overflow-y: scroll;
-  }
 </style>
 
 <div>
@@ -27,6 +19,9 @@
 <div>
 Query: <code><a href="{{ query }}">{{ query_unquote }}</a></code>
 </div>
+<div class="table-description">
+  {{ KNOWL_INC('tables.'+table, htag='h2', show_title=True) }}
+</div>
 <div>
   {% include "apischema.html" %}
 </div>

lmfdb/api/test_api.py

@@ -1,3 +1,5 @@
+import re
+
 from lmfdb.tests import LmfdbTest
 
 class ApiTest(LmfdbTest):
@@ -74,3 +76,59 @@ def test_api_usage(self):
             if '11a1' in query:
                 assert '"lmfdb_label": "11.a2"' in data
                 assert '"jinv": [\n        -122023936,\n        161051\n      ]' in data
+
+    def test_api_schema_display(self):
+        r"""
+        Check the schema display on table pages: an Example column filled
+        from a random row, and the table description no longer in the title
+        """
+        for tbl in ['mf_hecke_traces', 'nf_fields']:  # one small, one big table
+            data = self.tc.get("/api/{}".format(tbl), follow_redirects=True).get_data(as_text=True)
+            assert "<th>Example</th>" in data
+            assert 'class="schema-example"' in data
+            assert 'class="schema-holder {}-schema-holder"'.format(tbl) in data
+            # the table description is displayed as a knowl, not in the title
+            assert "Database - {} (".format(tbl) not in data
+        # The random row must be sampled with a projection that includes the
+        # search columns, otherwise the Example cells for search columns render
+        # blank.  degree is a search column of nf_fields that is never null, so
+        # its Example cell must carry an actual value, not an empty string.
+        data = self.tc.get("/api/nf_fields", follow_redirects=True).get_data(as_text=True)
+        m = re.search(r'columns\.nf_fields\.degree\b.*?'
+                      r'<td class="schema-example">(.*?)</td>', data, re.S)
+        assert m is not None, "degree row not found in nf_fields schema table"
+        assert m.group(1).strip() != "", "Example cell for search column 'degree' is blank"
+
+    def test_api_table_description(self):
+        r"""
+        Check that the tables.<table> knowl is shown with its own title as a
+        heading above the schema, rather than folded into the page title
+        """
+        from markupsafe import escape
+        from lmfdb.knowledge.knowl import Knowl
+        from lmfdb.knowledge.main import md, md_preprocess
+
+        tbl = 'nf_fields'
+        # The title and content live in the knowl database, so we ask for them
+        # rather than hard coding them here.
+        with self.app.test_request_context():
+            self.app.preprocess_request()
+            knowl = Knowl('tables.{}'.format(tbl))
+            assert knowl.exists(), "no tables.{} knowl to test against".format(tbl)
+            heading = "<h2>{}</h2>".format(escape(knowl.title))
+            content = md.convert(md_preprocess(knowl.content))
+
+        data = self.tc.get("/api/{}".format(tbl), follow_redirects=True).get_data(as_text=True)
+        assert heading in data, "table description heading missing"
+        assert content in data, "table description content missing"
+        # the heading introduces the description, and both precede the schema
+        assert data.index(heading) < data.index(content) < data.index("{}-schema-shower".format(tbl))
+
+    def test_api_schema_on_datapage(self):
+        r"""
+        Check that the schema display on underlying data pages still works
+        (with no Example column there)
+        """
+        data = self.tc.get("/EllipticCurve/Q/data/11.a1", follow_redirects=True).get_data(as_text=True)
+        assert 'class="schema-holder ec_curvedata-schema-holder"' in data
+        assert "<th>Example</th>" not in data

lmfdb/knowledge/knowl.py

@@ -526,14 +526,18 @@ def get_table_description(self, table):
     def set_table_description(self, table, description):
         uid = db.login()
         kid = f"tables.{table}"
+        old = self.get_knowl(kid, beta=True)
+        if old is None:
+            old = {'authors': []}
         data = {
             'content': description,
             'defines': table,
+            # Unlike a column description, the title here is set by editors,
+            # so carry it over: this updates the content only, and a missing
+            # title would be saved as the generated fallback, discarding it.
+            'title': old.get('title', ''),
         }
         kwl = Knowl(kid, data=data)
-        old = self.get_knowl(kid, beta=True)
-        if old is None:
-            old = {'authors': []}
         self.save(kwl, uid, most_recent=old)
 
     def drop_table(self, table):
@@ -1100,19 +1104,20 @@ def __init__(self, ID, template_kwargs=None, data=None, editing=False, showing=F
             self.type, self.source, self.source_name = extract_typ(ID)
         if self.type == 2:
             pieces = ID.split(".")
-            # Ignore the title passed in
             if len(pieces) == 3:
-                # Column
+                # Column: the title is generated, so ignore the title passed in
                 self.title = f"Column {pieces[2]} of table {pieces[1]}"
                 if pieces[1] in db.tablenames:
                     self.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT")
                 else:
                     self.coltype = "DEFUNCT"
             elif len(pieces) == 2:
-                # Table
-                self.title = f"Table {pieces[1]}"
+                # Table: unlike a column description, the title is editable,
+                # so it is only generated for records that don't have one yet
                 self.coltype = None
-                if pieces[1] not in db.tablenames:
+                if not self.title:
+                    self.title = f"Table {pieces[1]}"
+                if pieces[1] not in db.tablenames and not self.title.endswith(" (DEFUNCT)"):
                     self.title += " (DEFUNCT)"
 
         if showing:

lmfdb/knowledge/templates/knowl-edit.html

@@ -20,6 +20,13 @@
   {% else %}
     {% from "knowl-defs.html" import knowlbar with context %}
 
+    {# Column descriptions have a generated, read-only title and a Postgres
+       column type; table descriptions have neither, and their title is edited
+       like a normal knowl's.  They are told apart by the source, which is the
+       table name for a column description and none for a table description. #}
+    {% set column_description = (k.type == 2 and k.source is not none) %}
+    {% set table_description = (k.type == 2 and k.source is none) %}
+
     {% set renamable = (k.type == 0 and k.exists() and user_is_admin and k.source is none) %}
     {% set renaming_other_to_this = (k.type == 0 and k.source is not none and k.id != k.source) %}
     {% set renaming_this_to_other = (k.type == 0 and k.id == k.source) %}
@@ -81,14 +88,20 @@
         </tr>
         {% endif %} {# renamable #}
         <tr>
-          <td>{{KNOWL("doc.knowl.description",title="Description")}}</td>
-          {% if k.type == 2 %}
+          <td>
+            {%- if table_description -%}
+              Title
+            {%- else -%}
+              {{KNOWL("doc.knowl.description",title="Description")}}
+            {%- endif -%}
+          </td>
+          {% if column_description %}
             <td>{{ k.title }}<input name="title" id="ktitle" value="{{ k.title }}" type="hidden" /></td>
           {% else %}
             <td><input size="40" name="title" id="ktitle" value="{{ k.title }}" /></td>
           {% endif %}
         </tr>
-        {% if k.type == 2 %}
+        {% if column_description %}
           <tr>
             <td>Postgres column type</td>
             <td>{{ k.coltype }}</td>

lmfdb/templates/knowl-defs.html

@@ -55,13 +55,13 @@
 {%- endwith %}
 {%- endmacro %}
 
-{% macro KNOWL_INC( knowlid, title=none, htag="h1", show_missing=False, backupid=none) -%}
+{% macro KNOWL_INC( knowlid, title=none, htag="h1", show_missing=False, backupid=none, show_title=False) -%}
 {% with knowl = Knowl(knowlid) -%}
 {% if knowl.exists() -%}
-  {% if title is none -%}
-    {# <{{htag}}>{{knowl.title}}</{{htag}}> #}
-  {%- else -%}
+  {% if title is not none -%}
     <{{htag}}>{{title}}</{{htag}}>
+  {%- elif show_title -%}
+    <{{htag}}>{{knowl.title}}</{{htag}}>
   {%- endif %}
   {{knowl.content|render_knowl|safe }}
   {% if user_is_authenticated -%}

lmfdb/templates/style.css

@@ -702,6 +702,22 @@ table.ntdata tr.toplined {
   border-top: 2px solid {{ color.table_ntdata_border }};
 }
 
+/* schema display on API and underlying data pages */
+div.schema-holder {
+  max-height: 500px;
+  width: 97%;
+  overflow-y: auto;
+}
+table.schema-table {
+  width: 95%;
+}
+table.schema-table td.schema-example {
+  font-family: monospace;
+  white-space: normal;
+  overflow-wrap: anywhere;
+  max-width: 40em;
+}
+
 table.ntdata th {
   text-align: left;
   padding-right: 7px;