add a statistics page plotting model count over time

Author: SHINTEKI

biggr_models/handlers/home_handlers.py

@@ -1,7 +1,102 @@
-"""Handler for the BiGGr front page."""
+"""Handlers for the BiGGr front page and statistics page."""
 
 from biggr_models.handlers import utils
-from biggr_models.queries.summary_queries import get_summary_counts
+from biggr_models.queries.summary_queries import (
+    get_summary_counts,
+    get_summary_history,
+)
+
+
+def build_model_chart(points, width=720, height=320, pad_left=64, pad=32):
+    """Turn [{"date": ..., "count": ...}, ...] into SVG geometry for one line.
+
+    Returns None when there are fewer than two points, so the template can show
+    the cold-start message instead of an axis with nothing on it.
+
+    The y axis starts at zero: scaling to the minimum would turn a small
+    absolute change into a dramatic climb, which is the kind of misleading
+    chart this page exists to avoid.
+    """
+    if not points or len(points) < 2:
+        return None
+
+    counts = [p["count"] for p in points]
+    dates = [p["date"] for p in points]
+
+    plot_w = width - pad_left - pad
+    plot_h = height - 2 * pad
+
+    top = _nice_ceiling(max(counts))
+    span = (dates[-1] - dates[0]).total_seconds() or 1
+
+    def x_of(date):
+        return pad_left + plot_w * (date - dates[0]).total_seconds() / span
+
+    def y_of(count):
+        return pad + plot_h * (1 - count / top)
+
+    coords = [(x_of(d), y_of(c)) for d, c in zip(dates, counts)]
+
+    return {
+        "width": width,
+        "height": height,
+        "plot_left": pad_left,
+        "plot_right": pad_left + plot_w,
+        "plot_top": pad,
+        "plot_bottom": pad + plot_h,
+        "polyline": " ".join(f"{x:.1f},{y:.1f}" for x, y in coords),
+        # A filled area under the line reads as "accumulated total" far more
+        # clearly than a bare stroke.
+        "area": "{} {} {}".format(
+            f"{coords[0][0]:.1f},{pad + plot_h:.1f}",
+            " ".join(f"{x:.1f},{y:.1f}" for x, y in coords),
+            f"{coords[-1][0]:.1f},{pad + plot_h:.1f}",
+        ),
+        "dots": [
+            {"x": x, "y": y, "count": c, "date": d}
+            for (x, y), c, d in zip(coords, counts, dates)
+        ],
+        "y_ticks": [
+            {"value": v, "y": y_of(v)} for v in _tick_values(top)
+        ],
+        "x_ticks": [
+            {"label": dates[0].strftime("%b %Y"), "x": x_of(dates[0])},
+            {"label": dates[-1].strftime("%b %Y"), "x": x_of(dates[-1])},
+        ],
+        "first": {"count": counts[0], "date": dates[0]},
+        "last": {"count": counts[-1], "date": dates[-1]},
+        "added": counts[-1] - counts[0],
+    }
+
+
+def _nice_ceiling(value, divisions=4):
+    """Round up to an axis maximum that divides into readable ticks.
+
+    Rounds the tick *step* rather than the maximum, so the ticks land on round
+    numbers while the top stays close to the data: 4881 gives 1250-steps and a
+    top of 5000, not 8000, which would leave the line hugging the floor.
+    """
+    if value <= 0:
+        return divisions
+    # Try every round step across the plausible magnitudes and keep the
+    # smallest top that still covers the data, so the line fills the plot.
+    candidates = []
+    magnitude = 1
+    while magnitude <= max(value, 1):
+        for multiple in (1, 1.25, 2, 2.5, 5):
+            step = magnitude * multiple
+            top = step * divisions
+            # Keep only steps that give whole-number tick labels.
+            if top >= value and float(step).is_integer():
+                candidates.append(int(top))
+        magnitude *= 10
+    return min(candidates) if candidates else int(value)
+
+
+def _tick_values(top, count=4):
+    """Evenly spaced y-axis values from 0 to top, inclusive."""
+    step = top / count
+    return [int(round(step * i)) for i in range(count + 1)]
 
 
 class HomeHandler(utils.BaseHandler):
@@ -13,4 +108,22 @@ def get(self):
         counts = utils.do_safe_query(get_summary_counts)
         # Wrap in a dict so the result stays truthy even when no counts are
         # available; return_result skips the context entirely on a falsy result.
-        self.return_result({"counts": counts})
+        self.return_result({"counts": counts})
+
+
+class StatisticsHandler(utils.BaseHandler):
+    """Database growth over time."""
+
+    template = utils.env.get_template("statistics.html")
+
+    def get(self):
+        history = utils.do_safe_query(get_summary_history)
+        model_points = history.get("models", [])
+        self.return_result(
+            {
+                "history": history,
+                "model_points": model_points,
+                "chart": build_model_chart(model_points),
+                "breadcrumbs": [("Home", "/"), ("Statistics", "/statistics")],
+            }
+        )

biggr_models/queries/summary_queries.py

@@ -1,12 +1,13 @@
-"""Read precomputed whole-database counts for the front page.
+"""Read precomputed whole-database counts for the front page and statistics.
 
 The counts are written by the cobradb ETL (cobradb.summary_loading), which runs
-as the final step of bin/load_db. This module only reads them, so the front page
-costs one indexed query instead of a COUNT(*) per card on every request.
+as the final step of bin/load_db. Every run appends a snapshot, so the table is
+both the current totals and the history of how the database has grown. Reading
+it costs one indexed query instead of a COUNT(*) per card on every request.
 """
 
 import logging
-from typing import Dict
+from typing import Dict, List
 
 from sqlalchemy import select
 from sqlalchemy.exc import SQLAlchemyError
@@ -33,7 +34,9 @@ def get_summary_counts(session) -> Dict[str, int]:
     """
     try:
         rows = session.execute(
-            select(DatabaseSummaryCount.entity_type, DatabaseSummaryCount.count)
+            select(
+                DatabaseSummaryCount.entity_type, DatabaseSummaryCount.count
+            ).order_by(DatabaseSummaryCount.date_time)
         ).all()
     except SQLAlchemyError as e:
         # do_safe_query only maps NotFoundError and ValueError, so an error here
@@ -43,4 +46,34 @@ def get_summary_counts(session) -> Dict[str, int]:
         logging.warning("Could not read database summary counts: %s", e)
         return {}
 
-    return {entity_type: count for entity_type, count in rows}
+    # The table holds one row per entity per ETL run. Ordering by date_time
+    # means the newest row is assigned last, so the dict ends up holding the
+    # current totals rather than an arbitrary older snapshot.
+    return {entity_type: count for entity_type, count in rows}
+
+
+def get_summary_history(session) -> Dict[str, List[Dict]]:
+    """Return {entity_type: [{"date": ..., "count": ...}, ...]} oldest first.
+
+    Never raises; an empty dict lets the statistics page explain that no
+    history has been collected yet instead of returning a 500.
+    """
+    try:
+        rows = session.execute(
+            select(
+                DatabaseSummaryCount.entity_type,
+                DatabaseSummaryCount.date_time,
+                DatabaseSummaryCount.count,
+            ).order_by(DatabaseSummaryCount.date_time)
+        ).all()
+    except SQLAlchemyError as e:
+        session.rollback()
+        logging.warning("Could not read database summary history: %s", e)
+        return {}
+
+    history: Dict[str, List[Dict]] = {}
+    for entity_type, date_time, count in rows:
+        history.setdefault(entity_type, []).append(
+            {"date": date_time, "count": count}
+        )
+    return history

biggr_models/routes.py

@@ -49,6 +49,7 @@ def get_routes():
     routes = [
         (r"/", home_handlers.HomeHandler),
         (r"/about/?", utils.TemplateHandler, {"template_name": "about.html"}),
+        (r"/statistics/?", home_handlers.StatisticsHandler),
         (
             r"/api/%s/objects/?$" % api_v,
             object_handlers.ObjectHandler,

biggr_models/templates/general_navbar.html

@@ -29,6 +29,7 @@ <h5 class="offcanvas-title" id="offcanvasNavbar2Label">BiGGr</h5> <button type="
 					<li class="nav-item"> <a class="nav-link{% if homepage %} active" aria-current="page"{% else %}"{% endif %} href="/">Home</a> </li>
 					<li class="nav-item"> <a class="nav-link{% if aboutpage %} active" aria-current="page"{% else %}"{% endif %} href="/about">About</a> </li>
 					<li class="nav-item"> <a class="nav-link{% if dataaccesspage %} active" aria-current="page"{% else %}"{% endif %} href="/data_access">Data Access</a> </li>
+					<li class="nav-item"> <a class="nav-link{% if statisticspage %} active" aria-current="page"{% else %}"{% endif %} href="/statistics">Statistics</a> </li>
 				</ul>
 				<div>
 					<div class="input-group d-flex mt-3 mt-lg-0"><input id="navbar_search_input" name="search_query"

biggr_models/templates/statistics.html

@@ -0,0 +1,121 @@
+{% extends "general_main.html" %}
+{% block title %}BiGGr Statistics{% endblock %}
+{% set statisticspage = True %}
+{% block body %}
+<div class="row">
+  <div class="col-12 col-lg-10 offset-lg-1">
+
+    <h1 class="fs-3 mb-1">Database growth</h1>
+    <p class="text-body-secondary mb-4">
+      The number of metabolic models in BiGGr, recorded at each database update.
+    </p>
+
+    <div class="card mb-4">
+      <div class="card-body">
+        {% if chart %}
+        <div class="d-flex flex-wrap align-items-baseline gap-4 mb-3">
+          <div>
+            <div class="fs-2 fw-semibold lh-1">{{ "{:,}".format(chart.last.count) }}</div>
+            <div class="small text-body-secondary">models today</div>
+          </div>
+          <div>
+            <div class="fs-4 fw-semibold lh-1 text-success">
+              {% if chart.added >= 0 %}+{% endif %}{{ "{:,}".format(chart.added) }}
+            </div>
+            <div class="small text-body-secondary">
+              since {{ chart.first.date.strftime("%b %Y") }}
+            </div>
+          </div>
+        </div>
+
+        <div style="overflow-x: auto;">
+          <svg viewBox="0 0 {{ chart.width }} {{ chart.height }}"
+               width="100%" height="{{ chart.height }}"
+               role="img" aria-label="Number of models over time"
+               style="min-width: 480px; display: block;">
+
+            {# horizontal gridlines + y-axis labels #}
+            {% for tick in chart.y_ticks %}
+            <line x1="{{ chart.plot_left }}" y1="{{ "%.1f"|format(tick.y) }}"
+                  x2="{{ chart.plot_right }}" y2="{{ "%.1f"|format(tick.y) }}"
+                  stroke="currentColor" stroke-opacity="0.12" stroke-width="1"/>
+            <text x="{{ chart.plot_left - 10 }}" y="{{ "%.1f"|format(tick.y + 4) }}"
+                  text-anchor="end" font-size="12"
+                  fill="currentColor" fill-opacity="0.65">{{ "{:,}".format(tick.value) }}</text>
+            {% endfor %}
+
+            {# filled area reads as an accumulated total #}
+            <polygon points="{{ chart.area }}"
+                     fill="var(--bs-primary)" fill-opacity="0.12"/>
+
+            <polyline points="{{ chart.polyline }}"
+                      fill="none" stroke="var(--bs-primary)"
+                      stroke-width="2.5" stroke-linejoin="round" stroke-linecap="round"/>
+
+            {# one dot per ETL run, with the exact figure on hover #}
+            {% for dot in chart.dots %}
+            <circle cx="{{ "%.1f"|format(dot.x) }}" cy="{{ "%.1f"|format(dot.y) }}" r="4"
+                    fill="var(--bs-body-bg)" stroke="var(--bs-primary)" stroke-width="2"
+                    data-bs-toggle="tooltip"
+                    data-bs-title="{{ dot.date.strftime('%d %b %Y') }}: {{ '{:,}'.format(dot.count) }} models"/>
+            {% endfor %}
+
+            {# x-axis baseline + date labels #}
+            <line x1="{{ chart.plot_left }}" y1="{{ chart.plot_bottom }}"
+                  x2="{{ chart.plot_right }}" y2="{{ chart.plot_bottom }}"
+                  stroke="currentColor" stroke-opacity="0.3" stroke-width="1"/>
+            {% for tick in chart.x_ticks %}
+            <text x="{{ "%.1f"|format(tick.x) }}" y="{{ chart.plot_bottom + 20 }}"
+                  text-anchor="{% if loop.first %}start{% else %}end{% endif %}"
+                  font-size="12" fill="currentColor" fill-opacity="0.65">{{ tick.label }}</text>
+            {% endfor %}
+          </svg>
+        </div>
+
+        {% else %}
+        {# Cold start: the table is filled one snapshot per ETL run, so a new
+           deployment has nothing to plot until the second run. #}
+        <div class="text-center py-4">
+          {% if model_points %}
+          <div class="fs-2 fw-semibold lh-1">{{ "{:,}".format(model_points[-1].count) }}</div>
+          <div class="small text-body-secondary mb-3">models</div>
+          <p class="text-body-secondary mb-0">
+            Growth is recorded at each database update. The chart appears once
+            there are at least two updates to compare.
+          </p>
+          {% else %}
+          <p class="text-body-secondary mb-0">
+            No growth data has been recorded yet. It is collected at the end of
+            each database update.
+          </p>
+          {% endif %}
+        </div>
+        {% endif %}
+      </div>
+    </div>
+
+    {% if history %}
+    <h2 class="fs-5 mb-3">Current contents</h2>
+    <div class="row g-3 mb-4">
+      {% for label, key in [("Collections", "collections"), ("Models", "models"),
+                            ("Metabolites", "metabolites"), ("Reactions", "reactions"),
+                            ("Genomes", "genomes"), ("Compartments", "compartments")] %}
+      {% if history.get(key) %}
+      <div class="col-6 col-md-4 col-lg-2">
+        <div class="border rounded p-3 h-100">
+          <div class="fs-4 fw-semibold lh-1">{{ "{:,}".format(history[key][-1].count) }}</div>
+          <div class="small text-body-secondary">{{ label }}</div>
+        </div>
+      </div>
+      {% endif %}
+      {% endfor %}
+    </div>
+
+    <p class="small text-body-secondary">
+      Last updated {{ history["models"][-1].date.strftime("%d %B %Y") if history.get("models") }}.
+    </p>
+    {% endif %}
+
+  </div>
+</div>
+{% endblock %}