Add server round-trip drill-down with pagination

Author: mayagbarnes

e2e_playwright/e2e_utils.py

@@ -97,6 +97,8 @@
     "test_pivot_per_measure_row_totals",
     "test_pivot_per_measure_col_totals",
     "test_pivot_col_groups",
+    "test_pivot_drilldown_pagination",
+    "test_pivot_drilldown_pagination_hybrid",
 ]
 
 DATA_PIVOT_KEYS = [

e2e_playwright/pivot_table_interactions_app.py

@@ -17,6 +17,7 @@
 
 from __future__ import annotations
 
+import pandas as pd
 import streamlit as st
 
 from streamlit_pivot import st_pivot_table
@@ -30,6 +31,18 @@
 )
 
 
+def _make_drilldown_pagination_data() -> pd.DataFrame:
+    """Generate a dataset where one cell has >500 matching rows to trigger pagination."""
+    rows = []
+    for i in range(700):
+        rows.append({"Region": "Alpha", "Year": "2023", "Revenue": 10 + i})
+    for i in range(50):
+        rows.append({"Region": "Alpha", "Year": "2024", "Revenue": 100 + i})
+    for i in range(30):
+        rows.append({"Region": "Beta", "Year": "2023", "Revenue": 200 + i})
+    return pd.DataFrame(rows)
+
+
 def render_app(data):
     df = data["df"]
 
@@ -216,6 +229,36 @@ def render_app(data):
         on_config_change=noop,
     )
 
+    drill_df = _make_drilldown_pagination_data()
+
+    st.subheader("Drilldown Pagination (Client)")
+    st_pivot_table(
+        drill_df,
+        key="test_pivot_drilldown_pagination",
+        rows=["Region"],
+        columns=["Year"],
+        values=["Revenue"],
+        aggregation="sum",
+        enable_drilldown=True,
+        execution_mode="client_only",
+        interactive=True,
+        on_config_change=noop,
+    )
+
+    st.subheader("Drilldown Pagination (Hybrid)")
+    st_pivot_table(
+        drill_df,
+        key="test_pivot_drilldown_pagination_hybrid",
+        rows=["Region"],
+        columns=["Year"],
+        values=["Revenue"],
+        aggregation="sum",
+        enable_drilldown=True,
+        execution_mode="threshold_hybrid",
+        interactive=True,
+        on_config_change=noop,
+    )
+
 
 def main():
     init_page()

e2e_playwright/pivot_table_interactions_test.py

@@ -823,3 +823,81 @@ def test_child_toggle_disabled_when_parent_collapsed(page_at_app: Page):
 
     expect(category_toggle).not_to_have_attribute("role", "button", timeout=5000)
     expect(category_toggle).to_have_attribute("title", "Expand Region first")
+
+
+# ---------------------------------------------------------------------------
+# Drilldown Pagination (client-only & hybrid)
+# ---------------------------------------------------------------------------
+
+
+def _open_drilldown_with_pagination(page: Page, pivot_key: str):
+    """Click the first data cell (Alpha × 2023 → 700 records) and wait for pagination."""
+    container = get_pivot(page, pivot_key)
+    expect(container.get_by_test_id("pivot-table")).to_be_visible(timeout=15000)
+    container.get_by_test_id("pivot-data-cell").first.evaluate("el => el.click()")
+    panel = page.get_by_test_id("drilldown-panel")
+    expect(panel).to_be_visible(timeout=10000)
+    expect(page.get_by_test_id("drilldown-pagination")).to_be_visible(timeout=10000)
+    return container, panel
+
+
+def test_drilldown_client_pagination_shows_controls(page_at_app: Page):
+    """Client-only: clicking a cell with >500 records shows pagination controls."""
+    page = page_at_app
+    _, panel = _open_drilldown_with_pagination(page, "test_pivot_drilldown_pagination")
+    expect(panel.get_by_test_id("drilldown-prev")).to_be_visible()
+    expect(panel.get_by_test_id("drilldown-next")).to_be_visible()
+    expect(panel.locator("text=Page 1 of 2")).to_be_visible()
+    expect(panel.locator("text=1–500 of 700 records")).to_be_visible()
+
+
+def test_drilldown_client_pagination_navigates(page_at_app: Page):
+    """Client-only: Next navigates to page 2; Prev returns to page 1."""
+    page = page_at_app
+    _, panel = _open_drilldown_with_pagination(page, "test_pivot_drilldown_pagination")
+
+    expect(panel.get_by_test_id("drilldown-prev")).to_be_disabled()
+    panel.get_by_test_id("drilldown-next").click()
+
+    expect(panel.locator("text=Page 2 of 2")).to_be_visible(timeout=5000)
+    expect(panel.locator("text=501–700 of 700 records")).to_be_visible()
+    expect(panel.get_by_test_id("drilldown-next")).to_be_disabled()
+
+    panel.get_by_test_id("drilldown-prev").click()
+    expect(panel.locator("text=Page 1 of 2")).to_be_visible(timeout=5000)
+
+
+def test_drilldown_hybrid_pagination_shows_controls(page_at_app: Page):
+    """Hybrid mode: clicking a cell with >500 records shows pagination controls."""
+    page = page_at_app
+    _, panel = _open_drilldown_with_pagination(
+        page, "test_pivot_drilldown_pagination_hybrid"
+    )
+    expect(panel.get_by_test_id("drilldown-prev")).to_be_visible()
+    expect(panel.get_by_test_id("drilldown-next")).to_be_visible()
+    expect(panel.locator("text=Page 1 of 2")).to_be_visible()
+    expect(panel.locator("text=1–500 of 700 records")).to_be_visible()
+
+
+def test_drilldown_hybrid_pagination_navigates(page_at_app: Page):
+    """Hybrid mode: Next navigates to page 2; Prev returns to page 1."""
+    page = page_at_app
+    _, panel = _open_drilldown_with_pagination(
+        page, "test_pivot_drilldown_pagination_hybrid"
+    )
+
+    expect(panel.get_by_test_id("drilldown-prev")).to_be_disabled()
+    panel.get_by_test_id("drilldown-next").click()
+
+    # Hybrid page changes trigger a full Streamlit rerun; re-query the panel
+    # from the page to avoid stale references and allow extra time.
+    panel = page.get_by_test_id("drilldown-panel")
+    expect(panel).to_be_visible(timeout=30000)
+    expect(panel.locator("text=Page 2 of 2")).to_be_visible(timeout=30000)
+    expect(panel.locator("text=501–700 of 700 records")).to_be_visible()
+    expect(panel.get_by_test_id("drilldown-next")).to_be_disabled()
+
+    panel.get_by_test_id("drilldown-prev").click()
+    panel = page.get_by_test_id("drilldown-panel")
+    expect(panel).to_be_visible(timeout=30000)
+    expect(panel.locator("text=Page 1 of 2")).to_be_visible(timeout=30000)

streamlit_app.py

@@ -1079,6 +1079,105 @@
         language="python",
     )
 
+# ---------------------------------------------------------------------------
+# Section 14: Server-Side Drill-Down (Hybrid Mode)
+# ---------------------------------------------------------------------------
+st.divider()
+st.subheader("14. Server-Side Drill-Down (Hybrid Mode)")
+
+st.markdown(
+    """
+When datasets are large enough to trigger **threshold_hybrid** mode, the pivot
+data is pre-aggregated on the server before being sent to the browser. In this
+mode, drill-down works via a **server round-trip**: clicking a cell sends a
+request to Python, which filters the *original* un-aggregated DataFrame and
+returns the matching rows.
+
+Because large cells can match thousands of rows, the results are **paginated**
+(500 rows per page) with **Prev / Next** controls.
+
+**Try it:**
+- Click any data cell — after a brief round-trip the drill-down panel appears.
+- If the cell has more than 500 matching rows, use the **← Prev / Next →**
+  buttons at the bottom of the panel to page through all results.
+- The header shows a range like "1–500 of 2,340 records" and the current page.
+
+**API parameter used:** `execution_mode` (set to `"threshold_hybrid"` here to
+force hybrid mode on a smaller dataset for demonstration purposes)
+"""
+)
+
+import numpy as np  # noqa: E402
+
+_rng = np.random.default_rng(42)
+_n = 50_000
+df_hybrid = pd.DataFrame(
+    {
+        "Region": _rng.choice(["North", "South", "East", "West"], _n),
+        "Category": _rng.choice(
+            ["Electronics", "Clothing", "Food", "Furniture", "Toys"], _n
+        ),
+        "Year": _rng.choice([2022, 2023, 2024], _n),
+        "Channel": _rng.choice(["Online", "Retail", "Wholesale"], _n),
+        "Revenue": _rng.uniform(10, 5000, _n).round(2),
+        "Profit": _rng.uniform(-500, 2000, _n).round(2),
+    }
+)
+
+st_pivot_table(
+    df_hybrid,
+    key="hybrid_drilldown_demo",
+    rows=["Region", "Category"],
+    columns=["Year"],
+    values=["Revenue", "Profit"],
+    aggregation={"Revenue": "sum", "Profit": "sum"},
+    number_format={"Revenue": "$,.0f", "Profit": "$,.0f"},
+    show_totals=True,
+    show_subtotals=True,
+    enable_drilldown=True,
+    execution_mode="threshold_hybrid",
+)
+
+with st.expander("View Code"):
+    st.code(
+        """
+import numpy as np
+
+rng = np.random.default_rng(42)
+n = 50_000
+df_hybrid = pd.DataFrame({
+    "Region": rng.choice(["North", "South", "East", "West"], n),
+    "Category": rng.choice(["Electronics", "Clothing", "Food", "Furniture", "Toys"], n),
+    "Year": rng.choice([2022, 2023, 2024], n),
+    "Channel": rng.choice(["Online", "Retail", "Wholesale"], n),
+    "Revenue": rng.uniform(10, 5000, n).round(2),
+    "Profit": rng.uniform(-500, 2000, n).round(2),
+})
+
+st_pivot_table(
+    df_hybrid,
+    key="hybrid_drilldown_demo",
+    rows=["Region", "Category"],
+    columns=["Year"],
+    values=["Revenue", "Profit"],
+    aggregation={"Revenue": "sum", "Profit": "sum"},
+    number_format={"Revenue": "$,.0f", "Profit": "$,.0f"},
+    show_totals=True,
+    show_subtotals=True,
+    enable_drilldown=True,
+    execution_mode="threshold_hybrid",
+)
+# Click any cell to see paginated server-side drill-down.
+""",
+        language="python",
+    )
+
+st.caption(
+    f"Dataset: {len(df_hybrid):,} rows × {len(df_hybrid.columns)} columns — "
+    "forced to threshold_hybrid mode for demonstration."
+)
+
+
 # ---------------------------------------------------------------------------
 # Footer: Raw Data
 # ---------------------------------------------------------------------------

streamlit_pivot/__init__.py

@@ -117,6 +117,7 @@ class PivotConfig(TypedDict, total=False):
 
 _warned_keys: set[str] = set()
 _PYTHON_CONFIG_STATE_PREFIX = "__streamlit_pivot_python_config__:"
+_DRILLDOWN_PAGE_SIZE = 500
 
 
 def _estimate_group_count(df: Any, fields: list[str]) -> int:
@@ -235,6 +236,33 @@ def _prepare_threshold_hybrid_frame(df: Any, config: PivotConfig) -> Any:
     return out
 
 
+def _compute_hybrid_drilldown(
+    df: Any,
+    drilldown_request: dict[str, Any],
+    page_size: int = _DRILLDOWN_PAGE_SIZE,
+) -> tuple[list[dict[str, Any]], list[str], int, int]:
+    """Filter the original DataFrame for a hybrid-mode drill-down request.
+
+    Returns (records_list, column_names, total_matching_count, page).
+    """
+    filters: dict[str, str] = drilldown_request.get("filters", {})
+    page: int = max(0, int(drilldown_request.get("page", 0)))
+    mask = pd.Series(True, index=df.index)
+    for col, val in filters.items():
+        if col not in df.columns:
+            continue
+        if val == "(null)":
+            mask &= df[col].isna()
+        else:
+            mask &= df[col].astype(str) == str(val)
+    filtered = df[mask]
+    total_count = len(filtered)
+    offset = page * page_size
+    page_slice = filtered.iloc[offset : offset + page_size]
+    records = json.loads(page_slice.to_json(orient="records", date_format="iso"))
+    return records, list(page_slice.columns), total_count, page
+
+
 def _normalize_aggregation_config(
     aggregation: str | dict[str, str] | None,
     values: list[str],
@@ -1063,8 +1091,8 @@ def st_pivot_table(
     )
     if use_threshold_hybrid:
         drill_note = (
-            " Drill-down is unavailable in this mode because values are "
-            "pre-aggregated on the server rather than built from raw rows."
+            " Drill-down uses a server round-trip to fetch matching rows "
+            "from the original dataset."
         )
         if drill_note not in threshold_reason:
             threshold_reason = f"{threshold_reason}{drill_note}"
@@ -1108,17 +1136,45 @@ def st_pivot_table(
                 f"menu_limit must be a positive integer, got {menu_limit!r}"
             )
         data_payload["menu_limit"] = menu_limit
-    if not enable_drilldown or use_threshold_hybrid:
+    if not enable_drilldown:
         data_payload["enable_drilldown"] = False
     if export_filename is not None:
         data_payload["export_filename"] = export_filename
 
+    # Server-side drill-down for hybrid mode: read the pending request from
+    # session state, filter the *original* (un-aggregated) DataFrame, and
+    # ship the matching rows back as JSON records.
+    if use_threshold_hybrid and enable_drilldown:
+        drilldown_request: dict[str, Any] | None = None
+        try:
+            state = st.session_state.get(key, {})
+            drilldown_request = (
+                state.get("drilldown_request") if isinstance(state, dict) else None
+            )
+        except (AttributeError, TypeError):
+            drilldown_request = None
+
+        if isinstance(drilldown_request, dict) and drilldown_request.get("filters"):
+            records, columns, total, page = _compute_hybrid_drilldown(
+                data, drilldown_request
+            )
+            data_payload["drilldown_records"] = records
+            data_payload["drilldown_columns"] = columns
+            data_payload["drilldown_total_count"] = total
+            data_payload["drilldown_page"] = page
+            data_payload["drilldown_page_size"] = _DRILLDOWN_PAGE_SIZE
+
     mount_kwargs: dict[str, Any] = {
         "key": key,
-        "default": {"config": config_to_send, "perf_metrics": None},
+        "default": {
+            "config": config_to_send,
+            "perf_metrics": None,
+            "drilldown_request": None,
+        },
         "data": data_payload,
         "on_config_change": on_config_change or _noop_callback,
         "on_perf_metrics_change": _noop_callback,
+        "on_drilldown_request_change": _noop_callback,
     }
 
     if on_cell_click is not None: