feat(frontend): wide-column support with raised cardinality cap and horizontal overscan

Raises maxColumnCardinality from 200 to 1000 with a feature flag
(wideColumnMode) that controls whether the legacy 200-column truncation
or the new virtualization-first approach is used. Adds enhanced
horizontal overscan (min 6 columns) for tables with 200+ columns to
reduce flicker during horizontal scrolling.

Made-with: Cursor

Author: mayagbarnes

streamlit_pivot/frontend/src/engine/PivotData.bench.ts

@@ -110,4 +110,10 @@ describe("PivotData computation benchmarks", () => {
   bench("stress dataset (200K rows, 500x20 grid)", () => {
     new PivotData(stress.records, stress.config);
   });
+
+  // Wide columns: 50,000 rows, 20 regions, 1,000 years -> 20,000 unique cells
+  const wide = generateRecords(50000, 20, 1000);
+  bench("wide columns (50K rows, 20x1000 grid)", () => {
+    new PivotData(wide.records, wide.config);
+  });
 });

streamlit_pivot/frontend/src/engine/perf.test.ts

@@ -36,7 +36,7 @@ describe("DEFAULT_BUDGETS", () => {
   it("matches expected values", () => {
     expect(DEFAULT_BUDGETS).toMatchInlineSnapshot(`
       {
-        "maxColumnCardinality": 200,
+        "maxColumnCardinality": 1000,
         "maxComputeMs": 500,
         "maxRenderMs": 200,
         "maxVisibleCells": 5000,
@@ -78,7 +78,7 @@ describe("checkBudgets", () => {
   });
 
   it("warns when column cardinality exceeds cap", () => {
-    const warnings = checkBudgets({ ...okMetrics, totalCols: 250 });
+    const warnings = checkBudgets({ ...okMetrics, totalCols: 1200 });
     expect(warnings).toHaveLength(1);
     expect(warnings[0]).toContain("column values exceed cardinality cap");
   });
@@ -88,7 +88,7 @@ describe("checkBudgets", () => {
       pivotComputeMs: 800,
       renderMs: 400,
       totalRows: 5000,
-      totalCols: 300,
+      totalCols: 1200,
       totalCells: 10000,
     };
     const warnings = checkBudgets(badMetrics);

streamlit_pivot/frontend/src/engine/perf.ts

@@ -60,15 +60,26 @@ export interface PerfBudgets {
   maxRenderMs: number;
   /** Max visible cells before virtualization activates */
   maxVisibleCells: number;
-  /** Max unique column values before truncation */
+  /** Max unique column values before truncation (wide-column mode cap) */
   maxColumnCardinality: number;
 }
 
+/** Column count above which horizontal windowing is preferred (legacy cap when wideColumnMode is off). */
+export const COLUMN_VIRTUALIZATION_THRESHOLD = 200;
+
+/** Hard cap when {@link FEATURE_FLAGS.wideColumnMode} is false (backward compatible). */
+export const LEGACY_MAX_COLUMN_CARDINALITY = 200;
+
+export const FEATURE_FLAGS = {
+  /** When false, column cardinality uses {@link LEGACY_MAX_COLUMN_CARDINALITY} (200). */
+  wideColumnMode: true,
+};
+
 export const DEFAULT_BUDGETS: PerfBudgets = {
   maxComputeMs: 500,
   maxRenderMs: 200,
   maxVisibleCells: 5000,
-  maxColumnCardinality: 200,
+  maxColumnCardinality: 1000,
 };
 
 /**

streamlit_pivot/frontend/src/renderers/VirtualizedTableRenderer.test.tsx

@@ -163,6 +163,27 @@ describe("VirtualizedTableRenderer", () => {
     expect(cellText).not.toMatch(/^R\d+$/);
   });
 
+  it("wide column counts stay DOM-bounded (500+ columns)", () => {
+    const records = makeRecords(4, 600);
+    const config = makeConfig({ show_totals: false });
+    const pivotData = new PivotData(records, config);
+
+    render(
+      <VirtualizedTableRenderer
+        pivotData={pivotData}
+        config={config}
+        containerHeight={400}
+        columnWidth={120}
+      />,
+    );
+
+    const dataRows = screen.getAllByTestId("pivot-data-row");
+    const dataCells = screen.getAllByTestId("pivot-data-cell");
+    const cellsPerRow = dataCells.length / dataRows.length;
+    expect(cellsPerRow).toBeLessThan(600);
+    expect(dataCells.length).toBeLessThan(600 * dataRows.length);
+  });
+
   it("column windowing with multiple row dims still renders all row header columns", () => {
     // Create records with 2 row dimensions
     const records: DataRecord[] = [];

streamlit_pivot/frontend/src/shared/VirtualScroll.tsx

@@ -24,6 +24,7 @@ import {
   useEffect,
   useMemo,
 } from "react";
+import { COLUMN_VIRTUALIZATION_THRESHOLD } from "../engine/perf";
 
 export interface VirtualScrollProps {
   totalRows: number;
@@ -98,6 +99,14 @@ const VirtualScroll: FC<VirtualScrollProps> = ({
   const totalContentWidth = totalColumns * columnWidth;
   const viewportWidth = measuredWidth || totalContentWidth;
 
+  const effectiveOverscanColumns = useMemo(
+    () =>
+      totalColumns > COLUMN_VIRTUALIZATION_THRESHOLD
+        ? Math.max(overscanColumns, 6)
+        : overscanColumns,
+    [totalColumns, overscanColumns],
+  );
+
   const { startRow, endRow } = useMemo(() => {
     const start = Math.max(0, Math.floor(scrollTop / rowHeight) - overscanRows);
     const visibleCount = Math.ceil(bodyHeight / rowHeight);
@@ -108,15 +117,21 @@ const VirtualScroll: FC<VirtualScrollProps> = ({
   const { startCol, endCol } = useMemo(() => {
     const start = Math.max(
       0,
-      Math.floor(scrollLeft / columnWidth) - overscanColumns,
+      Math.floor(scrollLeft / columnWidth) - effectiveOverscanColumns,
     );
     const visibleCount = Math.ceil(viewportWidth / columnWidth);
     const end = Math.min(
       totalColumns,
-      start + visibleCount + overscanColumns * 2,
+      start + visibleCount + effectiveOverscanColumns * 2,
     );
     return { startCol: start, endCol: end };
-  }, [scrollLeft, columnWidth, viewportWidth, totalColumns, overscanColumns]);
+  }, [
+    scrollLeft,
+    columnWidth,
+    viewportWidth,
+    totalColumns,
+    effectiveOverscanColumns,
+  ]);
 
   const colRange: [number, number] = [startCol, endCol];
 

streamlit_pivot/frontend/src/shared/budgetCheck.test.ts

@@ -15,15 +15,24 @@
  * limitations under the License.
  */
 
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it } from "vitest";
 import { checkRenderBudget } from "./budgetCheck";
-import { DEFAULT_BUDGETS } from "../engine/perf";
+import {
+  DEFAULT_BUDGETS,
+  FEATURE_FLAGS,
+  LEGACY_MAX_COLUMN_CARDINALITY,
+} from "../engine/perf";
 
 describe("checkRenderBudget", () => {
+  afterEach(() => {
+    FEATURE_FLAGS.wideColumnMode = true;
+  });
+
   it("returns no warnings for small pivot", () => {
     const result = checkRenderBudget(10, 5, 1);
     expect(result.needsVirtualization).toBe(false);
     expect(result.columnsTruncated).toBe(false);
+    expect(result.needsColumnVirtualization).toBe(false);
     expect(result.warnings).toHaveLength(0);
   });
 
@@ -38,8 +47,8 @@ describe("checkRenderBudget", () => {
     expect(result.warnings.length).toBeGreaterThan(0);
   });
 
-  it("truncates columns when cardinality exceeds limit", () => {
-    const result = checkRenderBudget(10, 300, 1);
+  it("truncates columns when cardinality exceeds limit (wideColumnMode on)", () => {
+    const result = checkRenderBudget(10, 1500, 1);
     expect(result.columnsTruncated).toBe(true);
     expect(result.truncatedColumnCount).toBe(
       DEFAULT_BUDGETS.maxColumnCardinality,
@@ -49,6 +58,23 @@ describe("checkRenderBudget", () => {
     );
   });
 
+  it("does not truncate at 500 columns when wideColumnMode is true", () => {
+    const result = checkRenderBudget(10, 500, 1);
+    expect(result.columnsTruncated).toBe(false);
+    expect(result.truncatedColumnCount).toBe(500);
+    expect(result.needsColumnVirtualization).toBe(true);
+  });
+
+  it("truncates at 200 columns when wideColumnMode is false (backward compat)", () => {
+    FEATURE_FLAGS.wideColumnMode = false;
+    const result = checkRenderBudget(10, 500, 1);
+    expect(result.columnsTruncated).toBe(true);
+    expect(result.truncatedColumnCount).toBe(LEGACY_MAX_COLUMN_CARDINALITY);
+    expect(result.warnings.some((w) => w.includes("Column cardinality"))).toBe(
+      true,
+    );
+  });
+
   it("accounts for multiple values in cell count", () => {
     const result = checkRenderBudget(100, 10, 5);
     expect(result.needsVirtualization).toBe(false);

streamlit_pivot/frontend/src/shared/budgetCheck.ts

@@ -15,15 +15,33 @@
  * limitations under the License.
  */
 
-import { DEFAULT_BUDGETS } from "../engine/perf";
+import {
+  COLUMN_VIRTUALIZATION_THRESHOLD,
+  DEFAULT_BUDGETS,
+  FEATURE_FLAGS,
+  LEGACY_MAX_COLUMN_CARDINALITY,
+} from "../engine/perf";
 
 export interface BudgetResult {
   needsVirtualization: boolean;
+  /** True when column count exceeds the threshold where horizontal windowing is recommended. */
+  needsColumnVirtualization: boolean;
   columnsTruncated: boolean;
   truncatedColumnCount: number;
   warnings: string[];
 }
 
+function exceedsCellBudget(
+  rowCount: number,
+  colCount: number,
+  valueCount: number,
+): boolean {
+  return (
+    rowCount * colCount * Math.max(valueCount, 1) >
+    DEFAULT_BUDGETS.maxVisibleCells
+  );
+}
+
 /**
  * Check whether the pivot dimensions exceed rendering budgets.
  * Returns whether virtualization is needed and any warning messages.
@@ -34,32 +52,83 @@ export function checkRenderBudget(
   valueCount: number,
 ): BudgetResult {
   const warnings: string[] = [];
-  let needsVirtualization = false;
-  let columnsTruncated = false;
-  let truncatedColumnCount = colCount;
+  const needsColumnVirtualization = colCount > COLUMN_VIRTUALIZATION_THRESHOLD;
+
+  if (!FEATURE_FLAGS.wideColumnMode) {
+    let columnsTruncated = false;
+    let truncatedColumnCount = colCount;
+
+    if (colCount > LEGACY_MAX_COLUMN_CARDINALITY) {
+      columnsTruncated = true;
+      truncatedColumnCount = LEGACY_MAX_COLUMN_CARDINALITY;
+      warnings.push(
+        `Column cardinality (${colCount}) exceeds limit (${LEGACY_MAX_COLUMN_CARDINALITY}). ` +
+          `Showing first ${LEGACY_MAX_COLUMN_CARDINALITY} columns.`,
+      );
+    }
 
-  if (colCount > DEFAULT_BUDGETS.maxColumnCardinality) {
-    columnsTruncated = true;
-    truncatedColumnCount = DEFAULT_BUDGETS.maxColumnCardinality;
-    warnings.push(
-      `Column cardinality (${colCount}) exceeds limit (${DEFAULT_BUDGETS.maxColumnCardinality}). ` +
-        `Showing first ${DEFAULT_BUDGETS.maxColumnCardinality} columns.`,
+    const needsVirtualization = exceedsCellBudget(
+      rowCount,
+      truncatedColumnCount,
+      valueCount,
     );
+
+    if (needsVirtualization) {
+      warnings.push(
+        `Total cells (${(rowCount * truncatedColumnCount * Math.max(valueCount, 1)).toLocaleString()}) exceeds DOM budget ` +
+          `(${DEFAULT_BUDGETS.maxVisibleCells.toLocaleString()}). Virtualization enabled.`,
+      );
+    }
+
+    return {
+      needsVirtualization,
+      needsColumnVirtualization,
+      columnsTruncated,
+      truncatedColumnCount,
+      warnings,
+    };
   }
 
-  const effectiveCells =
-    rowCount * truncatedColumnCount * Math.max(valueCount, 1);
+  const maxColCap = DEFAULT_BUDGETS.maxColumnCardinality;
+  const colsAfterHardCap = Math.min(colCount, maxColCap);
+  const exceedsHardCap = colCount > maxColCap;
 
-  if (effectiveCells > DEFAULT_BUDGETS.maxVisibleCells) {
-    needsVirtualization = true;
-    warnings.push(
-      `Total cells (${effectiveCells.toLocaleString()}) exceeds DOM budget ` +
-        `(${DEFAULT_BUDGETS.maxVisibleCells.toLocaleString()}). Virtualization enabled.`,
-    );
+  const needsVirtualizationFromCells = exceedsCellBudget(
+    rowCount,
+    colsAfterHardCap,
+    valueCount,
+  );
+  const needsVirtualization =
+    needsVirtualizationFromCells || needsColumnVirtualization;
+
+  let columnsTruncated = false;
+  let truncatedColumnCount = colCount;
+
+  if (needsVirtualization) {
+    truncatedColumnCount = colsAfterHardCap;
+    columnsTruncated = exceedsHardCap;
+
+    if (exceedsHardCap) {
+      warnings.push(
+        `Column cardinality (${colCount}) exceeds limit (${maxColCap}). ` +
+          `Showing first ${maxColCap} columns.`,
+      );
+    }
+
+    if (needsVirtualizationFromCells) {
+      warnings.push(
+        `Total cells (${(rowCount * colsAfterHardCap * Math.max(valueCount, 1)).toLocaleString()}) exceeds DOM budget ` +
+          `(${DEFAULT_BUDGETS.maxVisibleCells.toLocaleString()}). Virtualization enabled.`,
+      );
+    }
+  } else {
+    truncatedColumnCount = colCount;
+    columnsTruncated = false;
   }
 
   return {
     needsVirtualization,
+    needsColumnVirtualization,
     columnsTruncated,
     truncatedColumnCount,
     warnings,