Fix: unblock synthetic measures in server-side mode and temporal/header gaps

Author: mayagbarnes

e2e_playwright/pivot_table_interactions_test.py

@@ -2027,7 +2027,9 @@ def _select_builder_option(
     panel.get_by_test_id(test_id).click()
     option = page.get_by_test_id(f"{test_id}-{value}")
     expect(option).to_be_visible(timeout=5000)
-    option.click()
+    option.evaluate(
+        "el => { el.scrollIntoView({ block: 'center', inline: 'nearest' }); el.click(); }"
+    )
 
 
 def test_formula_measures_render_in_table(page_at_app: Page):

streamlit_pivot/__init__.py

@@ -300,8 +300,6 @@ def _apply_source_filters(
 
 
 def _can_use_threshold_hybrid(config: PivotConfig) -> tuple[bool, str]:
-    if config.get("synthetic_measures"):
-        return False, "threshold_hybrid currently skips synthetic measures"
     filters = config.get("filters", {})
     if filters:
         dim_set = set(config.get("rows", []) + config.get("columns", []))
@@ -366,7 +364,13 @@ def _prepare_threshold_hybrid_frame(
     date_grains = config.get("date_grains", {})
     auto_date_hierarchy = config.get("auto_date_hierarchy", True)
     aggregation = dict(config.get("aggregation", {}))
-    value_fields = list(config.get("values", []))
+    synth_sources = _get_synthetic_source_fields(config.get("synthetic_measures"))
+    df_cols = set(df.columns)
+    value_fields = [
+        f
+        for f in dict.fromkeys(list(config.get("values", [])) + synth_sources)
+        if f in df_cols
+    ]
 
     if not value_fields:
         if not group_fields:
@@ -1033,19 +1037,25 @@ def _compute_hybrid_totals(
     aggregation = config.get("aggregation", {})
     rows = config.get("rows", [])
     columns = config.get("columns", [])
-    values = config.get("values", [])
+    synth_sources = _get_synthetic_source_fields(config.get("synthetic_measures"))
+    df_cols = set(df.columns)
+    all_value_fields = [
+        f
+        for f in dict.fromkeys(list(config.get("values", [])) + synth_sources)
+        if f in df_cols
+    ]
     show_subtotals = config.get("show_subtotals", False)
     date_grains = config.get("date_grains", {})
     auto_date_hierarchy = config.get("auto_date_hierarchy", True)
 
     sidecar_fields = {
         vf: agg
-        for vf in values
+        for vf in all_value_fields
         if (agg := aggregation.get(vf, "sum")) in _SIDECAR_REQUIRED_AGGS
     }
     remap_only_fields = {
         vf: agg
-        for vf in values
+        for vf in all_value_fields
         if (agg := aggregation.get(vf, "sum")) in ("count", "count_distinct")
         and vf not in sidecar_fields
     }

streamlit_pivot/frontend/src/engine/PivotData.test.ts

@@ -2547,34 +2547,30 @@ describe("PivotData - formula engine in hybrid mode", () => {
     expect(pd.getAggregator(["EU"], ["2024"], "margin").value()).toBe(200);
   });
 
-  it("formula row totals work alongside hybrid pre-computed totals", () => {
+  it("formula row totals use hybrid pre-computed source values when available", () => {
     const cfg = hybridFormulaConfig({ aggregation: "median" });
     const totals = makeHybridTotals(cfg, { revenue: 187.5, cost: 70 }, [
       { key: ["US"], values: { revenue: 150, cost: 60 } },
       { key: ["EU"], values: { revenue: 225, cost: 80 } },
     ]);
     const pd = new PivotData(HYBRID_DATA, cfg, { hybridTotals: totals });
-    // Source field row totals use hybrid pre-computed values
     expect(pd.getRowTotal(["US"], "revenue").value()).toBe(150);
-    // Formula evaluates from client-side median aggregators:
-    // US rows: revenue [100, 200] → median 150, cost [40, 80] → median 60
+    // Formula uses hybrid-precomputed row values: revenue=150, cost=60
     const margin = pd.getRowTotal(["US"], "margin").value();
     expect(margin).toBe(90); // 150 - 60
   });
 
-  it("formula grand total evaluates from client aggregators even with hybrid totals", () => {
+  it("formula grand total uses hybrid pre-computed source values when available", () => {
     const cfg = hybridFormulaConfig({ aggregation: "median" });
     const totals = makeHybridTotals(cfg, { revenue: 999, cost: 888 });
     const pd = new PivotData(HYBRID_DATA, cfg, { hybridTotals: totals });
-    // Hybrid grand total for regular field uses pre-computed
     expect(pd.getGrandTotal("revenue").value()).toBe(999);
-    // Formula grand total uses client-side median aggregators:
-    // revenue [100,200,150,300] → median 175, cost [40,80,60,100] → median 70
+    // Formula uses hybrid-precomputed grand values: revenue=999, cost=888
     const margin = pd.getGrandTotal("margin").value();
-    expect(margin).toBe(105); // 175 - 70
+    expect(margin).toBe(111); // 999 - 888
   });
 
-  it("formula col totals evaluate from client aggregators even with hybrid totals", () => {
+  it("formula col totals use hybrid pre-computed source values when available", () => {
     const cfg = hybridFormulaConfig({ aggregation: "median" });
     const totals = makeHybridTotals(
       cfg,
@@ -2586,10 +2582,8 @@ describe("PivotData - formula engine in hybrid mode", () => {
       ],
     );
     const pd = new PivotData(HYBRID_DATA, cfg, { hybridTotals: totals });
-    // Hybrid col total for regular field uses pre-computed
     expect(pd.getColTotal(["2023"], "revenue").value()).toBe(125);
-    // Formula col total uses client-side median aggregators:
-    // 2023 col: revenue [100, 150] → median 125, cost [40, 60] → median 50
+    // Formula uses hybrid-precomputed col values: revenue=125, cost=50
     const margin = pd.getColTotal(["2023"], "margin").value();
     expect(margin).toBe(75); // 125 - 50
   });
@@ -2615,6 +2609,30 @@ describe("PivotData - formula engine in hybrid mode", () => {
     expect(pd.getAggregator(["US"], ["2023"], "safe_ratio").value()).toBe(2.5);
   });
 
+  it("formula prefers hybrid values over client-side re-aggregated values", () => {
+    const cfg = hybridFormulaConfig({ aggregation: "median" });
+    // Provide hybrid grand values that differ from what client-side re-aggregation would produce
+    const totals = makeHybridTotals(
+      cfg,
+      { revenue: 500, cost: 200 },
+      [
+        { key: ["US"], values: { revenue: 300, cost: 100 } },
+        { key: ["EU"], values: { revenue: 400, cost: 150 } },
+      ],
+      [
+        { key: ["2023"], values: { revenue: 350, cost: 120 } },
+        { key: ["2024"], values: { revenue: 450, cost: 180 } },
+      ],
+    );
+    const pd = new PivotData(HYBRID_DATA, cfg, { hybridTotals: totals });
+    // Grand total formula should use hybrid values: 500 - 200 = 300
+    expect(pd.getGrandTotal("margin").value()).toBe(300);
+    // Row total formula should use hybrid values: 300 - 100 = 200
+    expect(pd.getRowTotal(["US"], "margin").value()).toBe(200);
+    // Col total formula should use hybrid values: 350 - 120 = 230
+    expect(pd.getColTotal(["2023"], "margin").value()).toBe(230);
+  });
+
   it("formula source fields not in values are aggregated correctly in hybrid mode", () => {
     const cfg = hybridFormulaConfig({
       values: ["revenue"],

streamlit_pivot/frontend/src/engine/PivotData.ts

@@ -1210,7 +1210,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._rowTotalSumAggs.get(prefix + sf)?.value() ?? null,
-        (sf) => this._rowTotalAggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridRowTotals?.get(prefix + sf) ??
+          this._rowTotalAggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1233,7 +1236,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._colTotalSumAggs.get(prefix + sf)?.value() ?? null,
-        (sf) => this._colTotalAggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridColTotals?.get(prefix + sf) ??
+          this._colTotalAggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1255,7 +1261,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._grandTotalSumAggs.get(sf)?.value() ?? null,
-        (sf) => this._grandTotalAggs.get(sf)?.value() ?? null,
+        (sf) =>
+          this._hybridGrand?.get(sf) ??
+          this._grandTotalAggs.get(sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1366,7 +1375,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._subtotalSumAggs?.get(prefix + sf)?.value() ?? null,
-        (sf) => this._subtotalAggs?.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridSubtotals?.get(prefix + sf) ??
+          this._subtotalAggs?.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1681,7 +1693,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._colSubtotalSumAggs?.get(prefix + sf)?.value() ?? null,
-        (sf) => this._colSubtotalAggs?.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridColPrefix?.get(prefix + sf) ??
+          this._colSubtotalAggs?.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1709,7 +1724,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._colSubtotalSumAggs?.get(prefix + sf)?.value() ?? null,
-        (sf) => this._colSubtotalAggs?.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridColPrefixGrand?.get(`${colPrefixStr}\x01${sf}`) ??
+          this._colSubtotalAggs?.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1741,7 +1759,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => this._crossSubtotalSumAggs?.get(prefix + sf)?.value() ?? null,
-        (sf) => this._crossSubtotalAggs?.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridCrossSubtotals?.get(prefix + sf) ??
+          this._crossSubtotalAggs?.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1869,7 +1890,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => aggs.get(prefix + sf)?.value() ?? null,
-        (sf) => aggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridTemporalParent?.get(prefix + sf) ??
+          aggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -1899,7 +1923,12 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => aggs.get(prefix + sf)?.value() ?? null,
-        (sf) => aggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridTemporalParentGrand?.get(
+            `${modifiedColKeyStr}\x01${sf}`,
+          ) ??
+          aggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -2067,7 +2096,10 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => aggs.get(prefix + sf)?.value() ?? null,
-        (sf) => aggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridTemporalRowParent?.get(prefix + sf) ??
+          aggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }
@@ -2096,7 +2128,12 @@ export class PivotData {
       const value = this._evaluateSynthetic(
         synthetic,
         (sf) => aggs.get(prefix + sf)?.value() ?? null,
-        (sf) => aggs.get(prefix + sf)?.value() ?? null,
+        (sf) =>
+          this._hybridTemporalRowParentGrand?.get(
+            `${modifiedRowKeyStr}\x01${sf}`,
+          ) ??
+          aggs.get(prefix + sf)?.value() ??
+          null,
       );
       return this._fixedAggregator(value);
     }

tests/test_component_mount.py

@@ -273,10 +273,10 @@ def test_threshold_hybrid_preaggregates_compatible_large_configs(
     assert "Drill-down" in payload["server_mode_reason"]
 
 
-def test_threshold_hybrid_falls_back_for_incompatible_configs(
+def test_threshold_hybrid_works_with_synthetic_measures(
     sample_df, pivot_module, mount_recorder
 ):
-    """Synthetic measures still force fallback to client_only."""
+    """Synthetic measures are now compatible with threshold_hybrid."""
     calls = mount_recorder()
     large_df = sample_df.loc[sample_df.index.repeat(20000)].reset_index(drop=True)
 
@@ -300,7 +300,7 @@ def test_threshold_hybrid_falls_back_for_incompatible_configs(
     )
 
     payload = calls[0]["data"]
-    assert payload["execution_mode"] == "client_only"
+    assert payload["execution_mode"] == "threshold_hybrid"
 
 
 def test_threshold_hybrid_median_no_longer_falls_back(