feat: add DeckGL gridded guidance and browser row-count guard

lumen/ai/agents/base_view.py

@@ -11,7 +11,8 @@
 from ..llm import Message
 from ..schemas import Metaset
 from ..utils import (
-    get_root_exception, get_schema, log_debug, report_error, retry_llm_output,
+    detect_gridded, get_root_exception, get_schema, log_debug, report_error,
+    retry_llm_output,
 )
 from .base_lumen import BaseLumenAgent
 
@@ -88,12 +89,14 @@ async def _generate_yaml_spec(
     ) -> dict[str, Any]:
         errors_context = self._build_errors_context(pipeline, context, errors)
         doc = self.view_type.__doc__.split("\n\n")[0] if self.view_type.__doc__ else self.view_type.__name__
+        gridded = detect_gridded(pipeline)
         response = await self._stream_prompt(
             "main",
             messages,
             context,
             table=pipeline.table,
             doc=doc,
+            gridded=gridded,
             model_kwargs=dict(schema=schema),
             **errors_context,
         )

lumen/ai/agents/deck_gl.py

@@ -21,7 +21,8 @@
 from ..editors import DeckGLEditor
 from ..models import BaseModel, EscapeBaseModel, RetrySpec
 from ..utils import (
-    get_data, get_schema, retry_llm_output, sanitize_column_names,
+    detect_gridded, get_data, get_schema, retry_llm_output,
+    sanitize_column_names,
 )
 from .base_code import BaseCodeAgent
 
@@ -152,6 +153,7 @@ async def _generate_yaml_spec(
     ) -> dict[str, Any]:
         """Generate DeckGL spec via YAML/JSON (declarative mode)."""
         errors_context = self._build_errors_context(pipeline, context, errors)
+        gridded = detect_gridded(pipeline)
 
         with self._add_step(title="Generating DeckGL specification", steps_layout=self._steps_layout) as step:
             response = self._stream_prompt(
@@ -160,6 +162,7 @@ async def _generate_yaml_spec(
                 context,
                 table=pipeline.table,
                 doc=doc,
+                gridded=gridded,
                 **errors_context,
             )
             async for output in response:
@@ -182,6 +185,7 @@ async def _generate_code_spec(
     ) -> dict[str, Any] | None:
         """Generate spec via PyDeck code execution."""
         errors_context = self._build_errors_context(pipeline, context, errors)
+        gridded = detect_gridded(pipeline)
 
         with self._add_step(title="Generating PyDeck code", steps_layout=self._steps_layout) as step:
             response = self._stream_prompt(
@@ -190,6 +194,7 @@ async def _generate_code_spec(
                 context,
                 table=pipeline.table,
                 doc=doc,
+                gridded=gridded,
                 **errors_context,
             )
 

lumen/ai/agents/hvplot.py

@@ -64,6 +64,9 @@ async def _extract_spec(self, context: TContext, spec: dict[str, Any]):
 
         # Add defaults
         spec["responsive"] = True
+        if spec.get("kind") == "heatmap":
+            spec.pop("by", None)
+            spec.pop("groupby", None)
         data = await get_data(pipeline)
         if len(data) > 20000 and spec["kind"] in ("line", "scatter", "points"):
             spec["rasterize"] = True

lumen/ai/prompts/BaseViewAgent/main.jinja2

@@ -9,6 +9,16 @@ The current SQL table name: {{ memory['pipeline'].table }}.
 Here is the current dataset to use:
 {{ memory['data'] }}
 
+{%- if gridded is defined and gridded %}
+
+Data originates from an xarray dataset.
+- dimensions: {{ gridded.dims | join(', ') }}
+- data variables: {{ gridded.data_vars | join(', ') }}
+Use a 2D gridded encoding with axes on the coordinate dimensions and
+colour/value on the data variable, using whichever syntax this agent
+accepts (see agent-specific instructions below).
+{%- endif %}
+
 {%- if "view" in memory and memory["view"].get("agent") and actor_name.startswith(memory["view"].get("agent")) %}
 Adapt from the previous view specification below as needed:
 ```yaml

lumen/ai/prompts/DeckGLAgent/main.jinja2

@@ -45,6 +45,15 @@ Column names: spaces→underscores, non-alphanumeric removed (e.g., "Plant latit
 - **Elevation/Radius:** Use simple math only: `"@@=value"` or `"@@=value * 100"` (NO Math.min, Math.max, etc.)
 - Zoom levels: 1-3 continent, 4-6 country, 7-10 city, 11+ street
 - Pitch: 45-60 for good 3D perspective
+{%- if gridded is defined and gridded %}
+
+**Gridded xarray data**: The data is a long-form grid from an xarray dataset.
+- dimensions: {{ gridded.dims | join(', ') }}
+- data variables: {{ gridded.data_vars | join(', ') }}
+Use HexagonLayer with `getPosition: "@@=[{{ gridded.dims[1] if gridded.dims | length > 1 else gridded.dims[0] }}, {{ gridded.dims[0] }}]"` for 3D aggregation.
+Alternatively use ColumnLayer with `getElevation: "@@={{ gridded.data_vars[0] }}"` for per-cell elevation.
+Large grids (>250,000 rows) require setting `limit` on the view to avoid browser OOM.
+{%- endif %}
 {% endblock %}
 
 {% block examples %}
@@ -125,4 +134,27 @@ ArcLayer (connections):
   "mapStyle": "https://basemaps.cartocdn.com/gl/dark-matter-gl-style/style.json"
 }
 ```
+
+{%- if gridded is defined and gridded %}
+Gridded data, ColumnLayer (per-cell elevation from value field):
+```json
+{
+  "initialViewState": {"latitude": 0, "longitude": 30, "zoom": 3, "pitch": 50, "bearing": 0},
+  "layers": [{
+    "@@type": "ColumnLayer",
+    "id": "grid-values",
+    "getPosition": "@@=[{{ gridded.dims[1] if gridded.dims | length > 1 else gridded.dims[0] }}, {{ gridded.dims[0] }}]",
+    "getElevation": "@@={{ gridded.data_vars[0] }}",
+    "getFillColor": [65, 182, 196, 220],
+    "diskResolution": 6,
+    "radius": 50000,
+    "elevationScale": 10,
+    "extruded": true,
+    "pickable": true
+  }],
+  "views": [{"@@type": "MapView", "controller": true}],
+  "mapStyle": "https://basemaps.cartocdn.com/gl/dark-matter-gl-style/style.json"
+}
+```
+{%- endif %}
 {% endblock %}

lumen/ai/prompts/DeckGLAgent/main_pydeck.jinja2

@@ -25,6 +25,13 @@ Generate PyDeck code. Data is `df`, assign to `deck`. NO I/O methods (.to_html()
 - `bearing`: 0 or slight angle (-15 to 15) for visual interest
 - `elevation_scale`: Controls the height of 3D elements
 - `extruded`: Set to True for 3D extrusion
+{%- if gridded is defined and gridded %}
+
+**Gridded xarray data**: The data is a long-form grid from an xarray dataset.
+- dimensions: {{ gridded.dims | join(', ') }}
+- data variables: {{ gridded.data_vars | join(', ') }}
+Use ColumnLayer with `get_position=["{{ gridded.dims[1] if gridded.dims | length > 1 else gridded.dims[0] }}", "{{ gridded.dims[0] }}"]` and `get_elevation="{{ gridded.data_vars[0] }}"`.
+{%- endif %}
 
 # Map Styles (no API key needed)
 - `"https://basemaps.cartocdn.com/gl/dark-matter-gl-style/style.json"` (dark - best for 3D)

lumen/ai/prompts/VegaLiteAgent/main.jinja2

@@ -58,6 +58,14 @@ Legends appear when you map data to `color`, `size`, or `shape` in encoding.
 **Geographic maps**: Use `longitude`/`latitude` encodings (NOT `x`/`y`) with `projection: {type: mercator}` at top level. Base map added automatically.
 
 **Choropleth maps**: Join data to map boundaries - `lookup: <map_field>`, `from: {data: {...}, key: <your_field>, fields: [...]}` (key/fields inside from!)
+{%- if gridded is defined and gridded %}
+
+**Gridded xarray data**: The data is a long-form grid from an xarray dataset.
+- dimensions: {{ gridded.dims | join(', ') }}
+- data variables: {{ gridded.data_vars | join(', ') }}
+Use `mark: rect` with x/y mapped to coordinate dimensions and color mapped to the value column.
+Do not use `longitude`/`latitude` encodings or geographic projections for this data.
+{%- endif %}
 {% endblock %}
 
 {% block examples %}
@@ -203,6 +211,26 @@ layer:
       color: {field: category, type: nominal, legend: {title: "Category", orient: "right"}}
 ```
 
+{%- if gridded is defined and gridded %}
+Gridded heatmap (xarray-derived long-form data):
+```yaml
+data:
+  name: <TABLE_NAME>
+layer:
+  - mark: rect
+    encoding:
+      x: {field: {{ gridded.dims[1] if gridded.dims | length > 1 else gridded.dims[0] }}, type: ordinal}
+      y: {field: {{ gridded.dims[0] }}, type: ordinal}
+      color:
+        field: {{ gridded.data_vars[0] }}
+        type: quantitative
+        scale: {scheme: viridis}
+        legend: {title: "{{ gridded.data_vars[0] }}"}
+config:
+  view: {stroke: null}
+```
+{%- endif %}
+
 Choropleth map:
 ```yaml
 data:

lumen/ai/prompts/hvPlotAgent/main.jinja2

@@ -5,4 +5,16 @@ Generate the plot the user requested. Note that `x`, `y`, `by` and `groupby` fie
 no repeated columns are allowed. Do not arbitrarily set `groupby` and `by` fields unless explicitly requested. If a
 histogram is requested, use `y` instead of `x`. If x is categorical or strings, prefer barh over bar, and use `y` for
 the values.
+{%- if gridded is defined and gridded %}
+
+For gridded xarray data:
+- Prefer kind='quadmesh'. It preserves continuous coordinate axes and is
+  the right default for xarray-derived grids. Use 'image' only for strictly
+  regular grids, 'contourf' for filled contours, or 'heatmap' if you
+  specifically want ordinal/categorical axes.
+- Set x and y to coordinate names (e.g. x='lon', y='lat').
+- Set z to the value column (e.g. z='air'). The view pivots long-form
+  pandas to xarray internally and maps z= to hvPlot's C= for heatmap only.
+- Do not set by or groupby.
+{%- endif %}
 {% endblock %}

lumen/ai/utils.py

@@ -40,8 +40,9 @@
 
 from ..pipeline import Pipeline
 from ..sources.base import Source
+from ..sources.xarray_sql import XArraySQLSource
 from ..transforms import SQLRemoveSourceSeparator
-from ..util import log
+from ..util import check_xarray_available, log
 from .config import (
     PROMPTS_DIR, SOURCE_TABLE_SEPARATOR, UNRECOVERABLE_ERRORS,
     MissingContextError, RetriesExceededError,
@@ -1409,3 +1410,23 @@ def result_to_dataframe(result) -> pd.DataFrame | None:
             return None
 
     return None
+
+
+def detect_gridded(pipeline: Pipeline) -> dict[str, Any] | None:
+    """Return gridded metadata if pipeline.source is an XArraySQLSource, else None.
+
+    Returns a dict with keys ``dims``, ``coords``, ``data_vars``, ``source_type``,
+    or ``None`` for tabular pipelines.
+    """
+    source = pipeline.source
+    if not isinstance(source, XArraySQLSource):
+        return None
+    if not check_xarray_available():
+        return None
+    ds = source.dataset
+    return {
+        'source_type': 'xarray',
+        'dims': list(ds.dims),
+        'coords': {k: list(ds.coords[k].shape) for k in ds.coords},
+        'data_vars': list(ds.data_vars),
+    }

lumen/tests/ai/test_deckgl_gridded.py

@@ -0,0 +1,114 @@
+import datetime
+
+import pandas as pd
+import pytest
+
+from lumen.ai.config import PROMPTS_DIR
+from lumen.ai.utils import render_template
+from lumen.pipeline import Pipeline
+from lumen.sources.base import InMemorySource
+from lumen.views.base import _DECKGL_ROW_LIMIT, DeckGLView
+
+
+def _base_context(gridded=None):
+    class _FakeSource:
+        dialect = None
+
+    class _FakePipeline:
+        table = "air"
+        source = _FakeSource()
+
+    class _FakeMemory(dict):
+        pass
+
+    memory = _FakeMemory({
+        "pipeline": _FakePipeline(),
+        "data": "lat,lon,air\n0,10,1.0",
+        "table": "air",
+    })
+    return dict(
+        memory=memory,
+        doc="DeckGLView description",
+        gridded=gridded,
+        current_datetime=datetime.datetime(2026, 1, 1),
+        actor_name="DeckGLAgent",
+    )
+
+
+def test_deckgl_prompt_includes_gridded_block():
+    gridded = {
+        "source_type": "xarray",
+        "dims": ["lat", "lon"],
+        "coords": {"lat": [3], "lon": [4]},
+        "data_vars": ["air"],
+    }
+    rendered = render_template(
+        PROMPTS_DIR / "DeckGLAgent" / "main.jinja2",
+        **_base_context(gridded=gridded),
+    )
+    assert "ColumnLayer" in rendered
+    assert "air" in rendered
+    assert "lat" in rendered
+    assert "lon" in rendered
+
+
+def test_deckgl_prompt_no_gridded_block_for_tabular():
+    rendered = render_template(
+        PROMPTS_DIR / "DeckGLAgent" / "main.jinja2",
+        **_base_context(gridded=None),
+    )
+    assert "Gridded xarray data" not in rendered
+
+
+def test_deckgl_pydeck_prompt_includes_gridded_block():
+    gridded = {
+        "source_type": "xarray",
+        "dims": ["lat", "lon"],
+        "coords": {"lat": [3], "lon": [4]},
+        "data_vars": ["air"],
+    }
+    rendered = render_template(
+        PROMPTS_DIR / "DeckGLAgent" / "main_pydeck.jinja2",
+        **_base_context(gridded=gridded),
+    )
+    assert "ColumnLayer" in rendered
+    assert "air" in rendered
+
+
+def test_deckglview_row_limit_raises():
+    n = _DECKGL_ROW_LIMIT + 1
+    df = pd.DataFrame({"lon": range(n), "lat": range(n), "value": range(n)})
+    source = InMemorySource(tables={"big": df})
+    pipeline = Pipeline(source=source, table="big")
+
+    view = DeckGLView(
+        pipeline=pipeline,
+        spec={
+            "initialViewState": {"latitude": 0, "longitude": 0, "zoom": 2},
+            "layers": [{"@@type": "ScatterplotLayer", "getPosition": "@@=[lon, lat]"}],
+            "views": [{"@@type": "MapView", "controller": True}],
+        },
+    )
+    with pytest.raises(ValueError, match="250,000"):
+        view._get_params()
+
+
+def test_deckglview_row_limit_passes_under_threshold():
+    df = pd.DataFrame({"lon": [0.0, 1.0], "lat": [0.0, 1.0], "value": [1.0, 2.0]})
+    source = InMemorySource(tables={"small": df})
+    pipeline = Pipeline(source=source, table="small")
+
+    view = DeckGLView(
+        pipeline=pipeline,
+        spec={
+            "initialViewState": {"latitude": 0, "longitude": 0, "zoom": 2},
+            "layers": [{"@@type": "ScatterplotLayer", "getPosition": "@@=[lon, lat]"}],
+            "views": [{"@@type": "MapView", "controller": True}],
+        },
+    )
+    params = view._get_params()
+    assert "object" in params
+    assert params["object"]["layers"][0]["data"] == [
+        {"lon": 0.0, "lat": 0.0, "value": 1.0},
+        {"lon": 1.0, "lat": 1.0, "value": 2.0},
+    ]