Merge branch 'main' into feat/smart-followup-suggestions

Author: ahuang11

.github/workflows/build.yaml

@@ -6,7 +6,6 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+"
       - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+"
       - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
-  # Dry-run only
   workflow_dispatch:
   schedule:
     - cron: "0 17 * * SUN"
@@ -16,8 +15,8 @@ defaults:
     shell: bash -e {0}
 
 env:
-  PYTHON_VERSION: "3.11"
   PACKAGE: "lumen"
+  PYTHON_VERSION: "3.11"
 
 jobs:
   waiting_room:
@@ -48,45 +47,35 @@ jobs:
           install: false
       - name: conda build
         run: pixi run -e build build-conda
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: conda
-          path: dist/*.tar.bz2
+          name: artifacts-conda
+          path: dist/*.conda
           if-no-files-found: error
 
   conda_publish:
     name: Publish Conda
     runs-on: ubuntu-latest
     needs: [conda_build, waiting_room]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
-    defaults:
-      run:
-        shell: bash -el {0}
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with:
-          name: conda
+          name: artifacts-conda
           path: dist/
-      - name: Set environment variables
+      - name: anaconda setup
         run: |
-          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV
-      - uses: conda-incubator/setup-miniconda@v3
-        with:
-          miniconda-version: "latest"
-          channels: "conda-forge"
-      - name: conda setup
+          pipx install anaconda-client
+          binstar --version
+      - name: anaconda dev upload
+        if: contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc')
         run: |
-          conda install -y anaconda-client
-      - name: conda dev upload
-        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
+          binstar --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev dist/*.conda
+      - name: anaconda upload
+        if: (!(contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc')))
         run: |
-          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
-      - name: conda main upload
-        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
-        run: |
-          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE
+          binstar --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main dist/*.conda
 
   pip_build:
     name: Build PyPI
@@ -100,10 +89,10 @@ jobs:
           install: false
       - name: Build package
         run: pixi run -e build build-pip
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: pip
+          name: artifacts-pip
           path: dist/
           if-no-files-found: error
 
@@ -112,12 +101,12 @@ jobs:
     runs-on: "ubuntu-latest"
     needs: [pip_build]
     steps:
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with:
-          name: pip
+          name: artifacts-pip
           path: dist/
       - name: Install package
         run: python -m pip install dist/*.whl
@@ -129,14 +118,46 @@ jobs:
     runs-on: ubuntu-latest
     needs: [pip_build, waiting_room]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with:
-          name: pip
+          name: artifacts-pip
           path: dist/
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
+
+  announce:
+    name: Announce GitHub
+    runs-on: ubuntu-latest
+    needs: [waiting_room]
+    permissions:
+      contents: write
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    env:
+      TAG: ${{ github.ref_name }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v8
         with:
-          user: ${{ secrets.PPU }}
-          password: ${{ secrets.PPP }}
-          repository-url: "https://upload.pypi.org/legacy/"
+          pattern: artifacts-*
+          path: artifacts/
+          merge-multiple: true
+      - name: Extract changelog section
+        run: |
+          awk -v ver="## Version ${TAG#v}" '
+            $0 == ver {flag=1; next}
+            /^#{2} Version / && flag {exit}
+            flag && !/^(\*\*.*\*\*)$/
+          ' CHANGELOG.md | tee RELEASE_BODY.md
+      - name: Create draft GitHub release with dist artifacts
+        if: (!(contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc')))
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "$TAG" \
+            --title "Version ${TAG#v}" \
+            --notes-file RELEASE_BODY.md \
+            --draft \
+            artifacts/*

.github/workflows/docs.yaml

@@ -46,38 +46,29 @@ jobs:
           environments: docs
       - name: Build documentation
         run: pixi run -e docs docs-build
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         if: always()
         with:
           name: docs
           if-no-files-found: error
           path: builtdocs
-      - name: Set output
-        id: vars
-        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
 
   docs_publish:
     name: Publish Documentation
     runs-on: "ubuntu-latest"
     needs: [docs_build]
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with:
           name: docs
           path: builtdocs/
-      - name: Set output
-        id: vars
-        run: echo "tag=${{ needs.docs_build.outputs.tag }}" >> $GITHUB_OUTPUT
       - name: upload dev
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         if: |
           (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'dev') ||
-          (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
+          (github.event_name == 'push' && (contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc')))
         with:
           personal_token: ${{ secrets.ACCESS_TOKEN }}
           external_repository: holoviz-dev/lumen
@@ -86,8 +77,8 @@ jobs:
       - name: upload main
         if: |
           (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'main') ||
-          (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        uses: peaceiris/actions-gh-pages@v3
+          (github.event_name == 'push' && !(contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc')))
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./builtdocs

.github/workflows/gallery.yaml

@@ -26,11 +26,11 @@ jobs:
         shell: bash -el {0}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: '3.11'
       - name: Set and echo git ref
@@ -40,7 +40,7 @@ jobs:
         run: |
           echo "Deploying from ref $HEAD_BRANCH"
           echo "tag=$HEAD_BRANCH" >> $GITHUB_OUTPUT
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: "latest"
           auto-update-conda: true

.github/workflows/lock.yaml

@@ -16,7 +16,7 @@ jobs:
   action:
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@v4
+      - uses: dessant/lock-threads@v6
         with:
           issue-inactive-days: "100"
           pr-inactive-days: "100"

.github/workflows/test.yaml

@@ -47,7 +47,7 @@ jobs:
         if: needs.setup.outputs.img_change == 'true'
         with:
           extra_args: -a --hook-stage manual oxipng || true --
-      - uses: stefanzweifel/git-auto-commit-action@v4
+      - uses: stefanzweifel/git-auto-commit-action@v7
         if: needs.setup.outputs.img_change == 'true'
         with:
           commit_message: "Optimize PNG images (lossless)"
@@ -60,10 +60,10 @@ jobs:
       img_change: ${{ steps.filter.outputs.img }}
       matrix: ${{ env.MATRIX }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         if: github.event_name != 'pull_request'
       - name: Check for code changes
-        uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v4
         id: filter
         with:
           filters: |
@@ -129,7 +129,7 @@ jobs:
       - name: Test Unit
         run: |
           pixi run -e ${{ matrix.environment }} test-unit $COV
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 

docs/configuration/sources.md

@@ -29,6 +29,7 @@ ui.servable()
 |--------|---------|
 | Files | CSV, Parquet, JSON (local or URL) |
 | DuckDB | Local SQL queries on files |
+| xarray | N-dimensional scientific data (NetCDF, Zarr, HDF5) |
 | Snowflake | Cloud data warehouse |
 | BigQuery | Google's data warehouse |
 | PostgreSQL | PostgreSQL via SQLAlchemy |
@@ -173,6 +174,45 @@ source = DuckDBSource(
 
 1. Required for HTTP/S3 access
 
+### xarray for scientific data
+
+Query N-dimensional scientific datasets (NetCDF, Zarr, HDF5) with SQL via Apache DataFusion:
+
+``` bash title="Install dependencies"
+pip install lumen[xarray]
+```
+
+``` py title="Query a NetCDF file"
+from lumen.sources.xarray_sql import XArraySQLSource
+import lumen.ai as lmai
+
+source = XArraySQLSource(uri='air_temperature.nc')
+
+ui = lmai.ExplorerUI(data=source)
+ui.servable()
+```
+
+Each data variable in the dataset becomes a separate SQL table with coordinate columns:
+
+``` py title="Direct SQL queries"
+source = XArraySQLSource(uri='air_temperature.nc')
+source.get_tables()  # ['air']
+
+df = source.execute(
+    "SELECT lat, lon, AVG(air) as avg_temp "
+    "FROM air GROUP BY lat, lon"
+)
+```
+
+**Select specific variables:**
+
+``` py hl_lines="3"
+source = XArraySQLSource(
+    uri='climate_data.nc',
+    variables=['temperature', 'pressure']
+)
+```
+
 ### Multiple sources
 
 ``` py title="Mix sources"

docs/extending.md

@@ -0,0 +1,37 @@
+# :material-puzzle-plus: Extending Lumen
+
+Lumen's core building blocks—[Sources](configuration/sources.md), [Agents](configuration/agents.md), [Tools](configuration/tools.md), and [Analyses](configuration/analyses.md)—are all subclassable Python classes. If you've built a custom component and want to share it as a reusable, installable Python package, the **Panel Extension Copier Template** gives you a fully configured project scaffold to do just that.
+
+### What is the Panel Extension Copier Template?
+
+The [`copier-template-panel-extension`](https://github.com/panel-extensions/copier-template-panel-extension) is a [Copier](https://copier.readthedocs.io/en/stable/) template that scaffolds a fully configured Python package for Panel and Lumen extensions. It gives you:
+
+- **pytest** for testing
+- **Zensical + mkdocstrings** for automatic API docs hosted on GitHub Pages
+- **GitHub Actions** CI/CD for automated testing and publishing
+- **Pixi** for reproducible environment management
+
+### Quickstart
+
+Make sure you have [Pixi](https://pixi.sh) installed, then run:
+
+```bash
+pixi exec --spec copier --spec ruamel.yaml -- \
+  copier copy --trust \
+  https://github.com/panel-extensions/copier-template-panel-extension \
+  lumen-name-of-extension
+```
+
+You'll be prompted for a few details such as extension type, project slug, and author info. Choose **Lumen** as the extension type when asked, and select `py311` or higher for the minimum Python version—Lumen requires Python 3.11+.
+
+From there, follow the template's [step-by-step guide](https://github.com/panel-extensions/copier-template-panel-extension#getting-started) to set up GitHub Pages docs, link to PyPI, and publish your first release with a git tag.
+
+---
+
+## Further Reading
+
+- [Data Sources](configuration/sources.md) — built-in sources and how to configure them
+- [Agents](configuration/agents.md) — customizing agent behavior
+- [Tools](configuration/tools.md) — adding domain-specific tools
+- [Analyses](configuration/analyses.md) — deterministic analysis functions
+- [Panel Extension Copier Template on GitHub](https://github.com/panel-extensions/copier-template-panel-extension)