Adding demos for SQLbits 2026

.devcontainer/devcontainer-lock.json

@@ -0,0 +1,14 @@
+{
+  "features": {
+    "ghcr.io/devcontainers/features/azure-cli:1": {
+      "version": "1.2.9",
+      "resolved": "ghcr.io/devcontainers/features/azure-cli@sha256:4549175fbfd3475d1d62e82f6e5425d03954a6ae06027b2515b0ba41a8206417",
+      "integrity": "sha256:4549175fbfd3475d1d62e82f6e5425d03954a6ae06027b2515b0ba41a8206417"
+    },
+    "ghcr.io/devcontainers/features/powershell:2.0.1": {
+      "version": "2.0.1",
+      "resolved": "ghcr.io/devcontainers/features/powershell@sha256:2db157a4f97465bb64e03596e0f795e9899e3c041b64a7a0ffd8faea4fc50c39",
+      "integrity": "sha256:2db157a4f97465bb64e03596e0f795e9899e3c041b64a7a0ffd8faea4fc50c39"
+    }
+  }
+}

.devcontainer/devcontainer.json

@@ -5,14 +5,15 @@
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
 	"image": "mcr.microsoft.com/devcontainers/base:noble",
 	"features": {
-		"ghcr.io/devcontainers/features/azure-cli:1": {}
+		"ghcr.io/devcontainers/features/azure-cli:1": {},
+		"ghcr.io/devcontainers/features/powershell:2.0.1": {}
 	},
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH=\"$HOME/.local/bin:$PATH\" && uv sync",
+	"postCreateCommand": "pwsh -File .devcontainer/post-create.ps1",
 
 	// Environment variables
 	"containerEnv": {
@@ -24,14 +25,23 @@
 			"extensions": [
 				"ms-toolsai.jupyter",
 				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"eliostruyf.vscode-demo-time",
+				"mechatroner.rainbow-csv",
 				"ms-python.pylint",
 				"ms-python.black-formatter",
-				"ms-toolsai.datawrangler"
+				"bierner.markdown-mermaid",
+				"Anthropic.claude-code",
+				"jimasp.behave-vsc",
+				"stevejpurves.cucumber",
+				"CucumberOpen.cucumber-official"
 			],
 			"settings": {
 				"files.exclude": {
 					".uv-cache": true
-				}
+				},
+				"python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv/bin/python",
+				"behave-vsc.featuresPath": "tests/bdd"
 			}
 		}
 	}
@@ -41,4 +51,4 @@
 
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	// "remoteUser": "root"
-}
+}

.devcontainer/post-create.ps1

@@ -0,0 +1,20 @@
+#!/usr/bin/env pwsh
+# Post-create setup script for the dev container
+
+# Install uv (Python package manager)
+Write-Host "Installing uv..."
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Add uv to PATH for this session
+$env:PATH = "$HOME/.local/bin:$env:PATH"
+
+# Sync Python dependencies
+Write-Host "Syncing Python dependencies..."
+uv sync --extra dev
+
+# Configure git to use the version-controlled hooks directory
+Write-Host "Configuring git hooks..."
+git config core.hooksPath .hooks
+chmod +x .hooks/pre-commit
+
+Write-Host "Post-create setup complete!"

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+exclude = .git,__pycache__,dist,.venv
+max-line-length = 120

.gitignore

@@ -1,4 +1,10 @@
+# Claude Code Assets
+CLAUDE.md
+.claude/
+
+# Ignore data files, which may contain sensitive information or be too large for version control.
 data/
+tests/bdd/test_output/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.hooks/pre-commit

@@ -0,0 +1,2 @@
+#!/bin/sh
+pwsh -File "$(git rev-parse --show-toplevel)/.hooks/pre-commit.ps1"

.hooks/pre-commit.ps1

@@ -0,0 +1,99 @@
+#!/usr/bin/env pwsh
+# Pre-commit hook: runs linting and tests, blocking the commit on any failure.
+
+$ErrorActionPreference = "Stop"
+$repoRoot = git rev-parse --show-toplevel
+Set-Location $repoRoot
+
+$fabricNotebookResourceScripts = @(
+    @{
+        script = 'src/data_wrangler/data_wrangler.py'
+        notebooks = @(
+            'fabric/Process To Silver.Notebook'
+            'fabric/Project to Gold.Notebook'
+        )
+    }
+)
+
+$failed = $false
+
+# Write-Host "==> black: checking formatting..." -ForegroundColor Cyan
+# uv run black --check src/ tests/
+# if ($LASTEXITCODE -ne 0) {
+#     Write-Host "FAILED: black found formatting issues. Run 'uv run black src/ tests/' to fix." -ForegroundColor Red
+#     $failed = $true
+# }
+
+# Write-Host "==> flake8: linting src/..." -ForegroundColor Cyan
+# uv run flake8 src/
+# if ($LASTEXITCODE -ne 0) {
+#     Write-Host "FAILED: flake8 reported errors." -ForegroundColor Red
+#     $failed = $true
+# }
+
+Write-Host "==> pytest: running unit tests..." -ForegroundColor Cyan
+uv run pytest tests/pytest/
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "FAILED: pytest tests did not pass." -ForegroundColor Red
+    $failed = $true
+}
+
+Write-Host "==> behave: running BDD tests..." -ForegroundColor Cyan
+uv run behave tests/bdd/
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "FAILED: behave tests did not pass." -ForegroundColor Red
+    $failed = $true
+}
+
+# Ensure the fabric notebook resources are updated
+Write-Host '==> Syncing Fabric Notebook Resources...' -ForegroundColor Cyan
+try {
+    foreach ($resource in $fabricNotebookResourceScripts) {
+        $srcFile = Join-Path $repoRoot $resource.script
+        foreach ($notebook in $resource.notebooks) {
+            $destNotebookDir = Join-Path $repoRoot $notebook
+            $destDir = Join-Path $destNotebookDir 'Resources/builtin/internal_classes'
+            $destFile = Join-Path $destDir (Split-Path $srcFile -Leaf)
+
+            $copied = $false
+            $tempFile = New-TemporaryFile
+            try {
+                # Ensure that the synced files are consistent with either the current
+                # committed version or a staged version (i.e. to include cny changes
+                # that will be committed if this hook is successful).
+                $stagedPath = $resource.script -replace '\\', '/'
+                $stagedContent = & git show ":$stagedPath" 2>$null
+                if ($LASTEXITCODE -eq 0) {
+                    $stagedContent | Out-File -FilePath $tempFile.FullName -Encoding utf8NoBOM
+                    if (-not (Test-Path $destFile) -or (Get-FileHash $tempFile.FullName).Hash -ne (Get-FileHash $destFile).Hash) {
+                        Copy-Item -Force $tempFile.FullName $destFile
+                        $copied = $true
+                    }
+                }
+            }
+            finally {
+                Remove-Item $tempFile.FullName -ErrorAction SilentlyContinue
+            }
+
+            if ($copied) {
+                & git add "$destFile"
+                Write-Host "==> Updated notebook resource: $destFile - commit will be blocked to allow staged changes to be reviewed" -ForegroundColor Cyan
+                # TODO: Should we auto-commit having made these git index changes?
+                # $failed = $true
+            }
+        }
+    }
+    Write-Host 'Complete.' -f Green
+}
+catch {
+    Write-Host "FAILED: Error whilst syncing Fabric Notebook Resources - $($_.Exception.Message)" -ForegroundColor Red
+    $failed = $true
+}
+
+if ($failed) {
+    Write-Host "`nPre-commit checks failed. Commit aborted." -ForegroundColor Red
+    exit 1
+}
+
+Write-Host "`nAll pre-commit checks passed." -ForegroundColor Green
+exit 0

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "eliostruyf.vscode-demo-time"
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+    "behave-vsc.featuresPath": "tests/bdd",
+    "demoTime.defaultFileType": "json",
+    "claudeCode.useTerminal": true
+}

README.md

@@ -1,33 +1,123 @@
 # endjin-polars-examples
-endjin-polars-examples is a series of runnable notebooks and companion scripts that explore how to work with the [Polars](https://www.pola.rs/) DataFrame engine. The goal is to highlight idiomatic ways to ingest, transform, inspect, and summarize data while leaning on Polars' lazy execution, expression system, and columnar performance.
 
-## Dataset
-We demonstrate the techniques using the publicly available World Bank Open Data API. Specifically, we download the **World Bank Indicators** dataset, which contains macroeconomic series (GDP, population, education, etc.) for every country and year. The raw exports live under `data/world_bank/` and include:
+Practical examples of [Polars](https://www.pola.rs/) in use for modern, cloud-native data engineering pipelines. The primary dataset is the **HM Land Registry Price Paid Data** — publicly available records of every residential property sale in England and Wales.
+
+## Contents
+
+| Folder | Description |
+|---|---|
+| `.hooks` | Git hook scripts (NOTE: Will be setup automatically when using the Devcontainer) |
+| `notebooks/sqlbits_2026/` | Demo notebooks for the **SQLbits 2026** conference — download Land Registry data and run the full wrangling pipeline |
+| `notebooks/polars_blog/` | Polars blog post series covering eager/lazy patterns, Polars vs pandas, and Microsoft Fabric |
+| `src/data_wrangler/` | `DataWrangler` class — transformation pipeline and Pandera schema |
+| `src/data_importers/` | `LandRegistryImporter` — downloads yearly price paid CSV files |
+| `tests/bdd/` | Behavioural tests (Gherkin + Behave) covering every pipeline step |
+| `skills/` | How-to guides for working in this codebase |
+| `data/land_registry_data/` | Downloaded CSVs (not committed — generated by the download notebook) |
+
+## SQLbits 2026 notebooks
+
+The `notebooks/sqlbits_2026/` folder contains two notebooks intended to be run in order:
+
+1. **`download_land_registry_data.ipynb`** — downloads up to 10 years of yearly price paid CSV files (~100 MB each) from the Land Registry S3 bucket into `data/land_registry_data/`. Already-downloaded files are skipped.
+
+2. **`summarise_land_registry_data.ipynb`** — runs the full `DataWrangler` pipeline against the downloaded data and plots median house price by year and property type using Plotly.
+
+## DataWrangler pipeline
 
-- CSV extracts that mirror the API's standard download format (`csv/countries.csv`, `csv/data.csv`, and `csv/indicators.csv`).
-- JSON files for the data snapshots per year (`json/data_XXXX.json`).
-- Parquet and DuckDB snapshots for faster reads (`parquet/`, `duckdb/world_bank.db.wal`).
+`DataWrangler` in `src/data_wrangler/data_wrangler.py` supports both a backward-compatible `run_pipeline(data_folder)` class method and a data-source-driven API (`run_pipeline_with_data_source(...)`).
 
-You can explore the original API documentation and download options here:
-- World Bank Open Data main page: https://data.worldbank.org/
-- Indicator query and API docs: https://datahelpdesk.worldbank.org/knowledgebase/articles/889386-developer-information-overview
+Both paths chain the following steps using Polars lazy execution (`scan_csv` → `collect`):
 
-## Project layout
-- `notebooks/` contains narrative Polars tutorials (the `polars-blog-part-*` series) that walk through DataFrame creation, cleaning, filtering, joins, and aggregation.
-- `src/data_importers/` holds reusable scripts for fetching or reshaping the World Bank data before feeding it into Polars-powered notebooks.
-- `docs/` documents conceptual guides and blog drafts describing implementation details and translation strategies.
-- `data/world_bank/` stores the dataset in multiple formats so readers can experiment with CSV, JSON, Parquet, and DuckDB access patterns.
+```
+load_data                          scan_csv glob, name columns, cast price/date
+drop_records_without_postcode      remove rows with null/empty postcode
+drop_records_without_date          remove rows with null date
+filter_other_property_types        remove property_type = "O" (Other)
+extract_year_from_date             add year column from date
+rename_property_type               D/S/T/F → Detached/Semi-Detached/Terraced/Flat
+rename_duration                    F/L/U → Freehold/Leasehold/Unknown
+rename_old_new                     Y/N → New/Old
+extract_postcode_area              add postcode_area (e.g. SW1A from SW1A 2AA)
+summarise_by_year_and_property_type  group_by year + property_type, agg sales/prices
+sort_by_year_and_property_type     sort ascending by year then property_type
+collect                            execute the lazy plan, return eager DataFrame
+```
 
-## Prerequisites
-To work comfortably with this repository you should have the following tools installed:
-1. Visual Studio Code (https://code.visualstudio.com/) with a Python or Polars-friendly extension (e.g., the Official Python extension) so you can edit notebooks and scripts interactively.
-2. Docker (https://www.docker.com/) to reproduce builds or run containerized environments if you prefer isolating tooling (the repo should be runnable without Docker, but it helps when matching the upstream development container).
-3. Git (https://git-scm.com/) to clone the repository, track changes, and share contributions.
+All transformation methods are also callable individually as static methods, accepting either `pl.DataFrame` or `pl.LazyFrame`.
 
 ## Getting started
-1. Clone the repo and open it in VS Code.
-2. Activate the provided Python environment (via `uv` in this notebook, or another virtual environment that includes Polars).
-3. Open `notebooks/polars-blog-part-4.ipynb` (and its siblings) to run the cells interactively; the notebook shows both eager and lazy patterns.
-4. Use the scripts under `src/data_importers/` to refresh the World Bank dataset if you need more recent data.
 
-That’s it—Polars’ next-generation performance makes these examples snappy even when the World Bank tables grow large, and the notebooks explain each operator in context.
+You can run this project in two ways:
+
+### Option 1: Run locally
+
+Prerequisites:
+
+- [Python 3.12+](https://www.python.org/downloads/)
+- [uv](https://docs.astral.sh/uv/) (Python package manager)
+- [VS Code](https://code.visualstudio.com/) with the Python and Behave VSC extensions
+- `az login` if reading/writing to Azure storage (local runs use the filesystem)
+
+Install dependencies:
+
+```bash
+uv sync
+```
+
+### Option 2: Run in a devcontainer
+
+Prerequisites:
+
+- [VS Code](https://code.visualstudio.com/)
+- Dev Containers extension (`ms-vscode-remote.remote-containers`)
+- Docker, or an equivalent Docker-compatible container runtime
+
+Notes:
+
+- You do not need to install `uv` on your host machine for this option.
+- Open the repository in VS Code and choose **Reopen in Container**.
+- The container provides the toolchain; run commands from inside the container terminal.
+
+### Download Land Registry data
+
+Run `notebooks/sqlbits_2026/download_land_registry_data.ipynb`, or use the importer directly:
+
+```python
+from data_importers import LandRegistryImporter
+
+importer = LandRegistryImporter(
+    raw_data_download_path="data/land_registry_data",
+    storage_options={},
+    number_of_years=5,
+)
+importer.download_land_registry_data()
+```
+
+### Run the pipeline
+
+```python
+from data_wrangler import DataWrangler, LocalCsvDataSource
+
+data_source = LocalCsvDataSource(
+    data_folder="data/land_registry_data",
+    column_names=DataWrangler.COLUMN_NAMES,
+)
+summary = DataWrangler.run_pipeline_with_data_source(data_source)
+print(summary)
+```
+
+### Run the tests
+
+```bash
+uv run behave --tags @unit    # fast in-memory unit tests
+uv run behave --tags @e2e     # end-to-end test against tests/bdd/test_data/
+uv run behave                 # all tests
+```
+
+## Dataset
+
+**HM Land Registry Price Paid Data** tracks residential property sales in England and Wales submitted for registration. Data runs from 1995 to present. Each yearly file is approximately 100 MB. Published under the [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/).
+
+Source: https://www.gov.uk/guidance/about-the-price-paid-data
+

behave.ini

@@ -0,0 +1,3 @@
+# behave.ini
+[behave]
+paths=tests/bdd