Merge pull request #132 from UNDO-project/hotspot_redesign_backend

Hotspot redesign backend

Author: jethronap

README.md

@@ -20,6 +20,19 @@ The pipeline consists of three main agents:
 - **Comprehensive visualizations**: Heatmaps, hotspots, route maps, and statistical charts
 - **Spatial optimization**: Efficient GeoDataFrame indexing for large camera datasets
 
+## Hotspot methodology
+
+`v2.4.0` ships a four-layer hotspot analysis. Each layer answers a different question — they complement rather than replace each other.
+
+1. **HDBSCAN clusters** (`<city>_hotspots.geojson` + `<city>_hotspot_polygons.geojson`)
+   Density-based clustering with locally-adaptive `ε`, computed in UTM metres so a "20-metre cluster" means the same thing at any latitude. Replaces the prior single-`ε` DBSCAN, which fused downtown into one blob and missed sparser suburban clusters (cf. GraphTrace 2025 urban-analytics benchmarks).
+2. **Planar KDE density surface** (`<city>_heatmap.html` + `<city>_density.geojson`)
+   FFT-based kernel density on a metric grid; the folium heatmap is *derived* from the surface rather than from folium's opaque built-in interpolation, and the same surface contours into a GeoJSON layer at the 50/75/90/95 percentiles for researcher-grade work.
+3. **Getis-Ord Gi\* hex grid** (`<city>_gi_star.geojson` + `<city>_gi_star.png`)
+   The spatial statistic researchers and journalists know from ArcGIS/QGIS "Hot Spot Analysis": per-hex z-score, FDR-adjusted p-value, and a five-class hot/cold classification — the defensible statistical layer per Amnesty's [*Decode Surveillance NYC*](https://decoders.amnesty.org/projects/decode-surveillance) methodology.
+4. **Cameras per road-km** (`<city>_density_metrics.json`)
+   The single citable headline number Stanford's [*Surveilling Surveillance*](https://reglab.stanford.edu/projects/surveilling-surveillance/) (2021) made canonical for cross-city comparison. Normalises by the pedestrian network humans actually use, neutralising the park/water/industrial-zone bias of cameras/km². Reuses the routing agent's cached OSMnx graph.
+
 # Installation
 
 ## Prerequisites

cli_local_test_pipeline.sh

@@ -70,6 +70,31 @@ pytest tests/tools/test_mapping_tools.py
 echo "Done..."
 echo "==============================================="
 
+echo "Running geo projection tests"
+pytest tests/tools/test_geo_projection.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running hotspot clustering tests"
+pytest tests/tools/test_hotspot_clustering.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running KDE density tests"
+pytest tests/tools/test_density_kde.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running Gi* spatial stats tests"
+pytest tests/tools/test_spatial_stats.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running density-metrics tests (cameras-per-road-km)"
+pytest tests/tools/test_density_metrics.py
+echo "Done..."
+echo "==============================================="
+
 echo "Running chart tools tests"
 pytest tests/tools/test_chart_tools.py
 echo "Done..."

main.py

@@ -61,7 +61,17 @@ def parse_args():
         default=None,
         help=(
             "Override the preset for hotspot generation. Toggles both the "
-            "DBSCAN clustering output and the matplotlib plot together."
+            "HDBSCAN clustering output and the matplotlib plot together."
+        ),
+    )
+    parser.add_argument(
+        "--gi-star",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        dest="gi_star",
+        help=(
+            "Override the preset for the Getis-Ord Gi* hex-grid layer. "
+            "Toggles both the GeoJSON artifact and the choropleth chart."
         ),
     )
     parser.add_argument(
@@ -84,6 +94,18 @@ def parse_args():
             "(--report / --no-report)."
         ),
     )
+    parser.add_argument(
+        "--density-metrics",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        dest="density_metrics",
+        help=(
+            "Override the preset for the cameras-per-road-km headline "
+            "metric (--density-metrics / --no-density-metrics). Defaults "
+            "on for both BASIC and FULL presets — opt out when you don't "
+            "want to touch the OSMnx graph cache."
+        ),
+    )
     parser.add_argument(
         "--output-dir",
         help="Output directory for results",
@@ -260,6 +282,12 @@ def display_results(results: dict):
         elif analyze.get("success"):
             elements = analyze.get("element_count", 0)
             table.add_row("Elements Analyzed", f"[green]{elements}[/green]")
+            metrics = analyze.get("density_metrics") or {}
+            if metrics.get("cameras_per_road_km") is not None:
+                table.add_row(
+                    "Cameras / road-km",
+                    f"[green]{metrics['cameras_per_road_km']:.3f}[/green]",
+                )
         else:
             table.add_row(
                 "Analysis", f"[red]Failed: {analyze.get('error', 'Unknown')}[/red]"
@@ -306,6 +334,12 @@ def display_results(results: dict):
             files_table.add_row("Hotspots Data", str(analyze["hotspots_path"]))
         if analyze.get("plot_hotspots"):
             files_table.add_row("Hotspots Plot", str(analyze["plot_hotspots"]))
+        if analyze.get("gi_star_path"):
+            files_table.add_row("Gi* GeoJSON", str(analyze["gi_star_path"]))
+        if analyze.get("gi_star_chart"):
+            files_table.add_row("Gi* Plot", str(analyze["gi_star_chart"]))
+        if analyze.get("density_metrics_path"):
+            files_table.add_row("Density Metrics", str(analyze["density_metrics_path"]))
         if analyze.get("chart_path"):
             files_table.add_row("Statistics Chart", str(analyze["chart_path"]))
 
@@ -372,6 +406,9 @@ def main():
     if args.hotspots is not None:
         config_kwargs["generate_hotspots"] = args.hotspots
         config_kwargs["plot_hotspots"] = args.hotspots
+    if args.gi_star is not None:
+        config_kwargs["generate_gi_star"] = args.gi_star
+        config_kwargs["plot_gi_star"] = args.gi_star
     if args.charts is not None:
         config_kwargs["generate_chart"] = args.charts
         config_kwargs["plot_zone_sensitivity"] = args.charts
@@ -381,6 +418,8 @@ def main():
         config_kwargs["plot_install_timeline"] = args.charts
     if args.report is not None:
         config_kwargs["generate_report"] = args.report
+    if args.density_metrics is not None:
+        config_kwargs["compute_density_metrics"] = args.density_metrics
 
     # Routing configuration
     if args.enable_routing:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentic-counter-surveillance"
-version = "2.3.0"
+version = "2.4.0"
 description = "Multi-agent pipeline that scrapes OpenStreetMap surveillance-camera data, enriches it with a local LLM, renders maps and charts, generates an LLM-written city report, and computes low-surveillance walking routes."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -9,6 +9,9 @@ dependencies = [
     "fastapi>=0.123.5",
     "folium>=0.19.6",
     "geopandas>=1.0.1",
+    "h3>=4.4.2",
+    "hdbscan>=0.8.42",
+    "kdepy>=1.1.12",
     "langchain>=0.3.27",
     "langchain-community>=0.3.30",
     "langchain-core>=0.3.77",
@@ -18,6 +21,8 @@ dependencies = [
     "osmnx>=2.0.6",
     "pre-commit>=4.2.0",
     "pydantic-settings>=2.9.1",
+    "pyproj>=3.7.1",
+    "pysal>=26.1",
     "pytest>=8.3.5",
     "python-multipart>=0.0.20",
     "requests>=2.32.3",

src/agents/langchain_analyzer.py

@@ -90,13 +90,15 @@ def analyze(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
             "generate_geojson": input_data.get("generate_geojson", True),
             "generate_heatmap": input_data.get("generate_heatmap", False),
             "generate_hotspots": input_data.get("generate_hotspots", False),
+            "generate_gi_star": input_data.get("generate_gi_star", False),
             "compute_stats": input_data.get("compute_stats", True),
             "generate_chart": input_data.get("generate_chart", False),
             "plot_zone_sensitivity": input_data.get("plot_zone_sensitivity", False),
             "plot_sensitivity_reasons": input_data.get(
                 "plot_sensitivity_reasons", False
             ),
             "plot_hotspots": input_data.get("plot_hotspots", False),
+            "plot_gi_star": input_data.get("plot_gi_star", False),
             "plot_operator_distribution": input_data.get(
                 "plot_operator_distribution", False
             ),
@@ -105,7 +107,12 @@ def analyze(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
             ),
             "plot_install_timeline": input_data.get("plot_install_timeline", False),
             "generate_report": input_data.get("generate_report", False),
+            "compute_density_metrics": input_data.get("compute_density_metrics", False),
             "force_rerender": input_data.get("force_rerender", False),
+            # Non-bool passthrough used by the density-metrics step to
+            # reach the routing agent's graph cache. ``None`` is fine —
+            # OSMnx will geocode the city name alone.
+            "country": input_data.get("country"),
         }
 
         try:
@@ -133,6 +140,12 @@ def analyze(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
                 response["heatmap_path"] = result["heatmap_path"]
             if "hotspots_path" in result:
                 response["hotspots_path"] = result["hotspots_path"]
+            if "hotspot_polygons_path" in result:
+                response["hotspot_polygons_path"] = result["hotspot_polygons_path"]
+            if "density_path" in result:
+                response["density_path"] = result["density_path"]
+            if "gi_star_path" in result:
+                response["gi_star_path"] = result["gi_star_path"]
             if "stats" in result:
                 response["stats"] = result["stats"]
             if "pie_chart_path" in result:
@@ -145,6 +158,8 @@ def analyze(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
                 ]
             if "hotspots_chart" in result:
                 response["hotspots_chart"] = result["hotspots_chart"]
+            if "gi_star_chart" in result:
+                response["gi_star_chart"] = result["gi_star_chart"]
             if "operator_chart_path" in result:
                 response["operator_chart_path"] = result["operator_chart_path"]
             if "manufacturer_chart_path" in result:
@@ -155,6 +170,10 @@ def analyze(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
                 ]
             if "report_path" in result:
                 response["report_path"] = result["report_path"]
+            if "density_metrics_path" in result:
+                response["density_metrics_path"] = result["density_metrics_path"]
+            if "density_metrics" in result:
+                response["density_metrics"] = result["density_metrics"]
 
             # Add visualization errors if any
             if "visualization_errors" in result:

src/api/main.py

@@ -60,7 +60,7 @@ async def lifespan(app: FastAPI):
     2. Monitor progress via GET /api/v1/pipeline/{task_id}
     3. Retrieve generated files via /api/v1/outputs/...
     """,
-    version="2.3.0",
+    version="2.4.0",
     docs_url="/docs",
     redoc_url="/redoc",
     lifespan=lifespan,

src/api/models/requests.py

@@ -34,7 +34,15 @@ class OutputOverrides(BaseModel):
         default=None, description="Render the folium heatmap (HTML)"
     )
     generate_hotspots: Optional[bool] = Field(
-        default=None, description="Compute DBSCAN hotspots GeoJSON"
+        default=None,
+        description="Compute HDBSCAN hotspots (centroids + convex-hull polygons)",
+    )
+    generate_gi_star: Optional[bool] = Field(
+        default=None,
+        description=(
+            "Compute the Getis-Ord Gi* hex-grid statistical layer "
+            "(<city>_gi_star.geojson)."
+        ),
     )
     plot_zone_sensitivity: Optional[bool] = Field(
         default=None, description="Render the zone-sensitivity stacked bar chart"
@@ -45,6 +53,10 @@ class OutputOverrides(BaseModel):
     plot_hotspots: Optional[bool] = Field(
         default=None, description="Render the hotspots scatter plot (PNG)"
     )
+    plot_gi_star: Optional[bool] = Field(
+        default=None,
+        description="Render the Gi* choropleth chart (<city>_gi_star.png)",
+    )
     plot_operator_distribution: Optional[bool] = Field(
         default=None,
         description="Render the top-N operator distribution bar chart",
@@ -61,6 +73,13 @@ class OutputOverrides(BaseModel):
         default=None,
         description="Generate an LLM-written markdown city report (<city>_report.md)",
     )
+    compute_density_metrics: Optional[bool] = Field(
+        default=None,
+        description=(
+            "Compute the cameras-per-road-km headline metric "
+            "(<city>_density_metrics.json)."
+        ),
+    )
 
 
 class ScrapeRequest(BaseModel):

src/api/models/responses.py

@@ -192,7 +192,7 @@ class VersionResponse(BaseModel):
         json_schema_extra={
             "examples": [
                 {
-                    "version": "2.3.0",
+                    "version": "2.4.0",
                     "api_version": "v1",
                     "description": "Agentic Surveillance Research API",
                 }

src/api/routes/health.py

@@ -38,7 +38,7 @@ async def version_info():
     :return: Version details
     """
     return VersionResponse(
-        version="2.3.0",
+        version="2.4.0",
         api_version="v1",
         description="Agentic Surveillance Research API",
     )

src/api/routes/outputs.py

@@ -209,6 +209,79 @@ async def get_city_charts(city: str, chart: str):
     )
 
 
+#: Hotspot-redesign artifacts addressable by name.
+#:
+#: Each entry maps the URL leaf the frontend uses to the on-disk filename
+#: the analyzer chain writes. Two reasons for keeping this as a single
+#: declarative dict rather than five hand-rolled handlers:
+#: 1. The dashboard's layer-toggle UI iterates the names server-side via
+#:    ``/list``; the names here must match those filenames byte-for-byte
+#:    or the toggle goes dead.
+#: 2. The set grows together keeping the registry literal avoids the next layer
+#:    being silently added to ``/list`` but missing a dedicated route.
+_HOTSPOT_ARTIFACTS: dict = {
+    "density.geojson": "_density.geojson",
+    "density_metrics.json": "_density_metrics.json",
+    "gi_star.geojson": "_gi_star.geojson",
+    "hotspots.geojson": "_hotspots.geojson",
+    "hotspot_polygons.geojson": "_hotspot_polygons.geojson",
+}
+
+
+def _serve_named_artifact(city: str, suffix: str) -> FileResponse:
+    """
+    Resolve ``<city><suffix>`` inside the per-city directory and serve it.
+
+    Centralises the validate-then-stream dance so each artifact handler
+    is one line. ``suffix`` is the literal trailing chunk after the
+    city stem (e.g. ``"_density.geojson"``), matching the analyzer
+    chain's filename construction at the call site.
+    """
+    base = resolve_city_base(city)
+    file_path = base / f"{city}{suffix}"
+    validate_path(file_path)
+    return FileResponse(
+        path=file_path,
+        media_type=get_mime_type(file_path),
+        filename=file_path.name,
+    )
+
+
+@router.get("/{city}/density.geojson")
+async def get_city_density_geojson(city: str):
+    """Serve the KDE density contours: ``<city>_density.geojson``."""
+    return _serve_named_artifact(city, _HOTSPOT_ARTIFACTS["density.geojson"])
+
+
+@router.get("/{city}/density_metrics.json")
+async def get_city_density_metrics(city: str):
+    """Serve the headline density metric: ``<city>_density_metrics.json``."""
+    return _serve_named_artifact(city, _HOTSPOT_ARTIFACTS["density_metrics.json"])
+
+
+@router.get("/{city}/gi_star.geojson")
+async def get_city_gi_star_geojson(city: str):
+    """Serve the Getis-Ord Gi* hex grid: ``<city>_gi_star.geojson``."""
+    return _serve_named_artifact(city, _HOTSPOT_ARTIFACTS["gi_star.geojson"])
+
+
+@router.get("/{city}/hotspots.geojson")
+async def get_city_hotspots_geojson(city: str):
+    """
+    Serve the HDBSCAN cluster centroids:
+    ``<city>_hotspots.geojson``. Same filename as the prior DBSCAN
+    artifact — the schema underneath is the new one
+    (``cluster_id``/``count``/``persistence``).
+    """
+    return _serve_named_artifact(city, _HOTSPOT_ARTIFACTS["hotspots.geojson"])
+
+
+@router.get("/{city}/hotspot_polygons.geojson")
+async def get_city_hotspot_polygons_geojson(city: str):
+    """Serve the HDBSCAN convex hulls: ``<city>_hotspot_polygons.geojson``."""
+    return _serve_named_artifact(city, _HOTSPOT_ARTIFACTS["hotspot_polygons.geojson"])
+
+
 @router.get("/{city}/report")
 async def get_city_report(city: str):
     """