Merge pull request #49 from jethronap/48_stats_visualization

48 stats visualization

Author: jethronap

README.md

@@ -14,9 +14,13 @@ city or municipality. The agent downloads data in json format and stores them lo
 The Analyzer agent's goal is to get scraped data from the Scraper agent and:
 
 - Enrich them by doing analysis using and LLM. The agent creates new file with structured information.
-- Produces a `.geojson` file in order to be used with mapping.
+- Produces a `.geojson` file to be used with mapping.
 - Creates a `heatmat` of the given area showing the spatial density of the surveillance infrastructure
 - Computes simple summary statistics from the available data.
+- Computes surveillance hotspots using DBSCAN and plots them overlaid on an OSM map
+- Plots sensitivity reasons distributions coming from a LLM
+- Plots camera counts for sensitive zones
+- Plots the distribution of private and public cameras using a donut-chart 
 
 # Installation
 

local_test_pipeline.sh

@@ -35,12 +35,21 @@ pytest tests/tools/test_ollama_client.py
 echo "Done..."
 echo "==============================================="
 
-
 echo "Running io tools tests"
 pytest tests/tools/test_io_tools.py
 echo "Done..."
 echo "==============================================="
 
+echo "Running mapping tools tests"
+pytest tests/tools/test_mapping_tools.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running chart tools tests"
+pytest tests/tools/test_chart_tools.py
+echo "Done..."
+echo "==============================================="
+
 echo "Running memory store tests"
 pytest tests/memory/test_memory_store.py
 echo "Done..."

main.py

@@ -25,18 +25,18 @@ def main():
     #     content="This is only a test of the memory system.",
     # )
     # logger.success(
-    #     f"  → Stored memory: id={test_mem.id}, "
+    #     f" Stored memory: id={test_mem.id}, "
     #     f"agent_id={test_mem.agent_id}, step={test_mem.step}"
     # )
     #
     # # 3. Load memories back
     # logger.info("Loading memories for TestAgent...")
     # records = memory.load(agent_id="TestAgent")
-    # logger.success(f"  → Loaded {len(records)} record(s):")
+    # logger.success(f": Loaded {len(records)} record(s):")
     # for rec in records:
     #     print(
     #         f"    • [{rec.id}] {rec.timestamp.isoformat()} "
-    #         f"{rec.agent_id}/{rec.step} → {rec.content}"
+    #         f"{rec.agent_id}/{rec.step}: {rec.content}"
     #     )
     # agent = ScraperAgent(name="ScraperAgent", memory=memory)
     # result_context = agent.achieve_goal({"city": "Hamburg"})
@@ -68,14 +68,25 @@ def main():
     ctx = analyzer.achieve_goal(
         {
             "path": "overpass_data/malmö/malmö.json",
-            "generate_geojson": True,
-            "generate_heatmap": True,
+            "generate_geojson": False,
+            "generate_heatmap": False,
+            "generate_hotspots": True,
+            "generate_chart": False,
+            "plot_zone_sensitivity": False,
+            "plot_sensitivity_reasons": False,
+            "plot_hotspots": True,
         }
     )
-    print("Enriched file →", ctx["output_path"])
-    print("GeoJSON →", ctx["geojson_path"])
-    print("Heatmap →", ctx["heatmap_path"])
+    # print("Context: ", ctx)
+    print("Enriched file:", ctx["output_path"])
+    # print("GeoJSON:", ctx["geojson_path"])
+    # print("Heatmap:", ctx["heatmap_path"])
+    print("hotspots file:", ctx["hotspots_path"])
     print("Summary stats:", ctx["stats"])
+    # print("Chart:", ctx["chart_path"])
+    # print("Zone‐sensitivity chart at", ctx["chart_zone_sens"])
+    # print("Sensitivity reasons:", ctx["sensitivity_reasons_chart"])
+    print("hotspots plot:", ctx["plot_hotspots"])
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -1,16 +1,20 @@
 [project]
 name = "agentic-counter-surveillance"
-version = "0.1.0"
+version = "0.3.1"
 description = "A software based on agentic architecture"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "contextily>=1.6.2",
     "folium>=0.19.6",
+    "geopandas>=1.0.1",
     "loguru>=0.7.3",
+    "matplotlib>=3.10.3",
     "pre-commit>=4.2.0",
     "pydantic-settings>=2.9.1",
     "pytest>=8.3.5",
     "requests>=2.32.3",
     "ruff>=0.11.7",
+    "scikit-learn>=1.6.1",
     "sqlmodel>=0.0.24",
 ]

src/agents/analyzer_agent.py

@@ -1,9 +1,7 @@
-from __future__ import annotations
-
 import json
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Any, Callable
+from typing import Dict, List, Any, Callable, Union
 
 
 from src.agents.base_agent import Agent
@@ -18,8 +16,14 @@
     save_enriched_elements as save_json,
     to_geojson,
 )
-from src.tools.mapping_tools import to_heatmap
+from src.tools.mapping_tools import to_heatmap, to_hotspots
 from src.tools.stat_tools import compute_statistics
+from src.tools.chart_tools import (
+    private_public_pie,
+    plot_zone_sensitivity,
+    plot_sensitivity_reasons,
+    plot_hotspots,
+)
 from src.utils.decorators import log_action
 
 Tool = Callable[..., Any]
@@ -35,16 +39,21 @@ def __init__(
         self,
         name: str,
         memory: MemoryStore,
-        llm: LocalLLM | None = None,
-        tools: Dict[str, Tool] | None = None,
+        llm: Union[LocalLLM, None] = None,
+        tools: Union[Dict[str, Tool], None] = None,
     ):
         default_tools: Dict[str, Tool] = {
             "load_json": load_json,
             "enrich": self._enrich_element,
             "save_json": save_json,
             "to_geojson": to_geojson,
             "to_heatmap": to_heatmap,
+            "to_hotspots": to_hotspots,
+            "plot_hotspots": plot_hotspots,
             "report": compute_statistics,
+            "plot_pie": private_public_pie,
+            "plot_zone_sensitivity": plot_zone_sensitivity,
+            "plot_sensitivity_reasons": plot_sensitivity_reasons,
         }
         super().__init__(name, tools or default_tools, memory)
         self.llm = llm or LocalLLM()
@@ -58,12 +67,22 @@ def perceive(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
             raise FileNotFoundError(path)
         generate_geojson = input_data.get("generate_geojson", True)
         generate_heatmap = input_data.get("generate_heatmap", False)
+        generate_hotspots = input_data.get("generate_hotspots", False)
         compute_stats = input_data.get("compute_stats", True)
+        generate_chart = input_data.get("generate_chart", False)
+        plot_zone = input_data.get("plot_zone_sensitivity", False)
+        plot_reasons = input_data.get("plot_sensitivity_reasons", False)
+        plot_hotspots = input_data.get("plot_hotspots", False)
         return {
             "path": path,
             "generate_geojson": generate_geojson,
             "generate_heatmap": generate_heatmap,
+            "generate_hotspots": generate_hotspots,
             "compute_stats": compute_stats,
+            "generate_chart": generate_chart,
+            "plot_zone_sensitivity": plot_zone,
+            "plot_sensitivity_reasons": plot_reasons,
+            "plot_hotspots": plot_hotspots,
         }
 
     def plan(self, observation: Dict[str, Any]) -> List[str]:
@@ -72,8 +91,18 @@ def plan(self, observation: Dict[str, Any]) -> List[str]:
             steps.append("to_geojson")
         if observation["generate_heatmap"]:
             steps.append("to_heatmap")
+        if observation["generate_hotspots"]:
+            steps.append("to_hotspots")
         if observation["compute_stats"]:
             steps.append("report")
+        if observation["generate_chart"]:
+            steps.append("plot_pie")
+        if observation["plot_zone_sensitivity"]:
+            steps.append("plot_zone_sensitivity")
+        if observation["plot_sensitivity_reasons"]:
+            steps.append("plot_sensitivity_reasons")
+        if observation["plot_hotspots"]:
+            steps.append("plot_hotspots")
         return steps
 
     @log_action
@@ -162,11 +191,54 @@ def act(self, action: str, context: Dict[str, Any]) -> Any:
             self.remember("heatmap_cache", f"{context['raw_hash']}|{html_path}")
             return str(html_path)
 
+        if action == "to_hotspots":
+            geojson_path = Path(context["geojson_path"])
+            hotspots_path = geojson_path.with_name(
+                geojson_path.stem + "_hotspots.geojson"
+            )
+            self.tools["to_hotspots"](geojson_path, hotspots_path)
+            context["hotspots_path"] = str(hotspots_path)
+            cache_val = f"{context['raw_hash']}|{hotspots_path}"
+            self.remember("hotspot_cache", cache_val)
+            return str(hotspots_path)
+
+        if action == "plot_hotspots":
+            hot = Path(context["hotspots_path"])
+            pic = hot.with_suffix(".png")
+            self.tools["plot_hotspots"](hot, pic)
+            context["hotspots_plot"] = str(pic)
+            return str(pic)
+
         if action == "report":
             stats: Dict[str, Any] = self.tools["report"](context["enriched"])
             self.remember("report", json.dumps(stats))
             return stats
 
+        if action == "plot_pie":
+            stats = context["stats"]
+            src_path = Path(context["path"])
+            chart_path = self.tools["plot_pie"](stats, src_path.parent)
+            context["chart_path"] = chart_path
+            self.remember("pie_chart", f"{context['raw_hash']}|{chart_path}")
+            return str(chart_path)
+
+        if action == "plot_zone_sensitivity":
+            stats = context["stats"]
+            src_path = Path(context["path"])
+            chart_path = self.tools["plot_zone_sensitivity"](stats, src_path.parent)
+            context["chart_path"] = chart_path
+            self.remember("chart_zone_sens", f"{context['raw_hash']}|{chart_path}")
+            return str(chart_path)
+
+        if action == "plot_sensitivity_reasons":
+            enriched_path = Path(context["output_path"])
+            chart_path = enriched_path.with_name(
+                f"{enriched_path.stem}_sensitivity.png"
+            )
+            self.tools["plot_sensitivity_reasons"](enriched_path, chart_path)
+            context["sensitivity_reasons_chart"] = str(chart_path)
+            return str(chart_path)
+
         logger.error(f"Unhandled action: {action}")
         raise NotImplementedError(f"Unhandled action: {action}")
 
@@ -188,8 +260,18 @@ def achieve_goal(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
                 context["geojson_path"] = result
             elif step == "to_heatmap":
                 context["heatmap_path"] = result
+            elif step == "to_hotspots":
+                context["hotspots_path"] = result
             elif step == "report":
                 context["stats"] = result
+            elif step == "plot_pie":
+                context["chart_path"] = result
+            elif step == "plot_zone_sensitivity":
+                context["chart_zone_sens"] = result
+            elif step == "plot_sensitivity_reasons":
+                context["chart_sens_reasons"] = result
+            elif step == "plot_hotspots":
+                context["plot_hotspots"] = result
 
         return context
 

src/agents/scraper_agent.py

@@ -51,7 +51,7 @@ def perceive(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
 
     def plan(self, observation: Dict[str, Any]) -> List[str]:
         """
-        Very simple two-step plan: (1) fetch → (2) persist.
+        Very simple two-step plan: (1) fetch: (2) persist.
         :param observation:
         :return: The available steps.
         """