Merge pull request #27 from jethronap/24_test_scraper_agent

24 test scraper agent

Author: jethronap

local_test_pipeline.sh

@@ -6,26 +6,46 @@ export PYTHONPATH
 cd "$(dirname "$0")" || exit
 
 echo "Running base agent tests"
-pytest tests/test_base_agent.py
+pytest tests/agents/test_base_agent.py
 echo "Done..."
 echo "==============================================="
 
 echo "Running dummy agent tests"
-pytest tests/test_dummy_agent.py
+pytest tests/agents/test_dummy_agent.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running scraper agent tests"
+pytest tests/agents/test_scraper_agent.py
 echo "Done..."
 echo "==============================================="
 
 echo "Running llm wrapper tests"
-pytest tests/test_llm_wrapper.py
+pytest tests/tools/test_llm_wrapper.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running ollama client tests"
+pytest tests/tools/test_ollama_client.py
 echo "Done..."
 echo "==============================================="
 
 echo "Running memory store tests"
-pytest tests/test_memory_store.py
+pytest tests/memory/test_memory_store.py
 echo "Done..."
 echo "==============================================="
 
-echo "Running ollama client tests"
-pytest tests/test_ollama_client.py
+echo "Running with_retry decorator tests"
+pytest tests/utils/test_retry.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running db utils tests"
+pytest tests/utils/test_db.py
+echo "Done..."
+echo "==============================================="
+
+echo "Running overpass utils tests"
+pytest tests/utils/test_overpass.py
 echo "Done..."
 echo "==============================================="

tests/agents/__init__.py

@@ -0,0 +1,96 @@
+import json
+
+import pytest
+
+from src.utils.db import query_hash, payload_hash
+from src.agents.scraper_agent import ScraperAgent
+
+
+PAYLOAD_OK = {"elements": [{"id": 1}, {"id": 2}]}
+PAYLOAD_EMPTY = {"elements": []}
+
+
+def make_agent(mem_fake, run_stub, save_stub):
+    """
+    Helper to build an agent with injected stubs
+    """
+    tools = {"run_query": run_stub, "save_json": save_stub}
+    return ScraperAgent(name="ScraperAgent", memory=mem_fake, tools=tools)
+
+
+def test_first_run_saves(mem_fake, tmp_path, monkeypatch):
+    saved_path = tmp_path / "lund.json"
+
+    def run_stub(query):
+        return PAYLOAD_OK
+
+    def save_stub(data, city, overpass_dir):
+        assert data is PAYLOAD_OK
+        assert city == "Lund"
+        return saved_path
+
+    agent = make_agent(mem_fake, run_stub, save_stub)
+    ctx = agent.achieve_goal({"city": "Lund"})
+
+    # context flags
+    assert ctx["cache_hit"] is False
+    assert ctx["elements_count"] == 2
+    assert ctx["save_json"] == str(saved_path)
+
+    # memory rows contain cache entry
+    cache_rows = [r for r in mem_fake.rows if r.step == "cache"]
+    assert len(cache_rows) == 1
+    qh = query_hash(ctx["query"])
+    ph = payload_hash(PAYLOAD_OK)
+    assert f"{qh}|{saved_path}|{ph}" in cache_rows[0].content
+
+
+def test_second_run_hits_cache(mem_fake, tmp_path, monkeypatch):
+    cached_path = tmp_path / "lund.json"
+    cached_path.write_text(json.dumps(PAYLOAD_OK))
+
+    # the exact query string we'll pretend the agent will use
+    q = "FAKE QUERY TEXT"
+
+    # pre‑seed cache row
+    mem_fake.store(
+        "ScraperAgent",
+        "cache",
+        f"{query_hash(q)}|{cached_path}|{payload_hash(PAYLOAD_OK)}",
+    )
+
+    # stub tools that must NOT be called
+    def run_stub(_):
+        pytest.fail("run_query should not be called on cache hit")
+
+    def save_stub(*_):
+        pytest.fail("save_json should not be called on cache hit")
+
+    # make build_query return the same fake string
+    monkeypatch.setattr("src.agents.scraper_agent.build_query", lambda *a, **k: q)
+
+    agent = make_agent(mem_fake, run_stub, save_stub)
+
+    ctx = agent.achieve_goal({"city": "Lund"})
+
+    assert ctx["cache_hit"] is True
+    assert ctx["run_query"] == PAYLOAD_OK
+    assert ctx["save_json"] == str(cached_path)
+
+
+def test_empty_result_skips_save(mem_fake, monkeypatch):
+    def run_stub(query):
+        return PAYLOAD_EMPTY
+
+    def save_stub(*_):
+        pytest.fail("save_json must not be called for empty result")
+
+    agent = make_agent(mem_fake, run_stub, save_stub)
+
+    ctx = agent.achieve_goal({"city": "Nowhere"})
+
+    assert ctx["elements_count"] == 0
+    assert ctx["save_json"] == "NO_DATA"
+
+    empty_rows = [r for r in mem_fake.rows if r.step == "empty"]
+    assert empty_rows, "empty marker row should be stored"

tests/test_base_agent.py renamed to tests/agents/test_base_agent.py

@@ -1,5 +1,10 @@
+import json
+from types import SimpleNamespace
+
 import pytest
-from src.config.settings import OllamaSettings, DatabaseSettings
+import requests
+
+from src.config.settings import OllamaSettings, DatabaseSettings, OverpassSettings
 
 
 @pytest.fixture
@@ -31,3 +36,86 @@ def __init__(self, settings):
 
     def __call__(self, prompt, **kwargs):
         return self._response
+
+
+@pytest.fixture
+def fake_sleep(monkeypatch):
+    """
+    Replace time.sleep so tests run instantly and record delays.
+    """
+    delays = []
+
+    def _fake_sleep(seconds: float):
+        delays.append(seconds)
+
+    monkeypatch.setattr("time.sleep", _fake_sleep)
+    return delays
+
+
+def http_error(status: int) -> requests.HTTPError:
+    r = requests.Response()
+    r.status_code = status
+    return requests.HTTPError(response=r)
+
+
+@pytest.fixture
+def ovp_settings(tmp_path):
+    return OverpassSettings(
+        endpoint="https://overpass.test/api",
+        headers={"User-Agent": "pytest"},
+        query_timeout=10,
+        timeout=2,
+        dir=str(tmp_path / "ovp"),
+        max_attempts=2,
+        base_delay=0.0,
+    )
+
+
+class _DummyResp(SimpleNamespace):
+    def raise_for_status(self):
+        if 400 <= self.status_code:
+            raise requests.HTTPError(response=self)
+
+    def json(self):
+        return json.loads(self._text) if isinstance(self._text, str) else self._text
+
+    text = property(lambda self: self._text)
+
+
+@pytest.fixture
+def patch_requests(monkeypatch):
+    """
+    Fixture that lets each test prepare responses:
+
+        store["get"]  = _DummyResp(...)
+        store["post"] = _DummyResp(...)
+    """
+    store = {"get": None, "post": None}
+
+    def fake_get(*_, **__):
+        return store["get"]
+
+    def fake_post(*_, **__):
+        return store["post"]
+
+    monkeypatch.setattr(requests, "get", fake_get)
+    monkeypatch.setattr(requests, "post", fake_post)
+    return store
+
+
+class MemoryStoreFake:
+    def __init__(self):
+        self.rows = []
+
+    def store(self, agent_id: str, step: str, content: str):
+        row = SimpleNamespace(agent_id=agent_id, step=step, content=content)
+        self.rows.append(row)
+        return row
+
+    def load(self, agent_id: str):
+        return [r for r in self.rows if r.agent_id == agent_id]
+
+
+@pytest.fixture
+def mem_fake():
+    return MemoryStoreFake()