fix: switch AVL data source to portal proxy (no token needed)

Author: jackharrhy

CHANGELOG.md

@@ -1,7 +1,15 @@
-<!-- changelog-id: 10 -->
+<!-- changelog-id: 11 -->
 # Changelog
 
-## 2026-02-25 - St. John's Data Restored
+## 2026-02-25 — St. John's Data Restored (Again)
+The city rebuilt their plow tracking page, which broke our data feed for the
+second time today. We lost about two hours of data, but we're back again. The
+new page no longer exposes an API token at all, so we've switched to using the
+city's portal proxy instead.
+
+[View changes](https://github.com/jackharrhy/where-the-plow/compare/5dc3f4b...HEAD)
+
+## 2026-02-25 — St. John's Data Restored
 The city added authentication to their plow tracking service, which broke our
 data feed. We've worked around it by scraping the token from the city's own
 public-facing map page and using that to authenticate. A great use of tax

src/where_the_plow/client.py

@@ -1,5 +1,4 @@
 import logging
-import re
 from datetime import datetime, timedelta, timezone
 
 import httpx
@@ -12,58 +11,6 @@
 # To get the real UTC time we must add the 3:30 offset back.
 _NST_CORRECTION = timedelta(hours=3, minutes=30)
 
-# The AVL page embeds an ArcGIS API key via esriId.registerToken().
-# We scrape it and cache it; the page is public so this is stable.
-_AVL_TOKEN_PAGE = "https://map.stjohns.ca/avl/"
-_AVL_TOKEN_RE = re.compile(r'token:\s*"(AAPT[^"]+)"')
-_AVL_TOKEN_TTL = timedelta(hours=1)
-
-
-class AvlTokenManager:
-    """Scrapes and caches the ArcGIS API token from the public AVL page."""
-
-    def __init__(self):
-        self._token: str | None = None
-        self._fetched_at: datetime | None = None
-
-    @property
-    def _expired(self) -> bool:
-        if self._token is None or self._fetched_at is None:
-            return True
-        return datetime.now(timezone.utc) - self._fetched_at > _AVL_TOKEN_TTL
-
-    async def get_token(self, client: httpx.AsyncClient) -> str:
-        """Return a cached token, refreshing from the AVL page if stale."""
-        if not self._expired:
-            return self._token  # type: ignore[return-value]
-        return await self._refresh(client)
-
-    async def invalidate_and_refresh(self, client: httpx.AsyncClient) -> str:
-        """Force a fresh token fetch (e.g. after a 499 Token Required)."""
-        self._token = None
-        self._fetched_at = None
-        return await self._refresh(client)
-
-    async def _refresh(self, client: httpx.AsyncClient) -> str:
-        logger.info("Fetching AVL token from %s", _AVL_TOKEN_PAGE)
-        resp = await client.get(_AVL_TOKEN_PAGE, follow_redirects=True, timeout=15)
-        resp.raise_for_status()
-        match = _AVL_TOKEN_RE.search(resp.text)
-        if not match:
-            raise RuntimeError(
-                "Could not extract ArcGIS token from AVL page — "
-                "the page format may have changed"
-            )
-        token = match.group(1)
-        self._token = token
-        self._fetched_at = datetime.now(timezone.utc)
-        logger.info("AVL token refreshed (%d chars)", len(token))
-        return token
-
-
-# Module-level singleton so all AVL poll loops share one cached token.
-avl_token_manager = AvlTokenManager()
-
 
 # ── AVL (St. John's) response models ────────────────────────────────
 
@@ -232,14 +179,15 @@ async def fetch_source(client: httpx.AsyncClient, source) -> dict | list:
     params = {}
 
     if source.parser == "avl":
-        token = await avl_token_manager.get_token(client)
+        # The AVL data is served through an ArcGIS Enterprise portal proxy
+        # that handles authentication server-side.  No API token is needed;
+        # only the Referer header must match the public AVL page.
         params = {
             "f": "json",
             "outFields": "*",
             "outSR": "4326",
             "returnGeometry": "true",
             "where": "1=1",
-            "token": token,
         }
         if source.referer:
             headers["Referer"] = source.referer
@@ -248,17 +196,4 @@ async def fetch_source(client: httpx.AsyncClient, source) -> dict | list:
     resp.raise_for_status()
     data = resp.json()
 
-    # ArcGIS returns 200 with an error body when the token is bad/expired.
-    if isinstance(data, dict) and "error" in data:
-        code = data["error"].get("code")
-        if code in (498, 499):
-            logger.warning("AVL token rejected (code %s), refreshing", code)
-            token = await avl_token_manager.invalidate_and_refresh(client)
-            params["token"] = token
-            resp = await client.get(
-                source.api_url, params=params, headers=headers, timeout=10
-            )
-            resp.raise_for_status()
-            data = resp.json()
-
     return data

src/where_the_plow/config.py

@@ -15,7 +15,8 @@ class Settings(BaseSettings):
 
     # Source API URLs
     avl_api_url: str = (
-        "https://map.stjohns.ca/mapsrv/rest/services/AVL/MapServer/0/query"
+        "https://map.stjohns.ca/portal/sharing/servers/"
+        "e99efa79b60948dda2939a7d08204a61/rest/services/AVL/MapServer/0/query"
     )
     mt_pearl_api_url: str = "https://gps5.aatracking.com/api/MtPearlPortal/GetPlows"
     provincial_api_url: str = (

tests/test_collector.py

@@ -337,11 +337,11 @@ async def test_fetch_source_raises_on_http_error():
         await fetch_source(client, config)
 
 
-async def test_fetch_source_avl_sends_referer_and_token():
-    """AVL sources should send the Referer header and token param."""
+async def test_fetch_source_avl_sends_referer():
+    """AVL sources should send the Referer header (no token needed — portal proxy)."""
     config = _test_source_config(
         parser="avl",
-        api_url="https://map.stjohns.ca/arcgis/rest/services/test",
+        api_url="https://map.stjohns.ca/portal/sharing/servers/abc/rest/services/AVL/MapServer/0/query",
         referer="https://map.stjohns.ca/avl/",
     )
 
@@ -353,16 +353,11 @@ async def test_fetch_source_avl_sends_referer_and_token():
     client = AsyncMock(spec=httpx.AsyncClient)
     client.get = AsyncMock(return_value=mock_response)
 
-    with patch(
-        "where_the_plow.client.avl_token_manager.get_token",
-        new_callable=AsyncMock,
-        return_value="fake-token-123",
-    ):
-        result = await fetch_source(client, config)
+    result = await fetch_source(client, config)
 
     # Verify Referer was sent
     call_kwargs = client.get.call_args
     assert call_kwargs.kwargs["headers"]["Referer"] == "https://map.stjohns.ca/avl/"
-    # Verify token was included in params
-    assert call_kwargs.kwargs["params"]["token"] == "fake-token-123"
+    # No token should be present in params
+    assert "token" not in call_kwargs.kwargs["params"]
     assert result == {"features": []}