Skip to content

Commit 5dc3f4b

Browse files
jackharrhyjackharrhy
committed
fix: work around ArcGIS token auth on St. John's AVL endpoint
The city added token-based authentication to their plow tracking API. Scrape the public API key from the AVL page HTML, cache it for 1 hour, and auto-refresh on 498/499 errors. Also adapts to the new field schema (OBJECTID replaces ID, Description and Speed fields removed).
1 parent c5a1ccb commit 5dc3f4b

7 files changed

Lines changed: 456 additions & 60 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
1-
<!-- changelog-id: 9 -->
1+
<!-- changelog-id: 10 -->
22
# Changelog
33

4-
## 2026-02-24 — Faster Coverage Rendering
4+
## 2026-02-25 - St. John's Data Restored
5+
The city added authentication to their plow tracking service, which broke our
6+
data feed. We've worked around it by scraping the token from the city's own
7+
public-facing map page and using that to authenticate. A great use of tax
8+
payers' dollars putting a login wall in front of a handful of snowplow GPS
9+
dots, but we're back in business.
10+
11+
[View changes](https://github.com/jackharrhy/where-the-plow/compare/58db35a...HEAD)
12+
13+
## 2026-02-24 - Faster Coverage Rendering
514
Coverage playback and the heatmap view are now powered by [deck.gl](https://deck.gl), a GPU-accelerated visualization library. Time-lapse playback is noticeably smoother — the map no longer rebuilds thousands of line segments every frame, it just tells the GPU what time it is. Coverage lines now have rounded caps and a fade trail. Most importantly, **playback now works with all sources enabled** instead of requiring you to select a single source first.
615

716
[View changes](https://github.com/jackharrhy/where-the-plow/compare/ff2cbce...58db35a)

‎scripts/extract_avl_token.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""Extract the ArcGIS API token from the St. John's AVL page.
2+
3+
The AVL page at https://map.stjohns.ca/avl/ embeds a token via
4+
esriId.registerToken({ ..., token: "AAPT..." }). This script fetches
5+
the page and extracts that token using a regex.
6+
7+
Usage:
8+
uv run python scripts/extract_avl_token.py
9+
"""
10+
11+
import re
12+
import httpx
13+
14+
15+
AVL_PAGE_URL = "https://map.stjohns.ca/avl/"
16+
17+
18+
def extract_token(html: str) -> str | None:
19+
"""Pull the token value from the registerToken() call in the page source."""
20+
# Match: token: "AAPT...stuff..."
21+
match = re.search(r'token:\s*"(AAPT[^"]+)"', html)
22+
if match:
23+
return match.group(1)
24+
return None
25+
26+
27+
def main():
28+
print(f"Fetching {AVL_PAGE_URL} ...")
29+
resp = httpx.get(AVL_PAGE_URL, follow_redirects=True, timeout=15)
30+
resp.raise_for_status()
31+
print(f" Status: {resp.status_code}, Content-Length: {len(resp.text)}")
32+
33+
token = extract_token(resp.text)
34+
if token:
35+
print(f"\nToken found ({len(token)} chars):")
36+
print(f" {token[:40]}...{token[-20:]}")
37+
print(f"\nFull token:\n{token}")
38+
else:
39+
print("\nNo token found in page source!")
40+
print("The page may have changed its auth mechanism.")
41+
# Dump a snippet around 'token' for debugging
42+
for i, line in enumerate(resp.text.splitlines(), 1):
43+
if "token" in line.lower():
44+
print(f" Line {i}: {line[:200]}")
45+
46+
47+
if __name__ == "__main__":
48+
main()

‎scripts/test_avl_query.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""Test AVL query with and without token, and compare field schemas.
2+
3+
Shows exactly what changed: which fields are gone, which are new,
4+
and what the response looks like with the new schema.
5+
6+
Usage:
7+
uv run python scripts/test_avl_query.py
8+
"""
9+
10+
import json
11+
import re
12+
import httpx
13+
14+
AVL_PAGE_URL = "https://map.stjohns.ca/avl/"
15+
AVL_QUERY_URL = "https://map.stjohns.ca/mapsrv/rest/services/AVL/MapServer/0/query"
16+
17+
18+
def extract_token(html: str) -> str | None:
19+
match = re.search(r'token:\s*"(AAPT[^"]+)"', html)
20+
return match.group(1) if match else None
21+
22+
23+
# Fields the collector currently requests
24+
OLD_FIELDS = "ID,Description,VehicleType,LocationDateTime,Bearing,Speed,isDriving"
25+
26+
27+
def query_avl(token: str | None = None, out_fields: str = "*") -> dict:
28+
params = {
29+
"f": "json",
30+
"outFields": out_fields,
31+
"outSR": "4326",
32+
"returnGeometry": "true",
33+
"where": "1=1",
34+
}
35+
if token:
36+
params["token"] = token
37+
38+
resp = httpx.get(AVL_QUERY_URL, params=params, timeout=15)
39+
resp.raise_for_status()
40+
return resp.json()
41+
42+
43+
def main():
44+
# Step 1: Get token
45+
print("=" * 60)
46+
print("Step 1: Extract token from AVL page")
47+
print("=" * 60)
48+
page = httpx.get(AVL_PAGE_URL, follow_redirects=True, timeout=15)
49+
token = extract_token(page.text)
50+
if not token:
51+
print("FAIL: Could not extract token")
52+
return
53+
print(f" Token: {token[:30]}...")
54+
55+
# Step 2: Try without token
56+
print()
57+
print("=" * 60)
58+
print("Step 2: Query WITHOUT token (should fail)")
59+
print("=" * 60)
60+
result = query_avl(token=None)
61+
if "error" in result:
62+
print(f" Expected error: {result['error']}")
63+
else:
64+
print(f" Unexpected success! {len(result.get('features', []))} features")
65+
66+
# Step 3: Try with token + old fields
67+
print()
68+
print("=" * 60)
69+
print("Step 3: Query with token + OLD field names")
70+
print("=" * 60)
71+
print(f" Requesting: {OLD_FIELDS}")
72+
result = query_avl(token=token, out_fields=OLD_FIELDS)
73+
if "error" in result:
74+
print(f" Error: {result['error']}")
75+
else:
76+
features = result.get("features", [])
77+
print(f" Got {len(features)} features")
78+
if features:
79+
print(
80+
f" Sample attributes: {json.dumps(features[0]['attributes'], indent=4)}"
81+
)
82+
83+
# Step 4: Try with token + wildcard fields
84+
print()
85+
print("=" * 60)
86+
print("Step 4: Query with token + ALL fields (outFields=*)")
87+
print("=" * 60)
88+
result = query_avl(token=token, out_fields="*")
89+
if "error" in result:
90+
print(f" Error: {result['error']}")
91+
return
92+
93+
features = result.get("features", [])
94+
print(f" Got {len(features)} features")
95+
96+
# Show schema
97+
fields = result.get("fields", [])
98+
print(f"\n Available fields ({len(fields)}):")
99+
for f in fields:
100+
print(f" {f['name']:25s} {f['type']:30s} (alias: {f.get('alias', '')})")
101+
102+
if features:
103+
print(f"\n Sample feature:")
104+
print(f" attributes: {json.dumps(features[0]['attributes'], indent=6)}")
105+
print(f" geometry: {json.dumps(features[0]['geometry'], indent=6)}")
106+
107+
# Step 5: Compare old vs new schema
108+
print()
109+
print("=" * 60)
110+
print("Step 5: Schema comparison")
111+
print("=" * 60)
112+
old_set = set(OLD_FIELDS.split(","))
113+
new_set = {f["name"] for f in fields}
114+
115+
missing = old_set - new_set
116+
added = new_set - old_set
117+
kept = old_set & new_set
118+
119+
print(f" Fields REMOVED (in old, not in new): {missing or 'none'}")
120+
print(f" Fields ADDED (in new, not in old): {added or 'none'}")
121+
print(f" Fields KEPT (in both): {kept or 'none'}")
122+
123+
# Step 6: Show what the collector will need to adapt to
124+
print()
125+
print("=" * 60)
126+
print("Step 6: Impact on collector")
127+
print("=" * 60)
128+
print(" The collector currently uses these fields from each feature:")
129+
print(" vehicle_id <- attrs.ID (REMOVED)")
130+
print(" description <- attrs.Description (REMOVED)")
131+
print(" vehicle_type <- attrs.VehicleType (still exists)")
132+
print(" timestamp <- attrs.LocationDateTime (still exists)")
133+
print(" bearing <- attrs.Bearing (still exists)")
134+
print(" speed <- attrs.Speed (REMOVED)")
135+
print(" is_driving <- attrs.isDriving (still exists)")
136+
print()
137+
print(" New fields available: OBJECTID, SymbolURL, Width, Height")
138+
print()
139+
print(" Suggested mapping:")
140+
print(" vehicle_id <- OBJECTID (integer, was string)")
141+
print(" description <- VehicleType (no separate description anymore)")
142+
print(" vehicle_type <- VehicleType")
143+
print(" speed <- not available (always None)")
144+
145+
146+
if __name__ == "__main__":
147+
main()

‎scripts/test_collector_flow.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
"""End-to-end test: scrape token, query AVL, parse into collector format.
2+
3+
Mimics what the collector will need to do after the auth change.
4+
Outputs the vehicles/positions dicts in the same shape as parse_avl_response().
5+
6+
Usage:
7+
uv run python scripts/test_collector_flow.py
8+
"""
9+
10+
import json
11+
import re
12+
from datetime import datetime, timedelta, timezone
13+
14+
import httpx
15+
16+
AVL_PAGE_URL = "https://map.stjohns.ca/avl/"
17+
AVL_QUERY_URL = "https://map.stjohns.ca/mapsrv/rest/services/AVL/MapServer/0/query"
18+
19+
# Same correction the real collector uses
20+
_NST_CORRECTION = timedelta(hours=3, minutes=30)
21+
22+
23+
def extract_token(html: str) -> str | None:
24+
match = re.search(r'token:\s*"(AAPT[^"]+)"', html)
25+
return match.group(1) if match else None
26+
27+
28+
def fetch_token() -> str:
29+
resp = httpx.get(AVL_PAGE_URL, follow_redirects=True, timeout=15)
30+
resp.raise_for_status()
31+
token = extract_token(resp.text)
32+
if not token:
33+
raise RuntimeError("Could not extract token from AVL page")
34+
return token
35+
36+
37+
def query_avl(token: str) -> dict:
38+
params = {
39+
"f": "json",
40+
"outFields": "*",
41+
"outSR": "4326",
42+
"returnGeometry": "true",
43+
"where": "1=1",
44+
"token": token,
45+
}
46+
resp = httpx.get(AVL_QUERY_URL, params=params, timeout=15)
47+
resp.raise_for_status()
48+
data = resp.json()
49+
if "error" in data:
50+
raise RuntimeError(f"AVL query error: {data['error']}")
51+
return data
52+
53+
54+
def parse_new_response(data: dict) -> tuple[list[dict], list[dict]]:
55+
"""Parse the new AVL schema into the same shape as parse_avl_response().
56+
57+
Key differences from old schema:
58+
- ID is gone -> use str(OBJECTID)
59+
- Description is gone -> use VehicleType as description
60+
- Speed is gone -> always None
61+
"""
62+
vehicles = []
63+
positions = []
64+
65+
for feature in data.get("features", []):
66+
attrs = feature.get("attributes", {})
67+
geom = feature.get("geometry", {})
68+
69+
# Timestamp: same NST correction as before
70+
raw_ts = attrs.get("LocationDateTime", 0)
71+
naive_ts = datetime.fromtimestamp(raw_ts / 1000, tz=timezone.utc)
72+
ts = naive_ts + _NST_CORRECTION
73+
74+
vehicle_id = str(attrs.get("OBJECTID", ""))
75+
vehicle_type = attrs.get("VehicleType", "")
76+
77+
vehicles.append(
78+
{
79+
"vehicle_id": vehicle_id,
80+
"description": vehicle_type, # no separate description anymore
81+
"vehicle_type": vehicle_type,
82+
}
83+
)
84+
85+
positions.append(
86+
{
87+
"vehicle_id": vehicle_id,
88+
"timestamp": ts.isoformat(),
89+
"longitude": geom.get("x", 0.0),
90+
"latitude": geom.get("y", 0.0),
91+
"bearing": attrs.get("Bearing", 0),
92+
"speed": None, # no longer available
93+
"is_driving": attrs.get("isDriving", ""),
94+
}
95+
)
96+
97+
return vehicles, positions
98+
99+
100+
def main():
101+
print("Step 1: Fetch token from AVL page")
102+
token = fetch_token()
103+
print(f" Got token: {token[:30]}...")
104+
105+
print("\nStep 2: Query AVL with token")
106+
data = query_avl(token)
107+
feature_count = len(data.get("features", []))
108+
print(f" Got {feature_count} features")
109+
110+
print("\nStep 3: Parse into collector format")
111+
vehicles, positions = parse_new_response(data)
112+
113+
print(f"\n Vehicles ({len(vehicles)}):")
114+
for v in vehicles[:5]:
115+
print(f" {json.dumps(v)}")
116+
if len(vehicles) > 5:
117+
print(f" ... and {len(vehicles) - 5} more")
118+
119+
print(f"\n Positions ({len(positions)}):")
120+
for p in positions[:5]:
121+
print(f" {json.dumps(p)}")
122+
if len(positions) > 5:
123+
print(f" ... and {len(positions) - 5} more")
124+
125+
# Verify the data looks reasonable
126+
print("\n" + "=" * 60)
127+
print("Validation")
128+
print("=" * 60)
129+
issues = []
130+
for p in positions:
131+
if p["longitude"] == 0.0 and p["latitude"] == 0.0:
132+
issues.append(f" Vehicle {p['vehicle_id']}: zero coordinates")
133+
if p["speed"] is not None:
134+
issues.append(f" Vehicle {p['vehicle_id']}: unexpected speed value")
135+
136+
if issues:
137+
print(" Issues found:")
138+
for issue in issues:
139+
print(f" {issue}")
140+
else:
141+
print(" All positions have valid coordinates")
142+
print(" Speed is None for all (as expected with new schema)")
143+
144+
print(f"\n Vehicle types seen: {sorted(set(v['vehicle_type'] for v in vehicles))}")
145+
print(f"\n SUCCESS: End-to-end flow works with token + new schema")
146+
147+
148+
if __name__ == "__main__":
149+
main()

0 commit comments

Comments
 (0)