Merge pull request #3778 from rommapp/posthog-code/fix-ss-invalid-json-escape

fix(ss): tolerate ScreenScraper's invalid JSON escapes

Author: gantoine

backend/adapters/services/screenscraper.py

@@ -1,6 +1,7 @@
 import asyncio
 import http
 import json
+import re
 from typing import Final, cast
 
 import aiohttp
@@ -22,6 +23,26 @@
 
 LOGIN_ERROR_CHECK: Final = "Erreur de login"
 
+# ScreenScraper occasionally returns malformed JSON with unescaped backslashes in
+# text fields (e.g. game synopses), which the strict parser rejects with
+# "Invalid \escape" and discards the whole response. Match any backslash that is
+# not part of a valid JSON escape so we can repair those before parsing.
+_INVALID_ESCAPE_RE: Final = re.compile(r'\\(?!["\\/bfnrt]|u[0-9a-fA-F]{4})')
+
+
+def _loads_lenient(text: str) -> dict:
+    """Parse a ScreenScraper JSON payload, repairing invalid escapes on failure.
+
+    A single unescaped backslash would otherwise sink an entire response (and thus
+    the match), so on a decode error we double any backslash that isn't a valid
+    JSON escape and try once more.
+    """
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        return json.loads(_INVALID_ESCAPE_RE.sub(r"\\\\", text))
+
+
 # ScreenScraper enforces a per-account *thread* (concurrency) cap rather than a
 # request rate. Because a request can take several seconds, spacing out request
 # starts is not enough, as overlapping requests would exceed the cap and get
@@ -140,7 +161,7 @@ async def _request(self, url: str, request_timeout: int = 120) -> dict:
                         status_code=status.HTTP_401_UNAUTHORIZED,
                         detail="Invalid ScreenScraper credentials",
                     )
-                data = await res.json()
+                data = await res.json(loads=_loads_lenient)
             _update_thread_allowance(data)
             return data
         except aiohttp.ServerTimeoutError:
@@ -213,7 +234,7 @@ async def _request(self, url: str, request_timeout: int = 120) -> dict:
                         status_code=status.HTTP_401_UNAUTHORIZED,
                         detail="Invalid ScreenScraper credentials",
                     )
-                data = await res.json()
+                data = await res.json(loads=_loads_lenient)
             _update_thread_allowance(data)
             return data
         except (aiohttp.ClientResponseError, aiohttp.ServerTimeoutError) as err:

backend/tests/adapters/services/test_screenscraper.py

@@ -11,6 +11,7 @@
 from adapters.services.screenscraper import (
     LOGIN_ERROR_CHECK,
     ScreenScraperService,
+    _loads_lenient,
     auth_middleware,
     is_daily_quota_exhausted,
     reset_daily_quota,
@@ -1124,3 +1125,23 @@ async def test_login_error_in_retry_attempt(self, service):
         assert exc_info.value.status_code == status.HTTP_401_UNAUTHORIZED
         assert "Invalid ScreenScraper credentials" in exc_info.value.detail
         assert mock_session.get.call_count == 2
+
+
+class TestLoadsLenient:
+    """Test tolerant parsing of ScreenScraper's occasionally malformed JSON."""
+
+    def test_parses_valid_json(self):
+        assert _loads_lenient('{"a": 1, "b": "x"}') == {"a": 1, "b": "x"}
+
+    def test_repairs_invalid_backslash_escape(self):
+        # ScreenScraper sometimes emits raw backslashes in text fields, which the
+        # strict parser rejects with "Invalid \escape".
+        raw = '{"synopsis": "path C:\\emu\\games"}'
+        with pytest.raises(json.JSONDecodeError):
+            json.loads(raw)
+        assert _loads_lenient(raw) == {"synopsis": "path C:\\emu\\games"}
+
+    def test_preserves_valid_escapes(self):
+        assert _loads_lenient('{"s": "line\\nbreak \\"quoted\\" \\u00e9"}') == {
+            "s": 'line\nbreak "quoted" é'
+        }