fix: avoid native rate limiters in Pyodide (#124)

Author: njzjz-bot

docs/webworker.js

@@ -5,15 +5,48 @@
 // and `.wasm` files as well:
 importScripts("https://cdn.jsdelivr.net/pyodide/v0.25.1/full/pyodide.js");
 
+function installLegacyWenxianBrowserShims() {
+  self.pyodide.runPython(`
+from importlib.metadata import version
+import sys
+import types
+
+from packaging.version import Version
+
+if Version(version("wenxian")) < Version("0.3.4"):
+    class _NativeOnly:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    pyrate_limiter = types.ModuleType("pyrate_limiter")
+    pyrate_limiter.Duration = types.SimpleNamespace(SECOND=1)
+    pyrate_limiter.Limiter = _NativeOnly
+    pyrate_limiter.Rate = _NativeOnly
+
+    requests_ratelimiter = types.ModuleType("requests_ratelimiter")
+    requests_ratelimiter.__path__ = []
+    requests_ratelimiter.LimiterAdapter = _NativeOnly
+
+    requests_ratelimiter_impl = types.ModuleType(
+        "requests_ratelimiter.requests_ratelimiter"
+    )
+    requests_ratelimiter_impl.HostBucketFactory = _NativeOnly
+    requests_ratelimiter.requests_ratelimiter = requests_ratelimiter_impl
+
+    sys.modules["pyrate_limiter"] = pyrate_limiter
+    sys.modules["requests_ratelimiter"] = requests_ratelimiter
+    sys.modules["requests_ratelimiter.requests_ratelimiter"] = (
+        requests_ratelimiter_impl
+    )
+`);
+}
+
 async function loadPyodideAndPackages() {
   self.pyodide = await loadPyodide();
   await self.pyodide.loadPackage("micropip");
   const micropip = self.pyodide.pyimport("micropip");
-  await micropip.install([
-    "pyrate-limiter>=3",
-    "wenxian",
-    "pylatexenc==3.0a21",
-  ]);
+  await micropip.install(["wenxian", "pylatexenc==3.0a21"]);
+  installLegacyWenxianBrowserShims();
   await self.pyodide.loadPackage("sqlite3");
 }
 let pyodideReadyPromise = loadPyodideAndPackages();

pyproject.toml

@@ -24,8 +24,8 @@ dependencies = [
     "pylatexenc",
     "unidecode",
     "pyiso4",
-    "requests-ratelimiter",
-    "pyrate-limiter>=3",
+    "requests-ratelimiter; sys_platform != 'emscripten'",
+    "pyrate-limiter>=3; sys_platform != 'emscripten'",
 ]
 requires-python = ">=3.10"
 readme = "README.md"

tests/test_browser_session.py

@@ -0,0 +1,34 @@
+"""Browser-runtime tests for HTTP session setup."""
+
+from __future__ import annotations
+
+import builtins
+import runpy
+import sys
+
+import pytest
+
+from wenxian.feeder import session
+
+
+def test_browser_session_skips_native_rate_limiters(monkeypatch):
+    """Test importing the browser path never loads native-only limiters."""
+    session_path = session.__file__
+    assert session_path is not None
+    native_import = builtins.__import__
+
+    def guarded_import(name, *args, **kwargs):
+        if name.split(".", maxsplit=1)[0] in {
+            "pyrate_limiter",
+            "requests_ratelimiter",
+        }:
+            raise AssertionError(f"browser path imported native package {name}")
+        return native_import(name, *args, **kwargs)
+
+    with monkeypatch.context() as browser:
+        browser.setattr(sys, "platform", "emscripten")
+        browser.setattr(builtins, "__import__", guarded_import)
+        namespace = runpy.run_path(session_path)
+
+    with pytest.raises(RuntimeError, match="Synchronous HTTP is unavailable"):
+        namespace["SESSION"].get("https://example.test")

wenxian/feeder/session.py

@@ -9,17 +9,18 @@
 from typing import TYPE_CHECKING
 from urllib.parse import urlencode
 
-from pyrate_limiter import Duration, Limiter, Rate
-from requests import Session
-from requests.adapters import HTTPAdapter, Retry
-from requests_ratelimiter import LimiterAdapter
-from requests_ratelimiter.requests_ratelimiter import HostBucketFactory
-
 if TYPE_CHECKING:
     from asyncio import AbstractEventLoop
     from collections.abc import Mapping
     from typing import Any
 
+if sys.platform != "emscripten":
+    from pyrate_limiter import Duration, Limiter, Rate
+    from requests import Session
+    from requests.adapters import HTTPAdapter, Retry
+    from requests_ratelimiter import LimiterAdapter
+    from requests_ratelimiter.requests_ratelimiter import HostBucketFactory
+
 
 @dataclass
 class _BrowserResponse:
@@ -63,48 +64,60 @@ async def wait(self) -> None:
             state.next_start = loop.time() + self.interval
 
 
-_DEFAULT_TIMEOUT = (5.0, 20.0)
-
-
-class _TimeoutSession(Session):
-    """Requests session that applies a bounded timeout by default."""
-
-    def request(self, method, url, **kwargs):
-        """Send a request with the shared default timeout unless overridden."""
-        kwargs.setdefault("timeout", _DEFAULT_TIMEOUT)
-        return super().request(method, url, **kwargs)
+class _BrowserSession:
+    """Reject synchronous HTTP calls in browser runtimes."""
+
+    def get(self, *args: Any, **kwargs: Any) -> Any:
+        """Raise because Pyodide networking must use the asynchronous transport."""
+        raise RuntimeError(
+            "Synchronous HTTP is unavailable in Pyodide; use async_get instead."
+        )
+
+
+if sys.platform != "emscripten":
+    _DEFAULT_TIMEOUT = (5.0, 20.0)
+
+    class _TimeoutSession(Session):
+        """Requests session that applies a bounded timeout by default."""
+
+        def request(self, method, url, **kwargs):
+            """Send a request with the shared default timeout unless overridden."""
+            kwargs.setdefault("timeout", _DEFAULT_TIMEOUT)
+            return super().request(method, url, **kwargs)
+
+    SESSION = _TimeoutSession()
+
+    # retry logic
+    retries = Retry(
+        total=5,
+        backoff_factor=0.1,
+        status_forcelist=[
+            429,
+            500,
+            502,
+            503,
+            504,
+        ],
+    )
+
+    adapter_ncbi = LimiterAdapter(per_second=3, max_retries=retries)
+    SESSION.mount("https://www.ncbi.nlm.nih.gov/pmc/utils/", adapter_ncbi)
+    SESSION.mount("https://eutils.ncbi.nlm.nih.gov/", adapter_ncbi)
+    adapter_crossref = LimiterAdapter(per_second=50, max_retries=retries)
+    SESSION.mount("https://api.crossref.org/", adapter_crossref)
+    adapter_arxiv = LimiterAdapter(
+        limiter=Limiter(HostBucketFactory([Rate(1, Duration.SECOND * 3)])),
+        burst=1,
+        max_retries=retries,
+    )
+    SESSION.mount("https://export.arxiv.org/api", adapter_arxiv)
+    adapter_semanticscholar = LimiterAdapter(per_second=1, max_retries=retries)
+    SESSION.mount("https://api.semanticscholar.org/", adapter_semanticscholar)
+    SESSION.mount("https://", HTTPAdapter(max_retries=retries))
+else:
+    SESSION = _BrowserSession()
 
 
-SESSION = _TimeoutSession()
-
-# retry logic
-retries = Retry(
-    total=5,
-    backoff_factor=0.1,
-    status_forcelist=[
-        429,
-        500,
-        502,
-        503,
-        504,
-    ],
-)
-
-adapter_ncbi = LimiterAdapter(per_second=3, max_retries=retries)
-SESSION.mount("https://www.ncbi.nlm.nih.gov/pmc/utils/", adapter_ncbi)
-SESSION.mount("https://eutils.ncbi.nlm.nih.gov/", adapter_ncbi)
-adapter_crossref = LimiterAdapter(per_second=50, max_retries=retries)
-SESSION.mount("https://api.crossref.org/", adapter_crossref)
-adapter_arxiv = LimiterAdapter(
-    limiter=Limiter(HostBucketFactory([Rate(1, Duration.SECOND * 3)])),
-    burst=1,
-    max_retries=retries,
-)
-SESSION.mount("https://export.arxiv.org/api", adapter_arxiv)
-adapter_semanticscholar = LimiterAdapter(per_second=1, max_retries=retries)
-SESSION.mount("https://api.semanticscholar.org/", adapter_semanticscholar)
-SESSION.mount("https://", HTTPAdapter(max_retries=retries))
-
 _BROWSER_RETRY_STATUSES = {429, 500, 502, 503, 504}
 _BROWSER_TIMEOUT = 20.0
 _BROWSER_RETRIES = 2