Pass effective proxy to XClIdGen (#319)

kn1l · vladkens · web-flow · commit 441f08843425 · 2026-07-21T18:22:14.000+03:00
Co-authored-by: vladkens &lt;v.pronsky@gmail.com&gt;
diff --git a/tests/test_http.py b/tests/test_http.py
@@ -461,7 +461,8 @@ def test_curl_client_strips_user_agent_from_session():
 def test_httpx_client_resolves_ua_hint_to_real_string():
     from twscrape.http import HttpxClient
 
-    client = HttpxClient(headers={"user-agent": "@firefox"})
+    expected_ua, _ = _resolve_browser("@chrome", seed=0)
+    client = HttpxClient(headers={"user-agent": "@chrome"}, seed=0)
     ua = dict(client._client.headers).get("user-agent", "")
-    assert "Firefox/" in ua
+    assert ua == expected_ua
     assert "@" not in ua
diff --git a/tests/test_queue_client.py b/tests/test_queue_client.py
@@ -2,9 +2,10 @@
 
 import pytest
 
+from twscrape.account import Account
 from twscrape.accounts_pool import AccountsPool
 from twscrape.http import ConnectError, NetworkError
-from twscrape.queue_client import QueueClient
+from twscrape.queue_client import QueueClient, XClIdGenStore
 from twscrape.utils import utc
 
 from .mock_http import MockClient
@@ -150,6 +151,48 @@ async def get_data_stream():
     assert client.ctx is None
 
 
+async def test_queue_client_passes_effective_proxy_to_xclid(pool_mock: AccountsPool, monkeypatch):
+    mock = MockClient()
+    seen = {}
+
+    class FakeXClIdGen:
+        def calc(self, *args, **kwargs):
+            return "mocked-clid"
+
+    async def fake_get(cls, username, proxy=None, fresh=False):
+        seen["username"] = username
+        seen["proxy"] = proxy
+        seen["fresh"] = fresh
+        return FakeXClIdGen()
+
+    monkeypatch.setattr(Account, "make_client", lambda self, proxy=None: mock)
+    monkeypatch.setattr(XClIdGenStore, "get", classmethod(fake_get))
+
+    await pool_mock.add_account(
+        "user1",
+        "pass1",
+        "email1",
+        "email_pass1",
+        proxy="127.0.0.1:7897",
+    )
+    await pool_mock.set_active("user1", True)
+
+    client = QueueClient(pool_mock, "SearchTimeline")
+    await client.__aenter__()
+
+    mock.add_response(json={"ok": True})
+    rep = await client.get(URL)
+
+    assert rep is not None
+    assert seen == {
+        "username": "user1",
+        "proxy": "http://127.0.0.1:7897",
+        "fresh": False,
+    }
+
+    await client.__aexit__(None, None, None)
+
+
 # --- ConnectError ---
 
 
diff --git a/tests/test_xclid.py b/tests/test_xclid.py
@@ -0,0 +1,43 @@
+import twscrape.xclid as xclid
+
+
+class FakeClient:
+    def __init__(self):
+        self.closed = False
+
+    async def aclose(self):
+        self.closed = True
+
+
+async def test_xclid_create_passes_proxy_to_client(monkeypatch):
+    seen = {}
+    fake_client = FakeClient()
+
+    def fake_make_client(*, headers=None, proxy=None):
+        seen["headers"] = headers
+        seen["proxy"] = proxy
+        return fake_client
+
+    async def fake_get_tw_page_text(url, clt):
+        seen["url"] = url
+        seen["client"] = clt
+        return "<html></html>"
+
+    async def fake_load_keys(soup, clt):
+        seen["load_client"] = clt
+        return [1, 2, 3], "anim-key"
+
+    monkeypatch.setattr(xclid, "_make_http_client", fake_make_client)
+    monkeypatch.setattr(xclid, "get_tw_page_text", fake_get_tw_page_text)
+    monkeypatch.setattr(xclid, "load_keys", fake_load_keys)
+
+    gen = await xclid.XClIdGen.create(proxy="http://127.0.0.1:7897")
+
+    assert gen.vk_bytes == [1, 2, 3]
+    assert gen.anim_key == "anim-key"
+    assert seen["headers"] == {"user-agent": "@chrome"}
+    assert seen["proxy"] == "http://127.0.0.1:7897"
+    assert seen["url"] == "https://x.com/tesla"
+    assert seen["client"] is fake_client
+    assert seen["load_client"] is fake_client
+    assert fake_client.closed is True
diff --git a/twscrape/account.py b/twscrape/account.py
@@ -51,11 +51,13 @@ def to_rs(self):
         rs["last_used"] = rs["last_used"].isoformat() if rs["last_used"] else None
         return rs
 
-    def make_client(self, proxy: str | None = None) -> HttpClient:
+    def resolve_proxy(self, proxy: str | None = None) -> str | None:
         proxies = [proxy, os.getenv("TWS_PROXY"), self.proxy]
         proxies = [x for x in proxies if x is not None]
-        proxy = parse_proxy(proxies[0]) if proxies else None
+        return parse_proxy(proxies[0]) if proxies else None
 
+    def make_client(self, proxy: str | None = None) -> HttpClient:
+        proxy = self.resolve_proxy(proxy)
         headers = {**self.headers}
         headers["user-agent"] = self.user_agent
         headers["content-type"] = "application/json"
diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py
@@ -22,18 +22,19 @@ class AbortReqError(Exception): ...
 
 
 class XClIdGenStore:
-    items: dict[str, XClIdGen] = {}  # username -> XClIdGen
+    items: dict[tuple[str, str | None], XClIdGen] = {}  # (username, proxy) -> XClIdGen
 
     @classmethod
-    async def get(cls, username: str, fresh=False) -> XClIdGen:
-        if username in cls.items and not fresh:
-            return cls.items[username]
+    async def get(cls, username: str, proxy: str | None = None, fresh=False) -> XClIdGen:
+        key = (username, proxy)
+        if key in cls.items and not fresh:
+            return cls.items[key]
 
         tries = 0
         while tries < 3:
             try:
-                clid_gen = await XClIdGen.create()
-                cls.items[username] = clid_gen
+                clid_gen = await XClIdGen.create(proxy=proxy)
+                cls.items[key] = clid_gen
                 return clid_gen
             except Exception as e:
                 tries += 1
@@ -48,10 +49,11 @@ async def get(cls, username: str, fresh=False) -> XClIdGen:
 
 
 class Ctx:
-    def __init__(self, acc: Account, clt: HttpClient):
+    def __init__(self, acc: Account, clt: HttpClient, proxy: str | None = None):
         self.req_count = 0
         self.acc = acc
         self.clt = clt
+        self.proxy = proxy
 
     async def aclose(self):
         await self.clt.aclose()
@@ -63,7 +65,7 @@ async def req(self, method: HttpMethod, url: str, params: ReqParams = None) -> R
 
         tries = 0
         while tries < 3:
-            gen = await XClIdGenStore.get(self.acc.username, fresh=tries > 0)
+            gen = await XClIdGenStore.get(self.acc.username, proxy=self.proxy, fresh=tries > 0)
             hdr = {"x-client-transaction-id": gen.calc(method, path)}
             rep = await self.clt.request(method, url, params=params, headers=hdr)
             if rep.status_code != 404:
@@ -157,7 +159,7 @@ async def _get_ctx(self):
             return None
 
         clt = acc.make_client(proxy=self.proxy)
-        self.ctx = Ctx(acc, clt)
+        self.ctx = Ctx(acc, clt, proxy=acc.resolve_proxy(self.proxy))
         return self.ctx
 
     async def _check_rep(self, rep: Response) -> None:
diff --git a/twscrape/xclid.py b/twscrape/xclid.py
@@ -13,8 +13,8 @@
 from .http import make_client as _make_http_client
 
 
-def _make_client() -> HttpClient:
-    return _make_http_client(headers={"user-agent": "@chrome"})
+def _make_client(proxy: str | None = None) -> HttpClient:
+    return _make_http_client(headers={"user-agent": "@chrome"}, proxy=proxy)
 
 
 async def get_tw_page_text(url: str, clt: HttpClient):
@@ -315,8 +315,8 @@ async def load_keys(soup: bs4.BeautifulSoup, clt: HttpClient) -> tuple[list[int]
 
 class XClIdGen:
     @staticmethod
-    async def create() -> "XClIdGen":
-        clt = _make_client()
+    async def create(proxy: str | None = None) -> "XClIdGen":
+        clt = _make_client(proxy=proxy)
         try:
             text = await get_tw_page_text("https://x.com/tesla", clt)
             soup = bs4.BeautifulSoup(text, "html.parser")
