Skip to content

Commit 8a63bce

Browse files
committed
refactor(typing): make mypy pydoll clean (136 → 0) without weakening the API
Unify the CDP/BiDi command model and tighten types end-to-end so the whole package type-checks, while improving (not regressing) the public typing. Core - Unify Command into one generic TypedDict (protocol/base.py) with a phantom `response: NotRequired[R]` field, so the response type is structurally recoverable (mypy can't infer a TypedDict type arg that appears in no field); re-exported from cdp/base and bidi/base. execute_command and create_command_future are now generic (Command[P, R] -> R), dropping a `# type: ignore`. - FindElementsMixin is generic over the element type: Tab.find()/query() return the concrete WebElement (CDP) / BiDiWebElement (BiDi) for full IDE autocomplete, while the shared protocols use Sequence[WebElementProtocol] so the concrete returns conform covariantly. BiDi typing - Discriminated unions via Literal `type` tags (input source actions, EvaluateResult); typed source-action construction for mouse/keyboard/scroll and element click. - Real types for locateNodes params/result, ElementClipRectangle, and cookie/proxy/intercept/window/download construction (no more dict placeholders). Dynamic deserialization isolated to _deserialize_remote_value(Mapping) -> object. WebElement conformance - WebElement and BiDiWebElement now satisfy WebElementProtocol; implemented BiDiWebElement.get_children_elements / get_siblings_elements. CDP↔generic convergence - get_cookies/set_cookies convert between the portable Cookie types and CDP (chromium/_cookies.py); CDP-only cookie fields stay reachable via the escape hatch. set_window_bounds maps WindowBounds→Bounds; expect_download keeps the CDP DEFAULT reset via the escape hatch. Misc: edge.py Options→ChromiumOptions; guard the browser-WS port; CDP event tracker stores raw dicts (drops two `# type: ignore`s). Tests: realistic CDP cookie fakes in the cookie round-trips; new BiDi relations integration test. Unit 407 green; BiDi + CDP cookie integration green.
1 parent e56fee2 commit 8a63bce

31 files changed

Lines changed: 690 additions & 392 deletions
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
"""Conversion between protocol-agnostic cookies and CDP cookie types.
2+
3+
The public surface (``Tab``/``Browser`` ``get_cookies``/``set_cookies``) speaks the
4+
portable ``pydoll.protocol.types`` cookie types. CDP commands use their own richer
5+
cookie types, so these helpers translate at the boundary. CDP-only fields (``session``,
6+
``priority``, ``partitionKey``, ...) are dropped on the way out; reach for them via the
7+
escape hatch ``execute_protocol_command(NetworkCommands.get_cookies())`` when needed.
8+
"""
9+
10+
from __future__ import annotations
11+
12+
from pydoll.protocol.cdp.network.types import Cookie as CDPCookie
13+
from pydoll.protocol.cdp.network.types import CookieParam as CDPCookieParam
14+
from pydoll.protocol.cdp.network.types import CookieSameSite
15+
from pydoll.protocol.types import Cookie, CookieParam
16+
17+
18+
def to_generic_cookies(cdp_cookies: list[CDPCookie]) -> list[Cookie]:
19+
"""Convert CDP cookies into protocol-agnostic cookies."""
20+
return [_to_generic_cookie(cookie) for cookie in cdp_cookies]
21+
22+
23+
def _to_generic_cookie(cookie: CDPCookie) -> Cookie:
24+
result = Cookie(
25+
name=cookie['name'],
26+
value=cookie['value'],
27+
domain=cookie['domain'],
28+
path=cookie['path'],
29+
size=cookie['size'],
30+
httpOnly=cookie['httpOnly'],
31+
secure=cookie['secure'],
32+
sameSite=cookie.get('sameSite', ''),
33+
)
34+
expires = cookie['expires']
35+
if expires >= 0:
36+
result['expiry'] = int(expires)
37+
return result
38+
39+
40+
def to_cdp_cookie_params(cookies: list[CookieParam]) -> list[CDPCookieParam]:
41+
"""Convert protocol-agnostic cookie params into CDP cookie params."""
42+
return [_to_cdp_cookie_param(cookie) for cookie in cookies]
43+
44+
45+
def _to_cdp_cookie_param(cookie: CookieParam) -> CDPCookieParam:
46+
result = CDPCookieParam(name=cookie['name'], value=cookie['value'])
47+
if 'domain' in cookie:
48+
result['domain'] = cookie['domain']
49+
if 'path' in cookie:
50+
result['path'] = cookie['path']
51+
if 'secure' in cookie:
52+
result['secure'] = cookie['secure']
53+
if 'httpOnly' in cookie:
54+
result['httpOnly'] = cookie['httpOnly']
55+
if 'expiry' in cookie:
56+
result['expires'] = float(cookie['expiry'])
57+
if 'sameSite' in cookie:
58+
result['sameSite'] = CookieSameSite(cookie['sameSite'])
59+
return result

pydoll/browser/chromium/base.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, overload
1515
from urllib.parse import urlsplit, urlunsplit
1616

17+
from pydoll.browser.chromium._cookies import to_cdp_cookie_params, to_generic_cookies
1718
from pydoll.browser.chromium.tab import Tab
1819
from pydoll.browser.intercepted_request import InterceptedRequest
1920
from pydoll.browser.managers import (
@@ -38,10 +39,10 @@
3839
InvalidWebSocketAddress,
3940
NoValidTabFound,
4041
)
42+
from pydoll.protocol.cdp.browser.types import Bounds, PermissionType
4143
from pydoll.protocol.cdp.browser.types import (
4244
DownloadBehavior as CDPDownloadBehavior,
4345
)
44-
from pydoll.protocol.cdp.browser.types import PermissionType
4546
from pydoll.protocol.cdp.fetch.events import FetchEvent
4647
from pydoll.protocol.cdp.fetch.types import AuthChallengeResponseType
4748
from pydoll.protocol.cdp.network.types import ErrorReason as CDPErrorReason
@@ -433,7 +434,9 @@ async def set_cookies(
433434
):
434435
"""Set multiple cookies in browser or context."""
435436
logger.debug(f'Setting {len(cookies)} cookies (context={browser_context_id})')
436-
return await self._execute_command(StorageCommands.set_cookies(cookies, browser_context_id))
437+
return await self._execute_command(
438+
StorageCommands.set_cookies(to_cdp_cookie_params(cookies), browser_context_id)
439+
)
437440

438441
async def get_cookies(self, browser_context_id: Optional[str] = None) -> list[Cookie]:
439442
"""Get all cookies from browser or context.
@@ -448,7 +451,7 @@ async def get_cookies(self, browser_context_id: Optional[str] = None) -> list[Co
448451
logger.debug(
449452
f'Retrieved {len(response["result"]["cookies"])} cookies (context={browser_context_id})'
450453
)
451-
return response['result']['cookies']
454+
return to_generic_cookies(response['result']['cookies'])
452455

453456
async def get_version(self) -> BrowserVersion:
454457
"""Get browser version information."""
@@ -501,7 +504,18 @@ async def set_window_bounds(self, bounds: WindowBounds):
501504
"""
502505
window_id = await self._get_window_id()
503506
logger.info(f'Setting window bounds: id={window_id}, bounds={bounds}')
504-
return await self._execute_command(BrowserCommands.set_window_bounds(window_id, bounds))
507+
cdp_bounds: Bounds = {}
508+
if 'width' in bounds:
509+
cdp_bounds['width'] = bounds['width']
510+
if 'height' in bounds:
511+
cdp_bounds['height'] = bounds['height']
512+
if 'x' in bounds:
513+
cdp_bounds['left'] = bounds['x']
514+
if 'y' in bounds:
515+
cdp_bounds['top'] = bounds['y']
516+
return await self._execute_command(
517+
BrowserCommands.set_window_bounds(window_id, cdp_bounds)
518+
)
505519

506520
async def grant_permissions(
507521
self,

pydoll/browser/chromium/edge.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from pydoll.utils import validate_browser_paths
1111

1212
if TYPE_CHECKING:
13-
from pydoll.browser.chromium.options import Options
13+
from pydoll.browser.chromium.options import ChromiumOptions
1414

1515
logger = logging.getLogger(__name__)
1616

@@ -20,7 +20,7 @@ class Edge(Browser):
2020

2121
def __init__(
2222
self,
23-
options: Optional[Options] = None,
23+
options: Optional[ChromiumOptions] = None,
2424
connection_port: Optional[int] = None,
2525
):
2626
"""

pydoll/browser/chromium/tab.py

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828

2929
import aiofiles
3030

31+
from pydoll.browser.chromium._cookies import to_cdp_cookie_params, to_generic_cookies
3132
from pydoll.browser.requests import Request
3233
from pydoll.commands import (
34+
BrowserCommands,
3335
DomCommands,
3436
FetchCommands,
3537
NetworkCommands,
@@ -61,7 +63,8 @@
6163
from pydoll.extractor.engine import ExtractionEngine
6264
from pydoll.interactions import KeyboardAPI, MouseAPI, ScrollAPI
6365
from pydoll.interactions.iframe import IFrameContext
64-
from pydoll.protocol.cdp.browser.types import DownloadBehavior, DownloadProgressState
66+
from pydoll.protocol.cdp.browser.types import DownloadBehavior as CDPDownloadBehavior
67+
from pydoll.protocol.cdp.browser.types import DownloadProgressState
6568
from pydoll.protocol.cdp.dom.types import Node, ShadowRootType
6669
from pydoll.protocol.cdp.network.types import ResourceType
6770
from pydoll.protocol.cdp.page.events import PageEvent
@@ -71,6 +74,7 @@
7174
)
7275
from pydoll.protocol.cdp.target.types import TargetInfo
7376
from pydoll.protocol.events import CDP_DOMAIN_MAP, CDP_EVENT_MAP, Event
77+
from pydoll.protocol.types import DownloadBehavior
7478
from pydoll.utils import (
7579
decode_base64_to_bytes,
7680
has_return_outside_function,
@@ -791,14 +795,14 @@ async def get_cookies(self) -> list[Cookie]:
791795
)
792796
cookies = response_storage['result']['cookies']
793797
logger.debug(f'Fetched {len(cookies)} cookies')
794-
return cookies
798+
return to_generic_cookies(cookies)
795799

796800
response_network: NetworkGetCookiesResponse = await self._execute_command(
797801
NetworkCommands.get_cookies()
798802
)
799803
cookies = response_network['result']['cookies']
800804
logger.debug(f'Fetched {len(cookies)} cookies')
801-
return cookies
805+
return to_generic_cookies(cookies)
802806

803807
async def get_network_response_body(self, request_id: str) -> str:
804808
"""
@@ -858,7 +862,7 @@ async def set_cookies(self, cookies: list[CookieParam]):
858862
"""
859863
logger.info(f'Setting {len(cookies)} cookies on current page')
860864
return await self._execute_command(
861-
StorageCommands.set_cookies(cookies, self._browser_context_id)
865+
StorageCommands.set_cookies(to_cdp_cookie_params(cookies), self._browser_context_id)
862866
)
863867

864868
async def delete_all_cookies(self):
@@ -1115,8 +1119,7 @@ async def _fetch_document_html(self, frame_tree: FrameResourceTree) -> str:
11151119
return html
11161120
except Exception:
11171121
logger.debug('getResourceContent failed for document, falling back to JS')
1118-
html = await self.execute_script('return document.documentElement.outerHTML')
1119-
return cast(str, html)
1122+
return str(await self.execute_script('return document.documentElement.outerHTML') or '')
11201123

11211124
async def _fetch_bundle_assets(
11221125
self,
@@ -1519,9 +1522,11 @@ async def _cleanup_download_context(
15191522
download_dir: str,
15201523
) -> None:
15211524
await self.remove_callback(cb_id_progress)
1522-
await self._browser.set_download_behavior(
1523-
behavior=DownloadBehavior.DEFAULT,
1524-
browser_context_id=self._browser_context_id,
1525+
await self._browser.execute_protocol_command(
1526+
BrowserCommands.set_download_behavior(
1527+
behavior=CDPDownloadBehavior.DEFAULT,
1528+
browser_context_id=self._browser_context_id,
1529+
)
15251530
)
15261531

15271532
if cleanup_dir:

0 commit comments

Comments
 (0)