Merge pull request #4466 from seleniumbase/cdp-mode-patch-129

CDP Mode: Patch 129

Author: mdmintz

README.md

@@ -11,6 +11,8 @@
 
 <p align="center" class="hero__title"><b>All-in-one Browser Automation Framework:<br />Web Crawling / Testing / Scraping / Stealth</b></p>
 
+<p align="center"><a href="https://trendshift.io/repositories/12493?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-12493" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/12493" alt="seleniumbase%2FSeleniumBase | Trendshift" width="250" height="55"/></a></p>
+
 <p align="center"><a href="https://pypi.python.org/pypi/seleniumbase" target="_blank"><img src="https://img.shields.io/pypi/v/seleniumbase.svg?color=3399EE" alt="PyPI version" /></a> <a href="https://pepy.tech/projects/seleniumbase?timeRange=threeMonths&category=version&includeCIDownloads=true&granularity=daily&viewType=line&versions=*" target="_blank"><img src="https://static.pepy.tech/badge/seleniumbase" alt="SeleniumBase PyPI downloads" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-22BBCC.svg" title="SeleniumBase" /></a> <a href="https://seleniumbase.io"><img src="https://img.shields.io/badge/docs-seleniumbase.io-11BBAA.svg" alt="SeleniumBase Docs" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/actions" target="_blank"><img src="https://github.com/seleniumbase/SeleniumBase/workflows/Tests/badge.svg" alt="SeleniumBase GitHub Actions" /></a> <a href="https://www.youtube.com/playlist?list=PLp9uKicxkBc5UIlGi2BuE3aWC7JyXpD3m"><img src="https://img.shields.io/badge/docs-📺-F12345.svg?style=flat&logo=YouTube&logoColor=white&label=YouTube" alt="YouTube Channel" /></a></p>
 
 <p align="center">

examples/cdp_mode/raw_cdp_quotes.py

@@ -0,0 +1,24 @@
+import json
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+sb.goto("https://quotes.toscrape.com/")
+all_quotes = []
+while True:
+    quotes = sb.find_elements("div.quote")
+    for quote in quotes:
+        all_quotes.append(
+            {
+                "author": quote.query_selector("span small").text,
+                "text": quote.query_selector("span.text").text,
+            }
+        )
+    if sb.is_element_visible("li.next"):
+        sb.click("li.next")
+    else:
+        break
+file_path = "downloaded_files/quotes.json"
+with open(file_path, "w", encoding="utf-8") as outfile:
+    json.dump(all_quotes, outfile)
+print(f'Quotes saved to "{file_path}".')
+sb.quit()

examples/cdp_mode/raw_cdp_yelp.py

@@ -0,0 +1,43 @@
+import json
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+sb.goto("https://www.yelp.com/")
+sb.sleep(3)
+restaurants_menu = 'button[aria-label*="Restaurants"]'
+cafes = 'span:contains("Cafes")'
+sb.wait_for_element(restaurants_menu)
+sb.sleep(1)
+sb.hover_and_click(restaurants_menu, cafes)
+sb.sleep(3)
+all_cafes = []
+counter = 0
+max_pages = 3
+while True:
+    dtci = '[data-traffic-crawl-id="SearchResult%s"]'
+    cafes = sb.find_elements('[data-testid="serp-ia-card"]')
+    for cafe in cafes:
+        name = cafe.query_selector(dtci % "BizName")
+        rating = cafe.query_selector(dtci % "BizRating")
+        snippet = cafe.query_selector(dtci % "ReviewSnippet")
+        if name and rating and snippet:
+            all_cafes.append(
+                {
+                    "name": name.text,
+                    "rating": rating.text,
+                    "snippet": snippet.text,
+                }
+            )
+    counter += 1
+    if counter >= max_pages:
+        break
+    if sb.is_element_visible('span:contains("Next Page")'):
+        sb.click('span:contains("Next Page")')
+        sb.sleep(1)
+    else:
+        break
+file_path = "downloaded_files/cafes.json"
+with open(file_path, "w", encoding="utf-8") as outfile:
+    json.dump(all_cafes, outfile)
+print(f'Cafes saved to "{file_path}".')
+sb.quit()

mkdocs_build/requirements.txt

@@ -3,7 +3,7 @@
 
 regex>=2026.7.19
 pymdown-extensions>=10.21.3
-pipdeptree>=4.2.0
+pipdeptree>=4.2.1
 python-dateutil>=2.8.2
 click>=8.4.2
 Markdown==3.10.3
@@ -15,7 +15,7 @@ Babel==2.18.0
 paginate==0.5.7
 mkdocs==1.6.1
 mkdocs-material==9.6.23
-mkdocs-exclude-search==0.6.6
+mkdocs-exclude-search==0.6.7
 mkdocs-simple-hooks==0.1.5
 mkdocs-get-deps==0.2.2
 mkdocs-material-extensions==1.3.1

requirements.txt

@@ -1,22 +1,17 @@
-pip>=26.0.1;python_version<"3.10"
-pip>=26.2.1;python_version>="3.10"
+pip>=26.2.1
 packaging>=26.3
-setuptools~=70.2;python_version<"3.10"
-setuptools>=84.0.0;python_version>="3.10"
-wheel>=0.47.0
+setuptools>=84.0.0
+wheel>=0.48.0
 attrs>=26.1.0
 certifi>=2026.7.22
 exceptiongroup>=1.3.1
-websockets~=15.0.1;python_version<"3.10"
 websockets~=16.1.1;python_version=="3.10"
-websockets>=17.0.1;python_version>="3.11"
-filelock~=3.19.1;python_version<"3.10"
-filelock>=3.32.2;python_version>="3.10"
+websockets>=16.1.1;python_version>="3.11"
+filelock>=3.32.3
 fasteners>=0.20
 mycdp>=1.4.0
 pynose>=1.5.5
-platformdirs~=4.4.0;python_version<"3.10"
-platformdirs>=4.11.2;python_version>="3.10"
+platformdirs>=4.11.3
 typing-extensions>=4.16.0
 sbvirtualdisplay>=1.4.0
 MarkupSafe>=3.0.3
@@ -27,62 +22,50 @@ parse-type>=0.6.6
 colorama>=0.4.6
 psutil>=7.2.2
 pyyaml>=6.0.3
-pygments>=2.20.0
+pygments>=2.20.1
 pyreadline3>=3.5.4;platform_system=="Windows"
 tabcompleter>=1.4.1
 pdbp>=1.8.2
-idna>=3.18
-charset-normalizer>=3.4.9,<4
-urllib3>=1.26.20,<2;python_version<"3.10"
-urllib3>=2.7.0,<3;python_version>="3.10"
-requests~=2.32.5;python_version<"3.10"
-requests~=2.34.2;python_version>="3.10"
+idna>=3.19
+charset-normalizer>=3.5.1,<4
+urllib3>=2.7.0,<3
+requests~=2.34.2
 sniffio==1.3.1
 h11==0.16.0
 outcome==1.3.0.post0
-trio>=0.31.0,<1;python_version<"3.10"
-trio>=0.33.0,<1;python_version>="3.10"
+trio>=0.34.0,<1
 trio-websocket~=0.12.2
-wsproto==1.2.0;python_version<"3.10"
-wsproto~=1.3.2;python_version>="3.10"
+wsproto~=1.3.2
 websocket-client~=1.9.0
-selenium==4.32.0;python_version<"3.10"
-selenium==4.46.0;python_version>="3.10"
-cssselect==1.3.0;python_version<"3.10"
-cssselect>=1.5.0,<2;python_version>="3.10"
+selenium==4.47.0
+cssselect>=1.5.0,<2
 sortedcontainers==2.4.0
-execnet==2.1.1;python_version<"3.10"
-execnet==2.1.2;python_version>="3.10"
-iniconfig==2.1.0;python_version<"3.10"
-iniconfig==2.3.0;python_version>="3.10"
+execnet==2.1.2
+iniconfig==2.3.0
 pluggy==1.6.0
 pytest==8.4.2;python_version<"3.11"
 pytest==9.1.1;python_version>="3.11"
 pytest-html==4.0.2
 pytest-metadata==3.1.1
 pytest-ordering==0.6
-pytest-rerunfailures==16.0.1;python_version<"3.10"
 pytest-rerunfailures==16.3;python_version=="3.10"
-pytest-rerunfailures==16.4;python_version>="3.11"
+pytest-rerunfailures==16.6;python_version>="3.11"
 pytest-xdist==3.8.0
 parameterized==0.9.0
 behave==1.2.6
-soupsieve~=2.8.4;python_version<"3.10"
-soupsieve~=2.9.2;python_version>="3.10"
+soupsieve~=2.9.2
 beautifulsoup4~=4.15.0
 pyotp~=2.10.0
 python-xlib==0.33;platform_system=="Linux"
 PyAutoGUI>=0.9.54;platform_system=="Linux"
-markdown-it-py==3.0.0;python_version<"3.10"
-markdown-it-py==4.2.0;python_version>="3.10"
+markdown-it-py==4.2.0
 mdurl==0.1.2
 rich>=15.0.0,<16
 
 # --- Testing Requirements --- #
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
-coverage>=7.10.7;python_version<"3.10"
-coverage>=7.15.4;python_version>="3.10"
+coverage>=7.15.4
 pytest-cov>=7.1.0
 flake8==7.3.0
 mccabe==0.7.0

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.51.12"
+__version__ = "4.52.0"

seleniumbase/console_scripts/sb_install.py

@@ -659,8 +659,8 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             else:
                 not_latest = c5 + "(" + c4 + "Legacy Version" + c5 + ")" + cr
                 p_version = p_version + " " + not_latest
-            msg = c2 + "chromedriver to download" + cr
-            log_d("\n*** %s = %s" % (msg, p_version))
+            msg = c2 + "chromedriver" + cr
+            log_d("\n*** Getting %s %s" % (msg, p_version))
         else:
             raise Exception("Could not find chromedriver to download!\n")
         if not get_latest:
@@ -704,9 +704,9 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             use_version = found_version
         if not use_version:
             use_version = get_cft_latest_version_from_milestone(major_version)
-        msg = c2 + "Chrome for Testing to download" + cr
+        msg = c2 + "Chrome for Testing" + cr
         p_version = c3 + use_version + cr
-        log_d("\n*** %s = %s" % (msg, p_version))
+        log_d("\n*** Getting %s %s" % (msg, p_version))
         if IS_MAC:
             if IS_ARM_MAC:
                 platform_code = "mac-arm64"
@@ -813,9 +813,9 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             use_version = found_version
         if not use_version:
             use_version = get_cft_latest_version_from_milestone(major_version)
-        msg = c2 + "Chrome-Headless-Shell to download" + cr
+        msg = c2 + "Chrome-Headless-Shell" + cr
         p_version = c3 + use_version + cr
-        log_d("\n*** %s = %s" % (msg, p_version))
+        log_d("\n*** Getting %s %s" % (msg, p_version))
         if IS_MAC:
             if IS_ARM_MAC:
                 platform_code = "mac-arm64"
@@ -892,9 +892,9 @@ def main(override=None, intel_for_uc=None, force_uc=None):
         if not found_geckodriver:
             url_request = requests_get(download_url)
         if found_geckodriver or url_request.ok:
-            msg = c2 + "geckodriver to download" + cr
+            msg = c2 + "geckodriver" + cr
             p_version = c3 + use_version + cr
-            log_d("\n*** %s = %s" % (msg, p_version))
+            log_d("\n*** Getting %s %s" % (msg, p_version))
         else:
             raise Exception(
                 "\nCould not find the specified geckodriver "
@@ -999,9 +999,9 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                 raise Exception(
                     "Could not find version [%s] of EdgeDriver!" % use_version
                 )
-        msg = c2 + "edgedriver to download" + cr
+        msg = c2 + "edgedriver" + cr
         p_version = c3 + use_version + cr
-        log_d("\n*** %s = %s" % (msg, p_version))
+        log_d("\n*** Getting %s %s" % (msg, p_version))
     elif name == "iedriver":
         full_version = "4.14.0"
         use_version = full_version
@@ -1029,9 +1029,9 @@ def main(override=None, intel_for_uc=None, force_uc=None):
         url_request = requests_get_with_retry(headless_ie_url)
         if url_request.ok:
             headless_ie_exists = True
-            msg = c2 + "HeadlessIEDriver to download" + cr
+            msg = c2 + "HeadlessIEDriver" + cr
             p_version = c3 + headless_ie_version + cr
-            log_d("\n*** %s = %s" % (msg, p_version))
+            log_d("\n*** Getting %s %s" % (msg, p_version))
     else:
         invalid_run_command()
 
@@ -1095,7 +1095,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                     os.remove(new_file)
         if not driver_file or not driver_path or not filename:
             raise Exception("headless_ie_selenium.exe missing from Zip file!")
-        log_d("Extracting %s from %s ..." % (filename, headless_ie_file_name))
+        log_d("Extracting %s from %s:" % (filename, headless_ie_file_name))
         zip_ref.extractall(downloads_folder)
         zip_ref.close()
         os.remove(zip_file_path)
@@ -1120,13 +1120,12 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             os.rmdir(os.path.join(downloads_folder, h_ie_fn))
         driver_path = os.path.join(downloads_folder, filename)
         log_d(
-            "The file [%s] was saved to:\n%s%s%s\n"
+            "['%s'] was saved to:\n%s%s%s\n"
             % (filename, c3, driver_path, cr)
         )
-        log_d("Making [%s %s] executable ..." % (driver_file, use_version))
         make_executable(driver_path)
         log_d(
-            "%s[%s %s] is now ready for use!%s"
+            "%s[%s %s] is ready for use!%s"
             % (c1, driver_file, use_version, cr)
         )
 
@@ -1180,7 +1179,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                         os.remove(new_file)  # Technically the old file now
             if driver_contents:
                 contents = driver_contents
-            log_d("Extracting %s from %s ..." % (contents, file_name))
+            log_d("Extracting %s from %s:" % (contents, file_name))
             if name == "uc_driver":
                 f_name = "uc_driver"
                 new_file = os.path.join(downloads_folder, f_name)
@@ -1232,11 +1231,10 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                 d_folder = os.sep.join(pr_file.split(os.sep)[:-1]) + os.sep
                 d_file = pr_file.split(os.sep)[-1]
                 d_ff = c3 + d_folder + cr + "\n" + c3 + d_file + cr
-                log_d("The file [%s] was saved to:\n%s\n" % (f_name, d_ff))
-                log_d("Making [%s %s] executable ..." % (f_name, use_version))
+                log_d("['%s'] was saved to:\n%s\n" % (f_name, d_ff))
                 make_executable(new_file)
                 log_d(
-                    "%s[%s %s] is now ready for use!%s" %
+                    "%s[%s %s] is ready for use!%s" %
                     (c1, f_name, use_version, cr)
                 )
                 if copy_to_path and os.path.exists(LOCAL_PATH):
@@ -1301,7 +1299,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                 if str_name == "IEDriverServer.exe":
                     raise Exception("IEDriverServer missing from Zip file!")
                 raise Exception("msedgedriver missing from Zip file!")
-            log_d("Extracting %s from %s ..." % (contents, file_name))
+            log_d("Extracting %s from %s:" % (contents, file_name))
             zip_ref.extractall(downloads_folder)
             zip_ref.close()
             os.remove(zip_file_path)
@@ -1323,13 +1321,12 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             pr_sep = c3 + os.sep + cr
             pr_driver_file = c3 + driver_file + cr
             log_d(
-                "The file [%s] was saved to:\n%s%s\n%s\n"
+                "['%s'] was saved to:\n%s%s\n%s\n"
                 % (driver_file, pr_driver_base, pr_sep, pr_driver_file)
             )
-            log_d("Making [%s %s] executable ..." % (driver_file, use_version))
             make_executable(driver_path)
             log_d(
-                "%s[%s %s] is now ready for use!%s"
+                "%s[%s %s] is ready for use!%s"
                 % (c1, driver_file, use_version, cr)
             )
             if copy_to_path and os.path.exists(LOCAL_PATH):
@@ -1462,7 +1459,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                 if "Driver" in new_file or "driver" in new_file:
                     if os.path.exists(new_file):
                         os.remove(new_file)  # Technically the old file now
-            log_d("Extracting %s from %s ..." % (contents, file_name))
+            log_d("Extracting %s from %s:" % (contents, file_name))
             if sys.version_info < (3, 12):
                 tar.extractall(downloads_folder)
             else:
@@ -1473,11 +1470,10 @@ def main(override=None, intel_for_uc=None, force_uc=None):
             for f_name in contents:
                 new_file = os.path.join(downloads_folder, str(f_name))
                 pr_file = c3 + new_file + cr
-                log_d("The file [%s] was saved to:\n%s\n" % (f_name, pr_file))
-                log_d("Making [%s %s] executable ..." % (f_name, use_version))
+                log_d("['%s'] was saved to:\n%s\n" % (f_name, pr_file))
                 make_executable(new_file)
                 log_d(
-                    "%s[%s %s] is now ready for use!%s"
+                    "%s[%s %s] is ready for use!%s"
                     % (c1, f_name, use_version, cr)
                 )
                 if copy_to_path and os.path.exists(LOCAL_PATH):
@@ -1494,9 +1490,8 @@ def main(override=None, intel_for_uc=None, force_uc=None):
         # Not a .zip file or a .tar.gz file. Just a direct download.
         if "Driver" in file_name or "driver" in file_name:
             log_d("%sDownload Complete!%s\n" % (c1, cr))
-            log_d("Making [%s] executable ..." % file_name)
             make_executable(file_path)
-            log_d("%s[%s] is now ready for use!%s" % (c1, file_name, cr))
+            log_d("%s[%s] is ready for use!%s" % (c1, file_name, cr))
             log_d("Location of [%s]:\n%s\n" % (file_name, file_path))