[py] retry safaridriver startup in tests (#17615)

* [py][rb] retry SafariDriver startup and remove Safari 26.5 test guards

* [py] retry driver start on connection errors, not just WebDriverException

Author: titusfortner

py/conftest.py

@@ -17,17 +17,20 @@
 
 import http.server
 import json
+import logging
 import os
 import socketserver
 import sys
 import threading
+import time
 import types
 from dataclasses import dataclass
 from pathlib import Path
 
 import pytest
 import rich.console
 import rich.traceback
+import urllib3
 
 try:
     from python.runfiles import Runfiles  # only exists when using bazel
@@ -41,6 +44,8 @@
 from test.selenium.webdriver.common.network import get_lan_ip
 from test.selenium.webdriver.common.webserver import SimpleWebServer
 
+logger = logging.getLogger(__name__)
+
 drivers = (
     "chrome",
     "edge",
@@ -296,6 +301,9 @@ class SupportedBidiDrivers(ContainerProtocol):
 
 
 class Driver:
+    DRIVER_START_RETRIES = 3
+    DRIVER_START_INTERVAL = 1
+
     def __init__(self, driver_class, request):
         self.driver_class = driver_class
         self._request = request
@@ -455,9 +463,25 @@ def _initialize_driver(self):
         if self.is_remote:
             kwargs["command_executor"] = self._server.status_url.removesuffix("/status")
             return webdriver.Remote(**kwargs)
-        if self.driver_path is not None:
-            kwargs["service"] = self.service
-        return getattr(webdriver, self.driver_class)(**kwargs)
+        return self._start_local_driver(kwargs)
+
+    def _start_local_driver(self, kwargs):
+        for attempt in range(1, self.DRIVER_START_RETRIES + 1):
+            if self.driver_path is not None:
+                kwargs["service"] = self.service
+            try:
+                return getattr(webdriver, self.driver_class)(**kwargs)
+            except (WebDriverException, urllib3.exceptions.HTTPError, OSError) as e:
+                if attempt == self.DRIVER_START_RETRIES:
+                    raise
+                logger.warning(
+                    "%s failed to start (attempt %s/%s); retrying. Error: %s",
+                    self.driver_class,
+                    attempt,
+                    self.DRIVER_START_RETRIES,
+                    e,
+                )
+                time.sleep(self.DRIVER_START_INTERVAL)
 
     def stop_driver(self):
         driver_to_stop = self._driver

py/test/selenium/webdriver/common/alerts_tests.py

@@ -38,7 +38,6 @@ def close_alert(driver):
         pass
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_override_the_window_alert_method(driver, pages):
     pages.load("alerts.html")
     driver.execute_script("window.alert = function(msg) { document.getElementById('text').innerHTML = msg; }")
@@ -55,7 +54,6 @@ def test_should_be_able_to_override_the_window_alert_method(driver, pages):
         raise e
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_users_to_accept_an_alert_manually(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="alert").click()
@@ -65,7 +63,6 @@ def test_should_allow_users_to_accept_an_alert_manually(driver, pages):
     assert "Testing Alerts" == driver.title
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_users_to_accept_an_alert_with_no_text_manually(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "empty-alert").click()
@@ -76,7 +73,6 @@ def test_should_allow_users_to_accept_an_alert_with_no_text_manually(driver, pag
     assert "Testing Alerts" == driver.title
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_get_text_of_alert_opened_in_set_timeout(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "slow-alert").click()
@@ -92,7 +88,6 @@ def test_should_get_text_of_alert_opened_in_set_timeout(driver, pages):
         alert.accept()
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_users_to_dismiss_an_alert_manually(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="alert").click()
@@ -102,7 +97,6 @@ def test_should_allow_users_to_dismiss_an_alert_manually(driver, pages):
     assert "Testing Alerts" == driver.title
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_auser_to_accept_aprompt(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="prompt").click()
@@ -113,7 +107,6 @@ def test_should_allow_auser_to_accept_aprompt(driver, pages):
     assert "Testing Alerts" == driver.title
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_auser_to_dismiss_aprompt(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="prompt").click()
@@ -124,7 +117,6 @@ def test_should_allow_auser_to_dismiss_aprompt(driver, pages):
     assert "Testing Alerts" == driver.title
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_auser_to_set_the_value_of_aprompt(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="prompt").click()
@@ -136,7 +128,6 @@ def test_should_allow_auser_to_set_the_value_of_aprompt(driver, pages):
     assert "cheese" == result
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 @pytest.mark.xfail_firefox
 @pytest.mark.xfail_remote
 def test_setting_the_value_of_an_alert_throws(driver, pages):
@@ -155,7 +146,6 @@ def test_setting_the_value_of_an_alert_throws(driver, pages):
 @pytest.mark.xfail_edge(
     condition=sys.platform == "darwin", reason="https://bugs.chromium.org/p/chromedriver/issues/detail?id=26", run=False
 )
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_alert_should_not_allow_additional_commands_if_dimissed(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "alert").click()
@@ -208,7 +198,6 @@ def test_should_throw_an_exception_if_an_alert_has_not_been_dealt_with_and_dismi
     # Alert would be dismissed automatically
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_prompt_should_use_default_value_if_no_keys_sent(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "prompt-with-default").click()
@@ -220,7 +209,6 @@ def test_prompt_should_use_default_value_if_no_keys_sent(driver, pages):
     assert "This is a default value" == txt
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_prompt_should_have_null_value_if_dismissed(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "prompt-with-default").click()
@@ -230,7 +218,6 @@ def test_prompt_should_have_null_value_if_dismissed(driver, pages):
     assert "null" == driver.find_element(By.ID, "text").text
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_handles_two_alerts_from_one_interaction(driver, pages):
     pages.load("alerts.html")
 
@@ -270,7 +257,6 @@ def test_should_handle_alert_on_page_load_using_get(driver, pages):
     )
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 @pytest.mark.xfail_chrome(reason="Non W3C conformant")
 @pytest.mark.xfail_edge(reason="Non W3C conformant")
 def test_should_handle_alert_on_page_before_unload(driver, pages):
@@ -281,7 +267,6 @@ def test_should_handle_alert_on_page_before_unload(driver, pages):
     WebDriverWait(driver, 3).until(EC.title_is("Testing Alerts"))
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_the_user_to_get_the_text_of_an_alert(driver, pages):
     pages.load("alerts.html")
     driver.find_element(by=By.ID, value="alert").click()
@@ -291,7 +276,6 @@ def test_should_allow_the_user_to_get_the_text_of_an_alert(driver, pages):
     assert "cheese" == value
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_allow_the_user_to_get_the_text_of_aprompt(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "prompt").click()
@@ -303,7 +287,6 @@ def test_should_allow_the_user_to_get_the_text_of_aprompt(driver, pages):
     assert "Enter something" == value
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_alert_should_not_allow_additional_commands_if_dismissed(driver, pages):
     pages.load("alerts.html")
     driver.find_element(By.ID, "alert").click()

py/test/selenium/webdriver/common/click_scrolling_tests.py

@@ -23,7 +23,6 @@
 from selenium.webdriver.support.ui import WebDriverWait
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_clicking_on_anchor_scrolls_page(driver, pages):
     scrollScript = """var pageY;
     if (typeof(window.pageYOffset) == 'number') {
@@ -54,7 +53,6 @@ def test_should_scroll_to_click_on_an_element_hidden_by_overflow(driver, pages):
         pytest.fail(f"Should not be out of bounds: {e.msg}")
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_click_on_an_element_hidden_by_overflow(driver, pages):
     pages.load("scroll.html")
 
@@ -64,7 +62,6 @@ def test_should_be_able_to_click_on_an_element_hidden_by_overflow(driver, pages)
     assert "line8" == driver.find_element(By.ID, "clicked").text
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 @pytest.mark.xfail_firefox
 @pytest.mark.xfail_remote
 def test_should_be_able_to_click_on_an_element_hidden_by_double_overflow(driver, pages):
@@ -74,7 +71,6 @@ def test_should_be_able_to_click_on_an_element_hidden_by_double_overflow(driver,
     WebDriverWait(driver, 3).until(EC.title_is("Clicked Successfully!"))
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_click_on_an_element_hidden_by_yoverflow(driver, pages):
     pages.load("scrolling_tests/page_with_y_overflow_auto.html")
 
@@ -125,7 +121,6 @@ def test_should_be_able_to_click_element_in_aframe_that_is_out_of_view(driver, p
     assert element.is_selected()
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_click_element_that_is_out_of_view_in_aframe(driver, pages):
     pages.load("scrolling_tests/page_with_scrolling_frame.html")
     driver.switch_to.frame(driver.find_element(By.NAME, "scrolling_frame"))
@@ -151,7 +146,6 @@ def test_should_be_able_to_click_element_that_is_out_of_view_in_aframe_that_is_o
     assert element.is_selected()
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 @pytest.mark.xfail_firefox
 @pytest.mark.xfail_chrome
 @pytest.mark.xfail_remote

py/test/selenium/webdriver/common/click_tests.py

@@ -27,13 +27,11 @@ def loadPage(pages):
     pages.load("clicks.html")
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_can_click_on_alink_that_overflows_and_follow_it(driver):
     driver.find_element(By.ID, "overflowLink").click()
     WebDriverWait(driver, 3).until(EC.title_is("XHTML Test Page"))
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_clicking_alink_made_up_of_numbers_is_handled_correctly(driver):
     driver.find_element(By.LINK_TEXT, "333333").click()
     WebDriverWait(driver, 3).until(EC.title_is("XHTML Test Page"))

py/test/selenium/webdriver/common/correct_event_firing_tests.py

@@ -94,7 +94,6 @@ def test_should_emit_on_change_events_when_changing_the_state_of_acheckbox(drive
     assert driver.find_element(By.ID, "result").text == "checkbox thing"
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_emit_click_event_when_clicking_on_atext_input_element(driver, pages):
     pages.load("javascriptPage.html")
     clicker = driver.find_element(By.ID, "clickField")

py/test/selenium/webdriver/common/element_attribute_tests.py

@@ -123,7 +123,6 @@ def test_should_indicate_when_aselect_is_disabled(driver, pages):
     assert not disabled.is_enabled()
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_return_the_value_of_checked_for_acheckbox_even_if_it_lacks_that_attribute(driver, pages):
     pages.load("formPage.html")
     checkbox = driver.find_element(By.XPATH, "//input[@id='checky']")
@@ -132,7 +131,6 @@ def test_should_return_the_value_of_checked_for_acheckbox_even_if_it_lacks_that_
     assert "true" == checkbox.get_attribute("checked")
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_return_the_value_of_selected_for_radio_buttons_even_if_they_lack_that_attribute(driver, pages):
     pages.load("formPage.html")
     neverSelected = driver.find_element(By.ID, "cheese")

py/test/selenium/webdriver/common/executing_async_javascript_tests.py

@@ -151,7 +151,6 @@ def test_should_catch_errors_when_executing_initial_script(driver, pages):
         driver.execute_async_script("throw Error('you should catch this!');")
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_execute_asynchronous_scripts(driver, pages):
     pages.load("ajaxy_page.html")
 

py/test/selenium/webdriver/common/form_handling_tests.py

@@ -23,7 +23,6 @@
 from selenium.webdriver.support.ui import WebDriverWait
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_click_on_submit_input_elements(driver, pages):
     pages.load("formPage.html")
     driver.find_element(By.ID, "submitButton").click()
@@ -35,7 +34,6 @@ def test_clicking_on_unclickable_elements_does_nothing(driver, pages):
     driver.find_element(By.XPATH, "//body").click()
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_click_image_buttons(driver, pages):
     pages.load("formPage.html")
     driver.find_element(By.ID, "imageButton").click()
@@ -100,7 +98,6 @@ def test_should_enter_data_into_form_fields(driver, pages):
     assert newFormValue == "some text"
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_select_acheck_box(driver, pages):
     pages.load("formPage.html")
     checkbox = driver.find_element(By.ID, "checky")
@@ -111,7 +108,6 @@ def test_should_be_able_to_select_acheck_box(driver, pages):
     assert checkbox.is_selected() is False
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_toggle_the_checked_state_of_acheckbox(driver, pages):
     pages.load("formPage.html")
     checkbox = driver.find_element(By.ID, "checky")
@@ -122,7 +118,6 @@ def test_should_toggle_the_checked_state_of_acheckbox(driver, pages):
     assert checkbox.is_selected() is False
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_toggling_acheckbox_should_return_its_current_state(driver, pages):
     pages.load("formPage.html")
     checkbox = driver.find_element(By.ID, "checky")
@@ -133,7 +128,6 @@ def test_toggling_acheckbox_should_return_its_current_state(driver, pages):
     assert checkbox.is_selected() is False
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_select_aradio_button(driver, pages):
     pages.load("formPage.html")
     radioButton = driver.find_element(By.ID, "peas")
@@ -142,7 +136,6 @@ def test_should_be_able_to_select_aradio_button(driver, pages):
     assert radioButton.is_selected() is True
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_should_be_able_to_select_aradio_button_by_clicking_on_it(driver, pages):
     pages.load("formPage.html")
     radioButton = driver.find_element(By.ID, "peas")
@@ -226,7 +219,6 @@ def test_should_be_able_to_clear_text_from_text_areas(driver, pages):
     assert len(value) == 0
 
 
-@pytest.mark.xfail_safari(reason="SafariDriver 26.5 regression")
 def test_radio_should_not_be_selected_after_selecting_sibling(driver, pages):
     pages.load("formPage.html")
     cheese = driver.find_element(By.ID, "cheese")