chore: make selenium container management more reliable

Author: ssbarnea

Taskfile.yml

@@ -230,7 +230,19 @@ tasks:
     generates:
       - out/junit/ui-selenium-test-results.xml
     cmds:
-      - pytest {{.CLI_ARGS}}
+      - >
+        pytest
+        -m=vscode
+        --cov-branch
+        --cov-context=test
+        --cov-report=term-missing
+        --cov-report=xml:out/coverage/ui-selenium/coverage.xml
+        --cov=test
+        --junit-xml=out/junit/ui-selenium-test-results.xml
+        --no-cov-on-fail
+        {{.CLI_ARGS}}
+      # --reruns-delay=120
+      # --reruns=3
       - defer: podman-compose down
     interactive: true
   unit:

pyproject.toml

@@ -62,21 +62,7 @@ exclude = "(.ansible|out|syntaxes).*"
 [tool.pytest]
 # do not add options here as this will likely break either console runs or IDE
 # integration like vscode or pycharm
-addopts = [
-  "--cov-branch",  # no parallel support with selenium
-  "--cov-context=test",
-  "--cov-report=term-missing",
-  "--cov-report=xml:out/coverage/ui-selenium/coverage.xml",
-  "--cov=test",
-  "--durations=10",
-  "--junit-xml=out/junit/ui-selenium-test-results.xml",
-  "--no-cov-on-fail",
-  "--reruns-delay=120",
-  "--reruns=3",
-  "-m=vscode",
-  "-n0",
-  "-v"
-]
+addopts = ["--durations-min=10", "--durations=10", "-n0", "-v"]
 filterwarnings = [
   "error",
   "once::pytest.PytestWarning",

test/selenium/conftest.py

@@ -13,13 +13,14 @@
 
 import pytest
 
+from test.selenium.const import CONTAINER_NAME
 from test.selenium.utils.ui_utils import LIGHTSPEED_PASSWORD, LIGHTSPEED_USER
 
 # Project root (vscode-ansible), assuming conftest at test/selenium/conftest.py
 _PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
 
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__package__)
+logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s")
 
 pytest_plugins = [
     # For screenshot_on_fail - must be loaded before fixtures that import it
@@ -83,8 +84,26 @@ def pytest_sessionfinish(
     session: pytest.Session, exitstatus: pytest.ExitCode
 ) -> None:  # pragma: no cover
     """Teardown the selenium server after tests on CI."""
-    subprocess.run(
-        "podman-compose stats --no-stream --no-reset", check=False, shell=True
+    result = subprocess.run(
+        f"podman-compose stats --no-stream --no-reset {CONTAINER_NAME}",
+        text=True,
+        capture_output=True,
+        check=False,
+        shell=True,
     )
+    if result.returncode:
+        logger.info("podman-compose stats :\n%s\n%s", result.stdout, result.stderr)
     if os.environ.get("CI"):
-        subprocess.run("podman-compose down", check=False, shell=True)
+        result = subprocess.run(
+            f"podman stop {CONTAINER_NAME} 2>/dev/null || true",
+            # Apparently compose down can fail with various errors but stopping container works
+            # "podman-compose down",
+            capture_output=True,
+            check=False,
+            shell=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            logger.error(
+                "Failed to stop selenium server:\n%s\n%s", result.stdout, result.stderr
+            )

test/selenium/const.py

@@ -0,0 +1,3 @@
+"""Constants for the selenium tests."""
+
+CONTAINER_NAME = "selenium-vscode"

test/selenium/fixtures/ui_fixtures.py

@@ -1,6 +1,6 @@
 """This module is a fixtures for the ui testing."""
 
-# cspell: ignore capmanager capturemanager pluginmanager getplugin
+# cspell: ignore capmanager capturemanager pluginmanager getplugin healthcheck
 # pylint: disable=E0401
 import contextlib
 import logging
@@ -16,6 +16,8 @@
 from selenium.common import WebDriverException
 from selenium.webdriver.remote.webdriver import WebDriver
 
+from test.selenium.const import CONTAINER_NAME
+
 if TYPE_CHECKING:
     from _pytest.capture import CaptureManager
     from selenium.webdriver.common.options import ArgOptions
@@ -31,7 +33,19 @@
 )
 
 # Initialize logging
-log = logging.getLogger(__name__)
+log = logging.getLogger(__package__)
+
+
+def is_container_healthy() -> bool:
+    """Check if the selenium container is healthy."""
+    result = subprocess.run(
+        f"podman healthcheck run {CONTAINER_NAME}",
+        shell=True,
+        check=False,
+        text=True,
+        capture_output=True,
+    )
+    return result.returncode == 0
 
 
 @pytest.fixture(scope="session")
@@ -46,15 +60,33 @@ def browser_setup(
     capmanager: CaptureManager = request.config.pluginmanager.getplugin(
         "capturemanager"
     )  # type: ignore[name-defined]
-    log.info(
-        "Starting selenium server at http://localhost:4444 and vnc://localhost:5999"
-    )
-    with capmanager.global_and_fixture_disabled():
+    if not is_container_healthy():
         subprocess.run(
-            "podman-compose up --remove-orphans --timeout 5 -d selenium-vscode",
-            check=True,
+            f"podman stop {CONTAINER_NAME} 2>/dev/null || true",
             shell=True,
+            check=False,
+            text=True,
+            capture_output=True,
+        )
+        log.info(
+            "Starting selenium server at http://localhost:4444 and vnc://localhost:5999"
         )
+        with capmanager.global_and_fixture_disabled():
+            subprocess.run(
+                f"podman-compose up --quiet-pull --remove-orphans --timeout 5 -d {CONTAINER_NAME}",
+                check=True,
+                shell=True,
+            )
+        count = 0
+        while True:
+            if is_container_healthy():
+                break
+            count += 1
+            time.sleep(1)
+            log.info(
+                "Waiting for container %s to be healthy: %s", CONTAINER_NAME, count
+            )
+
     browser = os.environ.get("BROWSER_TYPE")
     options: ArgOptions  # type: ignore[name-defined]
     if browser == "chrome":

test/selenium/ui/test_lightspeed.py

@@ -45,7 +45,7 @@ def vscode_login_wrapper(driver: Any) -> None:
     logged_in_flag = True
 
 
-@pytest.mark.flaky(reruns=6, reruns_delay=10)
+# @pytest.mark.flaky(reruns=6, reruns_delay=10)
 @pytest.mark.vscode
 def test_vscode_widget(
     browser_setup: Any,

test/selenium/utils/settings_utils.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import Any
 
-log = logging.getLogger(__name__)
+log = logging.getLogger(__package__)
 
 PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
 USER_SETTINGS_PATH = PROJECT_ROOT / ".vscode-test/user-data/User/settings.json"