Fix test failures caused by port 8787 already in use (#9296)

Author: crusaderky

distributed/cli/tests/test_dask_scheduler.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import os
-import re
 import shutil
 import signal
 import subprocess
@@ -26,16 +25,11 @@
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
     gen_test,
+    get_dashboard_port,
     popen,
 )
 
 
-def _get_dashboard_port(client: Client) -> int:
-    match = re.search(r":(\d+)\/status", client.dashboard_link)
-    assert match
-    return int(match.group(1))
-
-
 def test_defaults(loop, requires_default_ports):
     with popen([sys.executable, "-m", "dask", "scheduler"]):
 
@@ -45,7 +39,7 @@ async def f():
 
         with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop) as c:
             c.sync(f)
-            assert _get_dashboard_port(c) == 8787
+            assert get_dashboard_port(c) == 8787
 
 
 @pytest.mark.slow
@@ -75,9 +69,24 @@ async def f():
 def test_no_dashboard(loop, requires_default_ports):
     requests = pytest.importorskip("requests")
 
-    with popen([sys.executable, "-m", "dask", "scheduler", "--no-dashboard"]):
-        with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop):
-            response = requests.get("http://127.0.0.1:8787/status/")
+    port = open_port()
+
+    with popen(
+        [
+            sys.executable,
+            "-m",
+            "dask",
+            "scheduler",
+            "--host",
+            f"127.0.0.1:{port}",
+            "--dashboard-address",
+            "127.0.0.1:0",
+            "--no-dashboard",
+        ]
+    ):
+        with Client(f"127.0.0.1:{port}", loop=loop) as c:
+            dashboard_port = get_dashboard_port(c)
+            response = requests.get(f"http://127.0.0.1:{dashboard_port}/status/")
             assert response.status_code == 404
 
 
@@ -91,7 +100,7 @@ def test_dashboard(loop):
         [sys.executable, "-m", "dask", "scheduler", "--host", f"127.0.0.1:{port}"],
     ):
         with Client(f"127.0.0.1:{port}", loop=loop) as c:
-            dashboard_port = _get_dashboard_port(c)
+            dashboard_port = get_dashboard_port(c)
 
         names = ["localhost", "127.0.0.1", get_ip()]
         start = time()
@@ -172,10 +181,8 @@ def test_dashboard_allowlist(loop):
     pytest.importorskip("bokeh")
     requests = pytest.importorskip("requests")
 
-    with pytest.raises(requests.ConnectionError):
-        requests.get("http://localhost:8787/status/").ok
-
     port = open_port()
+
     with popen(
         [
             sys.executable,
@@ -184,15 +191,15 @@ def test_dashboard_allowlist(loop):
             "scheduler",
             f"--port={port}",
         ]
-    ) as proc:
-        with Client(f"127.0.0.1:{port}", loop=loop) as c:
+    ):
+        with Client(f"127.0.0.1:{port}", loop=loop):
             pass
 
         start = time()
         while True:
             try:
-                for name in ["127.0.0.2", "127.0.0.3"]:
-                    response = requests.get("http://%s:8787/status/" % name)
+                for ip in ["127.0.0.2", "127.0.0.3"]:
+                    response = requests.get(f"http://{ip}:8787/status/")
                     assert response.ok
                 break
             except Exception as f:
@@ -343,8 +350,7 @@ def test_dashboard_port_zero(loop):
         ],
     ):
         with Client(f"tcp://127.0.0.1:{port}", loop=loop) as c:
-            port = _get_dashboard_port(c)
-            assert port > 0
+            assert get_dashboard_port(c) > 0
 
 
 PRELOAD_TEXT = """

distributed/comm/tests/test_ws.py

@@ -15,6 +15,7 @@
 from distributed.comm.registry import backends, get_backend
 from distributed.comm.tests.test_comms import check_tls_extra
 from distributed.security import Security
+from distributed.utils import open_port
 from distributed.utils_test import (
     gen_cluster,
     gen_test,
@@ -105,9 +106,12 @@ async def test_expect_scheduler_ssl_when_sharing_server(tmp_path):
         "distributed.scheduler.dashboard.tls.key": key_path,
         "distributed.scheduler.dashboard.tls.cert": cert_path,
     }
+    port = open_port()
     with dask.config.set(c):
         with pytest.raises(RuntimeError):
-            async with Scheduler(protocol="ws://", dashboard=True, port=8787):
+            async with Scheduler(
+                protocol="ws://", port=port, dashboard_address=f":{port}"
+            ):
                 pass
 
 

distributed/diagnostics/tests/test_memray.py

@@ -12,7 +12,9 @@
 
 def test_all_workers(tmp_path, loop):
     n_workers = 7
-    with LocalCluster(n_workers=n_workers, loop=loop) as cluster:
+    with LocalCluster(
+        n_workers=n_workers, loop=loop, dashboard_address=":0"
+    ) as cluster:
         with Client(cluster, loop=loop) as client:
             with memray_workers(tmp_path):
                 pass

distributed/tests/test_jupyter.py

@@ -14,7 +14,7 @@
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import Status
 from distributed.utils import open_port
-from distributed.utils_test import gen_test, popen
+from distributed.utils_test import gen_test, get_dashboard_port, popen
 
 pytest.importorskip("jupyter_server")
 
@@ -47,22 +47,27 @@ def test_jupyter_cli(loop, requires_default_ports):
     requests = pytest.importorskip("requests")
 
     port = open_port()
+
     with popen(
         [
             sys.executable,
             "-m",
             "dask",
             "scheduler",
             "--jupyter",
-            "--no-dashboard",
+            "--dashboard-address",
+            "127.0.0.1:0",
             "--host",
             f"127.0.0.1:{port}",
         ],
         terminate_timeout=120,
         kill_timeout=60,
     ):
-        with Client(f"127.0.0.1:{port}", loop=loop):
-            response = requests.get("http://127.0.0.1:8787/jupyter/api/status")
+        with Client(f"127.0.0.1:{port}", loop=loop) as c:
+            dashboard_port = get_dashboard_port(c)
+            response = requests.get(
+                f"http://127.0.0.1:{dashboard_port}/jupyter/api/status"
+            )
             assert response.status_code == 200
 
 
@@ -129,7 +134,8 @@ def test_shutdowns_cleanly(requires_default_ports):
                 "dask",
                 "scheduler",
                 "--jupyter",
-                "--no-dashboard",
+                "--dashboard-address",
+                "127.0.0.1:0",
                 "--host",
                 f"127.0.0.1:{port}",
             ],
@@ -140,13 +146,15 @@ def test_shutdowns_cleanly(requires_default_ports):
         )
 
         # wait until scheduler is running
-        with Client(f"127.0.0.1:{port}"):
-            pass
+        with Client(f"127.0.0.1:{port}") as c:
+            dashboard_port = get_dashboard_port(c)
 
         with requests.Session() as session:
-            session.get("http://127.0.0.1:8787/jupyter/lab").raise_for_status()
+            session.get(
+                f"http://127.0.0.1:{dashboard_port}/jupyter/lab"
+            ).raise_for_status()
             session.post(
-                "http://127.0.0.1:8787/jupyter/api/shutdown",
+                f"http://127.0.0.1:{dashboard_port}/jupyter/api/shutdown",
                 headers={"X-XSRFToken": session.cookies["_xsrf"]},
             ).raise_for_status()
 

distributed/utils_test.py

@@ -11,6 +11,7 @@
 import logging
 import multiprocessing
 import os
+import re
 import signal
 import socket
 import ssl
@@ -2578,3 +2579,9 @@ async def padded_time(before=0.05, after=0.05):
     t = time()
     await asyncio.sleep(after)
     return t
+
+
+def get_dashboard_port(client: Client) -> int:
+    match = re.search(r":(\d+)\/status", client.dashboard_link)
+    assert match
+    return int(match.group(1))