Skip to content

Commit 40b7646

Browse files
cjwatsoncjwatson
authored
Run dask commands with a matching interpreter (#8975)
1 parent 1b92625 commit 40b7646

File tree

10 files changed

+334
-40
lines changed

10 files changed

+334
-40
lines changed

‎distributed/cli/tests/test_dask_scheduler.py‎

Lines changed: 104 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def _get_dashboard_port(client: Client) -> int:
4040

4141

4242
def test_defaults(loop, requires_default_ports):
43-
with popen(["dask", "scheduler"]):
43+
with popen([sys.executable, "-m", "dask", "scheduler"]):
4444

4545
async def f():
4646
# Default behaviour is to listen on all addresses
@@ -53,7 +53,17 @@ async def f():
5353

5454
def test_hostport(loop):
5555
port = open_port()
56-
with popen(["dask", "scheduler", "--no-dashboard", "--host", f"127.0.0.1:{port}"]):
56+
with popen(
57+
[
58+
sys.executable,
59+
"-m",
60+
"dask",
61+
"scheduler",
62+
"--no-dashboard",
63+
"--host",
64+
f"127.0.0.1:{port}",
65+
]
66+
):
5767

5868
async def f():
5969
# The scheduler's main port can't be contacted from the outside
@@ -65,7 +75,7 @@ async def f():
6575

6676

6777
def test_no_dashboard(loop, requires_default_ports):
68-
with popen(["dask", "scheduler", "--no-dashboard"]):
78+
with popen([sys.executable, "-m", "dask", "scheduler", "--no-dashboard"]):
6979
with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop):
7080
response = requests.get("http://127.0.0.1:8787/status/")
7181
assert response.status_code == 404
@@ -76,7 +86,7 @@ def test_dashboard(loop):
7686
port = open_port()
7787

7888
with popen(
79-
["dask", "scheduler", "--host", f"127.0.0.1:{port}"],
89+
[sys.executable, "-m", "dask", "scheduler", "--host", f"127.0.0.1:{port}"],
8090
):
8191
with Client(f"127.0.0.1:{port}", loop=loop) as c:
8292
dashboard_port = _get_dashboard_port(c)
@@ -109,6 +119,8 @@ def test_dashboard_non_standard_ports(loop):
109119
port2 = open_port()
110120
with popen(
111121
[
122+
sys.executable,
123+
"-m",
112124
"dask",
113125
"scheduler",
114126
f"--port={port1}",
@@ -137,6 +149,8 @@ def test_multiple_protocols(loop):
137149
port2 = open_port()
138150
with popen(
139151
[
152+
sys.executable,
153+
"-m",
140154
"dask",
141155
"scheduler",
142156
"--protocol=tcp,ws",
@@ -158,6 +172,8 @@ def test_dashboard_allowlist(loop):
158172
port = open_port()
159173
with popen(
160174
[
175+
sys.executable,
176+
"-m",
161177
"dask",
162178
"scheduler",
163179
f"--port={port}",
@@ -198,6 +214,8 @@ def test_interface(loop):
198214
port = open_port()
199215
with popen(
200216
[
217+
sys.executable,
218+
"-m",
201219
"dask",
202220
"scheduler",
203221
f"--port={port}",
@@ -208,6 +226,8 @@ def test_interface(loop):
208226
) as s:
209227
with popen(
210228
[
229+
sys.executable,
230+
"-m",
211231
"dask",
212232
"worker",
213233
f"127.0.0.1:{port}",
@@ -252,12 +272,24 @@ def check_pidfile(proc, pidfile):
252272
assert proc.pid == pid
253273

254274
with tmpfile() as s:
255-
with popen(["dask", "scheduler", "--pid-file", s, "--no-dashboard"]) as sched:
275+
with popen(
276+
[
277+
sys.executable,
278+
"-m",
279+
"dask",
280+
"scheduler",
281+
"--pid-file",
282+
s,
283+
"--no-dashboard",
284+
]
285+
) as sched:
256286
check_pidfile(sched, s)
257287

258288
with tmpfile() as w:
259289
with popen(
260290
[
291+
sys.executable,
292+
"-m",
261293
"dask",
262294
"worker",
263295
f"127.0.0.1:{port}",
@@ -273,6 +305,8 @@ def test_scheduler_port_zero(loop):
273305
with tmpfile() as fn:
274306
with popen(
275307
[
308+
sys.executable,
309+
"-m",
276310
"dask",
277311
"scheduler",
278312
"--no-dashboard",
@@ -292,6 +326,8 @@ def test_dashboard_port_zero(loop):
292326
port = open_port()
293327
with popen(
294328
[
329+
sys.executable,
330+
"-m",
295331
"dask",
296332
"scheduler",
297333
"--host",
@@ -329,6 +365,8 @@ def check_scheduler():
329365
with tmpfile() as fn:
330366
with popen(
331367
[
368+
sys.executable,
369+
"-m",
332370
"dask",
333371
"scheduler",
334372
"--scheduler-file",
@@ -359,6 +397,8 @@ def check_scheduler():
359397
with tmpfile() as fn:
360398
with popen(
361399
[
400+
sys.executable,
401+
"-m",
362402
"dask",
363403
"scheduler",
364404
"--scheduler-file",
@@ -382,6 +422,8 @@ def test_preload_remote_module(loop, tmp_path):
382422
):
383423
with popen(
384424
[
425+
sys.executable,
426+
"-m",
385427
"dask",
386428
"scheduler",
387429
"--scheduler-file",
@@ -408,7 +450,9 @@ def test_preload_config(loop):
408450
with tmpfile() as fn:
409451
env = os.environ.copy()
410452
env["DASK_DISTRIBUTED__SCHEDULER__PRELOAD"] = PRELOAD_TEXT
411-
with popen(["dask", "scheduler", "--scheduler-file", fn], env=env):
453+
with popen(
454+
[sys.executable, "-m", "dask", "scheduler", "--scheduler-file", fn], env=env
455+
):
412456
with Client(scheduler_file=fn, loop=loop) as c:
413457
assert (
414458
c.run_on_scheduler(lambda dask_scheduler: dask_scheduler.foo)
@@ -446,6 +490,8 @@ def check_passthrough():
446490
print(fn)
447491
with popen(
448492
[
493+
sys.executable,
494+
"-m",
449495
"dask",
450496
"scheduler",
451497
"--scheduler-file",
@@ -477,7 +523,16 @@ def check_passthrough():
477523
with tmpfile() as fn2:
478524
print(fn2)
479525
with popen(
480-
["dask", "scheduler", "--scheduler-file", fn2, "--preload", path],
526+
[
527+
sys.executable,
528+
"-m",
529+
"dask",
530+
"scheduler",
531+
"--scheduler-file",
532+
fn2,
533+
"--preload",
534+
path,
535+
],
481536
stdout=sys.stdout,
482537
stderr=sys.stderr,
483538
):
@@ -530,10 +585,20 @@ def dask_setup(worker):
530585
"""
531586
port = open_port()
532587
with popen(
533-
["dask", "scheduler", "--no-dashboard", "--host", f"127.0.0.1:{port}"]
588+
[
589+
sys.executable,
590+
"-m",
591+
"dask",
592+
"scheduler",
593+
"--no-dashboard",
594+
"--host",
595+
f"127.0.0.1:{port}",
596+
]
534597
) as s:
535598
with popen(
536599
[
600+
sys.executable,
601+
"-m",
537602
"dask",
538603
"worker",
539604
f"localhost:{port}",
@@ -555,9 +620,37 @@ def dask_setup(worker):
555620
def test_multiple_workers(loop):
556621
scheduler_address = f"127.0.0.1:{open_port()}"
557622
with (
558-
popen(["dask", "scheduler", "--no-dashboard", "--host", scheduler_address]),
559-
popen(["dask", "worker", scheduler_address, "--no-dashboard"]),
560-
popen(["dask", "worker", scheduler_address, "--no-dashboard"]),
623+
popen(
624+
[
625+
sys.executable,
626+
"-m",
627+
"dask",
628+
"scheduler",
629+
"--no-dashboard",
630+
"--host",
631+
scheduler_address,
632+
]
633+
),
634+
popen(
635+
[
636+
sys.executable,
637+
"-m",
638+
"dask",
639+
"worker",
640+
scheduler_address,
641+
"--no-dashboard",
642+
]
643+
),
644+
popen(
645+
[
646+
sys.executable,
647+
"-m",
648+
"dask",
649+
"worker",
650+
scheduler_address,
651+
"--no-dashboard",
652+
]
653+
),
561654
Client(scheduler_address, loop=loop) as c,
562655
):
563656
start = time()

‎distributed/cli/tests/test_dask_spec.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import json
55
import signal
66
import subprocess
7+
import sys
78

89
import pytest
910
import yaml
@@ -19,6 +20,8 @@ async def test_text():
1920
port = open_port()
2021
with popen(
2122
[
23+
sys.executable,
24+
"-m",
2225
"dask",
2326
"spec",
2427
"--spec",
@@ -27,6 +30,8 @@ async def test_text():
2730
):
2831
with popen(
2932
[
33+
sys.executable,
34+
"-m",
3035
"dask",
3136
"spec",
3237
"tcp://localhost:%d" % port,
@@ -55,6 +60,8 @@ async def test_file(c, s, tmp_path):
5560
)
5661
with popen(
5762
[
63+
sys.executable,
64+
"-m",
5865
"dask",
5966
"spec",
6067
s.address,
@@ -72,6 +79,8 @@ async def test_file(c, s, tmp_path):
7279
def test_errors():
7380
with popen(
7481
[
82+
sys.executable,
83+
"-m",
7584
"dask",
7685
"spec",
7786
"--spec",
@@ -85,7 +94,7 @@ def test_errors():
8594
assert "exactly one" in line
8695
assert "--spec" in line and "--spec-file" in line
8796

88-
with popen(["dask", "spec"], capture_output=True) as proc:
97+
with popen([sys.executable, "-m", "dask", "spec"], capture_output=True) as proc:
8998
line = proc.stdout.readline().decode()
9099
assert "exactly one" in line
91100
assert "--spec" in line and "--spec-file" in line
@@ -97,6 +106,8 @@ def test_errors():
97106
@gen_cluster(client=True, nthreads=[])
98107
async def test_signal_handling_worker(c, s, worker_type, sig):
99108
worker = await asyncio.create_subprocess_exec(
109+
sys.executable,
110+
"-m",
100111
"dask",
101112
"spec",
102113
"--spec",
@@ -136,6 +147,8 @@ async def test_signal_handling_worker(c, s, worker_type, sig):
136147
async def test_signal_handling_scheduler(sig):
137148
port = open_port()
138149
scheduler = await asyncio.create_subprocess_exec(
150+
sys.executable,
151+
"-m",
139152
"dask",
140153
"spec",
141154
"--spec",

0 commit comments

Comments
 (0)