Skip to content

vllm: set VLLM_NIXL_SIDE_CHANNEL_HOST to node's routable IP #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
esmeetu wants to merge 1 commit into
base: main
Choose a base branch
from
Open

vllm: set VLLM_NIXL_SIDE_CHANNEL_HOST to node's routable IP #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/srtctl/backends/vllm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,18 @@ def get_process_environment(self, process: Process) -> dict[str, str]:
vLLM with dynamo requires unique ports for each worker:
- DYN_VLLM_KV_EVENT_PORT: ZMQ port for KV events publishing
- VLLM_NIXL_SIDE_CHANNEL_PORT: Port for NIXL side channel transfers
- VLLM_NIXL_SIDE_CHANNEL_HOST: Routable IP for NIXL side channel
(vLLM defaults to ``0.0.0.0`` / ``localhost`` which breaks the
multi-node NIXL handshake)
"""
from srtctl.core.slurm import get_hostname_ip

env: dict[str, str] = {}
if process.kv_events_port is not None:
env["DYN_VLLM_KV_EVENT_PORT"] = str(process.kv_events_port)
if process.nixl_port is not None:
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(process.nixl_port)
env["VLLM_NIXL_SIDE_CHANNEL_HOST"] = get_hostname_ip(process.node)
return env

def get_served_model_name(self, default: str) -> str:
Expand Down
7 changes: 6 additions & 1 deletion tests/test_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1297,6 +1297,8 @@ def test_standard_tp_mode_still_works(self):

def test_vllm_get_process_environment(self):
"""Test vLLM sets port environment variables from process."""
from unittest.mock import patch

from srtctl.backends import VLLMProtocol
from srtctl.core.topology import Process

Expand All @@ -1315,10 +1317,12 @@ def test_vllm_get_process_environment(self):
nixl_port=6550,
)

env = backend.get_process_environment(process)
with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.1"):
env = backend.get_process_environment(process)

assert env["DYN_VLLM_KV_EVENT_PORT"] == "5550"
assert env["VLLM_NIXL_SIDE_CHANNEL_PORT"] == "6550"
assert env["VLLM_NIXL_SIDE_CHANNEL_HOST"] == "10.0.0.1"

def test_vllm_get_process_environment_none_ports(self):
"""Test vLLM handles None ports gracefully."""
Expand All @@ -1343,6 +1347,7 @@ def test_vllm_get_process_environment_none_ports(self):

assert "DYN_VLLM_KV_EVENT_PORT" not in env
assert "VLLM_NIXL_SIDE_CHANNEL_PORT" not in env
assert "VLLM_NIXL_SIDE_CHANNEL_HOST" not in env

def test_tp_mode_command_includes_multinode_flags(self):
"""Test standard TP mode includes multi-node coordination flags."""
Expand Down