Skip to content

Commit 2c5745a

Browse files
author
xingonzhang
committed
Pass explicit comp_mask=NUM_LAG_PORTS to query_mlx5_device
pyverbs's default comp_mask=-1 ORs only the mask bits it knows about, and at least some versions do not include MLX5DV_CONTEXT_MASK_NUM_LAG_PORTS in that list. The result is that num_lag_ports stays at 0 and we fall back to the legacy single-rail behaviour even on real LAG fabrics. Passing the bit explicitly (1 << 9, taken from rdma-core's mlx5dv.h) restores the intended behaviour. Verified on H800 + CX-7 2-rail LAG: comp_mask=-1 (default) -> num_lag_ports = 0 comp_mask=1<<9 (this patch) -> num_lag_ports = 2 get_rdma_gbs('mlx5_bond_1') = 50.0 GB/s (was 25.0)
1 parent c3cb61d commit 2c5745a

1 file changed

Lines changed: 9 additions & 1 deletion

File tree

deep_ep/utils/envs.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,13 @@ def check_fast_rdma_atomic_support(nic_name: str = _DEFAULT_NIC_NAME) -> bool:
242242
return False
243243

244244

245+
# MLX5DV_CONTEXT_MASK_NUM_LAG_PORTS from <infiniband/mlx5dv.h>; pyverbs does not
246+
# re-export this constant, and ``query_mlx5_device()`` 's default comp_mask=-1
247+
# (which ORs the masks pyverbs knows about) skips it on at least some versions,
248+
# leaving ``num_lag_ports`` at 0. Pass the bit explicitly.
249+
_MLX5DV_CONTEXT_MASK_NUM_LAG_PORTS = 1 << 9
250+
251+
245252
@functools.lru_cache()
246253
def _query_num_lag_ports(nic_name: str) -> int:
247254
"""
@@ -263,7 +270,8 @@ def _query_num_lag_ports(nic_name: str) -> int:
263270
try:
264271
ctx = Mlx5Context(attr=Mlx5DVContextAttr(), name=nic_name)
265272
try:
266-
num_lag_ports = int(ctx.query_mlx5_device().num_lag_ports or 0)
273+
dv = ctx.query_mlx5_device(comp_mask=_MLX5DV_CONTEXT_MASK_NUM_LAG_PORTS)
274+
num_lag_ports = int(dv.num_lag_ports or 0)
267275
finally:
268276
ctx.close()
269277
return max(num_lag_ports, 1)

0 commit comments

Comments
 (0)