Skip to content

cleanup(rdna): drop leftover bring-up debug probes #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
BlivionIaG wants to merge 1 commit into from
Closed

cleanup(rdna): drop leftover bring-up debug probes #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 0 additions & 31 deletions vllm/compilation/backends.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,23 +103,6 @@ def copy_and_call(*args: Any) -> Any:
input_buffers[i] = staged
staged.copy_(runtime_tensor)
list_args[index] = staged
if (
os.environ.get("VLLM_CG_REPLAY_LOG") == "1"
and runtime_tensor.dtype in (torch.int32, torch.int64)
and runtime_tensor.numel() <= 32
):
n = getattr(copy_and_call, "_id_log_n", 0)
if n < 24:
copy_and_call._id_log_n = n + 1
logger.warning(
"copy_and_call idx=%s runtime_ids=%s static_ids=%s "
"shape=%s stride=%s",
index,
runtime_tensor.flatten()[:8].tolist(),
staged.flatten()[:8].tolist(),
tuple(runtime_tensor.shape),
tuple(staged.stride()),
)
return callable_fn(*list_args)

return copy_and_call
Expand Down Expand Up @@ -1264,20 +1247,6 @@ def __call__(self, graph: fx.GraphModule, example_inputs: Sequence[Any]) -> Any:

self.split_gm, self.piecewise_graphs = split_graph(graph, fx_split_ops)

if os.environ.get("VLLM_PIECE_DUMP") == "1":
try:
os.makedirs("/tmp/piece_dump", exist_ok=True)
with open(f"/tmp/piece_dump/pieces_{os.getpid()}.txt", "w") as _f:
_f.write(f"TP={os.environ.get('VLLM_PIECE_TP', '?')} n_pieces={len(self.piecewise_graphs)}\n")
for name, gm in self.split_gm.named_children():
_f.write(f"piece {name}\n")
for node in gm.graph.nodes:
if node.op == "call_function":
_f.write(f" {node.target}\n")
except Exception as _e:
with open("/tmp/piece_dump/err.txt", "a") as _f:
_f.write(f"{_e}\n")

# keep a split_gm copy from BEFORE the interpreter replaces
# submodules with PiecewiseBackend -- used for serialization
original_split_gm = None
Expand Down
101 changes: 0 additions & 101 deletions vllm/compilation/cuda_graph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,35 +390,6 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any | None:
batch_descriptor = forward_context.batch_descriptor
cudagraph_runtime_mode = forward_context.cudagraph_runtime_mode

if os.environ.get("VLLM_PIECE_IN_DEBUG") == "1":
try:
_pn = getattr(self, "_piece_in_n", 0)
if _pn < 10:
self._piece_in_n = _pn + 1
_rid = (
getattr(self, "submod_name", None)
or getattr(self.runnable, "submod_name", None)
or repr(self.runnable)[:60]
)
_t = self._collect_input_tensors(args, kwargs)
with open(f"/tmp/piece_in_{torch.cuda.current_device()}.log", "a") as _f:
_f.write(f"\n[piece_in] call#{_pn} rank={torch.cuda.current_device()} "
f"bd={batch_descriptor} rid={_rid}\n")
for _i, _x in enumerate(_t):
try:
if _x.is_floating_point() and 0 < _x.numel() <= 2_000_000:
_nan = bool(_x.isnan().any().item())
_v = _x.flatten()[:4].tolist()
_f.write(f" in[{_i}] s={tuple(_x.shape)} d={_x.dtype} "
f"p=0x{_x.data_ptr():x} nan={_nan} v={_v}\n")
else:
_f.write(f" in[{_i}] s={tuple(_x.shape)} d={_x.dtype} "
f"p=0x{_x.data_ptr():x} (big)\n")
except Exception as _e:
_f.write(f" in[{_i}] err={_e}\n")
except Exception:
pass

if (
cudagraph_runtime_mode == CUDAGraphMode.NONE
or cudagraph_runtime_mode != self.runtime_mode
Expand Down Expand Up @@ -539,78 +510,6 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any | None:
and self._copy_tree(args, entry.static_args)
and self._copy_tree(kwargs, entry.static_kwargs)
)
# Opt-in debug scan: per-replay isnan().any() costs blocking host
# syncs every step (~10% of c=8 GPU time). Padded rows are
# row-parallel and arenas are zero-init, so real rows are safe.
if os.environ.get("VLLM_CG_NAN_INPUT_CHECK") == "1":
for t in self._collect_input_tensors(
entry.static_args or (), entry.static_kwargs or {}
):
if (
t.is_floating_point()
and 0 < t.numel() <= 2_000_000
and t.isnan().any()
):
t.nan_to_num_(0.0)
log_replay = os.environ.get("VLLM_CG_REPLAY_LOG") == "1"
if log_replay:
rt = self._collect_input_tensors(args, kwargs)
st = self._collect_input_tensors(
entry.static_args or (), entry.static_kwargs or {}
)
same = sum(
1
for a, b in zip(rt, st)
if a.data_ptr() == b.data_ptr()
)
ids_t = next(
(
t
for t in st
if t.dtype == torch.int32 and t.dim() == 1 and 0 < t.numel() <= 32
),
None,
)
ids = None if ids_t is None else ids_t.flatten()[:8].tolist()
dummy = bool(
ids is not None
and len(ids) >= 2
and ids == ([0, 1] * ((len(ids) + 1) // 2))[: len(ids)]
)
_elog = getattr(self, "_cg_embed_log_n", 0)
_nlog = getattr(self, "_cg_replay_log_n", 0)
if ids is not None and _elog < 24:
self._cg_embed_log_n = _elog + 1
logger.warning(
"cg-replay-embed copied=%s skip=%s n=%s same_ptr=%s "
"dummy=%s input_ids=%s",
copied,
skip_replay,
len(rt),
same,
dummy,
ids,
)
elif ids is None and _nlog < 8:
self._cg_replay_log_n = _nlog + 1
spec = [
(
tuple(t.shape),
str(t.dtype).replace("torch.", ""),
bool(t.isnan().any().item())
if t.is_floating_point() and t.numel() < 2_000_000
else None,
)
for t in rt
]
logger.warning(
"cg-replay copied=%s skip=%s n=%s same_ptr=%s tensors=%s",
copied,
skip_replay,
len(rt),
same,
spec[:12],
)
if skip_replay or not copied:
# Addresses/shapes no longer match the captured graph; do not
# replay warmup tokens. Fall back to the underlying runnable.
Expand Down
12 changes: 1 addition & 11 deletions vllm/distributed/communication_op.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,7 @@


def _tp_all_reduce(input_: torch.Tensor) -> torch.Tensor:
import os
out = get_tp_group().all_reduce(input_)
if os.environ.get("VLLM_W4A16_PTR_DEBUG"):
with open(f"/tmp/ar_ptrs_{torch.cuda.current_device()}.log", "a") as f:
iv = input_[0, :4].tolist() if input_.numel() >= 4 else []
vals = out[0, :4].tolist() if out.numel() >= 4 else []
f.write(
f"ar capt={torch.cuda.is_current_stream_capturing()} "
f"in={input_.data_ptr():#x} iv={iv} out={out.data_ptr():#x} "
f"vals={vals} shape={tuple(out.shape)} same={out.data_ptr() == input_.data_ptr()}\n")
return out
return get_tp_group().all_reduce(input_)


def _tp_all_reduce_fake(input_: torch.Tensor) -> torch.Tensor:
Expand Down
40 changes: 1 addition & 39 deletions vllm/model_executor/kernels/linear/mixed_precision/rdna2_w4a16.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,20 +49,6 @@
W4A8_MIN_ROWS = 33


def _awq_prefill_available() -> bool:
"""Check if the AWQ-native prefill kernel is registered.

hasattr(torch.ops._rocm_C, ...) is unreliable for torch ops because
dir() only shows 'name' for the namespace object. Use a direct
attribute access in a try/except instead.
"""
try:
torch.ops._rocm_C.awq_gemm_rdna2_prefill
return True
except AttributeError:
return False


def _w4a8_lds_fits(k: int, group_size: int) -> bool:
"""Mirror of pick_split_k: some group-aligned split <= 16 fits M_TILE=8
rows of K plus the per-(token, group) scales in 64 KiB of LDS.
Expand Down Expand Up @@ -166,10 +152,6 @@ def _rdna2_w4a16_gemm(
# gptq_gemm_rdna2_prefill. No post-call test, no env access, no
# logging in the traced region.
output = ops.w4a8_gemm_rdna2(x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format)
elif kernel_name == "awq_prefill" and hasattr(ops, "awq_gemm_rdna2_prefill"):
output = ops.awq_gemm_rdna2_prefill(
x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format
)
elif kernel_name == "prefill" and hasattr(ops, "gptq_gemm_rdna2_prefill"):
output = ops.gptq_gemm_rdna2_prefill(
x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format
Expand All @@ -179,29 +161,9 @@ def _rdna2_w4a16_gemm(
x_2d, w_q, w_zp, w_s, w_g_idx, True, use_v2_format, size_bits
)
elif kernel_name == "rdna2_decode" and hasattr(ops, "gptq_gemm_rdna2"):
if os.environ.get("VLLM_W4A16_PTR_DEBUG"):
dev = torch.cuda.current_device()
with open(f"/tmp/w4a16_ptrs_{dev}.log", "a") as f:
xv = x_2d[0, :4].tolist()
f.write(
f"decode m={m} k={k} n={n} "
f"capt={torch.cuda.is_current_stream_capturing()} "
f"x={x_2d.data_ptr():#x} xv={xv} shape={tuple(x_2d.shape)} "
f"stride={tuple(x_2d.stride())} wq={w_q.data_ptr():#x} "
f"wg={w_g_idx.data_ptr():#x} "
f"sz={w_g_idx.numel() if w_g_idx.numel() else 0}\n"
)
output = ops.gptq_gemm_rdna2(x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format)
if os.environ.get("VLLM_W4A16_PTR_DEBUG"):
with open("/tmp/w4a16_ptrs.log", "a") as f:
vals = output[0, :4].tolist()
f.write(f"decode out={output.data_ptr():#x} vals={vals}\n")
else:
if hasattr(ops, "awq_gemm_rdna2_prefill") and use_v2_format:
output = ops.awq_gemm_rdna2_prefill(
x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format
)
elif hasattr(ops, "gptq_gemm_rdna2_prefill"):
if hasattr(ops, "gptq_gemm_rdna2_prefill"):
output = ops.gptq_gemm_rdna2_prefill(
x_2d, w_q, w_zp, w_s, w_g_idx, use_v2_format
)
Expand Down
Loading
Loading