Skip to content

cleanup(rdna): drop leftover runtime debug probes #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BlivionIaG merged 1 commit into from
Oct 16, 2026
Merged

cleanup(rdna): drop leftover runtime debug probes #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions docs/rdna2/tuning.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,11 +212,9 @@ NaN tracing: `VLLM_{LAYER,MOE,HC,GDN,PLE}_NAN_DEBUG`, `VLLM_RDNA2_MOE_DEBUG_NAN`
Kernel launch logs: `VLLM_RDNA2_W4A8_DEBUG`, `VLLM_RDNA2_W4A8_MOE_DEBUG`,
`VLLM_RDNA2_PREFILL_DEBUG`, `VLLM_QSA_RDNA2_DEBUG`, `VLLM_CONV1D_DEBUG`,
`VLLM_EXL3_DEBUG`, `VLLM_EXL3_HADAMARD_DBG`, `VLLM_PAGED_MQA_DEBUG_OOB`.
Cache/scheduler traces: `VLLM_DEBUG_PREFIX_HIT`, `VLLM_TRACE_PREFIX_CACHE`,
`VLLM_BT_DEBUG`, `VLLM_MAMBA_COPY_DEBUG`. Timing:
`VLLM_RDNA_PHASE_TIMING=<every N steps>` (+ `VLLM_RDNA_OP_TIMING=1|2`),
`VLLM_RDNA_SYNC_DEBUG`, `DBG_VLLM_STEP_TIMING`. Graph-pool memory:
`VLLM_RDNA_POOLWATCH` (+ `_HISTORY`, `_DUMP_ON_REPLAY`). PLE sidecar:
Cache/scheduler traces: `VLLM_DEBUG_PREFIX_HIT`, `VLLM_TRACE_PREFIX_CACHE`.
Timing: `VLLM_RDNA_PHASE_TIMING=<every N steps>` (+ `VLLM_RDNA_OP_TIMING=1|2`),
`VLLM_RDNA_SYNC_DEBUG`. PLE sidecar:
`PLE_OFFLOAD_DEBUG_{HOPS,TRACE,DELAY_MS}`, `PLE_OFFLOAD_FUSED_CHECK`.
GPU profiling: the launchers set `ROCPROFILER_REGISTER_ENABLED=0`, so the HIP/HSA
runtimes never register with rocprofiler (no tool init, no queue interception)
Expand Down
4 changes: 0 additions & 4 deletions vllm/envs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1449,10 +1449,6 @@ def _resolve_rust_cli_path() -> str | None:
"VLLM_USE_RDNA2_FA": lambda: (
os.getenv("VLLM_USE_RDNA2_FA", "False").lower() in ("true", "1")
),
# Per-step phase timing in gpu_model_runner. Diagnostic only, off by default.
"DBG_VLLM_STEP_TIMING": lambda: (
os.getenv("DBG_VLLM_STEP_TIMING", "False").lower() in ("true", "1")
),
# Bypass the "no more than two PCIe-only GPUs" XGMI-topology gate in
# CustomAllreduce. For RDNA systems where PCIe P2P actually works
# (P2PDMA-enabled kernel); init fails loudly if P2P is broken.
Expand Down
106 changes: 0 additions & 106 deletions vllm/utils/rocm_poolwatch.py
View file
Open in desktop

This file was deleted.

24 changes: 0 additions & 24 deletions vllm/v1/worker/gpu/block_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable

import torch

from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.buffer_utils import (
Expand All @@ -15,8 +13,6 @@
_load_ptr,
)

logger = init_logger(__name__)


class BlockTables:
def __init__(
Expand Down Expand Up @@ -186,26 +182,6 @@ def gather_block_tables(
BLOCK_SIZE=1024, # type: ignore
)
result = tuple(bt[:num_reqs_padded] for bt in out)
if os.environ.get("VLLM_BT_DEBUG", "0") == "1":
for gi, bt in enumerate(result):
rows = bt[:num_reqs]
nz = rows.flatten()
nz = nz[nz > 0]
if nz.numel() == 0:
continue
uniq, counts = nz.unique(return_counts=True)
dup = uniq[counts > 1]
if dup.numel() > 0:
logger.warning(
"[bt-debug] group %d n_reqs=%d: %d shared block ids: %s",
gi,
num_reqs,
int(dup.numel()),
dup.tolist()[:10],
)
for bid in dup.tolist()[:5]:
owners = (rows == bid).any(dim=1).nonzero().flatten().tolist()
logger.warning("[bt-debug] block %d in rows %s", bid, owners)
return result

def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]:
Expand Down
74 changes: 6 additions & 68 deletions vllm/v1/worker/gpu/model_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import functools
import gc
import os
import time
from contextlib import AbstractContextManager
from copy import deepcopy
Expand Down Expand Up @@ -185,39 +184,6 @@

logger = init_logger(__name__)

# --- DEBUG: per-step phase timing (gated by DBG_VLLM_STEP_TIMING=1) ---
_DBG_STEP_TIMING = os.environ.get("DBG_VLLM_STEP_TIMING") == "1"
_dbg_phase_ns: dict[str, float] = {}


class _DbgPhase:
__slots__ = ("_name", "_t0")

def __init__(self, name: str):
self._name = name
self._t0 = 0.0

def __enter__(self):
if _DBG_STEP_TIMING:
self._t0 = time.perf_counter_ns()
return self

def __exit__(self, *args):
if _DBG_STEP_TIMING:
dt = time.perf_counter_ns() - self._t0
_dbg_phase_ns[self._name] = _dbg_phase_ns.get(self._name, 0) + dt


def _dbg_flush_step(rank: int, step: int) -> None:
if not _DBG_STEP_TIMING or not _dbg_phase_ns:
return
parts = " ".join(f"{p}={v / 1e3:.1f}us" for p, v in _dbg_phase_ns.items())
logger.info("[STEP_TIMING rank=%d step=%d] %s", rank, step, parts)
_dbg_phase_ns.clear()


_dbg_step_count = 0


class GPUModelRunner(LoRAModelRunnerMixin):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
Expand Down Expand Up @@ -1290,22 +1256,12 @@ def update_requests(self, scheduler_output: SchedulerOutput) -> None:
# Zero GPU memory for freshly allocated cache blocks to prevent
# stale NaN/data from corrupting attention or SSM computation.
if scheduler_output.new_block_ids_to_zero:
if os.environ.get("VLLM_BT_DEBUG", "0") == "1":
logger.warning(
"[zero-debug] zero=%s", scheduler_output.new_block_ids_to_zero
)
assert self.kv_block_zeroer is not None
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)

# Apply copy-on-write block copies for partial prefix-cache hits, after
# zeroing new blocks and before the forward pass reads them.
if scheduler_output.kv_cache_block_copies:
if os.environ.get("VLLM_BT_DEBUG", "0") == "1":
logger.warning(
"[cow-debug] copies=%s zero=%s",
scheduler_output.kv_cache_block_copies,
scheduler_output.new_block_ids_to_zero,
)
copy_kv_cache_blocks_inplace(
self.kv_caches,
self.kv_cache_config.num_blocks,
Expand Down Expand Up @@ -1873,13 +1829,12 @@ def execute_model(
# re-gathered every step, so under PP a deferred postprocess on a
# non-last rank would walk another step's batch mapping through
# freed/reallocated block ids. (Port of vllm-project/vllm#55506.)
with _DbgPhase("preprocess"):
self.model_state.preprocess_state(
input_batch,
tuple(bt.gpu for bt in self.block_tables.block_tables),
self.kv_cache_config,
self.req_states.num_computed_tokens.gpu,
)
self.model_state.preprocess_state(
input_batch,
tuple(bt.gpu for bt in self.block_tables.block_tables),
self.kv_cache_config,
self.req_states.num_computed_tokens.gpu,
)

if self.lora_config:
# Activate LoRA adapters.
Expand Down Expand Up @@ -2048,7 +2003,6 @@ def execute_model(
input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL
)
self.step_timing.forward_start()
_dbg_forward_t0 = time.perf_counter_ns() if _DBG_STEP_TIMING else 0.0
if phase_timer is not None and not dummy_run:
phase_timer.start(input_batch.num_tokens)

Expand Down Expand Up @@ -2152,10 +2106,6 @@ def execute_model(
aux_hidden_states = None
output_intermediate_tensors = model_output

if _DBG_STEP_TIMING:
_dbg_phase_ns["forward"] = _dbg_phase_ns.get("forward", 0) + (
time.perf_counter_ns() - _dbg_forward_t0
)
if phase_timer is not None and not dummy_run:
phase_timer.mark("forward")
if sync_debug is not None and not dummy_run:
Expand Down Expand Up @@ -2188,7 +2138,6 @@ def execute_model(
def sample_tokens(
self, grammar_output: GrammarOutput | None
) -> AsyncOutput | ModelRunnerOutput | None:
global _dbg_step_count
if self.execute_model_state is None:
# The prior execute_model call must have failed.
return None
Expand Down Expand Up @@ -2222,10 +2171,6 @@ def sample_tokens(
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
# The first PP rank holds the encoder cache, so pass its EC output on.
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
if _DBG_STEP_TIMING:
global _dbg_step_count
_dbg_step_count += 1
_dbg_flush_step(os.getpid(), _dbg_step_count)
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)

# Last rank: sample tokens
Expand Down Expand Up @@ -2387,9 +2332,6 @@ def sample_tokens(
model_runner_output.kv_connector_output = kv_connector_output
model_runner_output.ec_connector_output = ec_connector_output

if _DBG_STEP_TIMING:
_dbg_step_count += 1
_dbg_flush_step(os.getpid(), _dbg_step_count)
return async_output

def take_draft_token_ids(self) -> DraftTokenIds | None:
Expand All @@ -2414,10 +2356,6 @@ def pool(self) -> AsyncPoolingOutput | ModelRunnerOutput | None:
if not self.is_last_pp_rank:
self.postprocess_num_computed_tokens(input_batch)
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
if _DBG_STEP_TIMING:
global _dbg_step_count
_dbg_step_count += 1
_dbg_flush_step(os.getpid(), _dbg_step_count)
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)

assert self.pooling_runner is not None
Expand Down
Loading