[Usage]: Run multi-models

[Potabk]

Your current environment

The output of above commands

How would you like to use vllm on ascend

This script runs deepseek-ocr and paddle-ocr-vl and hunyuan-ocr

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""

import os
import random
from contextlib import contextmanager
from dataclasses import asdict
from typing import NamedTuple

from vllm import LLM, EngineArgs, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.assets.video import VideoAsset
from vllm.lora.request import LoRARequest
from vllm.multimodal.image import convert_image_mode
from vllm.utils.argparse_utils import FlexibleArgumentParser


class ModelRequestData(NamedTuple):
    engine_args: EngineArgs
    prompts: list[str]
    stop_token_ids: list[int] | None = None
    lora_requests: list[LoRARequest] | None = None
    sampling_params: list[SamplingParams] | None = None

def run_deepseek_ocr(questions: list[str], modality: str) -> ModelRequestData:
    from vllm.model_executor.models.deepseek_ocr import NGramPerReqLogitsProcessor

    assert modality == "image"

    model_name = "deepseek-ai/DeepSeek-OCR"

    engine_args = EngineArgs(
        model=model_name,
        limit_mm_per_prompt={modality: 1},
        logits_processors=[NGramPerReqLogitsProcessor],
    )

    # deepseek-ocr use plain prompt template
    prompts = [f"<image>\n{question}" for question in questions]

    # The following sampling params config is taken from
    # the official Deepseek-OCR inference example.
    # (IMPORTANT) Use the custom logits processor and avoid skipping
    # special tokens for this model for the optimal OCR performance.
    sampling_params = [
        SamplingParams(
            temperature=0.0,
            max_tokens=8192,
            # ngram logit processor args
            extra_args=dict(
                ngram_size=30,
                window_size=90,
                # whitelist: <td>, </td>
                whitelist_token_ids={128821, 128822},
            ),
            skip_special_tokens=False,
        )
        for _ in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        sampling_params=sampling_params,
    )

# PaddleOCR-VL
def run_paddleocr_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "PaddlePaddle/PaddleOCR-VL"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
    )

    placeholder = "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>"
    prompts = [
        (f"<|begin_of_sentence|>User: {question}{placeholder}\nAssistant: ")
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )

# HunyuanOCR
def run_hunyuan_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "Tencent-Hunyuan/HunyuanOCR"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        limit_mm_per_prompt={modality: 1},
    )

    placeholder = "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>"  # noqa: E501
    prompts = [
        f"<｜hy_begin▁of▁sentence｜>{placeholder}{question}<｜hy_User｜>"
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=None,
    )




model_example_map = {
    "paddleocr_vl": run_paddleocr_vl,
    "deepseek_ocr": run_deepseek_ocr,
    "hunyuan_vl": run_hunyuan_vl,
}


MODELS_NEED_VIDEO_METADATA = [
    "glm4_1v",
    "glm4_5v",
    "glm4_5v_fp8",
    "qwen3_vl",
    "qwen3_vl_moe",
]


def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
        img_questions = [
            "What is the content of this image?",
            "Describe the content of this image in detail.",
            "What's in the image?",
            "Where is this image taken?",
        ]

        return {
            "data": image,
            "questions": img_questions,
        }

    if args.modality == "video":
        # Input video and question
        needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
        video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
        metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
        vid_questions = ["Why is this video funny?"]

        return {
            "data": ([(video, metadata)] if needs_metadata else video),
            "questions": vid_questions,
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


def apply_image_repeat(
    image_repeat_prob, num_prompts, data, prompts: list[str], modality
):
    """Repeats images with provided probability of "image_repeat_prob".
    Used to simulate hit/miss for the MM preprocessor cache.
    """
    assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    inputs_with_empty_media = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

        uuid = "uuid_{}".format(i)

        inputs.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: cur_image},
                "multi_modal_uuids": {modality: uuid},
            }
        )

        inputs_with_empty_media.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: None},
                "multi_modal_uuids": {modality: uuid},
            }
        )

    return inputs, inputs_with_empty_media


@contextmanager
def time_counter(enable: bool):
    if enable:
        import time

        start_time = time.time()
        yield
        elapsed_time = time.time() - start_time
        print("-" * 50)
        print("-- generate time = {}".format(elapsed_time))
        print("-" * 50)
    else:
        yield


def parse_args():
    parser = FlexibleArgumentParser(
        description="Demo on using vLLM for offline inference with "
        "vision language models for text generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="paddleocr_vl",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--num-prompts", type=int, default=4, help="Number of prompts to run."
    )
    parser.add_argument(
        "--modality",
        type=str,
        default="image",
        choices=["image", "video"],
        help="Modality of the input.",
    )
    parser.add_argument(
        "--num-frames",
        type=int,
        default=16,
        help="Number of frames to extract from the video.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=0,
        help="Set the seed when initializing `vllm.LLM`.",
    )

    parser.add_argument(
        "--image-repeat-prob",
        type=float,
        default=None,
        help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
    )

    parser.add_argument(
        "--disable-mm-processor-cache",
        action="store_true",
        help="If True, disables caching of multi-modal processor.",
    )

    parser.add_argument(
        "--time-generate",
        action="store_true",
        help="If True, then print the total generate() call time",
    )

    parser.add_argument(
        "--use-different-prompt-per-request",
        action="store_true",
        help="If True, then use different prompt (with the same multi-modal "
        "data) for each request.",
    )

    parser.add_argument(
        "--verify-mm-cache-hit-with-uuids",
        action="store_true",
        help="If True, will send all requests in a second batch with empty mm "
        "data to verify cache hits with UUIDs.",
    )
    parser.add_argument(
        "--tensor-parallel-size",
        "-tp",
        type=int,
        default=None,
        help="Tensor parallel size to override the model's default setting. ",
    )
    return parser.parse_args()


def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

    if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:
        raise ValueError(
            f"tensor_parallel_size must be a positive integer, "
            f"got {args.tensor_parallel_size}"
        )

    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
    questions = mm_input["questions"]

    req_data = model_example_map[model](questions, modality)

    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
        req_data.engine_args.limit_mm_per_prompt or {}
    )

    engine_args = asdict(req_data.engine_args) | {
        "seed": args.seed,
        "mm_processor_cache_gb": 0 if args.disable_mm_processor_cache else 4,
    }
    if args.tensor_parallel_size is not None:
        engine_args["tensor_parallel_size"] = args.tensor_parallel_size
    llm = LLM(**engine_args)

    # Don't want to check the flag multiple times, so just hijack `prompts`.
    prompts = (
        req_data.prompts
        if args.use_different_prompt_per_request
        else [req_data.prompts[0]]
    )

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
    sampling_params = (
        SamplingParams(
            temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
        )
        if req_data.sampling_params is None
        else req_data.sampling_params
    )

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        uuid = "uuid_0"
        inputs = {
            "prompt": prompts[0],
            "multi_modal_data": {modality: data},
            "multi_modal_uuids": {modality: uuid},
        }
        inputs_with_empty_media = {
            "prompt": prompts[0],
            "multi_modal_data": {modality: None},
            "multi_modal_uuids": {modality: uuid},
        }
    else:
        # Batch inference
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
            inputs, inputs_with_empty_media = apply_image_repeat(
                args.image_repeat_prob,
                args.num_prompts,
                data,
                prompts,
                modality,
            )
        else:
            # Use the same image for all prompts
            inputs = []
            inputs_with_empty_media = []
            for i in range(args.num_prompts):
                uuid = "uuid_{}".format(i)
                inputs.append(
                    {
                        "prompt": prompts[i % len(prompts)],
                        "multi_modal_data": {modality: data},
                        "multi_modal_uuids": {modality: uuid},
                    }
                )
                inputs_with_empty_media.append(
                    {
                        "prompt": prompts[i % len(prompts)],
                        "multi_modal_data": {modality: None},
                        "multi_modal_uuids": {modality: uuid},
                    }
                )

    # Add LoRA request if applicable
    lora_request = (
        req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
    )

    with time_counter(args.time_generate):
        outputs = llm.generate(
            inputs,
            sampling_params=sampling_params,
            lora_request=lora_request,
        )

    print("-" * 50)
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
        print("-" * 50)

    if args.verify_mm_cache_hit_with_uuids:
        try:
            # Verify cache hits with UUIDs
            print(
                "Sending a second batch of requests with empty media"
                " and matching UUIDs."
            )
            outputs = llm.generate(
                inputs_with_empty_media,
                sampling_params=sampling_params,
                lora_request=lora_request,
            )
            print("-" * 50)
            for o in outputs:
                generated_text = o.outputs[0].text
                print(generated_text)
                print("-" * 50)
        except Exception as e:
            print(f"Failed to verify cache hits with UUIDs. Error: {e}")


if __name__ == "__main__":
    args = parse_args()
    main(args)

for example:

export VLLM_USE_MODELSCOPE=true
python run_vision_language.py --model-type paddleocr_vl

[Potabk]

some tiny tests:

• DeepSeek-OCR

python run_vision_language.py --model-type deepseek_ocr

result:

The image captures the majestic Tokyo Skytree, a renowned landmark in Japan. The tower, painted in a pristine white, stands tall against the backdrop of a clear blue sky. The perspective of the photo is from a low angle, giving the viewer a sense of the tower's impressive height. The tower is surrounded by a sea of pink cherry blossom trees, their delicate petals adding a splash of color to the scene. The blossoms are in full bloom, their pink hues contrasting beautifully with the blue of the sky. The image does not contain any discernible text. The relative positions of the objects suggest that the viewer is standing at a distance from the tower, looking up towards it. The cherry blossom trees are scattered around the base of the tower, creating a natural frame that draws the eye towards the tower.
--------------------------------------------------
The image captures the majestic Tokyo Skytree, a renowned landmark in Japan. The tower, painted in a pristine white, stands tall against the backdrop of a clear blue sky. The perspective of the photo is from a low angle, giving the viewer a sense of the tower's impressive height. The tower is surrounded by a sea of pink cherry blossom trees, their delicate petals adding a splash of color to the scene. The blossoms are in full bloom, their pink hues contrasting beautifully with the blue of the sky. The image does not contain any discernible text. The relative positions of the objects suggest that the viewer is standing at a distance from the tower, looking up towards it. The cherry blossom trees are scattered around the base of the tower, creating a natural frame that draws the eye towards the tower.
--------------------------------------------------
The image captures the majestic Tokyo Skytree, a renowned landmark in Japan. The tower, painted in a pristine white, stands tall against the backdrop of a clear blue sky. The perspective of the photo is from a low angle, giving the viewer a sense of the tower's impressive height. The tower is surrounded by a sea of pink cherry blossom trees, their delicate petals adding a splash of color to the scene. The blossoms are in full bloom, their pink hues contrasting beautifully with the blue of the sky. The image does not contain any discernible text. The relative positions of the objects suggest that the viewer is standing at a distance from the tower, looking up towards it. The cherry blossom trees are scattered around the base of the tower, creating a natural frame that draws the eye towards the tower.
--------------------------------------------------
The image captures the majestic Tokyo Skytree, a renowned landmark in Japan. The tower, painted in a pristine white, stands tall against the backdrop of a clear blue sky. The perspective of the photo is from a low angle, giving the viewer a sense of the tower's impressive height. The tower is surrounded by a sea of pink cherry blossom trees, their delicate petals adding a splash of color to the scene. The blossoms are in full bloom, their pink hues contrasting beautifully with the blue of the sky. The image does not contain any discernible text. The relative positions of the objects suggest that the viewer is standing at a distance from the tower, looking up towards it. The cherry blossom trees are scattered around the base of the tower, creating a natural frame that draws the eye towards the tower.
--------------------------------------------------

• HUNYUAN-OCR-VL

python run_vision_language.py --mode-type hunyuan_vl

result

[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:40 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
(EngineCore_DP0 pid=38356) Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
(EngineCore_DP0 pid=38356) `use_fast` is set to `True` but the image processor class does not have a fast version.  Falling back to the slow version.
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:43 [model_runner_v1.py:2336] Starting to load model Tencent-Hunyuan/HunyuanOCR...
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:43 [layer.py:500] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder.
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:43 [platform.py:234] PIECEWISE compilation enabled on NPU. use_inductor not supported - using only ACL Graph mode
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:43 [utils.py:524] Calculated maximum supported batch sizes for ACL graph: 72
(EngineCore_DP0 pid=38356) WARNING 12-13 12:48:43 [utils.py:527] Currently, communication is performed using FFTS+ method, which reduces the number of available streams and, as a result, limits the range of runtime shapes that can be handled. To both improve communication performance and increase the number of supported shapes, set HCCL_OP_EXPANSION_MODE=AIV.
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:43 [utils.py:557] No adjustment needed for ACL graph batch sizes: HunYuanDenseV1ForCausalLM model (layers: 24) with 51 sizes
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:44 [compilation.py:903] Using OOT custom backend for compilation.
(EngineCore_DP0 pid=38356) Downloading Model from https://www.modelscope.cn to directory: /root/.cache/modelscope/hub/models/Tencent-Hunyuan/HunyuanOCR
Loading safetensors checkpoint shards:   0% Completed | 0/4 [00:00<?, ?it/s]
Loading safetensors checkpoint shards:  25% Completed | 1/4 [00:01<00:03,  1.08s/it]
Loading safetensors checkpoint shards:  50% Completed | 2/4 [00:02<00:02,  1.07s/it]
Loading safetensors checkpoint shards:  75% Completed | 3/4 [00:03<00:01,  1.27s/it]
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00,  1.20s/it]
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00,  1.19s/it]
(EngineCore_DP0 pid=38356)
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:50 [default_loader.py:308] Loading weights took 4.79 seconds
(EngineCore_DP0 pid=38356) INFO 12-13 12:48:55 [model_runner_v1.py:2362] Loading model weights took 1.8976 GB
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843] EngineCore failed to start.
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843] Traceback (most recent call last):
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 834, in run_engine_core
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 610, in __init__
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     super().__init__(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 235, in _initialize_kv_caches
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     available_gpu_memory = self.model_executor.determine_available_memory()
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/executor/abstract.py", line 126, in determine_available_memory
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return self.collective_rpc("determine_available_memory")
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     result = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/v1/serial_utils.py", line 479, in run_method
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return func(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 241, in determine_available_memory
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     self.model_runner.profile_run()
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2285, in profile_run
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states = self._dummy_run(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                     ^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return func(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2252, in _dummy_run
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states = self._generate_dummy_run_hidden_states(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2033, in _generate_dummy_run_hidden_states
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states = self.model(input_ids=input_ids,
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return self._call_impl(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return forward_call(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_vision.py", line 999, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states = self.language_model(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                     ^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return self._call_impl(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return forward_call(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 946, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     model_output = self.model(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]                    ^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 514, in __call__
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     output = TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/compilation/wrapper.py", line 171, in __call__
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     return self._compiled_callable(*args, **kwargs)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 745, in compile_wrapper
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     raise e.with_traceback(None) from e.__cause__  # User compiler error
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843] torch._dynamo.exc.Unsupported: Data-dependent assertion failed (cannot compile partial graph)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Explanation: Dynamo has determined when encountering a data-dependent assert failure that it should not compile the partial graph.
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Hint: This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround.
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Hint: Use `torch._assert()` to raise a hard AssertionError when the check fails. This error will propagate back the user code that called the compiled function (i.e. Dynamo will not trace any exception handling).
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Hint: Remove the assert statement.
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Hint: Move the assert statement outside of any context managers in order to graph break with partial graph compilation (if fullgraph=False).
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   Developer debug context: value: ConstantVariable(bool: False)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843] from user code:
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]    File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 661, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states, residual, kv_states = layer(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 568, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     hidden_states, ori_kv_states = self.self_attn(
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 229, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     q, k = self.rotary_emb(positions, q, k)
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]   File "/vllm-workspace/vllm/vllm/model_executor/layers/rotary_embedding/xdrope.py", line 54, in forward
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]     assert positions.ndim == 2
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
(EngineCore_DP0 pid=38356) ERROR 12-13 12:48:57 [core.py:843]
(EngineCore_DP0 pid=38356) Process EngineCore_DP0:
(EngineCore_DP0 pid=38356) Traceback (most recent call last):
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
(EngineCore_DP0 pid=38356)     self.run()
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=38356)     self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 847, in run_engine_core
(EngineCore_DP0 pid=38356)     raise e
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 834, in run_engine_core
(EngineCore_DP0 pid=38356)     engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=38356)                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 610, in __init__
(EngineCore_DP0 pid=38356)     super().__init__(
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=38356)     num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=38356)                                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 235, in _initialize_kv_caches
(EngineCore_DP0 pid=38356)     available_gpu_memory = self.model_executor.determine_available_memory()
(EngineCore_DP0 pid=38356)                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/executor/abstract.py", line 126, in determine_available_memory
(EngineCore_DP0 pid=38356)     return self.collective_rpc("determine_available_memory")
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc
(EngineCore_DP0 pid=38356)     result = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=38356)              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/v1/serial_utils.py", line 479, in run_method
(EngineCore_DP0 pid=38356)     return func(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 241, in determine_available_memory
(EngineCore_DP0 pid=38356)     self.model_runner.profile_run()
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2285, in profile_run
(EngineCore_DP0 pid=38356)     hidden_states = self._dummy_run(
(EngineCore_DP0 pid=38356)                     ^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
(EngineCore_DP0 pid=38356)     return func(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2252, in _dummy_run
(EngineCore_DP0 pid=38356)     hidden_states = self._generate_dummy_run_hidden_states(
(EngineCore_DP0 pid=38356)                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2033, in _generate_dummy_run_hidden_states
(EngineCore_DP0 pid=38356)     hidden_states = self.model(input_ids=input_ids,
(EngineCore_DP0 pid=38356)                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
(EngineCore_DP0 pid=38356)     return self._call_impl(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
(EngineCore_DP0 pid=38356)     return forward_call(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_vision.py", line 999, in forward
(EngineCore_DP0 pid=38356)     hidden_states = self.language_model(
(EngineCore_DP0 pid=38356)                     ^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
(EngineCore_DP0 pid=38356)     return self._call_impl(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
(EngineCore_DP0 pid=38356)     return forward_call(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 946, in forward
(EngineCore_DP0 pid=38356)     model_output = self.model(
(EngineCore_DP0 pid=38356)                    ^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 514, in __call__
(EngineCore_DP0 pid=38356)     output = TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs)
(EngineCore_DP0 pid=38356)              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/compilation/wrapper.py", line 171, in __call__
(EngineCore_DP0 pid=38356)     return self._compiled_callable(*args, **kwargs)
(EngineCore_DP0 pid=38356)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356)   File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 745, in compile_wrapper
(EngineCore_DP0 pid=38356)     raise e.with_traceback(None) from e.__cause__  # User compiler error
(EngineCore_DP0 pid=38356)     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=38356) torch._dynamo.exc.Unsupported: Data-dependent assertion failed (cannot compile partial graph)
(EngineCore_DP0 pid=38356)   Explanation: Dynamo has determined when encountering a data-dependent assert failure that it should not compile the partial graph.
(EngineCore_DP0 pid=38356)   Hint: This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround.
(EngineCore_DP0 pid=38356)   Hint: Use `torch._assert()` to raise a hard AssertionError when the check fails. This error will propagate back the user code that called the compiled function (i.e. Dynamo will not trace any exception handling).
(EngineCore_DP0 pid=38356)   Hint: Remove the assert statement.
(EngineCore_DP0 pid=38356)   Hint: Move the assert statement outside of any context managers in order to graph break with partial graph compilation (if fullgraph=False).
(EngineCore_DP0 pid=38356)
(EngineCore_DP0 pid=38356)   Developer debug context: value: ConstantVariable(bool: False)
(EngineCore_DP0 pid=38356)
(EngineCore_DP0 pid=38356)
(EngineCore_DP0 pid=38356) from user code:
(EngineCore_DP0 pid=38356)    File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 661, in forward
(EngineCore_DP0 pid=38356)     hidden_states, residual, kv_states = layer(
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 568, in forward
(EngineCore_DP0 pid=38356)     hidden_states, ori_kv_states = self.self_attn(
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/model_executor/models/hunyuan_v1.py", line 229, in forward
(EngineCore_DP0 pid=38356)     q, k = self.rotary_emb(positions, q, k)
(EngineCore_DP0 pid=38356)   File "/vllm-workspace/vllm/vllm/model_executor/layers/rotary_embedding/xdrope.py", line 54, in forward
(EngineCore_DP0 pid=38356)     assert positions.ndim == 2
(EngineCore_DP0 pid=38356)
(EngineCore_DP0 pid=38356) Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
(EngineCore_DP0 pid=38356)
Traceback (most recent call last):
  File "/workspace/run_vision_language.py", line 452, in <module>
    main(args)
  File "/workspace/run_vision_language.py", line 346, in main
    llm = LLM(**engine_args)
          ^^^^^^^^^^^^^^^^^^
  File "/vllm-workspace/vllm/vllm/entrypoints/llm.py", line 334, in __init__
    self.llm_engine = LLMEngine.from_engine_args(
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/vllm-workspace/vllm/vllm/v1/engine/llm_engine.py", line 183, in from_engine_args
    return cls(
           ^^^^
  File "/vllm-workspace/vllm/vllm/v1/engine/llm_engine.py", line 109, in __init__
    self.engine_core = EngineCoreClient.make_client(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 93, in make_client
    return SyncMPClient(vllm_config, executor_class, log_stats)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 642, in __init__
    super().__init__(
  File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 471, in __init__
    with launch_core_engines(vllm_config, executor_class, log_stats) as (
  File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 144, in __exit__
    next(self.gen)
  File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 903, in launch_core_engines
    wait_for_engine_startup(
  File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup

• PaddlePaddle/PaddleOCR-VL

python run_vision_language.py --model-type paddleocr_vl

Adding requests: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:05<00:00,  1.45s/it]
Processed prompts:  75%|████████████████████████        | 3/4 [00:01<00:00,  2.27it/s, est. speed input: 5287.52 toks/s, output: 1.99 toks/s](EngineCore_DP0 pid=41353) INFO 12-13 13:01:13 [acl_graph.py:185] Replaying aclgraph
Processed prompts: 100%|███████████████████████████████| 4/4 [00:01<00:00,  2.25it/s, est. speed input: 5997.15 toks/s, output: 24.21 toks/s]
--------------------------------------------------
The provided image is a logo or a graphic design, not a chart or graph. It does not contain any data, axes, or labels that can be extracted and converted into a markdown table.
--------------------------------------------------

--------------------------------------------------

--------------------------------------------------

--------------------------------------------------