Skip to content

Commit 16bf7b5

Browse files
LuodianLuodian
authored
Update version to 0.5.0 and refine dependencies in pyproject.toml; add Qwen3-VL evaluation scripts for SGLang and vLLM backends. (EvolvingLMMs-Lab#849)
1 parent a6c27b0 commit 16bf7b5

3 files changed

Lines changed: 204 additions & 2 deletions

File tree

‎examples/models/sglang_qwen3vl.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
3+
# Qwen3-VL Evaluation Script with SGLang Backend
4+
# This script demonstrates how to evaluate Qwen3-VL models using SGLang for accelerated inference
5+
#
6+
# Requirements:
7+
# - sglang>=0.4.6
8+
# - qwen-vl-utils
9+
# - CUDA-enabled GPU(s)
10+
#
11+
# Installation:
12+
# uv add "sglang[all]" qwen-vl-utils
13+
# OR
14+
# pip install "sglang[all]>=0.4.6" qwen-vl-utils
15+
16+
# ============================================================================
17+
# Configuration
18+
# ============================================================================
19+
20+
# Model Configuration
21+
# Available Qwen3-VL models:
22+
# - Qwen/Qwen3-VL-30B-A3B-Instruct
23+
# - Qwen/Qwen3-VL-30B-A3B-Thinking
24+
# - Qwen/Qwen3-VL-235B-A22B-Instruct
25+
# - Qwen/Qwen3-VL-235B-A22B-Thinking
26+
MODEL="Qwen/Qwen3-VL-30B-A3B-Instruct"
27+
28+
# Parallelization Settings
29+
# Adjust based on your GPU configuration
30+
TENSOR_PARALLEL_SIZE=4 # Number of GPUs for tensor parallelism (tp_size in SGLang)
31+
32+
# Memory and Performance Settings
33+
GPU_MEMORY_UTILIZATION=0.85 # mem_fraction_static in SGLang (0.0 - 1.0)
34+
BATCH_SIZE=64 # Batch size for evaluation
35+
36+
# SGLang Specific Settings
37+
MAX_PIXELS=1605632 # Maximum pixels for image processing
38+
MIN_PIXELS=784 # Minimum pixels (28x28)
39+
MAX_FRAME_NUM=32 # Maximum number of video frames
40+
THREADS=16 # Number of threads for decoding visuals
41+
42+
# Task Configuration
43+
# Common tasks: mmmu_val, mme, mathvista, ai2d, etc.
44+
TASKS="mmmu_val,mme"
45+
46+
# Output Configuration
47+
OUTPUT_PATH="./logs/qwen3vl_sglang"
48+
LOG_SAMPLES=true
49+
LOG_SUFFIX="qwen3vl_sglang"
50+
51+
# Evaluation Limits (optional)
52+
# LIMIT=100 # Uncomment to limit number of samples (for testing)
53+
54+
# ============================================================================
55+
# Environment Configuration
56+
# ============================================================================
57+
export HF_HOME="${HF_HOME:-$HOME/.cache/huggingface}"
58+
59+
# ============================================================================
60+
# Run Evaluation
61+
# ============================================================================
62+
63+
echo "=========================================="
64+
echo "Qwen3-VL Evaluation with SGLang"
65+
echo "=========================================="
66+
echo "Model: $MODEL"
67+
echo "Tensor Parallel Size: $TENSOR_PARALLEL_SIZE"
68+
echo "Tasks: $TASKS"
69+
echo "Batch Size: $BATCH_SIZE"
70+
echo "Max Pixels: $MAX_PIXELS"
71+
echo "Output Path: $OUTPUT_PATH"
72+
echo "=========================================="
73+
74+
# Build the command
75+
CMD="uv run python -m lmms_eval \
76+
--model sglang_runtime \
77+
--model_args model=${MODEL},tensor_parallel_size=${TENSOR_PARALLEL_SIZE},gpu_memory_utilization=${GPU_MEMORY_UTILIZATION},max_pixels=${MAX_PIXELS},min_pixels=${MIN_PIXELS},max_frame_num=${MAX_FRAME_NUM},threads=${THREADS} \
78+
--tasks ${TASKS} \
79+
--batch_size ${BATCH_SIZE} \
80+
--output_path ${OUTPUT_PATH}"
81+
82+
# Add optional arguments
83+
if [ "$LOG_SAMPLES" = true ]; then
84+
CMD="$CMD --log_samples --log_samples_suffix ${LOG_SUFFIX}"
85+
fi
86+
87+
if [ ! -z "$LIMIT" ]; then
88+
CMD="$CMD --limit ${LIMIT}"
89+
fi
90+
91+
# Execute
92+
echo "Running command:"
93+
echo "$CMD"
94+
echo ""
95+
96+
eval $CMD
97+
98+
echo ""
99+
echo "=========================================="
100+
echo "Evaluation Complete!"
101+
echo "Results saved to: $OUTPUT_PATH"
102+
echo "=========================================="

‎examples/models/vllm_qwen3vl.sh‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/bin/bash
2+
3+
# Qwen3-VL Evaluation Script with vLLM Backend
4+
# This script demonstrates how to evaluate Qwen3-VL models using vLLM for accelerated inference
5+
#
6+
# Requirements:
7+
# - vllm>=0.11.0
8+
# - qwen-vl-utils
9+
# - CUDA-enabled GPU(s)
10+
#
11+
# Installation:
12+
# uv add vllm qwen-vl-utils
13+
# OR
14+
# pip install vllm>=0.11.0 qwen-vl-utils
15+
16+
# ============================================================================
17+
# Configuration
18+
# ============================================================================
19+
20+
# Model Configuration
21+
# Available Qwen3-VL models:
22+
# - Qwen/Qwen3-VL-30B-A3B-Instruct
23+
# - Qwen/Qwen3-VL-30B-A3B-Thinking
24+
# - Qwen/Qwen3-VL-235B-A22B-Instruct
25+
# - Qwen/Qwen3-VL-235B-A22B-Thinking
26+
MODEL="Qwen/Qwen3-VL-30B-A3B-Instruct"
27+
28+
# Parallelization Settings
29+
# Adjust based on your GPU configuration
30+
TENSOR_PARALLEL_SIZE=4 # Number of GPUs for tensor parallelism
31+
DATA_PARALLEL_SIZE=1 # Number of GPUs for data parallelism
32+
33+
# Memory and Performance Settings
34+
GPU_MEMORY_UTILIZATION=0.85 # Fraction of GPU memory to use (0.0 - 1.0)
35+
BATCH_SIZE=64 # Batch size for evaluation
36+
37+
# Task Configuration
38+
# Common tasks: mmmu_val, mme, mathvista, ai2d, etc.
39+
TASKS="mmmu_val,mme"
40+
41+
# Output Configuration
42+
OUTPUT_PATH="./logs/qwen3vl_vllm"
43+
LOG_SAMPLES=true
44+
LOG_SUFFIX="qwen3vl_vllm"
45+
46+
# Evaluation Limits (optional)
47+
# LIMIT=100 # Uncomment to limit number of samples (for testing)
48+
49+
# ============================================================================
50+
# NCCL Configuration (for multi-GPU setups)
51+
# ============================================================================
52+
export NCCL_BLOCKING_WAIT=1
53+
export NCCL_TIMEOUT=18000000
54+
# export NCCL_DEBUG=INFO # Uncomment for debugging
55+
56+
# ============================================================================
57+
# Run Evaluation
58+
# ============================================================================
59+
60+
echo "=========================================="
61+
echo "Qwen3-VL Evaluation with vLLM"
62+
echo "=========================================="
63+
echo "Model: $MODEL"
64+
echo "Tensor Parallel Size: $TENSOR_PARALLEL_SIZE"
65+
echo "Data Parallel Size: $DATA_PARALLEL_SIZE"
66+
echo "Tasks: $TASKS"
67+
echo "Batch Size: $BATCH_SIZE"
68+
echo "Output Path: $OUTPUT_PATH"
69+
echo "=========================================="
70+
71+
# Build the command
72+
CMD="uv run python -m lmms_eval \
73+
--model vllm_chat \
74+
--model_args model=${MODEL},tensor_parallel_size=${TENSOR_PARALLEL_SIZE},data_parallel_size=${DATA_PARALLEL_SIZE},gpu_memory_utilization=${GPU_MEMORY_UTILIZATION} \
75+
--tasks ${TASKS} \
76+
--batch_size ${BATCH_SIZE} \
77+
--output_path ${OUTPUT_PATH}"
78+
79+
# Add optional arguments
80+
if [ "$LOG_SAMPLES" = true ]; then
81+
CMD="$CMD --log_samples --log_samples_suffix ${LOG_SUFFIX}"
82+
fi
83+
84+
if [ ! -z "$LIMIT" ]; then
85+
CMD="$CMD --limit ${LIMIT}"
86+
fi
87+
88+
# Execute
89+
echo "Running command:"
90+
echo "$CMD"
91+
echo ""
92+
93+
eval $CMD
94+
95+
echo ""
96+
echo "=========================================="
97+
echo "Evaluation Complete!"
98+
echo "Results saved to: $OUTPUT_PATH"
99+
echo "=========================================="

‎pyproject.toml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
77

88
[project]
99
name = "lmms_eval"
10-
version = "0.4.1"
10+
version = "0.5.0"
1111
authors = [
1212
{ name = "LMMMs-Lab Evaluation Team", email = "lmms-lab@outlook.com" },
1313
]
@@ -43,7 +43,7 @@ dependencies = [
4343
"ftfy",
4444
"openai",
4545
"opencv-python-headless",
46-
"av",
46+
"av<16.0.0",
4747
"hf_transfer",
4848
"nltk",
4949
"sentencepiece",
@@ -74,6 +74,7 @@ dependencies = [
7474
"protobuf",
7575
"sentence_transformers",
7676
"python-dotenv",
77+
"qwen-vl-utils>=0.0.14",
7778
]
7879

7980
[project.optional-dependencies]

0 commit comments

Comments
 (0)