diff --git a/.github/config/model-tests/vllm-model-tests.yml b/.github/config/model-tests/vllm-model-tests.yml index 66323de0c1a4..6d9e3064cbf6 100644 --- a/.github/config/model-tests/vllm-model-tests.yml +++ b/.github/config/model-tests/vllm-model-tests.yml @@ -48,6 +48,14 @@ smoke-test: fleet: "x86-g6exl-runner" extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --trust-remote-code --mamba_ssm_cache_dtype float32" + # LightOnOCR-2-1B: efficient 1B VLM for document OCR (LightOnOcrForConditionalGeneration). + # Mistral3-based, bf16 (~2GB). Requires multimodal flags for image input. + # Fits a single L4 (g6.xl, 24GB) easily at tp=1. + - name: "lighton-ocr-2-1b" + s3_model: "lightonocr-2-1b.tar.gz" + fleet: "x86-g6xl-runner" + extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --no-enable-prefix-caching" + - name: "qwen3-embedding-0.6b" s3_model: "qwen3-embedding-0.6b.tar.gz" fleet: "x86-g6xl-runner" diff --git a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml index 46fb0963117f..b487c01c82be 100644 --- a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml +++ b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml @@ -28,6 +28,16 @@ benchmark: test_script: "vllm_embedding_benchmark_test.sh" min_rps: 3 + # LightOnOCR-2-1B: 1B VLM for document OCR (LightOnOcrForConditionalGeneration). + # bf16 ~2GB, fits L40S easily. Thresholds are provisional — calibrate after first run. + - name: "lighton-ocr-2-1b" + s3_model: "lightonocr-2-1b.tar.gz" + runner_label: "gpu-l40s-1gpu-runners" + extra_args: "--dtype bfloat16 --max-model-len 4096" + min_throughput: 30 + min_rps: 1 + benchmark_profiles: "baseline,high_concurrency" + # Realtime model — benchmark via /v1/audio/transcriptions; it does not # support chat audio_url (empty MM embeddings crash the engine). - name: "voxtral-mini-4b" diff --git a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml index 7ed7f85ca538..26781b91cc23 100644 --- a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml +++ b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml @@ -70,6 +70,32 @@ sagemaker: temperature: 0 validate: "json_field:choices.0.message.content" + # LightOnOCR-2-1B — efficient 1B VLM for document OCR, bf16 (~2GB). + # LightOnOcrForConditionalGeneration (Mistral3-based). Text-only chat completion + # validates serving; full OCR with image input requires multimodal request support. + - name: "lighton-ocr-2-1b" + s3_model: "lightonocr-2-1b.tar.gz" + instance_type: "ml.g6.xlarge" + required_image_pattern: + os_version: "amzn2023" + env: + SM_VLLM_MODEL: "/opt/ml/model" + SM_VLLM_MAX_MODEL_LEN: "4096" + SM_VLLM_DTYPE: "bfloat16" + test_cases: + - name: "chat-completion" + route: "/v1/chat/completions" + content_type: "application/json" + request: + body: + model: "/opt/ml/model" + messages: + - role: "user" + content: "What is optical character recognition?" + max_tokens: 256 + temperature: 0 + validate: "json_field:choices.0.message.content" + - name: "voxtral-mini-4b" s3_model: "voxtral-mini-4b.tar.gz" instance_type: "ml.g6.xlarge"