From 5c6fb197d582844fb1d41fbed76b906144587724 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Mon, 13 Jul 2026 10:31:45 -0700 Subject: [PATCH 1/9] LightonOCR-2-1B model verification --- .../config/model-tests/vllm-model-tests.yml | 8 ++++++ .../vllm-sagemaker-endpoint-tests.yml | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/config/model-tests/vllm-model-tests.yml b/.github/config/model-tests/vllm-model-tests.yml index 66323de0c1a4..2a98ef7d3a9a 100644 --- a/.github/config/model-tests/vllm-model-tests.yml +++ b/.github/config/model-tests/vllm-model-tests.yml @@ -48,6 +48,14 @@ smoke-test: fleet: "x86-g6exl-runner" extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --trust-remote-code --mamba_ssm_cache_dtype float32" + # LightOnOCR-2-1B: efficient 1B VLM for document OCR (LightOnOcrForConditionalGeneration). + # Mistral3-based, bf16 (~2GB). Requires multimodal flags for image input. + # Fits a single L4 (g6.xl, 24GB) easily at tp=1. + - name: "lighton-ocr-2-1b" + s3_model: "lighton-ocr-2-1b.tar.gz" + fleet: "x86-g6xl-runner" + extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --limit-mm-per-prompt image=1 --no-enable-prefix-caching" + - name: "qwen3-embedding-0.6b" s3_model: "qwen3-embedding-0.6b.tar.gz" fleet: "x86-g6xl-runner" diff --git a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml index 7ed7f85ca538..6a49d2ee8eb0 100644 --- a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml +++ b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml @@ -70,6 +70,34 @@ sagemaker: temperature: 0 validate: "json_field:choices.0.message.content" + # LightOnOCR-2-1B — efficient 1B VLM for document OCR, bf16 (~2GB). + # LightOnOcrForConditionalGeneration (Mistral3-based). Text-only chat completion + # validates serving; full OCR with image input requires multimodal request support. + - name: "lighton-ocr-2-1b" + s3_model: "lighton-ocr-2-1b.tar.gz" + instance_type: "ml.g6.xlarge" + required_image_pattern: + os_version: "amzn2023" + env: + SM_VLLM_MODEL: "/opt/ml/model" + SM_VLLM_MAX_MODEL_LEN: "4096" + SM_VLLM_DTYPE: "bfloat16" + SM_VLLM_LIMIT_MM_PER_PROMPT: '{"image": 1}' + SM_VLLM_NO_ENABLE_PREFIX_CACHING: "true" + test_cases: + - name: "chat-completion" + route: "/v1/chat/completions" + content_type: "application/json" + request: + body: + model: "/opt/ml/model" + messages: + - role: "user" + content: "What is optical character recognition?" + max_tokens: 256 + temperature: 0 + validate: "json_field:choices.0.message.content" + - name: "voxtral-mini-4b" s3_model: "voxtral-mini-4b.tar.gz" instance_type: "ml.g6.xlarge" From dc5718ec5be2a5f8c0c8275756b66cd29faa5dca Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 08:57:19 -0700 Subject: [PATCH 2/9] lightonocr-2-1b verification --- .../model-tests/vllm-multimodal-benchmark-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml index 46fb0963117f..e8bf9880ef29 100644 --- a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml +++ b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml @@ -28,6 +28,14 @@ benchmark: test_script: "vllm_embedding_benchmark_test.sh" min_rps: 3 + - name: "lightonocr-2-1b" + s3_model: "lightonocr-2-1b.tar.gz" + runner_label: "gpu-l40s-1gpu-runners" + extra_args: "--dtype bfloat16 --max-model-len 16384" + min_throughput: 30 + min_rps: 1 + benchmark_profiles: "baseline,high_concurrency" + # Realtime model — benchmark via /v1/audio/transcriptions; it does not # support chat audio_url (empty MM embeddings crash the engine). - name: "voxtral-mini-4b" From b791f980122dcf356a5c5e019d3baa11ded8b453 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 08:59:25 -0700 Subject: [PATCH 3/9] naming consistency --- .../config/model-tests/vllm-multimodal-benchmark-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml index e8bf9880ef29..c2730f5dcd3a 100644 --- a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml +++ b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml @@ -28,8 +28,8 @@ benchmark: test_script: "vllm_embedding_benchmark_test.sh" min_rps: 3 - - name: "lightonocr-2-1b" - s3_model: "lightonocr-2-1b.tar.gz" + - name: "lighton-ocr-2-1b" + s3_model: "lighton-ocr-2-1b.tar.gz" runner_label: "gpu-l40s-1gpu-runners" extra_args: "--dtype bfloat16 --max-model-len 16384" min_throughput: 30 From 153ff0ed267de0dfeb6e094fb2a7a1a8cba62646 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 11:07:50 -0700 Subject: [PATCH 4/9] renaming model name to match s3 --- .github/config/model-tests/vllm-model-tests.yml | 2 +- .github/config/model-tests/vllm-multimodal-benchmark-tests.yml | 2 +- .github/config/model-tests/vllm-sagemaker-endpoint-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/config/model-tests/vllm-model-tests.yml b/.github/config/model-tests/vllm-model-tests.yml index 2a98ef7d3a9a..1faff4d3b414 100644 --- a/.github/config/model-tests/vllm-model-tests.yml +++ b/.github/config/model-tests/vllm-model-tests.yml @@ -52,7 +52,7 @@ smoke-test: # Mistral3-based, bf16 (~2GB). Requires multimodal flags for image input. # Fits a single L4 (g6.xl, 24GB) easily at tp=1. - name: "lighton-ocr-2-1b" - s3_model: "lighton-ocr-2-1b.tar.gz" + s3_model: "lightonocr-2-1b.tar.gz" fleet: "x86-g6xl-runner" extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --limit-mm-per-prompt image=1 --no-enable-prefix-caching" diff --git a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml index c2730f5dcd3a..e623b8610a1a 100644 --- a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml +++ b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml @@ -29,7 +29,7 @@ benchmark: min_rps: 3 - name: "lighton-ocr-2-1b" - s3_model: "lighton-ocr-2-1b.tar.gz" + s3_model: "lightonocr-2-1b.tar.gz" runner_label: "gpu-l40s-1gpu-runners" extra_args: "--dtype bfloat16 --max-model-len 16384" min_throughput: 30 diff --git a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml index 6a49d2ee8eb0..1cf9b67a4c27 100644 --- a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml +++ b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml @@ -74,7 +74,7 @@ sagemaker: # LightOnOcrForConditionalGeneration (Mistral3-based). Text-only chat completion # validates serving; full OCR with image input requires multimodal request support. - name: "lighton-ocr-2-1b" - s3_model: "lighton-ocr-2-1b.tar.gz" + s3_model: "lightonocr-2-1b.tar.gz" instance_type: "ml.g6.xlarge" required_image_pattern: os_version: "amzn2023" From cbb5c9ce4f786fe90ac7cecce25305abcc2296e1 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 11:16:57 -0700 Subject: [PATCH 5/9] fixing parameters --- .github/config/model-tests/vllm-model-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/model-tests/vllm-model-tests.yml b/.github/config/model-tests/vllm-model-tests.yml index 1faff4d3b414..99ca76d7984b 100644 --- a/.github/config/model-tests/vllm-model-tests.yml +++ b/.github/config/model-tests/vllm-model-tests.yml @@ -54,7 +54,7 @@ smoke-test: - name: "lighton-ocr-2-1b" s3_model: "lightonocr-2-1b.tar.gz" fleet: "x86-g6xl-runner" - extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --limit-mm-per-prompt image=1 --no-enable-prefix-caching" + extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --limit-mm-per-prompt '{\"image\": 1}' --no-enable-prefix-caching" - name: "qwen3-embedding-0.6b" s3_model: "qwen3-embedding-0.6b.tar.gz" From f32e0607edefe24f9acf215a20cd9d1d4a7e0b62 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 11:23:18 -0700 Subject: [PATCH 6/9] removing image tag --- .github/config/model-tests/vllm-model-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/model-tests/vllm-model-tests.yml b/.github/config/model-tests/vllm-model-tests.yml index 99ca76d7984b..6d9e3064cbf6 100644 --- a/.github/config/model-tests/vllm-model-tests.yml +++ b/.github/config/model-tests/vllm-model-tests.yml @@ -54,7 +54,7 @@ smoke-test: - name: "lighton-ocr-2-1b" s3_model: "lightonocr-2-1b.tar.gz" fleet: "x86-g6xl-runner" - extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --limit-mm-per-prompt '{\"image\": 1}' --no-enable-prefix-caching" + extra_args: "--tensor-parallel-size 1 --max-model-len 4096 --dtype bfloat16 --no-enable-prefix-caching" - name: "qwen3-embedding-0.6b" s3_model: "qwen3-embedding-0.6b.tar.gz" From 0d480c86ac111fa25d8e6912566d0d5d6d9296c8 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 12:13:06 -0700 Subject: [PATCH 7/9] added comments after using skill --- .../config/model-tests/vllm-multimodal-benchmark-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml index e623b8610a1a..b487c01c82be 100644 --- a/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml +++ b/.github/config/model-tests/vllm-multimodal-benchmark-tests.yml @@ -28,10 +28,12 @@ benchmark: test_script: "vllm_embedding_benchmark_test.sh" min_rps: 3 + # LightOnOCR-2-1B: 1B VLM for document OCR (LightOnOcrForConditionalGeneration). + # bf16 ~2GB, fits L40S easily. Thresholds are provisional — calibrate after first run. - name: "lighton-ocr-2-1b" s3_model: "lightonocr-2-1b.tar.gz" runner_label: "gpu-l40s-1gpu-runners" - extra_args: "--dtype bfloat16 --max-model-len 16384" + extra_args: "--dtype bfloat16 --max-model-len 4096" min_throughput: 30 min_rps: 1 benchmark_profiles: "baseline,high_concurrency" From 7e46d6d665b34ecac50280be3e9b5c5c20828782 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 17:16:11 -0700 Subject: [PATCH 8/9] removing flags --- .github/config/model-tests/vllm-sagemaker-endpoint-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml index 1cf9b67a4c27..26781b91cc23 100644 --- a/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml +++ b/.github/config/model-tests/vllm-sagemaker-endpoint-tests.yml @@ -82,8 +82,6 @@ sagemaker: SM_VLLM_MODEL: "/opt/ml/model" SM_VLLM_MAX_MODEL_LEN: "4096" SM_VLLM_DTYPE: "bfloat16" - SM_VLLM_LIMIT_MM_PER_PROMPT: '{"image": 1}' - SM_VLLM_NO_ENABLE_PREFIX_CACHING: "true" test_cases: - name: "chat-completion" route: "/v1/chat/completions" From dad5538197b607b29ee354ef29ee37fe5b3a7a54 Mon Sep 17 00:00:00 2001 From: Eric Zhong Date: Tue, 14 Jul 2026 20:50:08 -0700 Subject: [PATCH 9/9] rerun