fix(embed): send images when an OpenAI-compatible row declares them (#4389)

An embedding row with spec.input image and no image_format or image_field
was treated as text-only on every vendor except generic and Jina. Default
that case to the vLLM messages format. An explicit object format without a
field stays refused.

Fixes #4343

Author: JSJ189

internal/models/embedding/image_wire_test.go

@@ -154,6 +154,8 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {
 		{"text model on the OpenAI shape", "openai", "text-embedding-3-small", nil, false},
 		{"generic endpoint, image input not declared", "generic", "my-clip", nil, false},
 		{"declared on a generic (vLLM) endpoint", "generic", "my-clip", declare, true},
+		{"declared on the OpenAI provider", "openai", "my-clip", declare, true},
+		{"declared on LiteLLM", "litellm", "my-clip", declare, true},
 		{
 			"declared, but the row switched to an object format without a field", "generic", "my-clip",
 			&types.ModelSpecOverride{
@@ -188,6 +190,35 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {
 // image both pass the chat template and the prompt ends in <|endoftext|>,
 // the token its last-token pooling reads. A plain `input` string skips the
 // template and lands in a different space from the images.
+// An OpenAI-compatible gateway has no image field of its own. Declaring
+// image input, and nothing else, must still send the picture: vLLM takes it
+// as a messages conversation. The text request stays on input.
+func TestDeclaredImageInputOnOpenAIGatewayUsesMessages(t *testing.T) {
+	up := newUpstream(t)
+	declare := &types.ModelSpecOverride{Input: []string{"text", "image"}}
+	e := imageEmbedder(t, up, "openai", "WeMM-Embedding-2B", "/v1", declare)
+	ie, ok := AsImageEmbedder(e)
+	require.True(t, ok)
+
+	_, err := e.BatchEmbed(context.Background(), []string{"bridge"})
+	require.NoError(t, err)
+	images, err := ie.BatchEmbedImages(context.Background(), pngs(1))
+	require.NoError(t, err)
+	assert.Equal(t, [][]float32{{1}}, images)
+
+	require.Len(t, up.requests, 2)
+	assert.Equal(t, map[string]any{
+		"model": "WeMM-Embedding-2B", "encoding_format": "float",
+		"input": []any{"bridge"},
+	}, up.requests[0].body)
+	assert.Equal(t, map[string]any{
+		"model": "WeMM-Embedding-2B", "encoding_format": "float",
+		"messages": []any{map[string]any{"role": "user", "content": []any{
+			map[string]any{"type": "image_url", "image_url": map[string]any{"url": pngs(1)[0].DataURI()}},
+		}}},
+	}, up.requests[1].body)
+}
+
 func TestSelfHostedQwen3VLEmbeddingSendsTextAndImagesThroughTheTemplate(t *testing.T) {
 	up := newUpstream(t)
 	e := imageEmbedder(t, up, "generic", "Qwen/Qwen3-VL-Embedding-8B", "/v1", nil)

internal/models/runtime/resolve.go

@@ -492,6 +492,17 @@ func resolveEmbeddings(
 			"catalog: unknown embedding image_format %q on %s/%s (expected %q or %q)",
 			settings.ImageFormat, vendor.ID, spec.ID, api.EmbeddingImageObject, api.EmbeddingImageMessages)
 	}
+	// A row that declares image input on the OpenAI embedding shape still
+	// needs a wire format: that shape has none of its own, and only Jina
+	// names one. vLLM, the usual server behind these gateways, takes one
+	// chat conversation in messages. Jina and SGLang stay explicit
+	// (image_format=object plus image_field). An object format without a
+	// field is left alone, so a half-filled row is refused instead of resent
+	// as messages. Text stays on input unless the row sets text_as_messages.
+	if protocol == api.EmbeddingOpenAI && spec.AcceptsImages() &&
+		settings.ImageFormat == "" && settings.ImageField == "" {
+		settings.ImageFormat = api.EmbeddingImageMessages
+	}
 
 	out := &Resolved{
 		Vendor:       configcopy.Clone(vendor.Definition),

website-docs/03-features/06-models.md

@@ -89,7 +89,7 @@
 | 需要附带厂商私有参数，如阿里云联网搜索 | `{"extra_body": {"enable_search": true}}` |
 | 多轮对话回传思考内容时报错 | `{"replay_reasoning_content": false}` |
 | 自建向量服务一次请求只接受 16 条 | `{"max_batch_size": 16}` |
-| vLLM 部署的多模态向量模型（VLM2Vec、GME 等） | 打开「图片输入」即可，自定义厂商默认按 vLLM 的 `messages` 格式发图片；模板需要指令时加 `{"image_prompt": "Represent the given image."}`；文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` |
+| vLLM 部署的多模态向量模型（VLM2Vec、GME 等） | 打开「图片输入」即可。未填 `image_format` 和 `image_field` 时，OpenAI 兼容协议按 vLLM 的 `messages` 格式发图片。模板需要指令时加 `{"image_prompt": "Represent the given image."}`；文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` |
 | vLLM 部署的 Qwen3-VL-Embedding | 无需填写：自定义厂商下模型名含 `qwen3-vl-embedding` 时已内置图片输入与 `{"text_as_messages": true, "extra_body": {"add_special_tokens": true}}`。文本走普通 `input` 会绕过对话模板，与图片不在同一空间；不加 `add_special_tokens` 时末尾缺少模型取向量用的 `<\|endoftext\|>`，与[模型卡](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)结果不符。已用该模型建立的索引需重建向量 |
 | SGLang 部署的多模态向量模型 | 打开「图片输入」，并填 `{"image_format": "object", "image_field": "image"}` |
 | 向量或重排服务响应慢，需要更长的超时 | `{"request_timeout_seconds": 120}` |
@@ -235,8 +235,8 @@ WeMM 的模板已经包含所需特殊 token，保留 vLLM 的 `add_special_toke
 | `accepts_truncate_prompt_tokens` | bool | false | 服务是否支持 vLLM 的 `truncate_prompt_tokens` |
 | `request_timeout_seconds` | int | 60 | 单次请求超时（秒） |
 | `extra_body` | object | 空 | 附加到每次请求的字段 |
-| `image_field` | string | 空 | 仅 `openai-embeddings`：`input` 里承载图片的对象键名（Jina、SGLang 为 `image`）。留空且 `image_format` 不是 `messages` 时表示该端点不收图片 |
-| `image_format` | string | 空（自定义厂商为 `messages`） | 仅 `openai-embeddings`：图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`；`messages` 把图片作为一轮对话放进 `messages`（vLLM 的多模态向量接口），每张图一个请求。留空时有 `image_field` 即按 `object` 处理 |
+| `image_field` | string | 空 | 仅 `openai-embeddings`：`input` 里承载图片的对象键名（Jina、SGLang 为 `image`）。模型未声明图片输入、留空、且 `image_format` 不是 `messages` 时，端点不收图片 |
+| `image_format` | string | 空 | 仅 `openai-embeddings`：图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`；`messages` 把图片作为一轮对话放进 `messages`（vLLM 的多模态向量接口）。模型声明了图片输入、本字段和 `image_field` 都留空时，按 `messages` 发送。只填了 `image_field` 时按 `object` 处理。写成 `object` 却没有 `image_field` 时仍不发图片 |
 | `image_prompt` | string | 空 | 仅 `messages` 格式：随图片一起发送的文字，部分模型的对话模板需要（如 VLM2Vec 的 `Represent the given image.`） |
 | `text_as_messages` | bool | false | 仅 `openai-embeddings`：文本也按 `messages` 格式发送，每条一个请求。适用于必须经过对话模板、文本和图片才在同一向量空间的模型（自定义厂商下的 Qwen3-VL-Embedding 已内置）；已有索引的模型改动后需要重建向量 |
 | `max_image_batch_size` | int | 1 | 单次请求的最大图片数，超出自动分批 |