Multimodal Support: Image and Screenshot Handling for Vision Models (#24)

* fix: enhance OpenAI chat message structure to support multimodal content

* test: add integration and unit tests for multimodal support in OpenRouter API

* chore: update version to 0.4.2 and add multimodal support improvements in changelog

Author: DimazzzZ

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to the OpenRouter IntelliJ Plugin will be documented in this
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.2] - 2026-01-20
+
+### Improvements
+
+- **Multimodal Support** - Added support for image, audio, and video content with AI models
+- **Test Coverage** - Added automated integration tests for multimodal capabilities (images, audio, video)
+- **Test Infrastructure** - Added TestMediaGenerator utility for automated test media generation
+
 ## [0.4.1] - 2025-12-23
 
 ### Bug Fixes

README.md

@@ -1,7 +1,7 @@
 # OpenRouter IntelliJ Plugin
 
 [![JetBrains Plugin](https://img.shields.io/badge/JetBrains-Plugin-orange.svg)](https://plugins.jetbrains.com/plugin/28520)
-[![Version](https://img.shields.io/badge/version-0.4.1-blue.svg)](https://github.com/DimazzzZ/openrouter-intellij-plugin/releases)
+[![Version](https://img.shields.io/badge/version-0.4.2-blue.svg)](https://github.com/DimazzzZ/openrouter-intellij-plugin/releases)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 An IntelliJ IDEA plugin for integrating with [OpenRouter.ai](https://openrouter.ai), providing access to 400+ AI models with usage monitoring, quota tracking, and seamless JetBrains AI Assistant integration.

gradle.properties

@@ -3,7 +3,7 @@
 pluginGroup = org.zhavoronkov
 pluginName = OpenRouter
 pluginRepositoryUrl = https://github.com/DimazzzZ/openrouter-intellij-plugin
-pluginVersion = 0.4.1
+pluginVersion = 0.4.2
 
 # Plugin metadata
 pluginId = org.zhavoronkov.openrouter

src/main/kotlin/org/zhavoronkov/openrouter/models/OpenRouterModels.kt

@@ -238,9 +238,15 @@ data class ChatCompletionRequest(
     val stream: Boolean? = false
 )
 
+/**
+ * Chat message that supports both text-only and multimodal content
+ * Content can be:
+ * - A simple string for text-only messages
+ * - An array of content parts for multimodal messages (text + images/files)
+ */
 data class ChatMessage(
     val role: String, // "system", "user", "assistant"
-    val content: String,
+    val content: com.google.gson.JsonElement, // Can be String or Array of content parts
     val name: String? = null
 )
 

src/main/kotlin/org/zhavoronkov/openrouter/proxy/models/OpenAIModels.kt

@@ -1,5 +1,6 @@
 package org.zhavoronkov.openrouter.proxy.models
 
+import com.google.gson.JsonElement
 import com.google.gson.annotations.SerializedName
 
 /**
@@ -21,12 +22,45 @@ data class OpenAIChatCompletionRequest(
     val user: String? = null
 )
 
+/**
+ * OpenAI chat message that supports both text-only and multimodal content
+ * Content can be:
+ * - A simple string for text-only messages
+ * - An array of content parts for multimodal messages (text + images/files)
+ */
 data class OpenAIChatMessage(
     val role: String, // "system", "user", "assistant"
-    val content: String,
+    val content: JsonElement, // Can be String or Array of ContentPart
     val name: String? = null
 )
 
+/**
+ * Content part for multimodal messages
+ * Used when content is an array
+ */
+data class OpenAIContentPart(
+    val type: String, // "text", "image_url", "input_audio", etc.
+    val text: String? = null, // For type="text"
+    @SerializedName("image_url") val imageUrl: OpenAIImageUrl? = null, // For type="image_url"
+    @SerializedName("input_audio") val inputAudio: OpenAIInputAudio? = null // For type="input_audio"
+)
+
+/**
+ * Image URL content for vision models
+ */
+data class OpenAIImageUrl(
+    val url: String, // URL or base64 data URI
+    val detail: String? = null // "auto", "low", "high"
+)
+
+/**
+ * Audio input content for audio models
+ */
+data class OpenAIInputAudio(
+    val data: String, // Base64 encoded audio data
+    val format: String // "wav", "mp3", etc.
+)
+
 // Chat Completion Response Models
 data class OpenAIChatCompletionResponse(
     val id: String,

src/main/kotlin/org/zhavoronkov/openrouter/proxy/translation/RequestTranslator.kt

@@ -78,7 +78,9 @@ object RequestTranslator {
             // Basic validation
             request.model.isNotBlank() &&
                 request.messages.isNotEmpty() &&
-                request.messages.all { it.role.isNotBlank() && it.content.isNotBlank() } &&
+                request.messages.all { message ->
+                    message.role.isNotBlank() && isValidContent(message.content)
+                } &&
                 (request.temperature == null || request.temperature in 0.0..2.0) &&
                 (request.maxTokens == null || request.maxTokens > 0) &&
                 (request.topP == null || request.topP in 0.0..1.0)
@@ -91,6 +93,19 @@ object RequestTranslator {
         }
     }
 
+    /**
+     * Validates that content is either a non-blank string or a non-empty array
+     */
+    private fun isValidContent(content: com.google.gson.JsonElement): Boolean {
+        return when {
+            content.isJsonPrimitive && content.asJsonPrimitive.isString ->
+                content.asString.isNotBlank()
+            content.isJsonArray ->
+                content.asJsonArray.size() > 0
+            else -> false
+        }
+    }
+
     /**
      * Gets available model mappings for documentation/debugging
      */

src/main/kotlin/org/zhavoronkov/openrouter/proxy/translation/ResponseTranslator.kt

@@ -1,5 +1,6 @@
 package org.zhavoronkov.openrouter.proxy.translation
 
+import com.google.gson.JsonPrimitive
 import org.zhavoronkov.openrouter.models.ChatCompletionResponse
 import org.zhavoronkov.openrouter.models.ProvidersResponse
 import org.zhavoronkov.openrouter.proxy.models.OpenAIChatChoice
@@ -43,7 +44,7 @@ object ResponseTranslator {
                     index = index,
                     message = OpenAIChatMessage(
                         role = choice.message?.role ?: "assistant",
-                        content = choice.message?.content ?: ""
+                        content = choice.message?.content ?: JsonPrimitive("")
                     ),
                     finishReason = choice.finishReason
                 )
@@ -192,7 +193,9 @@ object ResponseTranslator {
                 response.model.isNotBlank() &&
                 response.choices.isNotEmpty() &&
                 response.choices.all { choice ->
-                    choice.message.role.isNotBlank() && choice.message.content.isNotBlank()
+                    choice.message.role.isNotBlank() &&
+                    (choice.message.content.isJsonPrimitive &&
+                     choice.message.content.asString.isNotBlank())
                 }
         } catch (e: NullPointerException) {
             PluginLogger.Service.error("Response validation failed: null value encountered", e)

src/main/kotlin/org/zhavoronkov/openrouter/startup/WhatsNewNotificationActivity.kt

@@ -26,7 +26,7 @@ import java.io.IOException
 class WhatsNewNotificationActivity : ProjectActivity {
 
     companion object {
-        private const val CURRENT_VERSION = "0.4.1"
+        private const val CURRENT_VERSION = "0.4.2"
         private const val CHANGELOG_URL =
             "https://github.com/DimazzzZ/openrouter-intellij-plugin/blob/main/CHANGELOG.md"
     }
@@ -68,16 +68,9 @@ class WhatsNewNotificationActivity : ProjectActivity {
             .createNotification(
                 "OpenRouter Plugin Updated to v$CURRENT_VERSION",
                 """
-                <b>🔧 Bug Fixes:</b><br/>
-                • <b>Fixed AI Assistant Streaming</b> - SSE format now complies with specification<br/>
-                • <b>Stream Termination</b> - All streams properly end with [DONE] marker<br/>
-                • <b>Error Handling</b> - Fixed error response SSE format<br/>
-                <br/>
-                <b>📝 Logging Improvements:</b><br/>
-                • 78% reduction in log noise • Request duration metrics • Standardized request IDs<br/>
-                <br/>
-                <b>🧪 Testing:</b><br/>
-                • 11 new SSE format compliance tests • Regression tests for streaming bug
+                <b>Improvements:</b><br/>
+                • <b>Multimodal Support</b> - Added support for image, audio, and video content with AI models<br/>
+                • <b>Test Coverage</b> - Added automated tests for multimodal capabilities
                 """.trimIndent(),
                 NotificationType.INFORMATION
             )

src/main/resources/META-INF/plugin.xml

@@ -64,6 +64,15 @@
 
     <!-- Change notes for this version -->
     <change-notes><![CDATA[
+        <h3>Version 0.4.2 - Multimodal Support (2026-01-20)</h3>
+        <p><strong>Improvements:</strong></p>
+        <ul>
+            <li><strong>Multimodal Support</strong> - Added support for image, audio, and video content with AI models</li>
+            <li><strong>Test Coverage</strong> - Added automated tests for multimodal capabilities</li>
+        </ul>
+
+        <hr/>
+
         <h3>🔧 Version 0.4.1 - SSE Streaming Fix & Logging Improvements (2025-12-23)</h3>
         <p><strong>🐛 Bug Fixes:</strong></p>
         <ul>