Add VLM inference infrastructure: engine, protocol, and CLI support (#65)

* Add VLM inference infrastructure: engine, protocol, and CLI support

Runtime:
- MultimodalInferenceEngine protocol with encodeImage() and generate()
- CoreAISequentialVLMEngine: vision encoder + projector + embed_tokens +
  LLM decoder with scatter-merge of image embeddings at placeholder positions
- EmbeddedInput type wrapping NDArray embeddings with position metadata
- VisionConfig in LanguageConfig for image_size, patch_size, token count/id
- LanguageBundle parses top-level "vision" block from metadata.json

CLI (llm-runner):
- --image flag routes through VLM engine when bundle kind is .vlm
- Chat template detection with generic fallback for prompt construction
- Accumulated token decode for correct spacing
- Stop sequence support in VLM path

Supports any VLM that exports 3 components (vision.aimodel, embed.aimodel,
model.aimodel) with a vision config block in metadata.json. Model-family-
specific export code lives in internal/python.

* VisionConfig: add image normalization fields (mean, std, rescaleFactor)

Read per-channel normalization from metadata instead of hardcoding.
Fields are optional — bundles without them default to CLIP values
(the most common across VLMs). Gemma/SigLIP bundles specify their
own [0.5, 0.5, 0.5] values explicitly in metadata.json.

* VLM cleanup: enforce VisionConfig, generalize EmbeddedInput, support bfloat16

- LanguageBundle init throws if a .vlm bundle omits the vision block
- Rename imageTokenPositions → embeddingPositions (supports future
  audio/multi-modal embedding injection, not just images)
- Accept bfloat16 logits in addition to float16
- Scatter merge uses UInt16 view (type-agnostic for f16/bf16)

* Format changes

* Fix potential infinite loop in KV cache growth, suppress unused-arg warnings

- Guard against currentKVCapacity==0 in growKVCache (would loop forever)
- Prefix unused protocol args with _ in warmup()

* Update InferenceEngine.swift

Author: stikves

swift/Sources/CoreAILanguageModels/Bundle/LanguageBundle.swift

@@ -21,6 +21,7 @@ public struct LanguageBundle: Sendable {
     public let bundle: ModelBundle
     public let modelAssetPath: String
     public let language: LanguageConfig
+    public let visionConfig: VisionConfig?
 
     public init(from path: String) throws {
         let expanded = (path as NSString).expandingTildeInPath
@@ -45,6 +46,11 @@ public struct LanguageBundle: Sendable {
         }
         self.modelAssetPath = main
         self.language = language
+        self.visionConfig = payload.vision
+
+        if bundle.kind == .vlm && self.visionConfig == nil {
+            throw ModelBundle.BundleError.missingField("vision")
+        }
     }
 
     // MARK: - Convenience accessors
@@ -98,6 +104,7 @@ extension LanguageBundle {
     fileprivate struct LanguagePayload: Decodable {
         let assets: Assets
         let language: LanguageConfig?
+        let vision: VisionConfig?
 
         struct Assets: Decodable {
             let main: String?

swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift

@@ -22,18 +22,23 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
     /// known role conventions (`main`, `extend_<N>`, `load_embeddings`, ...).
     public let functionMap: FunctionMap?
 
+    /// Vision-specific configuration. Nil for text-only language models.
+    public let vision: VisionConfig?
+
     public init(
         tokenizer: String,
         vocabSize: Int,
         maxContextLength: Int,
         embeddedTokenizer: Bool = true,
-        functionMap: FunctionMap? = nil
+        functionMap: FunctionMap? = nil,
+        vision: VisionConfig? = nil
     ) {
         self.tokenizer = tokenizer
         self.vocabSize = vocabSize
         self.maxContextLength = maxContextLength
         self.embeddedTokenizer = embeddedTokenizer
         self.functionMap = functionMap
+        self.vision = vision
     }
 
     enum CodingKeys: String, CodingKey {
@@ -42,6 +47,7 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
         case maxContextLength = "max_context_length"
         case embeddedTokenizer = "embedded_tokenizer"
         case functionMap = "function_map"
+        case vision
     }
 
     public init(from decoder: Swift.Decoder) throws {
@@ -51,6 +57,7 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
         self.maxContextLength = try c.decode(Int.self, forKey: .maxContextLength)
         self.embeddedTokenizer = try c.decodeIfPresent(Bool.self, forKey: .embeddedTokenizer) ?? true
         self.functionMap = try c.decodeIfPresent(FunctionMap.self, forKey: .functionMap)
+        self.vision = try c.decodeIfPresent(VisionConfig.self, forKey: .vision)
     }
 
     // MARK: - Additional Stop Tokens
@@ -137,3 +144,71 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
         return Array(result)
     }
 }
+
+/// Vision-specific configuration for VLM bundles.
+/// Nil for text-only language models.
+public struct VisionConfig: Codable, Sendable, Equatable {
+    /// Input image size (square). Vision encoder expects this resolution.
+    public let imageSize: Int
+
+    /// Patch size for the vision transformer.
+    public let patchSize: Int
+
+    /// Number of embedding tokens produced per image after projection.
+    public let imageTokenCount: Int
+
+    /// Token ID used as a placeholder in the text sequence for image positions.
+    public let imageTokenId: Int32
+
+    /// Per-channel normalization mean (RGB). Defaults to CLIP values when omitted.
+    public let imageMean: [Double]
+
+    /// Per-channel normalization std (RGB). Defaults to CLIP values when omitted.
+    public let imageStd: [Double]
+
+    /// Pixel rescale factor applied before normalization. Defaults to 1.0 when omitted.
+    public let rescaleFactor: Double
+
+    /// CLIP normalization (Qwen VL, Pixtral, InternVL, Phi-3.5-vision).
+    public static let clipMean = [0.48145466, 0.4578275, 0.40821073]
+    public static let clipStd = [0.26862954, 0.26130258, 0.27577711]
+
+    public init(
+        imageSize: Int,
+        patchSize: Int,
+        imageTokenCount: Int,
+        imageTokenId: Int32,
+        imageMean: [Double]? = nil,
+        imageStd: [Double]? = nil,
+        rescaleFactor: Double? = nil
+    ) {
+        self.imageSize = imageSize
+        self.patchSize = patchSize
+        self.imageTokenCount = imageTokenCount
+        self.imageTokenId = imageTokenId
+        self.imageMean = imageMean ?? Self.clipMean
+        self.imageStd = imageStd ?? Self.clipStd
+        self.rescaleFactor = rescaleFactor ?? 1.0
+    }
+
+    enum CodingKeys: String, CodingKey {
+        case imageSize = "image_size"
+        case patchSize = "patch_size"
+        case imageTokenCount = "image_token_count"
+        case imageTokenId = "image_token_id"
+        case imageMean = "image_mean"
+        case imageStd = "image_std"
+        case rescaleFactor = "rescale_factor"
+    }
+
+    public init(from decoder: Swift.Decoder) throws {
+        let c = try decoder.container(keyedBy: CodingKeys.self)
+        self.imageSize = try c.decode(Int.self, forKey: .imageSize)
+        self.patchSize = try c.decode(Int.self, forKey: .patchSize)
+        self.imageTokenCount = try c.decode(Int.self, forKey: .imageTokenCount)
+        self.imageTokenId = try c.decode(Int32.self, forKey: .imageTokenId)
+        self.imageMean = try c.decodeIfPresent([Double].self, forKey: .imageMean) ?? Self.clipMean
+        self.imageStd = try c.decodeIfPresent([Double].self, forKey: .imageStd) ?? Self.clipStd
+        self.rescaleFactor = try c.decodeIfPresent(Double.self, forKey: .rescaleFactor) ?? 1.0
+    }
+}