Skip to content

Commit 1603661

Browse files
committed
VisionConfig: add image normalization fields (mean, std, rescaleFactor)
Read per-channel normalization from metadata instead of hardcoding. Fields are optional — bundles without them default to CLIP values (the most common across VLMs). Gemma/SigLIP bundles specify their own [0.5, 0.5, 0.5] values explicitly in metadata.json.
1 parent 244a4f0 commit 1603661

2 files changed

Lines changed: 40 additions & 10 deletions

File tree

swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,22 +160,55 @@ public struct VisionConfig: Codable, Sendable, Equatable {
160160
/// Token ID used as a placeholder in the text sequence for image positions.
161161
public let imageTokenId: Int32
162162

163+
/// Per-channel normalization mean (RGB). Defaults to CLIP values when omitted.
164+
public let imageMean: [Double]
165+
166+
/// Per-channel normalization std (RGB). Defaults to CLIP values when omitted.
167+
public let imageStd: [Double]
168+
169+
/// Pixel rescale factor applied before normalization. Defaults to 1.0 when omitted.
170+
public let rescaleFactor: Double
171+
172+
/// CLIP normalization (Qwen VL, Pixtral, InternVL, Phi-3.5-vision).
173+
public static let clipMean = [0.48145466, 0.4578275, 0.40821073]
174+
public static let clipStd = [0.26862954, 0.26130258, 0.27577711]
175+
163176
public init(
164177
imageSize: Int,
165178
patchSize: Int,
166179
imageTokenCount: Int,
167-
imageTokenId: Int32
180+
imageTokenId: Int32,
181+
imageMean: [Double]? = nil,
182+
imageStd: [Double]? = nil,
183+
rescaleFactor: Double? = nil
168184
) {
169185
self.imageSize = imageSize
170186
self.patchSize = patchSize
171187
self.imageTokenCount = imageTokenCount
172188
self.imageTokenId = imageTokenId
189+
self.imageMean = imageMean ?? Self.clipMean
190+
self.imageStd = imageStd ?? Self.clipStd
191+
self.rescaleFactor = rescaleFactor ?? 1.0
173192
}
174193

175194
enum CodingKeys: String, CodingKey {
176195
case imageSize = "image_size"
177196
case patchSize = "patch_size"
178197
case imageTokenCount = "image_token_count"
179198
case imageTokenId = "image_token_id"
199+
case imageMean = "image_mean"
200+
case imageStd = "image_std"
201+
case rescaleFactor = "rescale_factor"
202+
}
203+
204+
public init(from decoder: Swift.Decoder) throws {
205+
let c = try decoder.container(keyedBy: CodingKeys.self)
206+
self.imageSize = try c.decode(Int.self, forKey: .imageSize)
207+
self.patchSize = try c.decode(Int.self, forKey: .patchSize)
208+
self.imageTokenCount = try c.decode(Int.self, forKey: .imageTokenCount)
209+
self.imageTokenId = try c.decode(Int32.self, forKey: .imageTokenId)
210+
self.imageMean = try c.decodeIfPresent([Double].self, forKey: .imageMean) ?? Self.clipMean
211+
self.imageStd = try c.decodeIfPresent([Double].self, forKey: .imageStd) ?? Self.clipStd
212+
self.rescaleFactor = try c.decodeIfPresent(Double.self, forKey: .rescaleFactor) ?? 1.0
180213
}
181214
}

swift/Sources/CoreAILanguageModels/InferenceEngines/CoreAISequentialVLMEngine.swift

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -312,16 +312,13 @@ public final class CoreAISequentialVLMEngine: MultimodalInferenceEngine, @unchec
312312
}
313313
self.llmFunction = llmFn
314314

315-
// Select image preprocessor based on vision config
316-
// Default to gemma3/SigLIP preset; can be extended for other encoders
315+
// Build image preprocessor from vision config normalization fields.
316+
let vc = config.visionConfig
317317
self.imagePreprocessor = ImagePreprocessor(
318-
targetSize: CGSize(
319-
width: config.visionConfig.imageSize,
320-
height: config.visionConfig.imageSize
321-
),
322-
mean: (0.485, 0.456, 0.406),
323-
std: (0.229, 0.224, 0.225),
324-
rescaleFactor: 1.0
318+
targetSize: CGSize(width: vc.imageSize, height: vc.imageSize),
319+
mean: (CGFloat(vc.imageMean[0]), CGFloat(vc.imageMean[1]), CGFloat(vc.imageMean[2])),
320+
std: (CGFloat(vc.imageStd[0]), CGFloat(vc.imageStd[1]), CGFloat(vc.imageStd[2])),
321+
rescaleFactor: CGFloat(vc.rescaleFactor)
325322
)
326323

327324
InstrumentsProfiler.endCustomInterval(

0 commit comments

Comments
 (0)