Skip to content

Commit 002344a

Browse files
feat: VLM support with image attachment (#187)
## Summary Adds Vision Language Model (VLM) support to Private Mind, enabling users to attach images to messages when using a vision-capable model. ### Model layer - Added `vision` flag to `Model` type and DB schema - Added `imagePath` to `Message` type and DB schema - Added LFM 2.5 VL 1.6B Quantized (2.43 GB) to default models ### LLM store - `loadModel` passes `capabilities: ['vision']` via `fromModelName` - `sendChatMessage` converts messages with `mediaPath` to multimodal content array `[{ type: 'image' }, { type: 'text', text }]` ### Chat bar UI - `+` attach button in `ChatBarActions`, visible only for vision models - Tapping opens a `BottomSheetModal` with Camera and Photo Library options - Image preview with loading placeholder while HEIC→JPEG conversion runs in the background - Themed dismiss button using `close.svg` with app color palette ### Message bubble - Attached image renders above message text with correct `aspectRatio` - Tapping opens a full-screen lightbox with a close button ### Model selection - Vision chip always visible in `ModelCard` (both compact and full view) - Featured chip hidden in compact view to prioritize Vision ### iOS permissions - Added `NSCameraUsageDescription` and `NSPhotoLibraryUsageDescription` to `Info.plist` ## Test plan - [x] Download and load LFM 2.5 VL 1.6B model - [x] Verify `+` button appears in chat bar only for vision models - [ ] Attach image from photo library and from camera - [ ] Verify loading placeholder appears immediately while picker converts HEIC - [ ] Send image-only and image+text messages - [ ] Verify image renders in message bubble; tap opens lightbox; close button works - [ ] Switch to non-vision model — `+` button disappears and pending image is cleared - [ ] Verify Vision chip shows in both model selection sheet and model hub 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent bd22e42 commit 002344a

35 files changed

Lines changed: 928 additions & 84 deletions

__mocks__/expo-file-system.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1-
export const File = jest.fn().mockImplementation(() => ({
2-
text: jest.fn(),
3-
}));
1+
export const Paths = {
2+
cache: { uri: 'file:///cache/' },
3+
};
4+
5+
export const File = jest.fn().mockImplementation((uriOrDir: any, name?: string) => {
6+
const uri = name
7+
? `${typeof uriOrDir === 'string' ? uriOrDir : uriOrDir.uri}${name}`
8+
: typeof uriOrDir === 'string' ? uriOrDir : uriOrDir.uri;
9+
return {
10+
uri,
11+
text: jest.fn(),
12+
copy: jest.fn(),
13+
};
14+
});

__mocks__/react-native-executorch.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@ export const PHI_4_MINI_4B = makeModelConstants('phi4-mini-4b');
3232
export const PHI_4_MINI_4B_QUANTIZED = makeModelConstants('phi4-mini-4b-quantized');
3333
export const LFM2_5_1_2B_INSTRUCT = makeModelConstants('lfm2.5-1.2b-instruct');
3434
export const LFM2_5_1_2B_INSTRUCT_QUANTIZED = makeModelConstants('lfm2.5-1.2b-instruct-quantized');
35+
export const LFM2_VL_1_6B_QUANTIZED = makeModelConstants('lfm2-vl-1.6b-quantized');
3536
export const WHISPER_TINY_EN = 'whisper-tiny-en';
3637

3738
export const LLMModule = {
3839
fromCustomModel: jest.fn(),
40+
fromModelName: jest.fn(),
3941
};
4042

4143
export const SpeechToTextModule = {
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
const insets = { top: 0, bottom: 0, left: 0, right: 0 };
2+
3+
export const useSafeAreaInsets = () => insets;
4+
export const SafeAreaProvider = ({ children }: { children: unknown }) => children;
5+
export const SafeAreaView = 'SafeAreaView';

__tests__/ChatBar.test.tsx

Lines changed: 88 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,22 @@ jest.mock('../store/llmStore', () => ({
2222
})),
2323
}));
2424

25+
jest.mock('react-native-image-picker', () => ({
26+
launchImageLibrary: jest.fn(),
27+
launchCamera: jest.fn(),
28+
}));
29+
30+
jest.mock('../components/bottomSheets/ImageSourceSheet', () => {
31+
const { View, TouchableOpacity, Text } = require('react-native');
32+
return ({ onPickFromLibrary, onPickFromCamera }: any) => (
33+
<View testID="image-source-sheet">
34+
<TouchableOpacity testID="pick-library-btn" onPress={onPickFromLibrary}><Text>Library</Text></TouchableOpacity>
35+
<TouchableOpacity testID="pick-camera-btn" onPress={onPickFromCamera}><Text>Camera</Text></TouchableOpacity>
36+
</View>
37+
);
38+
});
39+
40+
2541
jest.mock('../components/chat-screen/ChatSpeechInput', () => {
2642
const { View, TouchableOpacity, Text } = require('react-native');
2743
return ({ onSubmit, onCancel }: any) => (
@@ -45,6 +61,7 @@ jest.mock('../components/chat-screen/ChatBarActions', () => {
4561
const { View, TouchableOpacity, Text } = require('react-native');
4662
return ({
4763
userInput,
64+
imagePath,
4865
onSend,
4966
isGenerating,
5067
isProcessingPrompt,
@@ -54,12 +71,22 @@ jest.mock('../components/chat-screen/ChatBarActions', () => {
5471
thinkingEnabled,
5572
activeSourcesCount,
5673
onSelectSource,
74+
isVisionModel,
75+
onAttachImage,
5776
}: any) => (
5877
<View testID="chat-bar-actions">
78+
{isVisionModel && (
79+
<TouchableOpacity testID="attach-image-btn" onPress={onAttachImage}><Text>+</Text></TouchableOpacity>
80+
)}
5981
{(isGenerating || isProcessingPrompt) ? (
6082
<TouchableOpacity testID="interrupt-btn" onPress={onInterrupt}><Text>Stop</Text></TouchableOpacity>
61-
) : userInput ? (
62-
<TouchableOpacity testID="send-btn" onPress={onSend}><Text>Send</Text></TouchableOpacity>
83+
) : (userInput || imagePath) ? (
84+
<>
85+
{imagePath && !userInput && (
86+
<TouchableOpacity testID="speech-btn" onPress={onSpeechInput}><Text>Mic</Text></TouchableOpacity>
87+
)}
88+
<TouchableOpacity testID="send-btn" onPress={onSend}><Text>Send</Text></TouchableOpacity>
89+
</>
6390
) : (
6491
<TouchableOpacity testID="speech-btn" onPress={onSpeechInput}><Text>Mic</Text></TouchableOpacity>
6592
)}
@@ -93,6 +120,7 @@ const downloadedModel = {
93120
tokenizerConfigPath: '',
94121
thinking: false,
95122
featured: false,
123+
vision: false,
96124
};
97125

98126
const defaultProps = {
@@ -174,7 +202,7 @@ describe('downloaded model — text input', () => {
174202
renderBar({ onSend });
175203
fireEvent.changeText(screen.getByPlaceholderText('Ask about anything...'), 'Hello');
176204
fireEvent.press(screen.getByTestId('send-btn'));
177-
expect(onSend).toHaveBeenCalledWith('Hello');
205+
expect(onSend).toHaveBeenCalledWith('Hello', undefined);
178206
});
179207

180208
it('shows prompt suggestions when hasMessages is false', () => {
@@ -279,10 +307,29 @@ describe('speech input', () => {
279307
fireEvent.press(screen.getByTestId('speech-btn'));
280308
});
281309
fireEvent.press(screen.getByTestId('speech-submit'));
282-
expect(onSend).toHaveBeenCalledWith('voice transcript');
310+
expect(onSend).toHaveBeenCalledWith('voice transcript', undefined);
283311
expect(screen.queryByTestId('speech-input')).toBeNull();
284312
});
285313

314+
it('forwards attached imagePath when submitting speech transcript', async () => {
315+
const { launchImageLibrary } = require('react-native-image-picker');
316+
launchImageLibrary.mockResolvedValue({ assets: [{ uri: 'file://test-image.jpg' }] });
317+
318+
const onSend = jest.fn();
319+
renderBar({ onSend, model: { ...downloadedModel, vision: true } });
320+
321+
await act(async () => {
322+
fireEvent.press(screen.getByTestId('pick-library-btn'));
323+
});
324+
325+
// speech-btn is shown alongside send-btn when image is attached with no text
326+
await act(async () => {
327+
fireEvent.press(screen.getByTestId('speech-btn'));
328+
});
329+
fireEvent.press(screen.getByTestId('speech-submit'));
330+
expect(onSend).toHaveBeenCalledWith('voice transcript', 'file://test-image.jpg');
331+
});
332+
286333
it('hides speech input without calling onSend when cancelled', async () => {
287334
const onSend = jest.fn();
288335
renderBar({ onSend });
@@ -316,3 +363,40 @@ describe('speech input', () => {
316363
expect(screen.queryByTestId('speech-input')).toBeNull();
317364
});
318365
});
366+
367+
// ─── vision model attachment button ──────────────────────────────────────────
368+
369+
describe('vision model attachment', () => {
370+
it('shows + button when loaded model has vision === true', () => {
371+
renderBar({ model: { ...downloadedModel, vision: true } });
372+
expect(screen.getByTestId('attach-image-btn')).toBeTruthy();
373+
});
374+
375+
it('does not show + button when loaded model has vision === false', () => {
376+
renderBar({ model: { ...downloadedModel, vision: false } });
377+
expect(screen.queryByTestId('attach-image-btn')).toBeNull();
378+
});
379+
380+
it('does not show + button when model has no vision flag', () => {
381+
renderBar({ model: downloadedModel });
382+
expect(screen.queryByTestId('attach-image-btn')).toBeNull();
383+
});
384+
385+
it('calls onSend with empty userInput and imagePath when send is pressed after attaching an image with no text', async () => {
386+
const { launchImageLibrary } = require('react-native-image-picker');
387+
388+
launchImageLibrary.mockResolvedValue({
389+
assets: [{ uri: 'file://test-image.jpg' }],
390+
});
391+
392+
const onSend = jest.fn();
393+
renderBar({ onSend, model: { ...downloadedModel, vision: true } });
394+
395+
await act(async () => {
396+
fireEvent.press(screen.getByTestId('pick-library-btn'));
397+
});
398+
399+
fireEvent.press(screen.getByTestId('send-btn'));
400+
expect(onSend).toHaveBeenCalledWith('', 'file://test-image.jpg');
401+
});
402+
});

__tests__/MessageItem.test.tsx

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,28 @@ describe('user messages', () => {
117117
});
118118
});
119119

120+
// ─── user messages with image ─────────────────────────────────────────────────
121+
122+
describe('user messages with image', () => {
123+
it('renders image above text for user messages with imagePath', () => {
124+
renderItem({ role: 'user', content: 'Check this out', imagePath: 'file://test.jpg' });
125+
const image = screen.getByTestId('message-image');
126+
expect(image.props.source).toEqual({ uri: 'file://test.jpg' });
127+
expect(screen.getByText('Check this out')).toBeTruthy();
128+
});
129+
130+
it('renders text-only bubble when imagePath is absent', () => {
131+
renderItem({ role: 'user', content: 'Hello there' });
132+
expect(screen.queryByTestId('message-image')).toBeNull();
133+
expect(screen.getByText('Hello there')).toBeTruthy();
134+
});
135+
136+
it('does not render image for assistant messages even if imagePath is provided', () => {
137+
renderItem({ role: 'assistant', content: 'Response', imagePath: 'file://test.jpg' });
138+
expect(screen.queryByTestId('message-image')).toBeNull();
139+
});
140+
});
141+
120142
// ─── thinking block parsing ───────────────────────────────────────────────────
121143

122144
describe('thinking block parsing', () => {

__tests__/ModelCard.test.tsx

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,18 @@ describe('display', () => {
103103
expect(screen.getByTestId('chip-2.50 GB')).toBeTruthy();
104104
});
105105

106-
it('shows Featured chip when model is featured', () => {
107-
renderCard({ featured: true });
106+
it('shows Featured chip when model is featured and compactView is false', () => {
107+
renderCard({ featured: true, compactView: false });
108108
expect(screen.getByTestId('chip-Featured')).toBeTruthy();
109109
});
110110

111+
it('does not show Featured chip in compact view even when featured', () => {
112+
renderCard({ featured: true });
113+
expect(screen.queryByTestId('chip-Featured')).toBeNull();
114+
});
115+
111116
it('does not show Featured chip for non-featured model', () => {
112-
renderCard({ featured: false });
117+
renderCard({ featured: false, compactView: false });
113118
expect(screen.queryByTestId('chip-Featured')).toBeNull();
114119
});
115120

__tests__/chatRepository.test.ts

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { persistMessage } from '../database/chatRepository';
2+
3+
jest.mock('expo-sqlite', () => ({
4+
useSQLiteContext: jest.fn(() => ({})),
5+
}));
6+
jest.mock('@react-native-async-storage/async-storage', () => ({
7+
getItem: jest.fn().mockResolvedValue(null),
8+
}));
9+
10+
describe('persistMessage with imagePath', () => {
11+
it('includes imagePath in INSERT when provided', async () => {
12+
const runAsync = jest.fn().mockResolvedValue({ lastInsertRowId: 1 });
13+
const mockDb = { runAsync } as any;
14+
15+
await persistMessage(mockDb, {
16+
role: 'user',
17+
content: 'Look at this',
18+
chatId: 1,
19+
imagePath: '/path/to/image.jpg',
20+
});
21+
22+
expect(runAsync).toHaveBeenCalledWith(
23+
expect.stringContaining('imagePath'),
24+
expect.arrayContaining(['/path/to/image.jpg'])
25+
);
26+
});
27+
28+
it('passes null imagePath when not provided', async () => {
29+
const runAsync = jest.fn().mockResolvedValue({ lastInsertRowId: 2 });
30+
const mockDb = { runAsync } as any;
31+
32+
await persistMessage(mockDb, {
33+
role: 'user',
34+
content: 'Hello',
35+
chatId: 1,
36+
});
37+
38+
expect(runAsync).toHaveBeenCalledWith(
39+
expect.stringContaining('imagePath'),
40+
expect.arrayContaining([null])
41+
);
42+
});
43+
});

0 commit comments

Comments
 (0)