Skip to content

Commit e890954

Browse files
pocketarcpocketarc
authored
feat(gemini): Add media_resolution provider option for images, videos, documents, and audio. (#832)
1 parent a0ebf4e commit e890954

File tree

6 files changed

+255
-22
lines changed

6 files changed

+255
-22
lines changed

‎docs/providers/gemini.md‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,36 @@ For complete streaming documentation, see [Streaming Output](/core-concepts/stre
492492

493493
## Media Support
494494

495-
Gemini has robust support for processing multimedia content:
495+
Gemini has robust support for processing multimedia content.
496+
497+
### Media Resolution
498+
499+
Gemini 3 models support the `mediaResolution` provider option to control the quality vs token usage tradeoff for images, videos, documents, and audio. Higher resolutions improve fine detail recognition but increase token consumption.
500+
501+
| Resolution | Image Tokens | Video Tokens (per frame) | PDF Tokens |
502+
|------------|--------------|--------------------------|------------|
503+
| `MEDIA_RESOLUTION_LOW` | 280 | 70 | 280 + text |
504+
| `MEDIA_RESOLUTION_MEDIUM` | 560 | 70 | 560 + text |
505+
| `MEDIA_RESOLUTION_HIGH` | 1120 | 280 | 1120 + text |
506+
507+
```php
508+
use Prism\Prism\ValueObjects\Messages\UserMessage;
509+
use Prism\Prism\ValueObjects\Media\Image;
510+
use Prism\Prism\Enums\Provider;
511+
512+
$response = Prism::text()
513+
->using(Provider::Gemini, 'gemini-3-flash-preview')
514+
->withMessages([
515+
new UserMessage(
516+
'Read the fine print in this document.',
517+
additionalContent: [
518+
Image::fromLocalPath('/path/to/document.png')
519+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_HIGH']),
520+
],
521+
),
522+
])
523+
->asText();
524+
```
496525

497526
### Video Analysis
498527

‎src/Providers/Gemini/Maps/AudioVideoMapper.php‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,25 @@ public function toPayload(): array
2222
$url = $this->media->url();
2323

2424
if ($this->media->isUrl() && $url !== null && MediaUrlDetector::shouldPassAsFileUri($url)) {
25-
return [
25+
$payload = [
2626
'file_data' => [
2727
'file_uri' => $url,
2828
],
2929
];
30+
} else {
31+
$payload = [
32+
'inline_data' => [
33+
'mime_type' => $this->media->mimeType(),
34+
'data' => $this->media->base64(),
35+
],
36+
];
37+
}
38+
39+
if ($mediaResolution = $this->media->providerOptions('mediaResolution')) {
40+
$payload['media_resolution'] = ['level' => $mediaResolution];
3041
}
3142

32-
return [
33-
'inline_data' => [
34-
'mime_type' => $this->media->mimeType(),
35-
'data' => $this->media->base64(),
36-
],
37-
];
43+
return $payload;
3844
}
3945

4046
protected function provider(): string|Provider

‎src/Providers/Gemini/Maps/DocumentMapper.php‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,25 @@ public function toPayload(): array
2222
$url = $this->media->url();
2323

2424
if ($this->media->isUrl() && $url !== null && MediaUrlDetector::shouldPassAsFileUri($url)) {
25-
return [
25+
$payload = [
2626
'file_data' => [
2727
'file_uri' => $url,
2828
],
2929
];
30+
} else {
31+
$payload = [
32+
'inline_data' => [
33+
'mime_type' => $this->media->mimeType(),
34+
'data' => $this->media->base64(),
35+
],
36+
];
37+
}
38+
39+
if ($mediaResolution = $this->media->providerOptions('mediaResolution')) {
40+
$payload['media_resolution'] = ['level' => $mediaResolution];
3041
}
3142

32-
return [
33-
'inline_data' => [
34-
'mime_type' => $this->media->mimeType(),
35-
'data' => $this->media->base64(),
36-
],
37-
];
43+
return $payload;
3844
}
3945

4046
protected function provider(): string|Provider

‎src/Providers/Gemini/Maps/ImageMapper.php‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,25 @@ public function toPayload(): array
2222
$url = $this->media->url();
2323

2424
if ($this->media->isUrl() && $url !== null && MediaUrlDetector::shouldPassAsFileUri($url)) {
25-
return [
25+
$payload = [
2626
'file_data' => [
2727
'file_uri' => $url,
2828
],
2929
];
30+
} else {
31+
$payload = [
32+
'inline_data' => [
33+
'mime_type' => $this->media->mimeType(),
34+
'data' => $this->media->base64(),
35+
],
36+
];
37+
}
38+
39+
if ($mediaResolution = $this->media->providerOptions('mediaResolution')) {
40+
$payload['media_resolution'] = ['level' => $mediaResolution];
3041
}
3142

32-
return [
33-
'inline_data' => [
34-
'mime_type' => $this->media->mimeType(),
35-
'data' => $this->media->base64(),
36-
],
37-
];
43+
return $payload;
3844
}
3945

4046
protected function provider(): string|Provider

‎tests/Providers/Gemini/GeminiMediaTest.php‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,4 +339,62 @@ function (Request $request): bool {
339339
});
340340
});
341341

342+
it('can send video with media_resolution provider option', function (): void {
343+
FixtureResponse::fakeResponseSequence('*', 'gemini/media-detection');
344+
345+
$videoContent = file_get_contents('tests/Fixtures/sample-video.mp4');
346+
347+
$response = Prism::text()
348+
->using(Provider::Gemini, 'gemini-1.5-flash')
349+
->withMessages([
350+
new UserMessage(
351+
'What is in this video',
352+
additionalContent: [
353+
Video::fromRawContent($videoContent, 'video/mp4')
354+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_HIGH']),
355+
],
356+
),
357+
])
358+
->asText();
359+
360+
Http::assertSent(function (Request $request): bool {
361+
$message = $request->data()['contents'][0]['parts'];
362+
363+
expect($message[1]['inline_data']['mime_type'])
364+
->toBe('video/mp4')
365+
->and($message[1])->toHaveKey('media_resolution')
366+
->and($message[1]['media_resolution'])->toBe(['level' => 'MEDIA_RESOLUTION_HIGH']);
367+
368+
return true;
369+
});
370+
});
371+
372+
it('can send audio with media_resolution provider option', function (): void {
373+
FixtureResponse::fakeResponseSequence('*', 'gemini/media-detection');
374+
375+
$audioContent = file_get_contents('tests/Fixtures/sample-audio.wav');
376+
377+
$response = Prism::text()
378+
->using(Provider::Gemini, 'gemini-1.5-flash')
379+
->withMessages([
380+
new UserMessage(
381+
'Transcribe this audio',
382+
additionalContent: [
383+
Audio::fromRawContent($audioContent, 'audio/wav')
384+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_LOW']),
385+
],
386+
),
387+
])
388+
->asText();
389+
390+
Http::assertSent(function (Request $request): bool {
391+
$message = $request->data()['contents'][0]['parts'];
392+
393+
expect($message[1])->toHaveKey('media_resolution')
394+
->and($message[1]['media_resolution'])->toBe(['level' => 'MEDIA_RESOLUTION_LOW']);
395+
396+
return true;
397+
});
398+
});
399+
342400
});

‎tests/Providers/Gemini/MediaMapperTest.php‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
use Prism\Prism\Providers\Gemini\Maps\AudioVideoMapper;
6+
use Prism\Prism\Providers\Gemini\Maps\DocumentMapper;
7+
use Prism\Prism\Providers\Gemini\Maps\ImageMapper;
8+
use Prism\Prism\ValueObjects\Media\Audio;
9+
use Prism\Prism\ValueObjects\Media\Document;
10+
use Prism\Prism\ValueObjects\Media\Image;
11+
use Prism\Prism\ValueObjects\Media\Video;
12+
13+
describe('ImageMapper', function (): void {
14+
it('maps images from base64', function (): void {
15+
$image = Image::fromBase64(base64_encode('image-content'), 'image/png');
16+
17+
$payload = (new ImageMapper($image))->toPayload();
18+
19+
expect($payload)->toBe([
20+
'inline_data' => [
21+
'mime_type' => 'image/png',
22+
'data' => base64_encode('image-content'),
23+
],
24+
]);
25+
});
26+
27+
it('includes media_resolution when provider option is set', function (): void {
28+
$image = Image::fromBase64(base64_encode('image-content'), 'image/png')
29+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_HIGH']);
30+
31+
$payload = (new ImageMapper($image))->toPayload();
32+
33+
expect($payload)->toBe([
34+
'inline_data' => [
35+
'mime_type' => 'image/png',
36+
'data' => base64_encode('image-content'),
37+
],
38+
'media_resolution' => [
39+
'level' => 'MEDIA_RESOLUTION_HIGH',
40+
],
41+
]);
42+
});
43+
44+
it('does not include media_resolution when provider option is not set', function (): void {
45+
$image = Image::fromBase64(base64_encode('image-content'), 'image/png');
46+
47+
$payload = (new ImageMapper($image))->toPayload();
48+
49+
expect($payload)->not->toHaveKey('media_resolution');
50+
});
51+
});
52+
53+
describe('DocumentMapper', function (): void {
54+
it('maps documents from base64', function (): void {
55+
$document = Document::fromBase64(base64_encode('pdf-content'), 'application/pdf', 'test.pdf');
56+
57+
$payload = (new DocumentMapper($document))->toPayload();
58+
59+
expect($payload)->toBe([
60+
'inline_data' => [
61+
'mime_type' => 'application/pdf',
62+
'data' => base64_encode('pdf-content'),
63+
],
64+
]);
65+
});
66+
67+
it('includes media_resolution when provider option is set', function (): void {
68+
$document = Document::fromBase64(base64_encode('pdf-content'), 'application/pdf', 'test.pdf')
69+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_MEDIUM']);
70+
71+
$payload = (new DocumentMapper($document))->toPayload();
72+
73+
expect($payload)->toBe([
74+
'inline_data' => [
75+
'mime_type' => 'application/pdf',
76+
'data' => base64_encode('pdf-content'),
77+
],
78+
'media_resolution' => [
79+
'level' => 'MEDIA_RESOLUTION_MEDIUM',
80+
],
81+
]);
82+
});
83+
});
84+
85+
describe('AudioVideoMapper', function (): void {
86+
it('maps video from base64', function (): void {
87+
$video = Video::fromBase64(base64_encode('video-content'), 'video/mp4');
88+
89+
$payload = (new AudioVideoMapper($video))->toPayload();
90+
91+
expect($payload)->toBe([
92+
'inline_data' => [
93+
'mime_type' => 'video/mp4',
94+
'data' => base64_encode('video-content'),
95+
],
96+
]);
97+
});
98+
99+
it('maps audio from base64', function (): void {
100+
$audio = Audio::fromBase64(base64_encode('audio-content'), 'audio/wav');
101+
102+
$payload = (new AudioVideoMapper($audio))->toPayload();
103+
104+
expect($payload)->toBe([
105+
'inline_data' => [
106+
'mime_type' => 'audio/wav',
107+
'data' => base64_encode('audio-content'),
108+
],
109+
]);
110+
});
111+
112+
it('includes media_resolution when provider option is set', function (): void {
113+
$video = Video::fromBase64(base64_encode('video-content'), 'video/mp4')
114+
->withProviderOptions(['mediaResolution' => 'MEDIA_RESOLUTION_LOW']);
115+
116+
$payload = (new AudioVideoMapper($video))->toPayload();
117+
118+
expect($payload)->toBe([
119+
'inline_data' => [
120+
'mime_type' => 'video/mp4',
121+
'data' => base64_encode('video-content'),
122+
],
123+
'media_resolution' => [
124+
'level' => 'MEDIA_RESOLUTION_LOW',
125+
],
126+
]);
127+
});
128+
});

0 commit comments

Comments
 (0)