feat(ai): ImageConfig and FinishReasons

.changeset/pb-add-image-config.md

@@ -0,0 +1,6 @@
+---
+"@firebase/ai": minor
+"firebase": minor
+---
+
+Added support for `ImageConfig` (aspect ratio and size) and expanded `FinishReason` values to achieve parity with the iOS SDK.

common/api-review/ai.api.md

@@ -358,6 +358,16 @@ export const FinishReason: {
     readonly PROHIBITED_CONTENT: "PROHIBITED_CONTENT";
     readonly SPII: "SPII";
     readonly MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL";
+    readonly IMAGE_SAFETY: "IMAGE_SAFETY";
+    readonly IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT";
+    readonly IMAGE_OTHER: "IMAGE_OTHER";
+    readonly NO_IMAGE: "NO_IMAGE";
+    readonly IMAGE_RECITATION: "IMAGE_RECITATION";
+    readonly LANGUAGE: "LANGUAGE";
+    readonly UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL";
+    readonly TOO_MANY_TOOL_CALLS: "TOO_MANY_TOOL_CALLS";
+    readonly MISSING_THOUGHT_SIGNATURE: "MISSING_THOUGHT_SIGNATURE";
+    readonly MALFORMED_RESPONSE: "MALFORMED_RESPONSE";
 };
 
 // @public
@@ -516,6 +526,7 @@ export interface GenerationConfig {
     candidateCount?: number;
     // (undocumented)
     frequencyPenalty?: number;
+    imageConfig?: ImageConfig;
     // (undocumented)
     maxOutputTokens?: number;
     // (undocumented)
@@ -744,6 +755,44 @@ export interface HybridParams {
     onDeviceParams?: OnDeviceParams;
 }
 
+// @public
+export interface ImageConfig {
+    aspectRatio?: ImageConfigAspectRatio;
+    imageSize?: ImageConfigImageSize;
+}
+
+// @public
+export const ImageConfigAspectRatio: {
+    readonly SQUARE_1x1: "1:1";
+    readonly PORTRAIT_9x16: "9:16";
+    readonly LANDSCAPE_16x9: "16:9";
+    readonly PORTRAIT_3x4: "3:4";
+    readonly LANDSCAPE_4x3: "4:3";
+    readonly PORTRAIT_2x3: "2:3";
+    readonly LANDSCAPE_3x2: "3:2";
+    readonly PORTRAIT_4x5: "4:5";
+    readonly LANDSCAPE_5x4: "5:4";
+    readonly PORTRAIT_1x4: "1:4";
+    readonly LANDSCAPE_4x1: "4:1";
+    readonly PORTRAIT_1x8: "1:8";
+    readonly LANDSCAPE_8x1: "8:1";
+    readonly ULTRAWIDE_21x9: "21:9";
+};
+
+// @public
+export type ImageConfigAspectRatio = (typeof ImageConfigAspectRatio)[keyof typeof ImageConfigAspectRatio];
+
+// @public
+export const ImageConfigImageSize: {
+    readonly SIZE_512: "512";
+    readonly SIZE_1K: "1K";
+    readonly SIZE_2K: "2K";
+    readonly SIZE_4K: "4K";
+};
+
+// @public
+export type ImageConfigImageSize = (typeof ImageConfigImageSize)[keyof typeof ImageConfigImageSize];
+
 // @public @deprecated
 export const ImagenAspectRatio: {
     readonly SQUARE: "1:1";

docs-devsite/_toc.yaml

@@ -106,6 +106,8 @@ toc:
     path: /docs/reference/js/ai.groundingsupport.md
   - title: HybridParams
     path: /docs/reference/js/ai.hybridparams.md
+  - title: ImageConfig
+    path: /docs/reference/js/ai.imageconfig.md
   - title: ImagenGCSImage
     path: /docs/reference/js/ai.imagengcsimage.md
   - title: ImagenGenerationConfig

docs-devsite/ai.generationconfig.md

@@ -24,6 +24,7 @@ export interface GenerationConfig
 |  --- | --- | --- |
 |  [candidateCount](./ai.generationconfig.md#generationconfigcandidatecount) | number |  |
 |  [frequencyPenalty](./ai.generationconfig.md#generationconfigfrequencypenalty) | number |  |
+|  [imageConfig](./ai.generationconfig.md#generationconfigimageconfig) | [ImageConfig](./ai.imageconfig.md#imageconfig_interface) | Configuration options for generating images with Gemini models. |
 |  [maxOutputTokens](./ai.generationconfig.md#generationconfigmaxoutputtokens) | number |  |
 |  [presencePenalty](./ai.generationconfig.md#generationconfigpresencepenalty) | number |  |
 |  [responseJsonSchema](./ai.generationconfig.md#generationconfigresponsejsonschema) | { \[key: string\]: unknown; } | Output schema of the generated response. This is an alternative to <code>responseSchema</code> that accepts \[JSON Schema\](https://json-schema.org/).<!-- -->If set, <code>responseSchema</code> must be omitted, but <code>responseMimeType</code> is required and must be set to <code>application/json</code>. |
@@ -52,6 +53,16 @@ candidateCount?: number;
 frequencyPenalty?: number;
 ```
 
+## GenerationConfig.imageConfig
+
+Configuration options for generating images with Gemini models.
+
+<b>Signature:</b>
+
+```typescript
+imageConfig?: ImageConfig;
+```
+
 ## GenerationConfig.maxOutputTokens
 
 <b>Signature:</b>

docs-devsite/ai.md

@@ -99,6 +99,7 @@ The Firebase AI Web SDK.
 |  [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.<!-- -->Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)<!-- -->).<!-- -->Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). |
 |  [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. |
 |  [HybridParams](./ai.hybridparams.md#hybridparams_interface) | <b><i>(Public Preview)</i></b> Configures hybrid inference. |
+|  [ImageConfig](./ai.imageconfig.md#imageconfig_interface) | Configuration options for generating images with Gemini models. |
 |  [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.<!-- -->This feature is not available yet. |
 |  [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for generating images with Imagen.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. |
 |  [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | The response from a request to generate images with Imagen. |
@@ -166,6 +167,8 @@ The Firebase AI Web SDK.
 |  [HarmCategory](./ai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. |
 |  [HarmProbability](./ai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. |
 |  [HarmSeverity](./ai.md#harmseverity) | Harm severity levels. |
+|  [ImageConfigAspectRatio](./ai.md#imageconfigaspectratio) | Aspect ratios for generated images. |
+|  [ImageConfigImageSize](./ai.md#imageconfigimagesize) | Sizes for generated images. |
 |  [ImagenAspectRatio](./ai.md#imagenaspectratio) | Aspect ratios for Imagen images.<!-- -->To specify an aspect ratio for generated images, set the <code>aspectRatio</code> property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface)<!-- -->.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. |
 |  [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | A filter level controlling whether generation of images containing people or faces is allowed.<!-- -->See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a> documentation for more details. |
 |  [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | A filter level controlling how aggressively to filter sensitive content.<!-- -->Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, <code>violence</code>, <code>sexual</code>, <code>derogatory</code>, and <code>toxic</code>). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. |
@@ -195,6 +198,8 @@ The Firebase AI Web SDK.
 |  [HarmCategory](./ai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. |
 |  [HarmProbability](./ai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. |
 |  [HarmSeverity](./ai.md#harmseverity) | Harm severity levels. |
+|  [ImageConfigAspectRatio](./ai.md#imageconfigaspectratio) | Aspect ratios for generated images. |
+|  [ImageConfigImageSize](./ai.md#imageconfigimagesize) | Sizes for generated images. |
 |  [ImagenAspectRatio](./ai.md#imagenaspectratio) | Aspect ratios for Imagen images.<!-- -->To specify an aspect ratio for generated images, set the <code>aspectRatio</code> property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface)<!-- -->.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. |
 |  [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | A filter level controlling whether generation of images containing people or faces is allowed.<!-- -->See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a> documentation for more details. |
 |  [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | A filter level controlling how aggressively to filter sensitive content.<!-- -->Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, <code>violence</code>, <code>sexual</code>, <code>derogatory</code>, and <code>toxic</code>). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. |
@@ -546,6 +551,16 @@ FinishReason: {
     readonly PROHIBITED_CONTENT: "PROHIBITED_CONTENT";
     readonly SPII: "SPII";
     readonly MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL";
+    readonly IMAGE_SAFETY: "IMAGE_SAFETY";
+    readonly IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT";
+    readonly IMAGE_OTHER: "IMAGE_OTHER";
+    readonly NO_IMAGE: "NO_IMAGE";
+    readonly IMAGE_RECITATION: "IMAGE_RECITATION";
+    readonly LANGUAGE: "LANGUAGE";
+    readonly UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL";
+    readonly TOO_MANY_TOOL_CALLS: "TOO_MANY_TOOL_CALLS";
+    readonly MISSING_THOUGHT_SIGNATURE: "MISSING_THOUGHT_SIGNATURE";
+    readonly MALFORMED_RESPONSE: "MALFORMED_RESPONSE";
 }
 ```
 
@@ -637,6 +652,46 @@ HarmSeverity: {
 }
 ```
 
+## ImageConfigAspectRatio
+
+Aspect ratios for generated images.
+
+<b>Signature:</b>
+
+```typescript
+ImageConfigAspectRatio: {
+    readonly SQUARE_1x1: "1:1";
+    readonly PORTRAIT_9x16: "9:16";
+    readonly LANDSCAPE_16x9: "16:9";
+    readonly PORTRAIT_3x4: "3:4";
+    readonly LANDSCAPE_4x3: "4:3";
+    readonly PORTRAIT_2x3: "2:3";
+    readonly LANDSCAPE_3x2: "3:2";
+    readonly PORTRAIT_4x5: "4:5";
+    readonly LANDSCAPE_5x4: "5:4";
+    readonly PORTRAIT_1x4: "1:4";
+    readonly LANDSCAPE_4x1: "4:1";
+    readonly PORTRAIT_1x8: "1:8";
+    readonly LANDSCAPE_8x1: "8:1";
+    readonly ULTRAWIDE_21x9: "21:9";
+}
+```
+
+## ImageConfigImageSize
+
+Sizes for generated images.
+
+<b>Signature:</b>
+
+```typescript
+ImageConfigImageSize: {
+    readonly SIZE_512: "512";
+    readonly SIZE_1K: "1K";
+    readonly SIZE_2K: "2K";
+    readonly SIZE_4K: "4K";
+}
+```
+
 ## ImagenAspectRatio
 
 > Warning: This API is now obsolete.
@@ -980,6 +1035,26 @@ Harm severity levels.
 export type HarmSeverity = (typeof HarmSeverity)[keyof typeof HarmSeverity];
 ```
 
+## ImageConfigAspectRatio
+
+Aspect ratios for generated images.
+
+<b>Signature:</b>
+
+```typescript
+export type ImageConfigAspectRatio = (typeof ImageConfigAspectRatio)[keyof typeof ImageConfigAspectRatio];
+```
+
+## ImageConfigImageSize
+
+Sizes for generated images.
+
+<b>Signature:</b>
+
+```typescript
+export type ImageConfigImageSize = (typeof ImageConfigImageSize)[keyof typeof ImageConfigImageSize];
+```
+
 ## ImagenAspectRatio
 
 > Warning: This API is now obsolete.

packages/ai/src/models/generative-model.test.ts

@@ -22,7 +22,9 @@ import {
   InferenceMode,
   AIErrorCode,
   ChromeAdapter,
-  ThinkingLevel
+  ThinkingLevel,
+  ImageConfigAspectRatio,
+  ImageConfigImageSize
 } from '../public-types';
 import * as request from '../requests/request';
 import { SinonStub, match, restore, stub } from 'sinon';
@@ -322,6 +324,23 @@ describe('GenerativeModel', () => {
     });
     restore();
   });
+  it('passes imageConfig through to ChatSession', async () => {
+    const genModel = new GenerativeModel(fakeAI, {
+      model: 'my-model',
+      generationConfig: {
+        imageConfig: {
+          aspectRatio: ImageConfigAspectRatio.SQUARE_1x1,
+          imageSize: ImageConfigImageSize.SIZE_512
+        }
+      }
+    });
+    const chatSession = genModel.startChat();
+    expect(chatSession.params?.generationConfig?.imageConfig).to.deep.equal({
+      aspectRatio: '1:1',
+      imageSize: '512'
+    });
+    restore();
+  });
   it('overrides base model params with startChatParams', () => {
     const genModel = new GenerativeModel(fakeAI, {
       model: 'my-model',

packages/ai/src/requests/response-helpers.ts

@@ -229,7 +229,24 @@ export function getInlineDataParts(
   }
 }
 
-const badFinishReasons = [FinishReason.RECITATION, FinishReason.SAFETY];
+const badFinishReasons = [
+  FinishReason.RECITATION,
+  FinishReason.SAFETY,
+  FinishReason.BLOCKLIST,
+  FinishReason.PROHIBITED_CONTENT,
+  FinishReason.SPII,
+  FinishReason.MALFORMED_FUNCTION_CALL,
+  FinishReason.IMAGE_SAFETY,
+  FinishReason.IMAGE_PROHIBITED_CONTENT,
+  FinishReason.IMAGE_OTHER,
+  FinishReason.NO_IMAGE,
+  FinishReason.IMAGE_RECITATION,
+  FinishReason.LANGUAGE,
+  FinishReason.UNEXPECTED_TOOL_CALL,
+  FinishReason.TOO_MANY_TOOL_CALLS,
+  FinishReason.MISSING_THOUGHT_SIGNATURE,
+  FinishReason.MALFORMED_RESPONSE
+];
 
 function hadBadFinishReason(candidate: GenerateContentCandidate): boolean {
   return (

packages/ai/src/types/enums.ts

@@ -238,7 +238,47 @@ export const FinishReason = {
   /**
    * The function call generated by the model was invalid.
    */
-  MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL'
+  MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL',
+  /**
+   * Token generation stopped because generated images contain safety violations.
+   */
+  IMAGE_SAFETY: 'IMAGE_SAFETY',
+  /**
+   * Image generation stopped because generated images have other prohibited content.
+   */
+  IMAGE_PROHIBITED_CONTENT: 'IMAGE_PROHIBITED_CONTENT',
+  /**
+   * Image generation stopped because of other miscellaneous issue.
+   */
+  IMAGE_OTHER: 'IMAGE_OTHER',
+  /**
+   * The model was expected to generate an image, but none was generated.
+   */
+  NO_IMAGE: 'NO_IMAGE',
+  /**
+   * Image generation stopped due to recitation.
+   */
+  IMAGE_RECITATION: 'IMAGE_RECITATION',
+  /**
+   * The response candidate content was flagged for using an unsupported language.
+   */
+  LANGUAGE: 'LANGUAGE',
+  /**
+   * Model generated a tool call but no tools were enabled in the request.
+   */
+  UNEXPECTED_TOOL_CALL: 'UNEXPECTED_TOOL_CALL',
+  /**
+   * Model called too many tools consecutively, thus the system exited execution.
+   */
+  TOO_MANY_TOOL_CALLS: 'TOO_MANY_TOOL_CALLS',
+  /**
+   * Request has at least one thought signature missing.
+   */
+  MISSING_THOUGHT_SIGNATURE: 'MISSING_THOUGHT_SIGNATURE',
+  /**
+   * Finished due to malformed response.
+   */
+  MALFORMED_RESPONSE: 'MALFORMED_RESPONSE'
 } as const;
 
 /**
@@ -247,6 +287,54 @@ export const FinishReason = {
  */
 export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason];
 
+/**
+ * Aspect ratios for generated images.
+ * @public
+ */
+/* eslint-disable camelcase */
+export const ImageConfigAspectRatio = {
+  SQUARE_1x1: '1:1',
+  PORTRAIT_9x16: '9:16',
+  LANDSCAPE_16x9: '16:9',
+  PORTRAIT_3x4: '3:4',
+  LANDSCAPE_4x3: '4:3',
+  PORTRAIT_2x3: '2:3',
+  LANDSCAPE_3x2: '3:2',
+  PORTRAIT_4x5: '4:5',
+  LANDSCAPE_5x4: '5:4',
+  PORTRAIT_1x4: '1:4',
+  LANDSCAPE_4x1: '4:1',
+  PORTRAIT_1x8: '1:8',
+  LANDSCAPE_8x1: '8:1',
+  ULTRAWIDE_21x9: '21:9'
+} as const;
+/* eslint-enable camelcase */
+
+/**
+ * Aspect ratios for generated images.
+ * @public
+ */
+export type ImageConfigAspectRatio =
+  (typeof ImageConfigAspectRatio)[keyof typeof ImageConfigAspectRatio];
+
+/**
+ * Sizes for generated images.
+ * @public
+ */
+export const ImageConfigImageSize = {
+  SIZE_512: '512',
+  SIZE_1K: '1K',
+  SIZE_2K: '2K',
+  SIZE_4K: '4K'
+} as const;
+
+/**
+ * Sizes for generated images.
+ * @public
+ */
+export type ImageConfigImageSize =
+  (typeof ImageConfigImageSize)[keyof typeof ImageConfigImageSize];
+
 /**
  * @public
  */