feat(vertexai): Gemini multimodal output

Author: dlarocque

common/api-review/vertexai.api.md

@@ -124,6 +124,7 @@ export { Date_2 as Date }
 export interface EnhancedGenerateContentResponse extends GenerateContentResponse {
     // (undocumented)
     functionCalls: () => FunctionCall[] | undefined;
+    inlineData: () => GenerativeContentBlob[] | undefined;
     text: () => string;
 }
 
@@ -304,6 +305,8 @@ export interface GenerationConfig {
     // (undocumented)
     presencePenalty?: number;
     responseMimeType?: string;
+    // @beta
+    responseModalities?: ResponseModality[];
     responseSchema?: TypedSchema | SchemaRequest;
     // (undocumented)
     stopSequences?: string[];
@@ -596,6 +599,16 @@ export interface RequestOptions {
     timeout?: number;
 }
 
+// @beta
+export const ResponseModality: {
+    readonly TEXT: "TEXT";
+    readonly IMAGE: "IMAGE";
+    readonly AUDIO: "AUDIO";
+};
+
+// @beta
+export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality];
+
 // @public (undocumented)
 export interface RetrievedContextAttribution {
     // (undocumented)

docs-devsite/vertexai.enhancedgeneratecontentresponse.md

@@ -24,6 +24,7 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [functionCalls](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () =&gt; [FunctionCall](./vertexai.functioncall.md#functioncall_interface)<!-- -->\[\] \| undefined |  |
+|  [inlineData](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedata) | () =&gt; [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface)<!-- -->\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->'s first candidate. |
 |  [text](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () =&gt; string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. |
 
 ## EnhancedGenerateContentResponse.functionCalls
@@ -34,6 +35,16 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
 functionCalls: () => FunctionCall[] | undefined;
 ```
 
+## EnhancedGenerateContentResponse.inlineData
+
+Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->'s first candidate.
+
+<b>Signature:</b>
+
+```typescript
+inlineData: () => GenerativeContentBlob[] | undefined;
+```
+
 ## EnhancedGenerateContentResponse.text
 
 Returns the text string from the response, if available. Throws if the prompt or candidate was blocked.

docs-devsite/vertexai.generationconfig.md

@@ -27,6 +27,7 @@ export interface GenerationConfig
 |  [maxOutputTokens](./vertexai.generationconfig.md#generationconfigmaxoutputtokens) | number |  |
 |  [presencePenalty](./vertexai.generationconfig.md#generationconfigpresencepenalty) | number |  |
 |  [responseMimeType](./vertexai.generationconfig.md#generationconfigresponsemimetype) | string | Output response MIME type of the generated candidate text. Supported MIME types are <code>text/plain</code> (default, text output), <code>application/json</code> (JSON response in the candidates), and <code>text/x.enum</code>. |
+|  [responseModalities](./vertexai.generationconfig.md#generationconfigresponsemodalities) | [ResponseModality](./vertexai.md#responsemodality)<!-- -->\[\] | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
 |  [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a <code>[Schema](./vertexai.schema.md#schema_class)</code> static method like <code>Schema.string()</code> or <code>Schema.object()</code> or it can be a plain JS object matching the <code>[SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface)</code> interface. <br/>Note: This only applies when the specified <code>responseMIMEType</code> supports a schema; currently this is limited to <code>application/json</code> and <code>text/x.enum</code>. |
 |  [stopSequences](./vertexai.generationconfig.md#generationconfigstopsequences) | string\[\] |  |
 |  [temperature](./vertexai.generationconfig.md#generationconfigtemperature) | number |  |
@@ -75,6 +76,21 @@ Output response MIME type of the generated candidate text. Supported MIME types
 responseMimeType?: string;
 ```
 
+## GenerationConfig.responseModalities
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Generation modalities to be returned in generation responses.
+
+- Multimodal response generation is only supported in `gemini-2.0-flash-exp`<!-- -->, not `gemini-2.0-flash`<!-- -->. - Only image generation (`ResponseModality.IMAGE`<!-- -->) is supported.
+
+<b>Signature:</b>
+
+```typescript
+responseModalities?: ResponseModality[];
+```
+
 ## GenerationConfig.responseSchema
 
 Output response schema of the generated candidate text. This value can be a class generated with a <code>[Schema](./vertexai.schema.md#schema_class)</code> static method like `Schema.string()` or `Schema.object()` or it can be a plain JS object matching the <code>[SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface)</code> interface. <br/>Note: This only applies when the specified `responseMIMEType` supports a schema; currently this is limited to `application/json` and `text/x.enum`<!-- -->.

docs-devsite/vertexai.md

@@ -125,12 +125,14 @@ The Vertex AI in Firebase Web SDK.
 |  Variable | Description |
 |  --- | --- |
 |  [POSSIBLE\_ROLES](./vertexai.md#possible_roles) | Possible roles. |
+|  [ResponseModality](./vertexai.md#responsemodality) | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
 
 ## Type Aliases
 
 |  Type Alias | Description |
 |  --- | --- |
 |  [Part](./vertexai.md#part) | Content part - includes text, image/video, or function call/response part types. |
+|  [ResponseModality](./vertexai.md#responsemodality) | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
 |  [Role](./vertexai.md#role) | Role is the producer of the content. |
 |  [Tool](./vertexai.md#tool) | Defines a tool that model can call to access external knowledge. |
 |  [TypedSchema](./vertexai.md#typedschema) | A type that includes all specific Schema types. |
@@ -223,6 +225,23 @@ Possible roles.
 POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]
 ```
 
+## ResponseModality
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Generation modalities to be returned in generation responses.
+
+<b>Signature:</b>
+
+```typescript
+ResponseModality: {
+    readonly TEXT: "TEXT";
+    readonly IMAGE: "IMAGE";
+    readonly AUDIO: "AUDIO";
+}
+```
+
 ## Part
 
 Content part - includes text, image/video, or function call/response part types.
@@ -233,6 +252,19 @@ Content part - includes text, image/video, or function call/response part types.
 export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart;
 ```
 
+## ResponseModality
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Generation modalities to be returned in generation responses.
+
+<b>Signature:</b>
+
+```typescript
+export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality];
+```
+
 ## Role
 
 Role is the producer of the content.

packages/vertexai/src/requests/response-helpers.ts

@@ -21,6 +21,7 @@ import {
   FunctionCall,
   GenerateContentCandidate,
   GenerateContentResponse,
+  GenerativeContentBlob,
   ImagenGCSImage,
   ImagenInlineImage,
   VertexAIErrorCode
@@ -89,6 +90,38 @@ export function addHelpers(
     }
     return '';
   };
+  (response as EnhancedGenerateContentResponse).inlineData = () => {
+    if (response.candidates && response.candidates.length > 0) {
+      if (response.candidates.length > 1) {
+        logger.warn(
+          `This response had ${response.candidates.length} ` +
+            `candidates. Returning data from the first candidate only. ` +
+            `Access response.candidates directly to use the other candidates.`
+        );
+      }
+      if (hadBadFinishReason(response.candidates[0])) {
+        throw new VertexAIError(
+          VertexAIErrorCode.RESPONSE_ERROR,
+          `Response error: ${formatBlockErrorMessage(
+            response
+          )}. Response body stored in error.response`,
+          {
+            response
+          }
+        );
+      }
+      return getInlineData(response);
+    } else if (response.promptFeedback) {
+      throw new VertexAIError(
+        VertexAIErrorCode.RESPONSE_ERROR,
+        `Data not available. ${formatBlockErrorMessage(response)}`,
+        {
+          response
+        }
+      );
+    }
+    return undefined;
+  };
   (response as EnhancedGenerateContentResponse).functionCalls = () => {
     if (response.candidates && response.candidates.length > 0) {
       if (response.candidates.length > 1) {
@@ -164,6 +197,31 @@ export function getFunctionCalls(
   }
 }
 
+/**
+ * Returns {@link GenerativeContentBlob}s in the first candidate if present.
+ *
+ * @internal
+ */
+export function getInlineData(
+  response: GenerateContentResponse
+): GenerativeContentBlob[] | undefined {
+  const data: GenerativeContentBlob[] = [];
+
+  if (response.candidates?.[0].content?.parts) {
+    for (const part of response.candidates?.[0].content?.parts) {
+      if (part.inlineData) {
+        data.push(part.inlineData);
+      }
+    }
+  }
+
+  if (data.length > 0) {
+    return data;
+  } else {
+    return undefined;
+  }
+}
+
 const badFinishReasons = [FinishReason.RECITATION, FinishReason.SAFETY];
 
 function hadBadFinishReason(candidate: GenerateContentCandidate): boolean {

packages/vertexai/src/types/enums.ts

@@ -240,3 +240,34 @@ export enum Modality {
    */
   DOCUMENT = 'DOCUMENT'
 }
+
+/**
+ * Generation modalities to be returned in generation responses.
+ *
+ * @beta
+ */
+export const ResponseModality = {
+  /**
+   * Text.
+   * @beta
+   */
+  TEXT: 'TEXT',
+  /**
+   * Image.
+   * @beta
+   */
+  IMAGE: 'IMAGE',
+  /**
+   * Audio.
+   * @alpha
+   */
+  AUDIO: 'AUDIO'
+} as const;
+
+/**
+ * Generation modalities to be returned in generation responses.
+ *
+ * @beta
+ */
+export type ResponseModality =
+  (typeof ResponseModality)[keyof typeof ResponseModality];

packages/vertexai/src/types/requests.ts

@@ -21,7 +21,9 @@ import {
   FunctionCallingMode,
   HarmBlockMethod,
   HarmBlockThreshold,
-  HarmCategory
+  HarmCategory,
+  Modality,
+  ResponseModality
 } from './enums';
 import { ObjectSchemaInterface, SchemaRequest } from './schema';
 
@@ -95,6 +97,16 @@ export interface GenerationConfig {
    * this is limited to `application/json` and `text/x.enum`.
    */
   responseSchema?: TypedSchema | SchemaRequest;
+  /**
+   * Generation modalities to be returned in generation responses.
+   *
+   * @remarks
+   *  - Multimodal response generation is only supported in `gemini-2.0-flash-exp`, not `gemini-2.0-flash`.
+   *  - Only image generation (`ResponseModality.IMAGE`) is supported.
+   *
+   * @beta
+   */
+  responseModalities?: ResponseModality[];
 }
 
 /**

packages/vertexai/src/types/responses.ts

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import { Content, FunctionCall } from './content';
+import { Content, FunctionCall, GenerativeContentBlob } from './content';
 import {
   BlockReason,
   FinishReason,
@@ -59,6 +59,15 @@ export interface EnhancedGenerateContentResponse
    * Throws if the prompt or candidate was blocked.
    */
   text: () => string;
+  /**
+   * Aggregates and returns all {@link InlineDataPart} from the {@link GenerateContentResponse}'s
+   * first candidate.
+   *
+   * @returns An array of {@link GenerativeContentBlob} containing data from the response, if available.
+   *
+   * @throws If the prompt or candidate was blocked.
+   */
+  inlineData: () => GenerativeContentBlob[] | undefined;
   functionCalls: () => FunctionCall[] | undefined;
 }