feat: add responseModalities support for Gemini image generation

src/llm/google/index.ts

@@ -21,6 +21,7 @@ import {
 
 export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
   thinkingConfig?: GeminiGenerationConfig['thinkingConfig'];
+  responseModalities?: ('TEXT' | 'IMAGE' | 'AUDIO')[];
 
   /**
    * Override to add gemini-3 model support for multimodal and function calling thought signatures
@@ -93,6 +94,8 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
     }
 
     this.thinkingConfig = fields.thinkingConfig ?? this.thinkingConfig;
+    this.responseModalities =
+      fields.responseModalities ?? this.responseModalities;
 
     this.streaming = fields.streaming ?? this.streaming;
     this.json = fields.json;
@@ -111,6 +114,9 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
           ...(this.json != null
             ? { responseMimeType: 'application/json' }
             : {}),
+          ...(this.responseModalities != null
+            ? { responseModalities: this.responseModalities }
+            : {}),
         },
       },
       {

src/llm/google/utils/common.ts

@@ -595,6 +595,11 @@ export function convertResponseContentToChatGenerationChunk(
             type: 'codeExecutionResult',
             codeExecutionResult: p.codeExecutionResult,
           };
+        } else if ('inlineData' in p && p.inlineData) {
+          return {
+            type: 'image_url' as const,
+            image_url: `data:${p.inlineData.mimeType};base64,${p.inlineData.data}`,
+          };
         }
         return p;
       })
@@ -753,6 +758,11 @@ export function mapGenerateContentResultToChatResult(
             type: 'codeExecutionResult',
             codeExecutionResult: p.codeExecutionResult,
           };
+        } else if ('inlineData' in p && p.inlineData) {
+          return {
+            type: 'image_url' as const,
+            image_url: `data:${p.inlineData.mimeType};base64,${p.inlineData.data}`,
+          };
         }
         return p;
       })

src/llm/vertexai/index.ts

@@ -23,6 +23,18 @@ class CustomChatConnection extends ChatConnection<VertexAIClientOptions> {
     ) {
       formattedData.generationConfig.thinkingConfig.includeThoughts = true;
     }
+
+    // Add responseModalities if present in fields
+    const fields = (
+      this as unknown as { fields: VertexAIClientOptions | undefined }
+    ).fields;
+    if (fields?.responseModalities) {
+      formattedData.generationConfig = {
+        ...formattedData.generationConfig,
+        responseModalities: fields.responseModalities,
+      };
+    }
+
     return formattedData;
   }
 }

src/types/llm.ts

@@ -59,6 +59,7 @@ export type AnthropicClientOptions = AnthropicInput;
 export type MistralAIClientOptions = ChatMistralAIInput;
 export type VertexAIClientOptions = ChatVertexAIInput & {
   includeThoughts?: boolean;
+  responseModalities?: ('TEXT' | 'IMAGE' | 'AUDIO')[];
 };
 export type BedrockAnthropicInput = ChatBedrockConverseInput & {
   additionalModelRequestFields?: ChatBedrockConverseInput['additionalModelRequestFields'] &
@@ -70,6 +71,7 @@ export type BedrockAnthropicClientOptions = BedrockAnthropicInput;
 export type GoogleClientOptions = GoogleGenerativeAIChatInput & {
   customHeaders?: RequestOptions['customHeaders'];
   thinkingConfig?: GeminiGenerationConfig['thinkingConfig'];
+  responseModalities?: ('TEXT' | 'IMAGE' | 'AUDIO')[];
 };
 export type DeepSeekClientOptions = ChatDeepSeekCallOptions;
 export type XAIClientOptions = ChatXAIInput;