feat(firestore-multimodal-genai): add responseMimeType param

firestore-multimodal-genai/README.md

@@ -140,6 +140,8 @@ This extension uses other Firebase and Google Cloud Platform services, which hav
 
 * Candidates field: The field in the message document into which to put the other candidate responses if the candidate count parameter is greater than one.
 
+* Response MIME Type: The MIME type of the response. This is used to determine how the response should be interpreted. For example, if you set this to 'application/json', the response will be treated as JSON.
+
 * Hate Speech Threshold: Threshold for hate speech content. Specify what probability level of hate speech content is blocked by the Gemini provider.
 
 * Dangerous Content Threshold: Threshold for dangerous content. Specify what probability level of dangerous content is blocked by the Gemini provider.

firestore-multimodal-genai/extension.yaml

@@ -291,6 +291,21 @@ params:
     required: false
     immutable: false
 
+  - param: RESPONSE_MIME_TYPE
+    label: Response MIME Type
+    description: >-
+      The MIME type of the response. This is used to determine how the response
+      should be interpreted. For example, if you set this to 'application/json',
+      the response will be treated as JSON.
+    type: select
+    options:
+      - label: Default (text/plain)
+        value: text/plain
+      - label: JSON (application/json)
+        value: application/json
+    default: text/plain
+    required: false
+
   # - param: RAG_HOOK_URL
   #   label: Custom RAG Hook URL
   #   description: >-

firestore-multimodal-genai/functions/package.json

@@ -11,8 +11,8 @@
   },
   "main": "lib/index.js",
   "dependencies": {
-    "@genkit-ai/googleai": "^1.3.0",
-    "@genkit-ai/vertexai": "^1.3.0",
+    "@genkit-ai/googleai": "^1.8.0",
+    "@genkit-ai/vertexai": "^1.8.0",
     "@google-ai/generativelanguage": "^3.1.0",
     "@google-cloud/aiplatform": "^4.1.0",
     "@google-cloud/vertexai": "^1.1.0",
@@ -21,7 +21,7 @@
     "@types/mustache": "^4.2.2",
     "firebase-admin": "^12.1.0",
     "firebase-functions": "^4.9.0",
-    "genkit": "^1.3.0",
+    "genkit": "^1.8.0",
     "google-auth-library": "^9.0.0",
     "mustache": "^4.2.0",
     "sharp": "^0.33.5",

firestore-multimodal-genai/functions/src/config.ts

@@ -47,6 +47,7 @@ export interface Config {
   };
   maxOutputTokens?: number;
   maxOutputTokensVertex?: number;
+  responseMimeType?: string;
   provider: string;
   apiKey?: string;
   safetySettings?: GoogleAISafetySetting[] | VertexSafetySetting[];
@@ -120,6 +121,7 @@ export default {
   topP: process.env.TOP_P ? parseFloat(process.env.TOP_P) : undefined,
   topK: process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined,
   candidates,
+  responseMimeType: process.env.RESPONSE_MIME_TYPE || 'text/plain',
   provider: process.env.GENERATIVE_AI_PROVIDER,
   maxOutputTokensVertex: process.env.MAX_OUTPUT_TOKENS
     ? parseInt(process.env.MAX_OUTPUT_TOKENS)

firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts

@@ -66,6 +66,7 @@ export class GeminiGenerativeClient extends GenerativeClient<
           temperature: options.temperature,
           candidateCount: options.candidateCount,
           maxOutputTokens: options.maxOutputTokens,
+          responseMimeType: options.responseMimeType,
         },
         safetySettings: options.safetySettings,
       });

firestore-multimodal-genai/functions/src/generative-client/genkit.ts

@@ -162,6 +162,11 @@ export class GenkitGenerativeClient extends GenerativeClient<
         maxOutputTokens: config.maxOutputTokens,
         safetySettings: config.safetySettings,
       },
+      output: {
+        contentType: config.responseMimeType || 'text/plain',
+        format: config.responseMimeType === 'text/plain' ? 'text' : 'json',
+        constrained: true,
+      },
     };
   }
 

firestore-multimodal-genai/functions/src/generative-client/vertex_ai.ts

@@ -56,6 +56,7 @@ export class VertexLanguageClient extends GenerativeClient<any, VertexAI> {
         temperature: options.temperature,
         candidateCount: options.candidateCount,
         maxOutputTokens: options.maxOutputTokens,
+        responseMimeType: options.responseMimeType,
       },
       safetySettings: options.safetySettings,
     };