diff --git a/firestore-multimodal-genai/README.md b/firestore-multimodal-genai/README.md index ee97e31b..05489f83 100644 --- a/firestore-multimodal-genai/README.md +++ b/firestore-multimodal-genai/README.md @@ -140,6 +140,8 @@ This extension uses other Firebase and Google Cloud Platform services, which hav * Candidates field: The field in the message document into which to put the other candidate responses if the candidate count parameter is greater than one. +* Response MIME Type: The MIME type of the response. This is used to determine how the response should be interpreted. For example, if you set this to 'application/json', the response will be treated as JSON. + * Hate Speech Threshold: Threshold for hate speech content. Specify what probability level of hate speech content is blocked by the Gemini provider. * Dangerous Content Threshold: Threshold for dangerous content. Specify what probability level of dangerous content is blocked by the Gemini provider. diff --git a/firestore-multimodal-genai/extension.yaml b/firestore-multimodal-genai/extension.yaml index 8c0e2269..3ec19854 100644 --- a/firestore-multimodal-genai/extension.yaml +++ b/firestore-multimodal-genai/extension.yaml @@ -291,6 +291,21 @@ params: required: false immutable: false + - param: RESPONSE_MIME_TYPE + label: Response MIME Type + description: >- + The MIME type of the response. This is used to determine how the response + should be interpreted. For example, if you set this to 'application/json', + the response will be treated as JSON. + type: select + options: + - label: Default (text/plain) + value: text/plain + - label: JSON (application/json) + value: application/json + default: text/plain + required: false + # - param: RAG_HOOK_URL # label: Custom RAG Hook URL # description: >- diff --git a/firestore-multimodal-genai/functions/package-lock.json b/firestore-multimodal-genai/functions/package-lock.json index 8920af65..05b4df09 100644 --- a/firestore-multimodal-genai/functions/package-lock.json +++ b/firestore-multimodal-genai/functions/package-lock.json @@ -6,8 +6,8 @@ "": { "name": "firestore-multimodal-genai", "dependencies": { - "@genkit-ai/googleai": "^1.3.0", - "@genkit-ai/vertexai": "^1.3.0", + "@genkit-ai/googleai": "^1.8.0", + "@genkit-ai/vertexai": "^1.8.0", "@google-ai/generativelanguage": "^3.1.0", "@google-cloud/aiplatform": "^4.1.0", "@google-cloud/vertexai": "^1.1.0", @@ -16,7 +16,7 @@ "@types/mustache": "^4.2.2", "firebase-admin": "^12.1.0", "firebase-functions": "^4.9.0", - "genkit": "^1.3.0", + "genkit": "^1.8.0", "google-auth-library": "^9.0.0", "mustache": "^4.2.0", "sharp": "^0.33.5", @@ -762,15 +762,15 @@ } }, "node_modules/@genkit-ai/ai": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@genkit-ai/ai/-/ai-1.3.0.tgz", - "integrity": "sha512-pocwD7/a6lXYiCkmTNNqINjLk44c8Mpgx0xpfAlC4PirMjACQSfPaPwrO1SdrmRM/0VV0WihcEOcirPcWOrV/Q==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@genkit-ai/ai/-/ai-1.11.1.tgz", + "integrity": "sha512-E1rlzaShKmvTHxyYGqVaXCsekK9r516wh4XoKv+pcWVG71aWxV05qoI8zMIKFEg8SDj5oLOBYMsyoDDgyfpFqQ==", "dependencies": { - "@genkit-ai/core": "1.3.0", + "@genkit-ai/core": "1.11.1", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.11.19", "colorette": "^2.0.20", - "dotprompt": "^1.0.0", + "dotprompt": "^1.1.1", "json5": "^2.2.3", "node-fetch": "^3.3.2", "partial-json": "^0.1.7", @@ -807,9 +807,9 @@ } }, "node_modules/@genkit-ai/core": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@genkit-ai/core/-/core-1.3.0.tgz", - "integrity": "sha512-vwoHaiKkA8VBgGbp7a2oDzBFCicCFOihwCup48PvOY8kA9VYCws/qdUkFnrDh5NqzL7urXmjKE2wNf/59UmU9g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@genkit-ai/core/-/core-1.11.1.tgz", + "integrity": "sha512-Dfp22tsx3HDA0ZAA5gRoLdWhZzwcoVC42WMPBYCV/0WAToTuXjuwI2cv1D3bMMNZcNuuTTk4uKLuUhILaT/7QQ==", "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "^1.25.0", @@ -823,7 +823,7 @@ "async-mutex": "^0.5.0", "body-parser": "^1.20.3", "cors": "^2.8.5", - "dotprompt": "^1.0.0", + "dotprompt": "^1.1.1", "express": "^4.21.0", "get-port": "^5.1.0", "json-schema": "^0.4.0", @@ -852,16 +852,16 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "node_modules/@genkit-ai/googleai": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@genkit-ai/googleai/-/googleai-1.3.0.tgz", - "integrity": "sha512-xeaN0i3mZAUozgG8NOIkCKUTwz75m9Ed5Y0+TbPTxdZK8dQYjX08oB/q4jBrAIxlLzKCXFzkam0dQBWe7ACtyw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@genkit-ai/googleai/-/googleai-1.11.1.tgz", + "integrity": "sha512-qRvOlcn3pRXl3V6qlH1+jQTNtd2r2fKpanC+mfwUfrEB1vI257I9GHH7tIxb8y7+5sJQ5lETr+T2WWf6vDznmQ==", "dependencies": { "@google/generative-ai": "^0.24.0", "google-auth-library": "^9.6.3", "node-fetch": "^3.3.2" }, "peerDependencies": { - "genkit": "^1.3.0" + "genkit": "^1.11.1" } }, "node_modules/@genkit-ai/googleai/node_modules/node-fetch": { @@ -882,9 +882,9 @@ } }, "node_modules/@genkit-ai/vertexai": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@genkit-ai/vertexai/-/vertexai-1.3.0.tgz", - "integrity": "sha512-2sg2DxhpMKxh8L64Nrg7Hpt+2kFT2Wr3PmPgvtJJakQA7o4rpgtDAhnqDtpovLmViWPCEJxCckT51HPeXK+KfA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@genkit-ai/vertexai/-/vertexai-1.11.1.tgz", + "integrity": "sha512-vJzz6pi0lHygnxvfdx2fD75bqHwXT/CPUc7PJOiO+3etldTxiqXubf7gOoFJ0U5NDKtHdoluWCRbASsrIPssjg==", "dependencies": { "@anthropic-ai/sdk": "^0.24.3", "@anthropic-ai/vertex-sdk": "^0.4.0", @@ -901,7 +901,7 @@ "firebase-admin": ">=12.2" }, "peerDependencies": { - "genkit": "^1.3.0" + "genkit": "^1.11.1" } }, "node_modules/@genkit-ai/vertexai/node_modules/@fastify/busboy": { @@ -5385,13 +5385,13 @@ } }, "node_modules/dotprompt": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dotprompt/-/dotprompt-1.0.1.tgz", - "integrity": "sha512-mruM6m+pWe4t41InRDRchNLSl3x+q7iIBukVuUfb7vvN7aEOwP+BuONACAdaEeAqlMDtWHcTsuqqBdAAjGwamg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/dotprompt/-/dotprompt-1.1.1.tgz", + "integrity": "sha512-xll31JxDiE7FaF030t0Dx4EMSV60Qn/pONDn6Hs5bBBeEANbtqIu6fPfaAOoSNbF1Y9TK+pj9Xnvud7G7GHpaA==", "dependencies": { "@types/handlebars": "^4.1.0", "handlebars": "^4.7.8", - "yaml": "^2.5.0" + "yaml": "^2.7.0" } }, "node_modules/duplexify": { @@ -6615,12 +6615,12 @@ } }, "node_modules/genkit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/genkit/-/genkit-1.3.0.tgz", - "integrity": "sha512-byJOPXPSmD+weiCnXhe2Kw4wLVlnMalpznOvosaoF/aCqdDChikZ+SQJESZi+m/91YssrV4zGZeqcoN5jSHD7A==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/genkit/-/genkit-1.11.1.tgz", + "integrity": "sha512-yexdH+MXO3W+Ne1G3HnsO5o1D4dz4PxKZOiaIuWdoxZe57ZV6w5rFB3zqHwP3CqqSYXMmjmkwYhZ4hoYIPDDag==", "dependencies": { - "@genkit-ai/ai": "1.3.0", - "@genkit-ai/core": "1.3.0", + "@genkit-ai/ai": "1.11.1", + "@genkit-ai/core": "1.11.1", "uuid": "^10.0.0" } }, @@ -7167,9 +7167,9 @@ } }, "node_modules/import-in-the-middle": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.13.1.tgz", - "integrity": "sha512-k2V9wNm9B+ysuelDTHjI9d5KPc4l8zAZTGqj+pcynvWkypZd857ryzN8jNC7Pg2YZXNMJcHRPpaDyCBbNyVRpA==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.14.0.tgz", + "integrity": "sha512-g5zLT0HaztRJWysayWYiUq/7E5H825QIiecMD2pI5QO7Wzr847l6GDvPvmZaDIdrDtS2w7qRczywxiK6SL5vRw==", "dependencies": { "acorn": "^8.14.0", "acorn-import-attributes": "^1.9.5", @@ -8893,9 +8893,9 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/module-details-from-path": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", - "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==" }, "node_modules/ms": { "version": "2.1.3", @@ -11482,14 +11482,14 @@ "dev": true }, "node_modules/yaml": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", - "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", + "integrity": "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ==", "bin": { "yaml": "bin.mjs" }, "engines": { - "node": ">= 14" + "node": ">= 14.6" } }, "node_modules/yargs": { diff --git a/firestore-multimodal-genai/functions/package.json b/firestore-multimodal-genai/functions/package.json index e2281b43..6884eb48 100644 --- a/firestore-multimodal-genai/functions/package.json +++ b/firestore-multimodal-genai/functions/package.json @@ -11,8 +11,8 @@ }, "main": "lib/index.js", "dependencies": { - "@genkit-ai/googleai": "^1.3.0", - "@genkit-ai/vertexai": "^1.3.0", + "@genkit-ai/googleai": "^1.8.0", + "@genkit-ai/vertexai": "^1.8.0", "@google-ai/generativelanguage": "^3.1.0", "@google-cloud/aiplatform": "^4.1.0", "@google-cloud/vertexai": "^1.1.0", @@ -21,7 +21,7 @@ "@types/mustache": "^4.2.2", "firebase-admin": "^12.1.0", "firebase-functions": "^4.9.0", - "genkit": "^1.3.0", + "genkit": "^1.8.0", "google-auth-library": "^9.0.0", "mustache": "^4.2.0", "sharp": "^0.33.5", diff --git a/firestore-multimodal-genai/functions/src/config.ts b/firestore-multimodal-genai/functions/src/config.ts index e51595bf..62b1db68 100644 --- a/firestore-multimodal-genai/functions/src/config.ts +++ b/firestore-multimodal-genai/functions/src/config.ts @@ -47,6 +47,7 @@ export interface Config { }; maxOutputTokens?: number; maxOutputTokensVertex?: number; + responseMimeType?: string; provider: string; apiKey?: string; safetySettings?: GoogleAISafetySetting[] | VertexSafetySetting[]; @@ -120,6 +121,7 @@ export default { topP: process.env.TOP_P ? parseFloat(process.env.TOP_P) : undefined, topK: process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined, candidates, + responseMimeType: process.env.RESPONSE_MIME_TYPE || 'text/plain', provider: process.env.GENERATIVE_AI_PROVIDER, maxOutputTokensVertex: process.env.MAX_OUTPUT_TOKENS ? parseInt(process.env.MAX_OUTPUT_TOKENS) diff --git a/firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts b/firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts index ed7e30d4..7933c8e2 100644 --- a/firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts +++ b/firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts @@ -66,6 +66,7 @@ export class GeminiGenerativeClient extends GenerativeClient< temperature: options.temperature, candidateCount: options.candidateCount, maxOutputTokens: options.maxOutputTokens, + responseMimeType: options.responseMimeType, }, safetySettings: options.safetySettings, }); diff --git a/firestore-multimodal-genai/functions/src/generative-client/genkit.ts b/firestore-multimodal-genai/functions/src/generative-client/genkit.ts index cc0a9ab2..659e4bcc 100644 --- a/firestore-multimodal-genai/functions/src/generative-client/genkit.ts +++ b/firestore-multimodal-genai/functions/src/generative-client/genkit.ts @@ -162,6 +162,11 @@ export class GenkitGenerativeClient extends GenerativeClient< maxOutputTokens: config.maxOutputTokens, safetySettings: config.safetySettings, }, + output: { + contentType: config.responseMimeType || 'text/plain', + format: config.responseMimeType === 'text/plain' ? 'text' : 'json', + constrained: true, + }, }; } diff --git a/firestore-multimodal-genai/functions/src/generative-client/vertex_ai.ts b/firestore-multimodal-genai/functions/src/generative-client/vertex_ai.ts index 0fdfc9ad..fd112489 100644 --- a/firestore-multimodal-genai/functions/src/generative-client/vertex_ai.ts +++ b/firestore-multimodal-genai/functions/src/generative-client/vertex_ai.ts @@ -56,6 +56,7 @@ export class VertexLanguageClient extends GenerativeClient { temperature: options.temperature, candidateCount: options.candidateCount, maxOutputTokens: options.maxOutputTokens, + responseMimeType: options.responseMimeType, }, safetySettings: options.safetySettings, };