From b718308cfde2ef71108ac7c26ae5ad67aa4f6ab9 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Wed, 21 Feb 2024 10:26:24 +0000 Subject: [PATCH 1/4] docs(*): add gemini docs on regions --- firestore-genai-chatbot/CHANGELOG.md | 4 ++++ firestore-genai-chatbot/PREINSTALL.md | 6 +++++- firestore-genai-chatbot/README.md | 8 ++++++-- firestore-genai-chatbot/extension.yaml | 8 ++++++-- firestore-multimodal-genai/CHANGELOG.md | 4 ++++ firestore-multimodal-genai/PREINSTALL.md | 7 ++++++- firestore-multimodal-genai/README.md | 9 +++++++-- firestore-multimodal-genai/extension.yaml | 8 ++++++-- 8 files changed, 44 insertions(+), 10 deletions(-) diff --git a/firestore-genai-chatbot/CHANGELOG.md b/firestore-genai-chatbot/CHANGELOG.md index 09f16368..151bbf4c 100644 --- a/firestore-genai-chatbot/CHANGELOG.md +++ b/firestore-genai-chatbot/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.0.10 + +- Add docs on regional support for Gemini APIs + ## Version 0.0.9 - fix typo in documentation diff --git a/firestore-genai-chatbot/PREINSTALL.md b/firestore-genai-chatbot/PREINSTALL.md index 060fd529..76e3d577 100644 --- a/firestore-genai-chatbot/PREINSTALL.md +++ b/firestore-genai-chatbot/PREINSTALL.md @@ -2,7 +2,11 @@ Use this extension to easily deploy a chatbot using Gemini models, stored and ma On install you will be asked to provide: -- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs here. +- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs [here](https://cloud.google.com/vertex-ai/docs/generative-ai/migrate/migrate-google-ai). + +Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + +A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). - **Gemini Model**: Input the name of the Gemini model you would like to use. To view available models for each provider, see: - [Vertex AI Gemini models](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models) diff --git a/firestore-genai-chatbot/README.md b/firestore-genai-chatbot/README.md index 634e70c2..c3f4a1a8 100644 --- a/firestore-genai-chatbot/README.md +++ b/firestore-genai-chatbot/README.md @@ -10,7 +10,11 @@ On install you will be asked to provide: -- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs here. +- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs [here](https://cloud.google.com/vertex-ai/docs/generative-ai/migrate/migrate-google-ai). + +Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + +A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). - **Gemini Model**: Input the name of the Gemini model you would like to use. To view available models for each provider, see: - [Vertex AI Gemini models](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models) @@ -83,7 +87,7 @@ This extension uses other Firebase and Google Cloud Platform services, which hav * Order Field: The field by which to order when fetching conversation history. If absent when processing begins, the current timestamp will be written to this field. Sorting will be in descending order. -* Cloud Functions location: Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). +* Cloud Functions location: Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). * Context: Contextual preamble for the generative AI model. A string giving context for the discussion. diff --git a/firestore-genai-chatbot/extension.yaml b/firestore-genai-chatbot/extension.yaml index b5656ccf..99162420 100644 --- a/firestore-genai-chatbot/extension.yaml +++ b/firestore-genai-chatbot/extension.yaml @@ -1,5 +1,5 @@ name: firestore-genai-chatbot -version: 0.0.9 +version: 0.0.10 specVersion: v1beta icon: icon.png @@ -161,7 +161,11 @@ params: description: >- Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection - guide](https://firebase.google.com/docs/functions/locations). + guide](https://firebase.google.com/docs/functions/locations). Note that + Generative AI on Vertex AI is only available in the regions listed + [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + A list of languages and regions supported by the Gemini API on Google AI + is [here](https://ai.google.dev/available_regions). type: select options: - label: Iowa (us-central1) diff --git a/firestore-multimodal-genai/CHANGELOG.md b/firestore-multimodal-genai/CHANGELOG.md index 501cd720..948c97d6 100644 --- a/firestore-multimodal-genai/CHANGELOG.md +++ b/firestore-multimodal-genai/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.0.9 + +- Add docs on regional support for Gemini APIs + ## Version 0.0.8 - fix typo in documentation diff --git a/firestore-multimodal-genai/PREINSTALL.md b/firestore-multimodal-genai/PREINSTALL.md index 68425387..95ee0c84 100644 --- a/firestore-multimodal-genai/PREINSTALL.md +++ b/firestore-multimodal-genai/PREINSTALL.md @@ -2,7 +2,12 @@ This extension allows you to perform generative tasks using the Gemini API, a cu On installation, you will be asked to provide the following information: -**Gemini API Provider**: This extension makes use of the Gemini family of large language models. Currently the extension supports the Google AI Gemini API (for developers) and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs here. +- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs [here](https://cloud.google.com/vertex-ai/docs/generative-ai/migrate/migrate-google-ai). + +Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + +A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). + **Gemini Model**: Input the name of which Gemini model you would like to use. To view available models for each provider, see: - [Vertex AI Gemini models](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models) diff --git a/firestore-multimodal-genai/README.md b/firestore-multimodal-genai/README.md index 74dae248..287c33d1 100644 --- a/firestore-multimodal-genai/README.md +++ b/firestore-multimodal-genai/README.md @@ -10,7 +10,12 @@ On installation, you will be asked to provide the following information: -**Gemini API Provider**: This extension makes use of the Gemini family of large language models. Currently the extension supports the Google AI Gemini API (for developers) and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs here. +- **Gemini API Provider** This extension makes use of the Gemini family of models. Currently the extension supports the Google AI Gemini API and the Vertex AI Gemini API. Learn more about the differences between the Google AI and Vertex AI Gemini APIs [here](https://cloud.google.com/vertex-ai/docs/generative-ai/migrate/migrate-google-ai). + +Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + +A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). + **Gemini Model**: Input the name of which Gemini model you would like to use. To view available models for each provider, see: - [Vertex AI Gemini models](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models) @@ -112,7 +117,7 @@ This extension uses other Firebase and Google Cloud Platform services, which hav * Response Field: The field in the message document into which to put the response. -* Cloud Functions location: Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). +* Cloud Functions location: Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). Note that Generative AI on Vertex AI is only available in the regions listed [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). A list of languages and regions supported by the Gemini API on Google AI is [here](https://ai.google.dev/available_regions). * Temperature: Controls the randomness of the output. Values can range over [0,1], inclusive. A value closer to 1 will produce responses that are more varied, while a value closer to 0 will typically result in less surprising responses from the model. diff --git a/firestore-multimodal-genai/extension.yaml b/firestore-multimodal-genai/extension.yaml index 05206c71..a3c70edb 100644 --- a/firestore-multimodal-genai/extension.yaml +++ b/firestore-multimodal-genai/extension.yaml @@ -1,5 +1,5 @@ name: firestore-multimodal-genai -version: 0.0.8 +version: 0.0.9 specVersion: v1beta icon: icon.png @@ -170,7 +170,11 @@ params: description: >- Where do you want to deploy the functions created for this extension? For help selecting a location, refer to the [location selection - guide](https://firebase.google.com/docs/functions/locations). + guide](https://firebase.google.com/docs/functions/locations). Note that + Generative AI on Vertex AI is only available in the regions listed + [here](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/locations-genai). + A list of languages and regions supported by the Gemini API on Google AI + is [here](https://ai.google.dev/available_regions). type: select options: - label: Iowa (us-central1) From 9a50afa199d144d365bd012b4a013af51b46dc25 Mon Sep 17 00:00:00 2001 From: Pavel Ryabov <105740958+jauntybrain@users.noreply.github.com> Date: Tue, 5 Mar 2024 08:31:52 -0800 Subject: [PATCH 2/4] fix(firestore-semantic-search): neighbor_count parameter location (#391) * fix(firestore-semantic-search): neighbor_count parameter incorrect location --- .../functions/__tests__/types/query.test.ts | 4 +++- .../functions/__tests__/vertex/index.test.ts | 2 +- firestore-semantic-search/functions/src/common/vertex.ts | 3 +-- firestore-semantic-search/functions/src/types/query.ts | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/firestore-semantic-search/functions/__tests__/types/query.test.ts b/firestore-semantic-search/functions/__tests__/types/query.test.ts index e7bf3ea9..4a41a56f 100644 --- a/firestore-semantic-search/functions/__tests__/types/query.test.ts +++ b/firestore-semantic-search/functions/__tests__/types/query.test.ts @@ -13,7 +13,9 @@ describe('Query class tests', () => { test('toVertexQuery() should return the proper vertex query format', () => { const id = 'testId'; const featureVector = [0.5, 0.6, 0.7]; + const neighborCount = 10; const expectedVertexQuery = { + neighbor_count: neighborCount, datapoint: { datapoint_id: id, feature_vector: featureVector, @@ -21,7 +23,7 @@ describe('Query class tests', () => { }; const query = new Query(id, featureVector); - const result = query.toVertexQuery(); + const result = query.toVertexQuery(neighborCount); expect(result).toEqual(expectedVertexQuery); }); }); diff --git a/firestore-semantic-search/functions/__tests__/vertex/index.test.ts b/firestore-semantic-search/functions/__tests__/vertex/index.test.ts index 43b9f0d4..1bdca87e 100644 --- a/firestore-semantic-search/functions/__tests__/vertex/index.test.ts +++ b/firestore-semantic-search/functions/__tests__/vertex/index.test.ts @@ -290,9 +290,9 @@ describe('queryIndex', () => { expectedUrl, { deployed_index_id: 'ext_test_instance_index', - neighbor_count: 1, queries: [ { + neighbor_count: 1, datapoint: { datapoint_id: 'test-id', feature_vector: [1, 2, 3], diff --git a/firestore-semantic-search/functions/src/common/vertex.ts b/firestore-semantic-search/functions/src/common/vertex.ts index 65eaeff4..90a176a1 100644 --- a/firestore-semantic-search/functions/src/common/vertex.ts +++ b/firestore-semantic-search/functions/src/common/vertex.ts @@ -259,9 +259,8 @@ export async function queryIndex( const response = await axios.post( `https://${endpoint}/v1beta1/projects/${config.projectId}/locations/${config.location}/indexEndpoints/${indexEndpoint}:findNeighbors`, { - queries: queries.map(query => query.toVertexQuery()), + queries: queries.map(query => query.toVertexQuery(searchResults)), deployed_index_id: `ext_${config.instanceId.replace(/-/g, '_')}_index`, - neighbor_count: searchResults, }, { headers: { diff --git a/firestore-semantic-search/functions/src/types/query.ts b/firestore-semantic-search/functions/src/types/query.ts index fa6debd6..843ec7da 100644 --- a/firestore-semantic-search/functions/src/types/query.ts +++ b/firestore-semantic-search/functions/src/types/query.ts @@ -23,8 +23,9 @@ export class Query { this.featureVector = featureVector; } - toVertexQuery() { + toVertexQuery(neighborCount: number) { return { + neighbor_count: neighborCount, datapoint: { datapoint_id: this.id, feature_vector: this.featureVector, From 97d0560dcecf2da4b7d17d561f72e960de3f9da5 Mon Sep 17 00:00:00 2001 From: Jacob Cable <32874567+cabljac@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:54:57 +0000 Subject: [PATCH 3/4] feat(firestore-multimodal-genai): add callable function (#422) * feat(firestore-multimodal-genai): add callable function * refactor(firestore-multimodal-genai): tidy some code up a little --- firestore-multimodal-genai/PREINSTALL.md | 2 + firestore-multimodal-genai/README.md | 6 +- firestore-multimodal-genai/extension.yaml | 11 +- .../__tests__/google_ai/index.test.ts | 5 + .../__tests__/vertex_ai/index.test.ts | 5 + .../functions/package-lock.json | 38 +++--- .../functions/package.json | 8 +- .../functions/src/__mocks__/config.ts | 9 +- .../functions/src/config.ts | 40 +++--- .../functions/src/custom_hook.test.ts | 120 ---------------- .../functions/src/custom_hook.ts | 50 ------- .../functions/src/errors.ts | 15 ++ .../src/generative-client/base_client.ts | 2 +- .../functions/src/generative-client/index.ts | 2 + .../functions/src/index.ts | 128 ++++++++++++------ 15 files changed, 177 insertions(+), 264 deletions(-) delete mode 100644 firestore-multimodal-genai/functions/src/custom_hook.test.ts delete mode 100644 firestore-multimodal-genai/functions/src/custom_hook.ts diff --git a/firestore-multimodal-genai/PREINSTALL.md b/firestore-multimodal-genai/PREINSTALL.md index 95ee0c84..fb197a0c 100644 --- a/firestore-multimodal-genai/PREINSTALL.md +++ b/firestore-multimodal-genai/PREINSTALL.md @@ -21,6 +21,8 @@ This extension will listen to the specified collection for new documents. When s 2. Query the Gemini API to generate a response based on the prompt. 3. Write the response from the Model API back to the triggering document in the response field. +Additionally the extension deploys a callable function, which may be called with data containing the values for handlebar substitution. + Note that the extension only supports top-level handlebars variables, substitution into nested handlebar templates is not supported. Each instance of the extension should be configured to perform one particular task. If you have multiple tasks, you can install multiple instances. diff --git a/firestore-multimodal-genai/README.md b/firestore-multimodal-genai/README.md index 287c33d1..47a75f62 100644 --- a/firestore-multimodal-genai/README.md +++ b/firestore-multimodal-genai/README.md @@ -29,6 +29,8 @@ This extension will listen to the specified collection for new documents. When s 2. Query the Gemini API to generate a response based on the prompt. 3. Write the response from the Model API back to the triggering document in the response field. +Additionally the extension deploys a callable function, which may be called with data containing the values for handlebar substitution. + Note that the extension only supports top-level handlebars variables, substitution into nested handlebar templates is not supported. Each instance of the extension should be configured to perform one particular task. If you have multiple tasks, you can install multiple instances. @@ -141,7 +143,9 @@ This extension uses other Firebase and Google Cloud Platform services, which hav **Cloud Functions:** -* **generateText:** Listens to Firestore data writes to generate conversations. +* **generateOnCall:** A callable function to perform generative AI tasks. + +* **generateText:** Listens to Firestore data writes to perform generative AI tasks. diff --git a/firestore-multimodal-genai/extension.yaml b/firestore-multimodal-genai/extension.yaml index a3c70edb..ceae9111 100644 --- a/firestore-multimodal-genai/extension.yaml +++ b/firestore-multimodal-genai/extension.yaml @@ -64,10 +64,19 @@ roles: Vertex AI if this provider is chosen. resources: + - name: generateOnCall + type: firebaseextensions.v1beta.function + description: >- + A callable function to perform generative AI tasks. + properties: + location: ${LOCATION} + httpsTrigger: {} + runtime: nodejs18 + - name: generateText type: firebaseextensions.v1beta.function description: >- - Listens to Firestore data writes to generate conversations. + Listens to Firestore data writes to perform generative AI tasks. properties: availableMemoryMb: 2048 timeout: 540s diff --git a/firestore-multimodal-genai/functions/__tests__/google_ai/index.test.ts b/firestore-multimodal-genai/functions/__tests__/google_ai/index.test.ts index 5ef146e1..c07a5810 100644 --- a/firestore-multimodal-genai/functions/__tests__/google_ai/index.test.ts +++ b/firestore-multimodal-genai/functions/__tests__/google_ai/index.test.ts @@ -31,6 +31,11 @@ jest.mock('../../src/config', () => ({ projectId: 'demo-test', instanceId: 'demo-test', provider: 'google-ai', + candidates: { + field: 'candidates', + count: 1, + shouldIncludeCandidatesField: false, + }, }, })); diff --git a/firestore-multimodal-genai/functions/__tests__/vertex_ai/index.test.ts b/firestore-multimodal-genai/functions/__tests__/vertex_ai/index.test.ts index adfbad4e..baafe6c7 100644 --- a/firestore-multimodal-genai/functions/__tests__/vertex_ai/index.test.ts +++ b/firestore-multimodal-genai/functions/__tests__/vertex_ai/index.test.ts @@ -30,6 +30,11 @@ jest.mock('../../src/config', () => ({ projectId: 'demo-test', instanceId: 'demo-test', provider: 'vertex-ai', + candidates: { + field: 'candidates', + count: 1, + shouldIncludeCandidatesField: false, + }, }, })); diff --git a/firestore-multimodal-genai/functions/package-lock.json b/firestore-multimodal-genai/functions/package-lock.json index d2e5b368..12fe9478 100644 --- a/firestore-multimodal-genai/functions/package-lock.json +++ b/firestore-multimodal-genai/functions/package-lock.json @@ -6,10 +6,10 @@ "": { "name": "firestore-multimodal-genai", "dependencies": { - "@google-ai/generativelanguage": "^2.0.0", - "@google-cloud/aiplatform": "^3.0.0", - "@google-cloud/vertexai": "^0.2.1", - "@google/generative-ai": "^0.1.3", + "@google-ai/generativelanguage": "^2.1.0", + "@google-cloud/aiplatform": "^3.15.0", + "@google-cloud/vertexai": "^0.5.0", + "@google/generative-ai": "^0.3.1", "@types/jest": "^29.5.12", "@types/mustache": "^4.2.2", "firebase-admin": "^11.5.0", @@ -857,9 +857,9 @@ } }, "node_modules/@google-ai/generativelanguage": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@google-ai/generativelanguage/-/generativelanguage-2.0.1.tgz", - "integrity": "sha512-a4qPKLB96p6irCEuyy3sikWQXg6OoRBgd3rz4GBXKdLoD0nJwc1LmneH6gerPFnfOtDT5ujqswKikR+jgDG0sQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@google-ai/generativelanguage/-/generativelanguage-2.1.0.tgz", + "integrity": "sha512-SwK8TFU4HZH/27QKVU6VTAo8OWh59DYOuG3Sy/K+uuw8baPdza26xQ4bOanNRq/2aWXuHKl9BCG6V3HjDBKUqg==", "dependencies": { "google-gax": "^4.0.3" }, @@ -868,9 +868,9 @@ } }, "node_modules/@google-cloud/aiplatform": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/@google-cloud/aiplatform/-/aiplatform-3.10.1.tgz", - "integrity": "sha512-UopqzotYe9MUKqln5w3Nc1+uJRjw/NBsKJ1r1YWTU+NQLeg4Vo5nUxNdAzkHAmicIIv8WrJ02tbyCxxeqKugHg==", + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/@google-cloud/aiplatform/-/aiplatform-3.15.0.tgz", + "integrity": "sha512-u0erPKRwGGqaoo5bmeMKyPyu4gt8fjGrde11QvmwNr4Dip5B7T0L5z8vKj7MK6HVOYcI5UGUmUoSdexcT/n2yQ==", "dependencies": { "google-gax": "^4.0.3", "protobuf.js": "^1.1.2" @@ -1250,9 +1250,9 @@ "optional": true }, "node_modules/@google-cloud/vertexai": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@google-cloud/vertexai/-/vertexai-0.2.1.tgz", - "integrity": "sha512-5oJwLfoRO/Oh61KZnXA9jtuOiSPCQW3tgCa9BhmfOE702F6jMqb65TSbVIMJbzJwjCZIhoRdnrh0rgTkKQXTDg==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/vertexai/-/vertexai-0.5.0.tgz", + "integrity": "sha512-qIFHYTXA5UCLdm9JG+Xf1suomCXxRqa1PKdYjqXuhZsCm8mn37Rb0Tf8djlhDzuRVWyWoQTmsWpsk28ZTmbqJg==", "dependencies": { "google-auth-library": "^9.1.0" }, @@ -1261,9 +1261,9 @@ } }, "node_modules/@google/generative-ai": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.1.3.tgz", - "integrity": "sha512-Cm4uJX1sKarpm1mje/MiOIinM7zdUUrQp/5/qGPAgznbdd/B9zup5ehT6c1qGqycFcSopTA1J1HpqHS5kJR8hQ==", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.3.1.tgz", + "integrity": "sha512-Zh1EK5nCWqIhxPm3K1KOM2mOwwgBvp6lkze74yTj6+Zqibtf55Db1q87cbbzWZeET3ZbNgHMYjVf2JyMM6pI7A==", "engines": { "node": ">=18.0.0" } @@ -6522,9 +6522,9 @@ } }, "node_modules/jose": { - "version": "4.15.4", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.4.tgz", - "integrity": "sha512-W+oqK4H+r5sITxfxpSU+MMdr/YSWGvgZMQDIsNoBDGGy4i7GBPTtvFKibQzW06n3U3TqHjhvBJsirShsEJ6eeQ==", + "version": "4.15.5", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz", + "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==", "funding": { "url": "https://github.com/sponsors/panva" } diff --git a/firestore-multimodal-genai/functions/package.json b/firestore-multimodal-genai/functions/package.json index 9ba72db6..7c855230 100644 --- a/firestore-multimodal-genai/functions/package.json +++ b/firestore-multimodal-genai/functions/package.json @@ -11,10 +11,10 @@ }, "main": "lib/index.js", "dependencies": { - "@google-ai/generativelanguage": "^2.0.0", - "@google-cloud/aiplatform": "^3.0.0", - "@google-cloud/vertexai": "^0.2.1", - "@google/generative-ai": "^0.1.3", + "@google-ai/generativelanguage": "^2.1.0", + "@google-cloud/aiplatform": "^3.15.0", + "@google-cloud/vertexai": "^0.5.0", + "@google/generative-ai": "^0.3.1", "@types/jest": "^29.5.12", "@types/mustache": "^4.2.2", "firebase-admin": "^11.5.0", diff --git a/firestore-multimodal-genai/functions/src/__mocks__/config.ts b/firestore-multimodal-genai/functions/src/__mocks__/config.ts index d6dad11c..f139e54b 100644 --- a/firestore-multimodal-genai/functions/src/__mocks__/config.ts +++ b/firestore-multimodal-genai/functions/src/__mocks__/config.ts @@ -12,12 +12,13 @@ export default { collectionName: 'discussions', prompt: 'test prompt', responseField: 'output', - candidateCount: 1, provider: 'vertex-ai', apiKey: process.env.API_KEY, bucketName: 'demo-gcp.appspot.com', imageField: 'image', - customRagHookUrl: 'https://www.test.com', - ragHookInputFields: ['test'], - ragHookOutputFields: ['foo'], + candidates: { + field: 'candidates', + count: 1, + shouldIncludeCandidatesField: false, + }, }; diff --git a/firestore-multimodal-genai/functions/src/config.ts b/firestore-multimodal-genai/functions/src/config.ts index b8da10b3..45201006 100644 --- a/firestore-multimodal-genai/functions/src/config.ts +++ b/firestore-multimodal-genai/functions/src/config.ts @@ -39,8 +39,11 @@ export interface Config { temperature?: number; topP?: number; topK?: number; - candidateCount?: number; - candidatesField?: string; + candidates: { + field: string; + count: number; + shouldIncludeCandidatesField: boolean; + }; maxOutputTokens?: number; maxOutputTokensVertex?: number; provider?: string; @@ -48,12 +51,6 @@ export interface Config { safetySettings?: GoogleAISafetySetting[] | VertexSafetySetting[]; bucketName?: string; imageField: string; - // ragConfig: { - // customRagHookUrl?: string; - // customRagHookApiKey?: string; - // ragHookInputFields?: string[]; - // ragHookOutputFields?: string[]; - // }; } function getSafetySettings(): GoogleAISafetySetting[] | VertexSafetySetting[] { @@ -87,6 +84,18 @@ function getSafetySettings(): GoogleAISafetySetting[] | VertexSafetySetting[] { const defaultBucketName = `${process.env.PROJECT_ID}.appspot.com`; +const candidates = { + field: process.env.CANDIDATES_FIELD || 'candidates', + count: process.env.CANDIDATE_COUNT + ? parseInt(process.env.CANDIDATE_COUNT) + : 1, + shouldIncludeCandidatesField: + process.env.GENERATIVE_AI_PROVIDER === 'generative' && + process.env.CANDIDATES_FIELD && + process.env.CANDIDATE_COUNT && + parseInt(process.env.CANDIDATE_COUNT) > 1, +}; + export default { vertex: { model: process.env.MODEL!, @@ -108,10 +117,7 @@ export default { : undefined, topP: process.env.TOP_P ? parseFloat(process.env.TOP_P) : undefined, topK: process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined, - candidateCount: process.env.CANDIDATE_COUNT - ? parseInt(process.env.CANDIDATE_COUNT) - : 1, - candidatesField: process.env.CANDIDATES_FIELD || 'candidates', + candidates, provider: process.env.GENERATIVE_AI_PROVIDER, maxOutputTokensVertex: process.env.MAX_OUTPUT_TOKENS ? parseInt(process.env.MAX_OUTPUT_TOKENS) @@ -120,14 +126,4 @@ export default { safetySettings: getSafetySettings(), bucketName: process.env.BUCKET_NAME || defaultBucketName, imageField: process.env.IMAGE_FIELD || 'image', - // ragConfig: { - // customRagHookUrl: process.env.CUSTOM_RAG_HOOK_URL, - // ragHookInputFields: process.env.RAG_HOOK_INPUT_FIELDS - // ? process.env.RAG_HOOK_INPUT_FIELDS.split(',') - // : undefined, - // ragHookOutputFields: process.env.RAG_HOOK_OUTPUT_FIELDS - // ? process.env.RAG_HOOK_OUTPUT_FIELDS.split(',') - // : undefined, - // // customRagHookApiKey: process.env.RAG_HOOK_API_KEY, - // }, }; diff --git a/firestore-multimodal-genai/functions/src/custom_hook.test.ts b/firestore-multimodal-genai/functions/src/custom_hook.test.ts deleted file mode 100644 index 3f1dcbb4..00000000 --- a/firestore-multimodal-genai/functions/src/custom_hook.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import {fetchCustomHookData} from './custom_hook'; -const utils = require('./utils'); -const mock = jest.spyOn(utils, 'extractFields'); - -describe.skip('fetchCustomHookData', () => { - const globalFetch = global.fetch; - - let mockData = {}; - let mockOK = true; - - beforeEach(() => { - mockOK = true; - mockData = {}; - jest.clearAllMocks(); // Clear mocks between tests - - // Mock fetch inside beforeEach to ensure it's reset for each test - global.fetch = jest.fn().mockImplementation(async () => ({ - ok: mockOK, - json: async () => mockData, - })); - }); - - afterAll(() => { - global.fetch = globalFetch; - }); - - test('successfully fetches and processes data', async () => { - mockData = { - field3: 'test', - }; - const mockConfig = { - customRagHookUrl: 'http://example.com', - customRagHookApiKey: 'test-api-key', - ragHookInputFields: ['field1', 'field2'], - ragHookOutputFields: ['field3', 'field4'], - }; - const docData = {field1: 'value1', field2: 'value2'}; - const result = await fetchCustomHookData(docData, mockConfig); - - expect(fetch).toHaveBeenCalledWith(mockConfig.customRagHookUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': mockConfig.customRagHookApiKey, - }, - body: JSON.stringify(docData), // Assuming extractFields returns docData directly in this mock - }); - - expect(mock).toHaveBeenCalledTimes(2); // Called for both input and output processing - expect(result).toEqual({field3: 'test'}); // Assuming extractFields returns the fetched data directly in this mock - }); - - test('throws error when custom hook URL is not provided', async () => { - const mockConfig = { - customRagHookUrl: 'http://example.com', - customRagHookApiKey: 'test-api-key', - ragHookInputFields: ['field1', 'field2'], - ragHookOutputFields: ['field3', 'field4'], - }; - await expect( - fetchCustomHookData({}, {...mockConfig, customRagHookUrl: ''}) - ).rejects.toThrow('Custom hook URL is not provided in the configuration.'); - }); - - // Test for handling fetch failure due to network error - test('handles fetch failure due to network error', async () => { - global.fetch = jest.fn().mockRejectedValue(new Error('Network error')); - - const mockConfig = { - customRagHookUrl: 'http://example.com', - customRagHookApiKey: 'test-api-key', - ragHookInputFields: ['field1', 'field2'], - ragHookOutputFields: ['field3', 'field4'], - }; - const docData = {field1: 'value1', field2: 'value2'}; - - await expect(fetchCustomHookData(docData, mockConfig)).rejects.toThrow( - 'Network error' - ); - }); - - // Test for validating response with `ok` status false - test('validates response with `ok` status false', async () => { - mockOK = false; // Simulate an unsuccessful response - global.fetch = jest.fn().mockImplementation(async () => ({ - ok: mockOK, - statusText: 'Internal Server Error', - json: async () => mockData, - })); - - const mockConfig = { - customRagHookUrl: 'http://example.com', - customRagHookApiKey: 'test-api-key', - ragHookInputFields: ['field1', 'field2'], - ragHookOutputFields: ['field3', 'field4'], - }; - const docData = {field1: 'value1', field2: 'value2'}; - - await expect(fetchCustomHookData(docData, mockConfig)).rejects.toThrow( - 'Internal Server Error' - ); - }); - - // Test for processing empty or unexpected data returned from the custom hook - test('processes empty or unexpected data returned from the custom hook', async () => { - mockData = {}; // Simulate an empty or unexpected response - - const mockConfig = { - customRagHookUrl: 'http://example.com', - customRagHookApiKey: 'test-api-key', - ragHookInputFields: ['field1', 'field2'], - ragHookOutputFields: ['field3', 'field4'], - }; - const docData = {field1: 'value1', field2: 'value2'}; - - const result = await fetchCustomHookData(docData, mockConfig); - - expect(result).toEqual({}); // Expect an empty object or handle unexpected data gracefully - }); -}); diff --git a/firestore-multimodal-genai/functions/src/custom_hook.ts b/firestore-multimodal-genai/functions/src/custom_hook.ts deleted file mode 100644 index 8e0ea8c8..00000000 --- a/firestore-multimodal-genai/functions/src/custom_hook.ts +++ /dev/null @@ -1,50 +0,0 @@ -import {extractFields} from './utils'; - -export const fetchCustomHookData = async ( - docData: Record, - { - customRagHookUrl, - customRagHookApiKey, - ragHookInputFields, - ragHookOutputFields, - }: { - customRagHookUrl?: string; - customRagHookApiKey?: string; - ragHookInputFields?: string[]; - ragHookOutputFields?: string[]; - } -): Promise> => { - if (!customRagHookUrl) { - throw new Error('Custom hook URL is not provided in the configuration.'); - } - - const params = extractFields(docData, ragHookInputFields); - - const headers: Record = { - 'Content-Type': 'application/json', - ...(customRagHookApiKey && {'x-api-key': customRagHookApiKey}), - }; - - try { - const response = await fetch(customRagHookUrl, { - method: 'POST', - headers, - body: JSON.stringify(params), - }); - - if (!response.ok) { - // Handle non-200 responses - throw new Error( - `RAG Hook Network response was not ok: ${response.statusText}` - ); - } - - const data = await response.json(); - - return extractFields(data, ragHookOutputFields); - } catch (error) { - // Handle network errors or JSON parsing errors - console.error('Failed to fetch custom hook data:', error); - throw error; // Rethrow after logging or handle accordingly - } -}; diff --git a/firestore-multimodal-genai/functions/src/errors.ts b/firestore-multimodal-genai/functions/src/errors.ts index 05f54838..975d861c 100644 --- a/firestore-multimodal-genai/functions/src/errors.ts +++ b/firestore-multimodal-genai/functions/src/errors.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import {GoogleError} from 'google-gax'; +import * as functions from 'firebase-functions'; export const missingVariableError = (field: string) => new Error( @@ -52,3 +53,17 @@ export function createErrorMessage(e: unknown): string { } return `An error occurred while processing the provided message, ${e.message}`; } + +export const throwUnauthenticatedError = () => { + throw new functions.https.HttpsError( + 'unauthenticated', + 'User is not authenticated' + ); +}; + +export const throwInvalidArgumentError = () => { + throw new functions.https.HttpsError( + 'invalid-argument', + 'Data must be an object' + ); +}; diff --git a/firestore-multimodal-genai/functions/src/generative-client/base_client.ts b/firestore-multimodal-genai/functions/src/generative-client/base_client.ts index fb0259b3..0cf96ad5 100644 --- a/firestore-multimodal-genai/functions/src/generative-client/base_client.ts +++ b/firestore-multimodal-genai/functions/src/generative-client/base_client.ts @@ -1,4 +1,4 @@ -interface GenerativeResponse { +export interface GenerativeResponse { candidates: string[]; safetyMetadata?: { blocked: boolean; diff --git a/firestore-multimodal-genai/functions/src/generative-client/index.ts b/firestore-multimodal-genai/functions/src/generative-client/index.ts index 3fc215ef..0389e5f2 100644 --- a/firestore-multimodal-genai/functions/src/generative-client/index.ts +++ b/firestore-multimodal-genai/functions/src/generative-client/index.ts @@ -30,3 +30,5 @@ export const getGenerativeClient = (): GenerativeClient => { throw new Error('Invalid provider'); } }; + +export {GenerativeResponse} from './base_client'; diff --git a/firestore-multimodal-genai/functions/src/index.ts b/firestore-multimodal-genai/functions/src/index.ts index c8b24afc..348d08ea 100644 --- a/firestore-multimodal-genai/functions/src/index.ts +++ b/firestore-multimodal-genai/functions/src/index.ts @@ -29,15 +29,11 @@ import { variableTypeError, } from './errors'; -const {prompt, responseField, collectionName, candidateCount, candidatesField} = - config; +const {prompt, responseField, collectionName} = config; -import {getGenerativeClient} from './generative-client'; +import {getGenerativeClient, GenerativeResponse} from './generative-client'; import {extractHandlebarsVariables} from './utils'; -// import {fetchCustomHookData} from './custom_hook'; - -// TODO: make sure we redact API KEY -// logs.init(config); +import {throwInvalidArgumentError, throwUnauthenticatedError} from './errors'; export const generateText = functions.firestore .document(collectionName) @@ -79,34 +75,8 @@ export const generateText = functions.firestore }); try { - const view: Record = {}; - - // if (config.ragConfig?.customRagHookUrl) { - // // TODO: extract the variable field values before passing to hook - // const customHookData = await fetchCustomHookData( - // data, - // config.ragConfig - // ); - // data = { - // ...data, - // ...customHookData, - // }; - // } - - const variableFields = extractHandlebarsVariables(prompt); - - for (const field of variableFields || []) { - if (!data[field]) { - throw missingVariableError(field); - } - if (typeof data[field] !== 'string') { - throw variableTypeError(field); - } - view[field] = data[field]; - } - // if prompt contains handlebars for variable substitution, do it: - const substitutedPrompt = Mustache.render(prompt, view); + const substitutedPrompt = getSubstitutedPrompt(data, prompt); const t0 = performance.now(); let requestOptions = {}; @@ -143,17 +113,11 @@ export const generateText = functions.firestore } } - const addCandidatesField = - config.provider === 'generative' && - candidatesField && - candidateCount && - candidateCount > 1; - - if (addCandidatesField) { + if (config.candidates.shouldIncludeCandidatesField) { return ref.update({ ...metadata, [responseField]: result.candidates[0], - [candidatesField]: result.candidates, + [config.candidates.field]: result.candidates, 'status.error': null, }); } @@ -173,3 +137,83 @@ export const generateText = functions.firestore }); } }); + +export const generateOnCall = functions.https.onCall(async (data, context) => { + if (!context.auth) throwUnauthenticatedError(); + if (typeof data !== 'object') throwInvalidArgumentError(); + + const {image, safetySettings} = data; + + const substitutedPrompt = getSubstitutedPrompt(data, prompt); + + let requestOptions = {}; + if (config.googleAi.model === 'gemini-pro-vision') { + requestOptions = { + ...requestOptions, + image, + safetySettings: safetySettings || config.safetySettings, + }; + } + + const generativeClient = getGenerativeClient(); + + const result = await generativeClient.generate( + substitutedPrompt, + requestOptions + ); + + const metadata = extractMetadata(result); + + if (config.candidates.shouldIncludeCandidatesField) { + return { + ...metadata, + [responseField]: result.candidates[0], + [config.candidates.field]: result.candidates, + }; + } else { + return { + ...metadata, + [responseField]: result.candidates[0], + }; + } +}); + +function getSubstitutedPrompt(data: any, prompt: string) { + const view: Record = {}; + + const variableFields = extractHandlebarsVariables(prompt); + + for (const field of variableFields || []) { + if (!data[field]) { + throw missingVariableError(field); + } + if (typeof data[field] !== 'string') { + throw variableTypeError(field); + } + view[field] = data[field]; + } + + // if prompt contains handlebars for variable substitution, do it: + const substitutedPrompt = Mustache.render(prompt, view); + + return substitutedPrompt; +} + +interface Metadata { + safetyMetadata?: Record; +} + +// Extract metadata from the result +function extractMetadata(result: GenerativeResponse): Metadata { + const metadata: Metadata = {}; + if (result.safetyMetadata) { + metadata.safetyMetadata = {}; + + for (const key of Object.keys(result.safetyMetadata)) { + if (result.safetyMetadata[key] !== undefined) { + metadata.safetyMetadata[key] = result.safetyMetadata[key]; + } + } + } + return metadata; +} From 412ebf60f47423dd8805280d94d18511067a2155 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 18 Mar 2024 09:29:29 +0000 Subject: [PATCH 4/4] docs: remove reference to VARIABLE_FIELDS --- firestore-multimodal-genai/POSTINSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore-multimodal-genai/POSTINSTALL.md b/firestore-multimodal-genai/POSTINSTALL.md index e81438a2..1099de10 100644 --- a/firestore-multimodal-genai/POSTINSTALL.md +++ b/firestore-multimodal-genai/POSTINSTALL.md @@ -13,7 +13,7 @@ Note: You can also use the Firebase Admin SDK to add a document: const ref = await admin .firestore() .collection("${param:COLLECTION_NAME}") - .add({ ... }) // Include values for ${param:VARIABLE_FIELDS} fields + .add({ ... }) ref.onSnapshot(snap => { if (snap.get('${param:RESPONSE_FIELD}')) console.log(