feat(voice,search): Added 'voice agent' mode using Deepgram and Cartesia, and web search tool via Tavily

.env.example

@@ -21,3 +21,39 @@ POSTGRES_URL=****
 # Instructions to create a Redis store here:
 # https://vercel.com/docs/redis
 REDIS_URL=****
+
+# ================================
+# WEB SEARCH (OPTIONAL)
+# ================================
+# Tavily API enables the AI to search the web for real-time information.
+# Get your API key from https://tavily.com
+# If not configured, the AI will indicate that web search is unavailable
+# when users request current information. The chat will continue to work
+# normally for all other queries.
+# TAVILY_API_KEY="..."
+
+
+# ================================
+# VOICE FEATURES (OPTIONAL)
+# ================================
+# The voice agent feature requires several external services.
+# If these are not configured, voice features will be disabled
+# but the rest of the app will work normally.
+
+# Deepgram API (Speech-to-Text)
+# Get your API key from https://deepgram.com
+# DEEPGRAM_API_KEY="..."
+
+# Cartesia API (Text-to-Speech)
+# Get your API key from https://cartesia.ai
+# Cartesia is used to synthesize the text response into speech.
+# https://play.cartesia.ai/console
+# CARTESIA_API_KEY="..."
+
+
+# NLP Worker Service (End-of-Turn Detection)
+# This is a custom service for detecting when users finish speaking.
+# If not configured, the app will fall back to simple heuristics.
+# See VOICE_FEATURES.md for setup instructions.
+# NLP_WORKER_URL=http://localhost:8097
+# NLP_WORKER_API_KEY=your-nlp-worker-api-key

.gitignore

@@ -1,5 +1,11 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+# Generated AI artifacts
+.ai-docs/
+.playwright-mcp/
+.cursor/plans/
+.commit.msg.md
+
 # dependencies
 node_modules
 .pnp
@@ -42,3 +48,12 @@ yarn-error.log*
 /playwright-report/
 /blob-report/
 /playwright/*
+
+# VAD model files (large binaries, served from public/)
+/public/*.onnx
+/public/*.wasm
+/public/*.mjs
+/public/vad.worklet.bundle.min.js
+
+*.disabled*
+**/*.disabled*

README.md

@@ -33,6 +33,11 @@
   - [Vercel Blob](https://vercel.com/storage/blob) for efficient file storage
 - [Auth.js](https://authjs.dev)
   - Simple and secure authentication
+- Voice Agent (Optional)
+  - Real-time speech-to-text with Deepgram
+  - Natural text-to-speech with Cartesia
+  - Smart end-of-turn detection with graceful fallbacks
+  - See [VOICE_FEATURES.md](VOICE_FEATURES.md) for setup
 
 ## Model Providers
 
@@ -54,9 +59,9 @@ You can deploy your own version of the Next.js AI Chatbot to Vercel with one cli
 
 ## Running locally
 
-You will need to use the environment variables [defined in `.env.example`](.env.example) to run Next.js AI Chatbot. It's recommended you use [Vercel Environment Variables](https://vercel.com/docs/projects/environment-variables) for this, but a `.env` file is all that is necessary.
+You will need to use the environment variables [defined in `.env.example`](.env.example) to run Next.js AI Chatbot. It's recommended you use [Vercel Environment Variables](https://vercel.com/docs/projects/environment-variables) for this, but a `.env.local` file is all that is necessary.
 
-> Note: You should not commit your `.env` file or it will expose secrets that will allow others to control access to your various AI and authentication provider accounts.
+> Note: You should not commit your `.env.local` file or it will expose secrets that will allow others to control access to your various AI and authentication provider accounts.
 
 1. Install Vercel CLI: `npm i -g vercel`
 2. Link local instance with Vercel and GitHub accounts (creates `.vercel` directory): `vercel link`
@@ -69,3 +74,19 @@ pnpm dev
 ```
 
 Your app template should now be running on [localhost:3000](http://localhost:3000).
+
+### Optional: Voice Features
+
+Voice features require additional API keys. See [VOICE_FEATURES.md](VOICE_FEATURES.md) for detailed setup instructions.
+
+```bash
+# Add to your .env.local file
+NEXT_PUBLIC_DEEPGRAM_API_KEY=your-deepgram-key
+NEXT_PUBLIC_CARTESIA_API_KEY=your-cartesia-key
+
+# Optional: Enhanced end-of-turn detection
+NLP_WORKER_URL=http://localhost:8097
+NLP_WORKER_API_KEY=your-api-key
+```
+
+The app works perfectly fine without these - voice features are completely optional.

app/(auth)/login/page.tsx

@@ -41,8 +41,7 @@ export default function Page() {
       updateSession();
       router.refresh();
     }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [state.status]);
+  }, [state.status, updateSession, router]);
 
   const handleSubmit = (formData: FormData) => {
     setEmail(formData.get("email") as string);

app/(auth)/register/page.tsx

@@ -35,14 +35,16 @@ export default function Page() {
         description: "Failed validating your submission!",
       });
     } else if (state.status === "success") {
-      toast({ type: "success", description: "Account created successfully!" });
+      toast({
+        type: "success",
+        description: "Account created successfully!",
+      });
 
       setIsSuccessful(true);
       updateSession();
       router.refresh();
     }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [state.status]);
+  }, [state.status, updateSession, router]);
 
   const handleSubmit = (formData: FormData) => {
     setEmail(formData.get("email") as string);

app/(chat)/actions.ts

@@ -3,8 +3,8 @@
 import { generateText, type UIMessage } from "ai";
 import { cookies } from "next/headers";
 import type { VisibilityType } from "@/components/visibility-selector";
-import { myProvider } from "@/lib/ai/providers";
 import { titlePrompt } from "@/lib/ai/prompts";
+import { myProvider } from "@/lib/ai/providers";
 import {
   deleteMessagesByChatIdAfterTimestamp,
   getMessageById,

app/(chat)/api/chat/route.ts

@@ -1,3 +1,4 @@
+/** biome-ignore-all lint/correctness/noUnusedImports: refactor testing */
 import { geolocation } from "@vercel/functions";
 import {
   convertToModelMessages,
@@ -25,6 +26,7 @@ import { myProvider } from "@/lib/ai/providers";
 import { createDocument } from "@/lib/ai/tools/create-document";
 import { getWeather } from "@/lib/ai/tools/get-weather";
 import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
+import { searchWeb } from "@/lib/ai/tools/search-web";
 import { updateDocument } from "@/lib/ai/tools/update-document";
 import { isProductionEnvironment } from "@/lib/constants";
 import {
@@ -47,6 +49,8 @@ import { type PostRequestBody, postRequestBodySchema } from "./schema";
 
 export const maxDuration = 60;
 
+const ENABLE_RATE_LIMITING = true;
+
 let globalStreamContext: ResumableStreamContext | null = null;
 
 const getTokenlensCatalog = cache(
@@ -116,13 +120,15 @@ export async function POST(request: Request) {
 
     const userType: UserType = session.user.type;
 
-    const messageCount = await getMessageCountByUserId({
-      id: session.user.id,
-      differenceInHours: 24,
-    });
+    if (ENABLE_RATE_LIMITING) {
+      const messageCount = await getMessageCountByUserId({
+        id: session.user.id,
+        differenceInHours: 24,
+      });
 
-    if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
-      return new ChatSDKError("rate_limit:chat").toResponse();
+      if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
+        return new ChatSDKError("rate_limit:chat").toResponse();
+      }
     }
 
     const chat = await getChatById({ id });
@@ -189,13 +195,15 @@ export async function POST(request: Request) {
               ? []
               : [
                   "getWeather",
+                  "searchWeb",
                   "createDocument",
                   "updateDocument",
                   "requestSuggestions",
                 ],
           experimental_transform: smoothStream({ chunking: "word" }),
           tools: {
             getWeather,
+            searchWeb,
             createDocument: createDocument({ session, dataStream }),
             updateDocument: updateDocument({ session, dataStream }),
             requestSuggestions: requestSuggestions({
@@ -230,13 +238,27 @@ export async function POST(request: Request) {
                 return;
               }
 
-              const summary = getUsage({ modelId, usage, providers });
-              finalMergedUsage = { ...usage, ...summary, modelId } as AppUsage;
-              dataStream.write({ type: "data-usage", data: finalMergedUsage });
+              const summary = getUsage({
+                modelId,
+                usage,
+                providers,
+              });
+              finalMergedUsage = {
+                ...usage,
+                ...summary,
+                modelId,
+              } as AppUsage;
+              dataStream.write({
+                type: "data-usage",
+                data: finalMergedUsage,
+              });
             } catch (err) {
               console.warn("TokenLens enrichment failed", err);
               finalMergedUsage = usage;
-              dataStream.write({ type: "data-usage", data: finalMergedUsage });
+              dataStream.write({
+                type: "data-usage",
+                data: finalMergedUsage,
+              });
             }
           },
         });

app/(chat)/api/history/route.ts

@@ -1,6 +1,6 @@
 import type { NextRequest } from "next/server";
 import { auth } from "@/app/(auth)/auth";
-import { getChatsByUserId, deleteAllChatsByUserId } from "@/lib/db/queries";
+import { deleteAllChatsByUserId, getChatsByUserId } from "@/lib/db/queries";
 import { ChatSDKError } from "@/lib/errors";
 
 export async function GET(request: NextRequest) {

app/(chat)/api/voice/deepgram-token/route.ts

@@ -0,0 +1,86 @@
+/**
+ * Deepgram Temporary Token Generation API Route
+ *
+ * **Endpoint**: POST /api/voice/deepgram-token
+ *
+ * **Purpose**: Generates short-lived JWT tokens for secure client-side
+ * Deepgram API access without exposing permanent API keys.
+ *
+ * **Security**:
+ * - Token TTL: 30 seconds (Deepgram default)
+ * - Scopes: usage:write only (can't create keys, access billing, etc.)
+ * - WebSocket connections stay open beyond token expiry
+ * - Never exposes main API key to client
+ *
+ * **Response**:
+ * ```json
+ * {
+ *   "token": "eyJ0eXAiOiJKV1QiLCJhbGc...",
+ *   "expires_in": 30
+ * }
+ * ```
+ *
+ * **Usage in Frontend**:
+ * ```typescript
+ * const { token } = await fetch('/api/voice/deepgram-token', { method: 'POST' });
+ * const deepgram = createClient(token);
+ * const connection = deepgram.listen.live({ ... });
+ * ```
+ *
+ * **Environment Variables**:
+ * - DEEPGRAM_API_KEY: Main Deepgram API key (server-side only)
+ *
+ * @see hooks/use-deepgram-stream.ts - Frontend Deepgram connection
+ * @see https://developers.deepgram.com/docs/token-based-authentication
+ */
+
+import { NextResponse } from "next/server";
+
+export async function POST() {
+  try {
+    const apiKey = process.env.DEEPGRAM_API_KEY;
+
+    if (!apiKey) {
+      console.error("Missing DEEPGRAM_API_KEY environment variable");
+      return NextResponse.json(
+        { error: "Deepgram API key not configured" },
+        { status: 500 }
+      );
+    }
+
+    // Generate temporary JWT token via /v1/auth/grant endpoint
+    // This creates a 30-second token that allows usage:write access
+    const response = await fetch("https://api.deepgram.com/v1/auth/grant", {
+      method: "POST",
+      headers: {
+        Authorization: `Token ${apiKey}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      console.error("Deepgram token generation failed:", error);
+      return NextResponse.json(
+        { error: "Failed to generate token" },
+        { status: response.status }
+      );
+    }
+
+    const data = await response.json();
+
+    return NextResponse.json({
+      token: data.access_token,
+      expires_in: data.expires_in,
+    });
+  } catch (error) {
+    console.error("Deepgram token generation error:", error);
+    return NextResponse.json(
+      {
+        error:
+          error instanceof Error ? error.message : "Failed to generate token",
+      },
+      { status: 500 }
+    );
+  }
+}