small-stone · small-stone · Apr 3, 2025 · Apr 3, 2025
diff --git a/components/Chat/Chat.tsx b/components/Chat/Chat.tsx
@@ -1,15 +1,9 @@
 import { IconClearAll, IconSettings } from '@tabler/icons-react';
-import {
-  MutableRefObject,
-  memo,
-  useCallback,
-  useContext,
-  useEffect,
-  useRef,
-  useState,
-} from 'react';
+import { MutableRefObject, memo, useCallback, useContext, useEffect, useRef, useState } from 'react';
 import toast from 'react-hot-toast';
 
+
+
 import { useTranslation } from 'next-i18next';
 
 import { getEndpoint } from '@/utils/app/api';
@@ -29,10 +23,11 @@ import Spinner from '../Spinner';
 import { ChatInput } from './ChatInput';
 import { ChatLoader } from './ChatLoader';
 import { ErrorMessageDiv } from './ErrorMessageDiv';
+import { MaxTokensSlider } from './MaxTokens';
+import { MemoizedChatMessage } from './MemoizedChatMessage';
 import { ModelSelect } from './ModelSelect';
 import { SystemPrompt } from './SystemPrompt';
 import { TemperatureSlider } from './Temperature';
-import { MemoizedChatMessage } from './MemoizedChatMessage';
 
 interface Props {
   stopConversationRef: MutableRefObject<boolean>;
@@ -71,6 +66,11 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
   const handleSend = useCallback(
     async (message: Message, deleteCount = 0, plugin: Plugin | null = null) => {
       if (selectedConversation) {
+        let startTime: number = 0;
+        let firstTokenTime: number = 0;
+        let endTime: number = 0;
+        let totalTokens = 0;
+
         let updatedConversation: Conversation;
         if (deleteCount) {
           const updatedMessages = [...selectedConversation.messages];
@@ -99,6 +99,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
           key: apiKey,
           prompt: updatedConversation.prompt,
           temperature: updatedConversation.temperature,
+          maxTokens: updatedConversation.maxTokens,
         };
         const endpoint = getEndpoint(plugin);
         let body;
@@ -152,6 +153,9 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
           let done = false;
           let isFirst = true;
           let text = '';
+
+          startTime = performance.now();
+
           while (!done) {
             if (stopConversationRef.current === true) {
               controller.abort();
@@ -162,7 +166,9 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
             done = doneReading;
             const chunkValue = decoder.decode(value);
             text += chunkValue;
+
             if (isFirst) {
+              firstTokenTime = performance.now();
               isFirst = false;
               const updatedMessages: Message[] = [
                 ...updatedConversation.messages,
@@ -177,6 +183,8 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                 value: updatedConversation,
               });
             } else {
+              totalTokens += chunkValue.split(/\s+/).length;
+
               const updatedMessages: Message[] =
                 updatedConversation.messages.map((message, index) => {
                   if (index === updatedConversation.messages.length - 1) {
@@ -197,6 +205,17 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
               });
             }
           }
+
+          endTime = performance.now();
+          const ttft = ((firstTokenTime - startTime) / 1000).toFixed(2);
+          const totalTime = (endTime - firstTokenTime) / 1000;
+          const tps = Math.round(totalTokens / totalTime);
+
+          const performanceMetrics = `\nTTFT: ${ttft} seconds, TPS: ${tps} tokens/second`;
+          const lastMessageIndex = updatedConversation.messages.length - 1;
+          updatedConversation.messages[lastMessageIndex].content +=
+            performanceMetrics;
+
           saveConversation(updatedConversation);
           const updatedConversations: Conversation[] = conversations.map(
             (conversation) => {
@@ -251,6 +270,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
       pluginKeys,
       selectedConversation,
       stopConversationRef,
+      homeDispatch,
     ],
   );
 
@@ -433,15 +453,25 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                           })
                         }
                       />
+
+                      <MaxTokensSlider
+                        label={t('Max Tokens')}
+                        onChangeMaxTokens={(maxTokens) =>
+                          handleUpdateConversation(selectedConversation, {
+                            key: 'maxTokens',
+                            value: maxTokens,
+                          })
+                        }
+                      />
                     </div>
                   )}
                 </div>
               </>
             ) : (
               <>
                 <div className="sticky top-0 z-10 flex justify-center border border-b-neutral-300 bg-neutral-100 py-2 text-sm text-neutral-500 dark:border-none dark:bg-[#444654] dark:text-neutral-200">
-                  {t('Model')}: {selectedConversation?.model?.name} | {t('Temp')}
-                  : {selectedConversation?.temperature} |
+                  {t('Model')}: {selectedConversation?.model?.name} |{' '}
+                  {t('Temp')}: {selectedConversation?.temperature} |
                   <button
                     className="ml-2 cursor-pointer hover:opacity-50"
                     onClick={handleSettings}
@@ -509,4 +539,4 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
     </div>
   );
 });
-Chat.displayName = 'Chat';
+Chat.displayName = 'Chat';
diff --git a/components/Chat/MaxTokens.tsx b/components/Chat/MaxTokens.tsx
@@ -0,0 +1,64 @@
+import { FC, useContext, useState } from 'react';
+
+import { useTranslation } from 'next-i18next';
+
+import { DEFAULT_MAX_TOKENS } from '@/utils/app/const';
+
+import HomeContext from '@/pages/api/home/home.context';
+
+interface Props {
+  label: string;
+  onChangeMaxTokens: (temperature: number) => void;
+}
+
+export const MaxTokensSlider: FC<Props> = ({ label, onChangeMaxTokens }) => {
+  const {
+    state: { conversations },
+  } = useContext(HomeContext);
+  const lastConversation = conversations[conversations.length - 1];
+  const [maxTokens, setMaxTokens] = useState(
+    lastConversation?.maxTokens ?? DEFAULT_MAX_TOKENS,
+  );
+  const { t } = useTranslation('chat');
+  const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const newValue = parseFloat(event.target.value);
+    setMaxTokens(newValue);
+    onChangeMaxTokens(newValue);
+  };
+
+  return (
+    <div className="flex flex-col">
+      <label className="mb-2 text-left text-neutral-700 dark:text-neutral-400">
+        {label}
+      </label>
+      <span className="text-[12px] text-black/50 dark:text-white/50 text-sm">
+        {t(
+          'Higher values for max_tokens will allow the model to generate longer responses, while lower values will restrict the output to be more concise and constrained.',
+        )}
+      </span>
+      <span className="mt-2 mb-1 text-center text-neutral-900 dark:text-neutral-100">
+        {maxTokens}
+      </span>
+      <input
+        className="cursor-pointer"
+        type="range"
+        min={100}
+        max={2048}
+        step={1}
+        value={maxTokens}
+        onChange={handleChange}
+      />
+      <ul className="w mt-2 pb-8 flex justify-between px-[24px] text-neutral-900 dark:text-neutral-100">
+        <li className="flex justify-center">
+          <span className="absolute">{t('Concise')}</span>
+        </li>
+        <li className="flex justify-center">
+          <span className="absolute">{t('Moderate')}</span>
+        </li>
+        <li className="flex justify-center">
+          <span className="absolute">{t('Extended')}</span>
+        </li>
+      </ul>
+    </div>
+  );
+};
diff --git a/pages/api/chat.ts b/pages/api/chat.ts
@@ -1,19 +1,23 @@
-import { DEFAULT_SYSTEM_PROMPT, DEFAULT_TEMPERATURE } from '@/utils/app/const';
+import {
+  DEFAULT_MAX_TOKENS,
+  DEFAULT_SYSTEM_PROMPT,
+  DEFAULT_TEMPERATURE,
+} from '@/utils/app/const';
 import { OpenAIError, OpenAIStream } from '@/utils/server';
 
 import { ChatBody, Message } from '@/types/chat';
 
 // @ts-expect-error
 import wasm from '../../node_modules/@dqbd/tiktoken/lite/tiktoken_bg.wasm?module';
 
-
 export const config = {
   runtime: 'edge',
 };
 
 const handler = async (req: Request): Promise<Response> => {
   try {
-    const { model, messages, key, prompt, temperature } = (await req.json()) as ChatBody;
+    const { model, messages, key, prompt, temperature, maxTokens } =
+      (await req.json()) as ChatBody;
 
     let promptToSend = prompt;
     if (!promptToSend) {
@@ -24,11 +28,24 @@ const handler = async (req: Request): Promise<Response> => {
     if (temperatureToUse == null) {
       temperatureToUse = DEFAULT_TEMPERATURE;
     }
+
+    let maxTokensToUse = maxTokens;
+    if (maxTokensToUse == null) {
+      maxTokensToUse = DEFAULT_MAX_TOKENS;
+    }
+
     if (model == null) {
       throw new Error('No model specified');
     }
-
-    const stream = await OpenAIStream(model, promptToSend, temperatureToUse, key, messages);
+
+    const stream = await OpenAIStream(
+      model,
+      promptToSend,
+      temperatureToUse,
+      maxTokensToUse,
+      key,
+      messages,
+    );
 
     return new Response(stream);
   } catch (error) {

diff --git a/pages/api/home/home.state.tsx b/pages/api/home/home.state.tsx
@@ -5,6 +5,7 @@ import { OpenAIModel } from '@/types/openai';
 import { PluginKey } from '@/types/plugin';
 import { Prompt } from '@/types/prompt';
 
+
 export interface HomeInitialState {
   apiKey: string;
   pluginKeys: PluginKey[];
@@ -19,6 +20,7 @@ export interface HomeInitialState {
   currentMessage: Message | undefined;
   prompts: Prompt[];
   temperature: number;
+  maxTokens: number;
   showChatbar: boolean;
   showPromptbar: boolean;
   currentFolder: FolderInterface | undefined;
@@ -49,4 +51,5 @@ export const initialState: HomeInitialState = {
   searchTerm: '',
   serverSideApiKeyIsSet: false,
   serverSidePluginKeysSet: false,
-};
+  maxTokens: 1024,
+};
diff --git a/public/locales/zh/chat.json b/public/locales/zh/chat.json
@@ -28,9 +28,14 @@
   "Chatbot UI is an advanced chatbot kit for OpenAI's chat models aiming to mimic ChatGPT's interface and functionality.": "Chatbot UI 是一个高级聊天机器人工具包，旨在模仿 OpenAI 聊天模型的 ChatGPT 界面和功能。",
   "Are you sure you want to clear all messages?": "你确定要清除所有的消息吗？",
   "Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.": "较高的数值（例如0.8）会使输出更随机，而较低的数值（例如0.2）会使输出更加聚焦和确定性更强。",
+  "Max Tokens": "最大令牌数",
+  "Higher values for max_tokens will allow the model to generate longer responses, while lower values will restrict the output to be more concise and constrained.": "较高的 max_tokens 值将允许模型生成更长的响应，而较低的值将限制输出更加简洁和受限。",
   "View Account Usage": "查阅账户用量",
   "Temperature": "生成温度",
   "Precise": "保守",
   "Neutral": "中立",
-  "Creative": "随性"
+  "Creative": "随性",
+  "Concise": "简洁",
+  "Moderate": "适中",
+  "Extended": "较长"
 }
diff --git a/types/chat.ts b/types/chat.ts
@@ -1,5 +1,6 @@
 import { OpenAIModel } from './openai';
 
+
 export interface Message {
   role: Role;
   content: string;
@@ -13,6 +14,7 @@ export interface ChatBody {
   key: string;
   prompt: string;
   temperature: number;
+  maxTokens: number;
 }
 
 export interface Conversation {
@@ -22,5 +24,6 @@ export interface Conversation {
   model: OpenAIModel | null;
   prompt: string;
   temperature: number;
+  maxTokens: number;
   folderId: string | null;
-}
+}
diff --git a/utils/app/const.ts b/utils/app/const.ts
@@ -19,3 +19,7 @@ export const OPENAI_ORGANIZATION =
 
 export const AZURE_DEPLOYMENT_ID =
   process.env.AZURE_DEPLOYMENT_ID || '';
+
+export const DEFAULT_MAX_TOKENS = parseInt(
+  process.env.NEXT_PUBLIC_DEFAULT_MAX_TOKENS || '1024',
+);