From da682f42ce4dee29a9b971a6cf6400fc7acf1dad Mon Sep 17 00:00:00 2001 From: stevenksmith Date: Wed, 13 Dec 2023 07:53:24 +0100 Subject: [PATCH 1/4] fix: in oai chat completion, use msg name field instead of prepended to msg content --- srv/adapter/chat-completion.ts | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/srv/adapter/chat-completion.ts b/srv/adapter/chat-completion.ts index bc14af88a..b40852c50 100644 --- a/srv/adapter/chat-completion.ts +++ b/srv/adapter/chat-completion.ts @@ -249,15 +249,23 @@ export async function toChatCompletionPayload( const line = all[i] - const obj: CompletionItem = { - role: 'assistant', - content: line.trim().replace(BOT_REPLACE, replyAs.name).replace(SELF_REPLACE, handle), - } - const isSystem = line.startsWith('System:') const isUser = line.startsWith(handle) const isBot = !isUser && !isSystem + const name = isBot ? replyAs.name : isUser ? handle : undefined + + const content = line + .trim() + .replace(BOT_REPLACE, replyAs.name) + .replace(SELF_REPLACE, handle) + .replace(name ? `${name}: ` : '', '') + + const obj: CompletionItem = { + role: 'assistant', + content, + name, + } const insert = inserts.get(distanceFromBottom) if (insert) history.push({ role: 'system', content: insert }) @@ -291,7 +299,7 @@ export async function toChatCompletionPayload( obj.role = 'user' } - const length = await encoder()(obj.content) + const length = (await encoder()(obj.content)) + (obj.name ? 1 + (await encoder()(obj.name)) : 0) if (tokens + length > maxBudget) { --i break @@ -339,12 +347,15 @@ export async function splitSampleChat(opts: SplitSampleChatProps) { ? 'user' : 'system' - const msg: CompletionItem = { - role: role, - content: sample.replace(BOT_REPLACE, char).replace(SELF_REPLACE, sender), - } + const name = role === 'assistant' ? char : role === 'user' ? sender : undefined + + const content = sample + .replace(BOT_REPLACE, char) + .replace(SELF_REPLACE, sender) + .replace(name ? `${name}: ` : '', '') - const length = await encoder()(msg.content) + const msg: CompletionItem = { role, content, name } + const length = (await encoder()(msg.content)) + (msg.name ? 1 + (await encoder()(msg.name)) : 0) if (budget && tokens + length > budget) break additions.push(msg) From 88a89c45f263aa1c196bf68d9386ad636818f8ea Mon Sep 17 00:00:00 2001 From: stevenksmith Date: Wed, 13 Dec 2023 08:09:55 +0100 Subject: [PATCH 2/4] refactor for readability --- srv/adapter/chat-completion.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/srv/adapter/chat-completion.ts b/srv/adapter/chat-completion.ts index b40852c50..fd7cfc443 100644 --- a/srv/adapter/chat-completion.ts +++ b/srv/adapter/chat-completion.ts @@ -299,7 +299,8 @@ export async function toChatCompletionPayload( obj.role = 'user' } - const length = (await encoder()(obj.content)) + (obj.name ? 1 + (await encoder()(obj.name)) : 0) + const nameCost = obj.name ? 1 + (await encoder()(obj.name)) : 0 + const length = nameCost + (await encoder()(obj.content)) if (tokens + length > maxBudget) { --i break @@ -355,7 +356,8 @@ export async function splitSampleChat(opts: SplitSampleChatProps) { .replace(name ? `${name}: ` : '', '') const msg: CompletionItem = { role, content, name } - const length = (await encoder()(msg.content)) + (msg.name ? 1 + (await encoder()(msg.name)) : 0) + const nameCost = msg.name ? 1 + (await encoder()(msg.name)) : 0 + const length = nameCost + (await encoder()(msg.content)) if (budget && tokens + length > budget) break additions.push(msg) From c4eefd5c3819d96c884a413d35d1eb68b5e7268a Mon Sep 17 00:00:00 2001 From: stevenksmith Date: Wed, 13 Dec 2023 08:29:21 +0100 Subject: [PATCH 3/4] fix: determine correct author name, fixing past impersonations & example messages --- srv/adapter/chat-completion.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/srv/adapter/chat-completion.ts b/srv/adapter/chat-completion.ts index fd7cfc443..b906a0725 100644 --- a/srv/adapter/chat-completion.ts +++ b/srv/adapter/chat-completion.ts @@ -253,7 +253,8 @@ export async function toChatCompletionPayload( const isUser = line.startsWith(handle) const isBot = !isUser && !isSystem - const name = isBot ? replyAs.name : isUser ? handle : undefined + const nameInLine: string | undefined = line.split(':')[0] + const name = isBot ? nameInLine ?? replyAs.name : isUser ? handle : undefined const content = line .trim() From 18ee08af1aebe13c265e776e1c5e9b56f31df8e1 Mon Sep 17 00:00:00 2001 From: stevenksmith Date: Sat, 16 Dec 2023 05:03:09 +0100 Subject: [PATCH 4/4] fix: make OpenAI name field follow API constraint --- srv/adapter/chat-completion.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/srv/adapter/chat-completion.ts b/srv/adapter/chat-completion.ts index b906a0725..6c2805645 100644 --- a/srv/adapter/chat-completion.ts +++ b/srv/adapter/chat-completion.ts @@ -166,6 +166,17 @@ export const streamCompletion: CompletionGenerator = async function* ( } } +/** + * OpenAI API enforces the following: + * "'Firstname Lastname' does not match '^[a-zA-Z0-9_-]{1,64}$' - 'messages.1.name'" + */ +function openAiName(inputName?: string): string | undefined { + return inputName + ?.replace(/ /g, '_') + .replace(/[^a-zA-Z0-9_-]/g, '') + .substring(0, 64) +} + /** * This function contains the inserts logic for Chat models (Turbo, GPT4...) * This logic also exists in other places: @@ -254,18 +265,18 @@ export async function toChatCompletionPayload( const isBot = !isUser && !isSystem const nameInLine: string | undefined = line.split(':')[0] - const name = isBot ? nameInLine ?? replyAs.name : isUser ? handle : undefined + const speakerName = isBot ? nameInLine ?? replyAs.name : isUser ? handle : undefined const content = line .trim() .replace(BOT_REPLACE, replyAs.name) .replace(SELF_REPLACE, handle) - .replace(name ? `${name}: ` : '', '') + .replace(speakerName ? `${speakerName}: ` : '', '') const obj: CompletionItem = { role: 'assistant', content, - name, + name: openAiName(speakerName), } const insert = inserts.get(distanceFromBottom) if (insert) history.push({ role: 'system', content: insert }) @@ -349,14 +360,14 @@ export async function splitSampleChat(opts: SplitSampleChatProps) { ? 'user' : 'system' - const name = role === 'assistant' ? char : role === 'user' ? sender : undefined + const speakerName = role === 'assistant' ? char : role === 'user' ? sender : undefined const content = sample .replace(BOT_REPLACE, char) .replace(SELF_REPLACE, sender) - .replace(name ? `${name}: ` : '', '') + .replace(speakerName ? `${speakerName}: ` : '', '') - const msg: CompletionItem = { role, content, name } + const msg: CompletionItem = { role, content, name: openAiName(speakerName) } const nameCost = msg.name ? 1 + (await encoder()(msg.name)) : 0 const length = nameCost + (await encoder()(msg.content)) if (budget && tokens + length > budget) break