From a7d76facf6a8c28bf69ef16a9416b13d0755876b Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:04:36 -0800 Subject: [PATCH 01/10] wip langchain nextjs template adaptation --- ui/.vscode/launch.json | 9 +- ui/app/api/rag-url-with-langchain/route.ts | 170 ++++++++++++ .../langchain-template/ChatMessageBubble.tsx | 39 +++ .../langchain-template/ChatWindow.tsx | 247 ++++++++++++++++++ .../langchain-template/IntermediateStep.tsx | 36 +++ .../UploadDocumentsForm.tsx | 65 +++++ .../stacks/rag-url-with-langchain.tsx | 235 +++++++++++++++++ ui/package-lock.json | 21 ++ ui/package.json | 1 + 9 files changed, 822 insertions(+), 1 deletion(-) create mode 100644 ui/app/api/rag-url-with-langchain/route.ts create mode 100644 ui/app/components/langchain-template/ChatMessageBubble.tsx create mode 100644 ui/app/components/langchain-template/ChatWindow.tsx create mode 100644 ui/app/components/langchain-template/IntermediateStep.tsx create mode 100644 ui/app/components/langchain-template/UploadDocumentsForm.tsx create mode 100644 ui/app/components/stacks/rag-url-with-langchain.tsx diff --git a/ui/.vscode/launch.json b/ui/.vscode/launch.json index f1d6dd45..46df60fc 100644 --- a/ui/.vscode/launch.json +++ b/ui/.vscode/launch.json @@ -1,11 +1,18 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Launch Chrome", + "request": "launch", + "type": "chrome", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}" + }, { "name": "Next.js", "type": "node-terminal", "request": "launch", - "command": "npm run dev", + "command": "npm run dev --debug-brk", "skipFiles": ["/**"] } ] diff --git a/ui/app/api/rag-url-with-langchain/route.ts b/ui/app/api/rag-url-with-langchain/route.ts new file mode 100644 index 00000000..204da37c --- /dev/null +++ b/ui/app/api/rag-url-with-langchain/route.ts @@ -0,0 +1,170 @@ +import type { NextRequest } from 'next/server'; +import { NextResponse } from 'next/server'; +import { createClient } from '@supabase/supabase-js'; +import type { Message as VercelChatMessage } from 'ai'; +import { StreamingTextResponse } from 'ai'; +import { ChatOpenAI } from 'langchain/chat_models/openai'; +import type { Document } from 'langchain/document'; +import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; +import { PromptTemplate } from 'langchain/prompts'; +import { + BytesOutputParser, + StringOutputParser, +} from 'langchain/schema/output_parser'; +import { RunnableSequence } from 'langchain/schema/runnable'; +import { SupabaseVectorStore } from 'langchain/vectorstores/supabase'; + +export const runtime = 'edge'; + +const combineDocumentsFn = (docs: Document[]) => { + const serializedDocs = docs.map((doc) => doc.pageContent); + return serializedDocs.join('\n\n'); +}; + +const formatVercelMessages = (chatHistory: VercelChatMessage[]) => { + const formattedDialogueTurns = chatHistory.map((message) => { + if (message.role === 'user') { + return `Human: ${message.content}`; + } else if (message.role === 'assistant') { + return `Assistant: ${message.content}`; + } else { + return `${message.role}: ${message.content}`; + } + }); + return formattedDialogueTurns.join('\n'); +}; + +const CONDENSE_QUESTION_TEMPLATE = `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. + + + {chat_history} + + +Follow Up Input: {question} +Standalone question:`; +const condenseQuestionPrompt = PromptTemplate.fromTemplate( + CONDENSE_QUESTION_TEMPLATE, +); + +const ANSWER_TEMPLATE = `You are an energetic talking puppy named Dana, and must answer all questions like a happy, talking dog would. +Use lots of puns! + +Answer the question based only on the following context and chat history: + + {context} + + + + {chat_history} + + +Question: {question} +`; +const answerPrompt = PromptTemplate.fromTemplate(ANSWER_TEMPLATE); + +/** + * This handler initializes and calls a retrieval chain. It composes the chain using + * LangChain Expression Language. See the docs for more information: + * + * https://js.langchain.com/docs/guides/expression_language/cookbook#conversational-retrieval-chain + */ +export async function POST(req: NextRequest) { + try { + const body = await req.json(); + const messages = body.messages ?? []; + const previousMessages = messages.slice(0, -1); + const currentMessageContent = messages[messages.length - 1].content; + + const model = new ChatOpenAI({ + modelName: 'gpt-3.5-turbo', + temperature: 0.2, + }); + + const client = createClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PRIVATE_KEY!, + ); + const vectorstore = new SupabaseVectorStore(new OpenAIEmbeddings(), { + client, + tableName: 'documents', + queryName: 'match_documents', + }); + + /** + * We use LangChain Expression Language to compose two chains. + * To learn more, see the guide here: + * + * https://js.langchain.com/docs/guides/expression_language/cookbook + */ + const standaloneQuestionChain = RunnableSequence.from([ + condenseQuestionPrompt, + model, + new StringOutputParser(), + ]); + + let resolveWithDocuments: (value: Document[]) => void; + const documentPromise = new Promise((resolve) => { + resolveWithDocuments = resolve; + }); + + const retriever = vectorstore.asRetriever({ + callbacks: [ + { + handleRetrieverEnd(documents) { + resolveWithDocuments(documents); + }, + }, + ], + }); + + const retrievalChain = retriever.pipe(combineDocumentsFn); + + const answerChain = RunnableSequence.from([ + { + context: RunnableSequence.from([ + (input) => input.question, + retrievalChain, + ]), + chat_history: (input) => input.chat_history, + question: (input) => input.question, + }, + answerPrompt, + model, + ]); + + const conversationalRetrievalQAChain = RunnableSequence.from([ + { + question: standaloneQuestionChain, + chat_history: (input) => input.chat_history, + }, + answerChain, + new BytesOutputParser(), + ]); + + const stream = await conversationalRetrievalQAChain.stream({ + question: currentMessageContent, + chat_history: formatVercelMessages(previousMessages), + }); + + const documents = await documentPromise; + const serializedSources = Buffer.from( + JSON.stringify( + documents.map((doc) => { + return { + pageContent: doc.pageContent.slice(0, 50) + '...', + metadata: doc.metadata, + }; + }), + ), + ).toString('base64'); + + return new StreamingTextResponse(stream, { + headers: { + 'x-message-index': (previousMessages.length + 1).toString(), + 'x-sources': serializedSources, + }, + }); + } catch (e: any) { + return NextResponse.json({ error: e.message }, { status: 500 }); + } +} diff --git a/ui/app/components/langchain-template/ChatMessageBubble.tsx b/ui/app/components/langchain-template/ChatMessageBubble.tsx new file mode 100644 index 00000000..8d1beca8 --- /dev/null +++ b/ui/app/components/langchain-template/ChatMessageBubble.tsx @@ -0,0 +1,39 @@ +import type { Message } from "ai/react"; + +export function ChatMessageBubble(props: { message: Message, aiEmoji?: string, sources: any[] }) { + const colorClassName = + props.message.role === "user" ? "bg-sky-600" : "bg-slate-50 text-black"; + const alignmentClassName = + props.message.role === "user" ? "ml-auto" : "mr-auto"; + const prefix = props.message.role === "user" ? "๐Ÿง‘" : props.aiEmoji; + return ( +
+
+ {prefix} +
+
+ {props.message.content} + {props.sources && props.sources.length ? <> + +

+ ๐Ÿ” Sources: +

+ + + {props.sources?.map((source, i) => ( +
+ {i + 1}. "{source.pageContent}"{ + source.metadata?.loc?.lines !== undefined + ?

Lines {source.metadata?.loc?.lines?.from} to {source.metadata?.loc?.lines?.to}
+ : "" + } +
+ ))} + + : ""} +
+
+ ); +} \ No newline at end of file diff --git a/ui/app/components/langchain-template/ChatWindow.tsx b/ui/app/components/langchain-template/ChatWindow.tsx new file mode 100644 index 00000000..34babd31 --- /dev/null +++ b/ui/app/components/langchain-template/ChatWindow.tsx @@ -0,0 +1,247 @@ +'use client'; + +import { toast, ToastContainer } from 'react-toastify'; + +import 'react-toastify/dist/ReactToastify.css'; + +import type { FormEvent, ReactElement } from 'react'; +import { useRef, useState } from 'react'; +import { useChat } from 'ai/react'; +import type { AgentStep } from 'langchain/schema'; + +import { ChatMessageBubble } from './ChatMessageBubble'; +import { IntermediateStep } from './IntermediateStep'; +import { UploadDocumentsForm } from './UploadDocumentsForm'; + +export function ChatWindow(props: { + endpoint: string; + emptyStateComponent: ReactElement; + placeholder?: string; + titleText?: string; + emoji?: string; + showIngestForm?: boolean; + showIntermediateStepsToggle?: boolean; +}) { + const messageContainerRef = useRef(null); + + const { + endpoint, + emptyStateComponent, + placeholder, + titleText = 'An LLM', + showIngestForm, + showIntermediateStepsToggle, + emoji, + } = props; + + const [showIntermediateSteps, setShowIntermediateSteps] = useState(false); + const [intermediateStepsLoading, setIntermediateStepsLoading] = + useState(false); + const ingestForm = showIngestForm && ( + + ); + const intemediateStepsToggle = showIntermediateStepsToggle && ( +
+ setShowIntermediateSteps(e.target.checked)} + > + +
+ ); + + const [sourcesForMessages, setSourcesForMessages] = useState< + Record + >({}); + + const { + messages, + input, + setInput, + handleInputChange, + handleSubmit, + isLoading: chatEndpointIsLoading, + setMessages, + } = useChat({ + api: endpoint, + onResponse(response) { + const sourcesHeader = response.headers.get('x-sources'); + const sources = sourcesHeader ? JSON.parse(atob(sourcesHeader)) : []; + const messageIndexHeader = response.headers.get('x-message-index'); + if (sources.length && messageIndexHeader !== null) { + setSourcesForMessages({ + ...sourcesForMessages, + [messageIndexHeader]: sources, + }); + } + }, + onError: (e) => { + toast(e.message, { + theme: 'dark', + }); + }, + }); + + async function sendMessage(e: FormEvent) { + e.preventDefault(); + if (messageContainerRef.current) { + messageContainerRef.current.classList.add('grow'); + } + if (!messages.length) { + await new Promise((resolve) => setTimeout(resolve, 300)); + } + if (chatEndpointIsLoading ?? intermediateStepsLoading) { + return; + } + if (!showIntermediateSteps) { + handleSubmit(e); + // Some extra work to show intermediate steps properly + } else { + setIntermediateStepsLoading(true); + setInput(''); + const messagesWithUserReply = messages.concat({ + id: messages.length.toString(), + content: input, + role: 'user', + }); + setMessages(messagesWithUserReply); + const response = await fetch(endpoint, { + method: 'POST', + body: JSON.stringify({ + messages: messagesWithUserReply, + show_intermediate_steps: true, + }), + }); + const json = await response.json(); + setIntermediateStepsLoading(false); + if (response.status === 200) { + // Represent intermediate steps as system messages for display purposes + const intermediateStepMessages = (json.intermediate_steps ?? []).map( + (intermediateStep: AgentStep, i: number) => { + return { + id: (messagesWithUserReply.length + i).toString(), + content: JSON.stringify(intermediateStep), + role: 'system', + }; + }, + ); + const newMessages = messagesWithUserReply; + for (const message of intermediateStepMessages) { + newMessages.push(message); + setMessages([...newMessages]); + await new Promise((resolve) => + setTimeout(resolve, 1000 + Math.random() * 1000), + ); + } + setMessages([ + ...newMessages, + { + id: ( + newMessages.length + intermediateStepMessages.length + ).toString(), + content: json.output, + role: 'assistant', + }, + ]); + } else { + if (json.error) { + toast(json.error, { + theme: 'dark', + }); + throw new Error(json.error); + } + } + } + } + + return ( +
0 ? 'border' : '' + }`} + > +

0 ? '' : 'hidden'} text-2xl`}> + {emoji} {titleText} +

+ {messages.length === 0 ? emptyStateComponent : ''} +
+ {messages.length > 0 + ? [...messages].reverse().map((m, i) => { + const sourceKey = (messages.length - 1 - i).toString(); + return m.role === 'system' ? ( + + ) : ( + + ); + }) + : ''} +
+ + {messages.length === 0 && ingestForm} + +
+
{intemediateStepsToggle}
+
+ + +
+
+ +
+ ); +} diff --git a/ui/app/components/langchain-template/IntermediateStep.tsx b/ui/app/components/langchain-template/IntermediateStep.tsx new file mode 100644 index 00000000..6bffaeba --- /dev/null +++ b/ui/app/components/langchain-template/IntermediateStep.tsx @@ -0,0 +1,36 @@ +import { useState } from "react"; +import type { Message } from "ai/react"; +import type { AgentStep } from "langchain/schema"; + +export function IntermediateStep(props: { message: Message }) { + const parsedInput: AgentStep = JSON.parse(props.message.content); + const action = parsedInput.action; + const observation = parsedInput.observation; + const [expanded, setExpanded] = useState(false) + return ( +
+
setExpanded(!expanded)}> + + ๐Ÿ› ๏ธ {action.tool} + + ๐Ÿ”ฝ + ๐Ÿ”ผ +
+
+
+ + Tool Input: +

+

+ {JSON.stringify(action.toolInput)} + +
+
+ {observation} +
+
+
+ ); +} \ No newline at end of file diff --git a/ui/app/components/langchain-template/UploadDocumentsForm.tsx b/ui/app/components/langchain-template/UploadDocumentsForm.tsx new file mode 100644 index 00000000..e0bf6629 --- /dev/null +++ b/ui/app/components/langchain-template/UploadDocumentsForm.tsx @@ -0,0 +1,65 @@ +'use client'; + +import type { FormEvent } from 'react'; +import { useState } from 'react'; + +export function UploadDocumentsForm() { + const [isLoading, setIsLoading] = useState(false); + const [document, setDocument] = useState(''); + const ingest = async (e: FormEvent) => { + e.preventDefault(); + setIsLoading(true); + const response = await fetch('/api/retrieval/ingest', { + method: 'POST', + body: JSON.stringify({ + text: document, + }), + }); + if (response.status === 200) { + setDocument('Uploaded!'); + } else { + const json = await response.json(); + if (json.error) { + setDocument(json.error); + } + } + setIsLoading(false); + }; + return ( +
+ + +
+ ); +} diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx new file mode 100644 index 00000000..ac5308e4 --- /dev/null +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -0,0 +1,235 @@ +'use client'; + +import { useEffect, useMemo, useRef, useState } from 'react'; +import type { Message } from 'ai/react'; +import { useChat } from 'ai/react'; +import type { AgentStep } from 'langchain/schema'; +import { toast } from 'react-toastify'; + +import 'react-toastify/dist/ReactToastify.css'; + +interface ChatHistoryProps { + chatHistory: Message[]; + handleCancel: () => void; +} +type SourcesForMessages = Record; + +const placeholderAnswering = 'A: Answeringโ€ฆ'; + +const ensureHttpProtocol = (urlString: string) => { + if (!/^(?:f|ht)tps?\:\/\//.test(urlString)) { + return `https://${urlString}`; + } + return urlString; +}; + +const isValidUrl = (urlString: string) => { + const urlPattern = + /^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$/; + const prefixedUrlString = ensureHttpProtocol(urlString); + return urlPattern.test(prefixedUrlString); +}; + +const ChatHistory: React.FC = ({ + chatHistory, + handleCancel, +}) => ( +
0 && 'border-t' + } border-gray-200`} + > +
    + {chatHistory.map((entry, index) => ( +
  • + + {entry.content} + {entry.content === placeholderAnswering && ( + + )} + +
    • + ))} +
+
+); + +const RAGURLWithLangchain = () => { + const [sourcesForMessages, setSourcesForMessages] = + useState({}); + const [inputUrl, setInputUrl] = useState( + 'https://www.waggledance.ai', + ); + const [showIntermediateSteps, setShowIntermediateSteps] = useState(false); + const [intermediateStepsLoading, setIntermediateStepsLoading] = + useState(false); + const abortControllerRef = useRef(null); + const inputRef = useRef(null); + const inputUrlRef = useRef(null); + const messageContainerRef = useRef(null); + + const hasValidUrl = useMemo(() => isValidUrl(inputUrl), [inputUrl]); + + const chatHistory = useMemo(() => { + return Object.entries(sourcesForMessages).map( + ([index, sources]) => + ({ + id: index, + content: sources.join(' '), + role: Number.parseInt(index) % 2 === 0 ? 'user' : 'assistant', + }) as Message, + ); + }, [sourcesForMessages]); + + const { + messages, + input, + error, + setInput, + handleInputChange, + handleSubmit, + isLoading, + setMessages, + } = useChat({ + api: '/api/rag-url-with-langchain', + onResponse(response) { + const sourcesHeader = response.headers.get('x-sources'); + const sources = sourcesHeader ? JSON.parse(atob(sourcesHeader)) : []; + const messageIndexHeader = response.headers.get('x-message-index'); + if (sources.length && messageIndexHeader !== null) { + setSourcesForMessages({ + ...sourcesForMessages, + [messageIndexHeader]: sources, + }); + } + }, + onError: (e) => { + toast(e.message, { + theme: 'dark', + }); + }, + }); + + useEffect(() => { + if (inputRef.current) { + inputRef.current.focus(); + } + }, []); + + const sendMessage = async (event: React.FormEvent) => { + event.preventDefault(); + if (!hasValidUrl) { + toast('Please enter a valid URL.'); + return; + } + if (messageContainerRef.current) { + messageContainerRef.current.classList.add('grow'); + } + if (!messages.length) { + await new Promise((resolve) => setTimeout(resolve, 300)); + } + + if (isLoading ?? intermediateStepsLoading) { + return; + } + + if (!showIntermediateSteps) { + handleSubmit(event); + } else { + setIntermediateStepsLoading(true); + abortControllerRef.current = new AbortController(); + + try { + const response = await fetch('/api/rag-url-with-langchain', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ url: inputUrl, chat: input }), + signal: abortControllerRef.current.signal, + }); + + if (response.status === 200) { + const json = await response.json(); + if (showIntermediateSteps) { + const intermediateStepMessages = ( + json.intermediate_steps ?? [] + ).map((intermediateStep: AgentStep, i: number) => ({ + id: (chatHistory.length + i).toString(), + content: JSON.stringify(intermediateStep), + role: 'system', + })); + setMessages([...messages, ...intermediateStepMessages]); + } else { + setMessages([ + ...messages, + { + id: Date.now().toString(), + content: json.answer, + role: 'assistant', + }, + ]); + } + } else { + toast('An error occurred while fetching data.'); + } + } catch (error) { + if (error.name !== 'AbortError') { + toast('An error occurred while sending the message.'); + } + } finally { + setIntermediateStepsLoading(false); + abortControllerRef.current = null; + } + } + }; + + return ( +
+
+ setInputUrl(e.target.value)} + placeholder="Enter URL" + className="w-full rounded border p-2" + /> + + +
+ abortControllerRef.current?.abort()} + /> + {error &&

{error.message}

} +
+ ); +}; + +export default RAGURLWithLangchain; diff --git a/ui/package-lock.json b/ui/package-lock.json index b83582d3..83e60c00 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -36,6 +36,7 @@ "react-markdown": "^9.0.1", "react-sketch-canvas": "^6.2.0", "react-syntax-highlighter": "^15.5.0", + "react-toastify": "^9.1.3", "replicate": "^0.18.1", "supabase": "^1.123.4", "tailwind-styled-components": "^2.2.0", @@ -2963,6 +2964,14 @@ "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" }, + "node_modules/clsx": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz", + "integrity": "sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==", + "engines": { + "node": ">=6" + } + }, "node_modules/cmd-shim": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/cmd-shim/-/cmd-shim-6.0.2.tgz", @@ -7686,6 +7695,18 @@ "react": ">= 0.14.0" } }, + "node_modules/react-toastify": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-9.1.3.tgz", + "integrity": "sha512-fPfb8ghtn/XMxw3LkxQBk3IyagNpF/LIKjOBflbexr2AWxAH1MJgvnESwEwBn9liLFXgTKWgBSdZpw9m4OTHTg==", + "dependencies": { + "clsx": "^1.1.1" + }, + "peerDependencies": { + "react": ">=16", + "react-dom": ">=16" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", diff --git a/ui/package.json b/ui/package.json index 5f5f047b..f3c1fae8 100644 --- a/ui/package.json +++ b/ui/package.json @@ -38,6 +38,7 @@ "react-markdown": "^9.0.1", "react-sketch-canvas": "^6.2.0", "react-syntax-highlighter": "^15.5.0", + "react-toastify": "^9.1.3", "replicate": "^0.18.1", "supabase": "^1.123.4", "tailwind-styled-components": "^2.2.0", From 55c25dca80dc093b55377f32c80a6bf3f938d65e Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:56:42 -0800 Subject: [PATCH 02/10] wip rag url --- .../rag-url-with-langchain/puppeteer/route.ts | 44 ++ ui/app/api/rag-url-with-langchain/route.ts | 277 ++++++- .../langchain-template/ChatMessageBubble.tsx | 39 - .../langchain-template/ChatWindow.tsx | 247 ------ .../langchain-template/IntermediateStep.tsx | 36 - .../UploadDocumentsForm.tsx | 65 -- .../stacks/rag-url-with-langchain.tsx | 252 +++++- ui/package-lock.json | 742 ++++++++++++++++++ ui/package.json | 2 + 9 files changed, 1267 insertions(+), 437 deletions(-) create mode 100644 ui/app/api/rag-url-with-langchain/puppeteer/route.ts delete mode 100644 ui/app/components/langchain-template/ChatMessageBubble.tsx delete mode 100644 ui/app/components/langchain-template/ChatWindow.tsx delete mode 100644 ui/app/components/langchain-template/IntermediateStep.tsx delete mode 100644 ui/app/components/langchain-template/UploadDocumentsForm.tsx diff --git a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts new file mode 100644 index 00000000..20b3da90 --- /dev/null +++ b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts @@ -0,0 +1,44 @@ +import type { NextRequest } from 'next/server'; +import type { + Browser, + Page, + PuppeteerEvaluate, + PuppeteerGotoOptions, +} from 'langchain/document_loaders/web/puppeteer'; +import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer'; + +export const runtime = 'nodejs'; + +export async function POST(req: NextRequest) { + const htmlOrUrl = await req.json(); + console.debug('POST rag-url-with-langchain crawl', htmlOrUrl); + const docs = await run(htmlOrUrl); + return new Response(JSON.stringify(docs), { + headers: { + 'content-type': 'application/json; charset=UTF-8', + }, + }); +} + +export const run = async ({ html, url }: { html?: string; url?: string }) => { + /** Loader use evaluate function ` await page.evaluate(() => document.body.innerHTML);` as default evaluate */ + url = url ?? 'about://about'; + const gotoOptions: PuppeteerGotoOptions | undefined = url + ? { waitUntil: 'domcontentloaded' } + : undefined; + const evaluate: PuppeteerEvaluate | undefined = html + ? async (page: Page, browser: Browser) => { + return await page.evaluate(() => html); + } + : undefined; + const loaderWithOptions = new PuppeteerWebBaseLoader(url, { + launchOptions: { + headless: 'new', + }, + gotoOptions, + /** Pass custom evaluate , in this case you get page and browser instances */ + evaluate, + }); + const docs = await loaderWithOptions.loadAndSplit(); + return docs; +}; diff --git a/ui/app/api/rag-url-with-langchain/route.ts b/ui/app/api/rag-url-with-langchain/route.ts index 204da37c..5746a501 100644 --- a/ui/app/api/rag-url-with-langchain/route.ts +++ b/ui/app/api/rag-url-with-langchain/route.ts @@ -1,21 +1,41 @@ import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; -import { createClient } from '@supabase/supabase-js'; import type { Message as VercelChatMessage } from 'ai'; import { StreamingTextResponse } from 'ai'; +import type { BaseCallbackConfig } from 'langchain/callbacks'; +import { + collapseDocs, + splitListOfDocs, +} from 'langchain/chains/combine_documents/reduce'; import { ChatOpenAI } from 'langchain/chat_models/openai'; -import type { Document } from 'langchain/document'; +import { Document } from 'langchain/document'; +import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio'; import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; import { PromptTemplate } from 'langchain/prompts'; +import { ContextualCompressionRetriever } from 'langchain/retrievers/contextual_compression'; +import { DocumentCompressorPipeline } from 'langchain/retrievers/document_compressors'; +import { EmbeddingsFilter } from 'langchain/retrievers/document_compressors/embeddings_filter'; import { BytesOutputParser, StringOutputParser, } from 'langchain/schema/output_parser'; -import { RunnableSequence } from 'langchain/schema/runnable'; -import { SupabaseVectorStore } from 'langchain/vectorstores/supabase'; +import { formatDocument } from 'langchain/schema/prompt_template'; +import { + RunnablePassthrough, + RunnableSequence, +} from 'langchain/schema/runnable'; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; +import { MemoryVectorStore } from 'langchain/vectorstores/memory'; export const runtime = 'edge'; +enum CrawlMethod { + FETCH = 'FETCH', + EDGE_BROWSER = 'EDGE_BROWSER', + PUPPETEER = 'PUPPETEER', + APIFY = 'APIFY', +} + const combineDocumentsFn = (docs: Document[]) => { const serializedDocs = docs.map((doc) => doc.pageContent); return serializedDocs.join('\n\n'); @@ -46,10 +66,7 @@ const condenseQuestionPrompt = PromptTemplate.fromTemplate( CONDENSE_QUESTION_TEMPLATE, ); -const ANSWER_TEMPLATE = `You are an energetic talking puppy named Dana, and must answer all questions like a happy, talking dog would. -Use lots of puns! - -Answer the question based only on the following context and chat history: +const ANSWER_TEMPLATE = `Answer the question based only on the following context and chat history: {context} @@ -62,6 +79,72 @@ Question: {question} `; const answerPrompt = PromptTemplate.fromTemplate(ANSWER_TEMPLATE); +const docsForCrawl = async ({ + origin, + url, + crawlMethod, +}: { + origin: string | null; + url: string; + crawlMethod: CrawlMethod | undefined; +}): Promise => { + const headers = new Headers({ + 'User-Agent': 'Stackwise/1.0 (https://stackwise.io)', + }); + if (!crawlMethod) { + throw new Error('crawlMethod is required'); + } + switch (crawlMethod) { + case CrawlMethod.EDGE_BROWSER: + return await (async () => { + // const browser = new WebBrowser({ + // model, + // embeddings, + // headers, + // }); + // const webBrowsePrompt = currentMessageContent; + // const response = await browser.call(`"${url}","${webBrowsePrompt}"`); + + const loader = new CheerioWebBaseLoader(url, { + selector: 'body', + }); + const docs = await loader.load(); + return docs; + })(); + case CrawlMethod.FETCH: + return await (async () => { + const fetched = await fetch(url, { + headers, + }); + const body = await fetched.text(); + const contentType = fetched.headers.get('Content-Type'); + // TODO: handle content types differently + switch ( + contentType + // case 'text/html': + // docs = await run({ html: body, url }); + // break; + // case 'application/json': + ) { + } + return [new Document({ pageContent: body })]; + })(); + + case CrawlMethod.PUPPETEER: + case CrawlMethod.APIFY: + return await (async () => { + const fetched = await fetch( + `${origin}/api/rag-url-with-langchain/${crawlMethod.toLowerCase()}`, + { + method: 'POST', + body: JSON.stringify({ url, html: undefined }), + }, + ); + const body = await fetched.json(); + return body as Document[]; + })(); + } +}; /** * This handler initializes and calls a retrieval chain. It composes the chain using * LangChain Expression Language. See the docs for more information: @@ -72,22 +155,29 @@ export async function POST(req: NextRequest) { try { const body = await req.json(); const messages = body.messages ?? []; + const url = body.url; + const crawlMethod: CrawlMethod | undefined = body.crawlMethod; + console.debug('POST rag-url-with-langchain', { messages, url }); const previousMessages = messages.slice(0, -1); const currentMessageContent = messages[messages.length - 1].content; + const html: string | undefined = undefined; + // const html = await (await fetch(url)).text(); + + // console.debug('POST rag-url-with-langchain html', { html }); + const model = new ChatOpenAI({ - modelName: 'gpt-3.5-turbo', - temperature: 0.2, + modelName: 'gpt-3.5-turbo-16k', + temperature: 0, + verbose: true, }); + // const embeddings = new OpenAIEmbeddings(); - const client = createClient( - process.env.SUPABASE_URL!, - process.env.SUPABASE_PRIVATE_KEY!, - ); - const vectorstore = new SupabaseVectorStore(new OpenAIEmbeddings(), { - client, - tableName: 'documents', - queryName: 'match_documents', + const origin = req.headers.get('origin'); + const docs = await docsForCrawl({ + origin, + url, + crawlMethod, }); /** @@ -107,7 +197,17 @@ export async function POST(req: NextRequest) { resolveWithDocuments = resolve; }); - const retriever = vectorstore.asRetriever({ + // const retriever = await compressedRelevantDocsRetriever(docs, [ + // { + // handleRetrieverEnd(documents) { + // resolveWithDocuments(documents); + // }, + // }, + // ]); + + const baseRetriever = ( + await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) + ).asRetriever({ callbacks: [ { handleRetrieverEnd(documents) { @@ -117,7 +217,7 @@ export async function POST(req: NextRequest) { ], }); - const retrievalChain = retriever.pipe(combineDocumentsFn); + const retrievalChain = baseRetriever.pipe(combineDocumentsFn); const answerChain = RunnableSequence.from([ { @@ -168,3 +268,140 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: e.message }, { status: 500 }); } } + +const compressedRelevantDocsRetriever = async (docs: Document[], callbacks) => { + const embeddingsFilter = new EmbeddingsFilter({ + embeddings: new OpenAIEmbeddings(), + similarityThreshold: 0.8, + k: 20, + }); + + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 400, + chunkOverlap: 40, + }); + + const compressorPipeline = new DocumentCompressorPipeline({ + transformers: [textSplitter, embeddingsFilter], + }); + + // const baseRetriever = new TavilySearchAPIRetriever({ + // includeRawContent: true, + // }); + const baseRetriever = ( + await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) + ).asRetriever({ callbacks }); + + const retriever = new ContextualCompressionRetriever({ + baseCompressor: compressorPipeline, + baseRetriever, + }); + + // const retrievedDocs = await retriever.getRelevantDocuments( + // "What did the speaker say about Justice Breyer in the 2022 State of the Union?" + // ); + // console.log({ retrievedDocs }); + return retriever; +}; + +const mapReduceDocumentsChain = (() => { + // Initialize the OpenAI model + const model = new ChatOpenAI({ + modelName: 'gpt-3.5-turbo', + temperature: 0, + verbose: true, + }); + + // Define prompt templates for document formatting, summarizing, collapsing, and combining + const documentPrompt = PromptTemplate.fromTemplate('{pageContent}'); + const summarizePrompt = PromptTemplate.fromTemplate( + 'Summarize this content:\n\n{context}', + ); + const collapsePrompt = PromptTemplate.fromTemplate( + 'Collapse this content:\n\n{context}', + ); + const combinePrompt = PromptTemplate.fromTemplate( + 'Combine these summaries:\n\n{context}', + ); + + // Wrap the `formatDocument` util so it can format a list of documents + const formatDocs = async (documents: Document[]): Promise => { + const formattedDocs = await Promise.all( + documents.map((doc) => formatDocument(doc, documentPrompt)), + ); + return formattedDocs.join('\n\n'); + }; + + // Define a function to get the number of tokens in a list of documents + const getNumTokens = async (documents: Document[]): Promise => + model.getNumTokens(await formatDocs(documents)); + + // Initialize the output parser + const outputParser = new StringOutputParser(); + + // Define the map chain to format, summarize, and parse the document + const mapChain = RunnableSequence.from([ + { context: async (i: Document) => formatDocument(i, documentPrompt) }, + summarizePrompt, + model, + outputParser, + ]); + + // Define the collapse chain to format, collapse, and parse a list of documents + const collapseChain = RunnableSequence.from([ + { context: async (documents: Document[]) => formatDocs(documents) }, + collapsePrompt, + model, + outputParser, + ]); + + // Define a function to collapse a list of documents until the total number of tokens is within the limit + const collapse = async ( + documents: Document[], + options?: { + config?: BaseCallbackConfig; + }, + tokenMax = 4000, + ) => { + const editableConfig = options?.config; + let docs = documents; + let collapseCount = 1; + while ((await getNumTokens(docs)) > tokenMax) { + if (editableConfig) { + editableConfig.runName = `Collapse ${collapseCount}`; + } + const splitDocs = splitListOfDocs(docs, getNumTokens, tokenMax); + docs = await Promise.all( + splitDocs.map((doc) => collapseDocs(doc, collapseChain.invoke)), + ); + collapseCount += 1; + } + return docs; + }; + + // Define the reduce chain to format, combine, and parse a list of documents + const reduceChain = RunnableSequence.from([ + { context: formatDocs }, + combinePrompt, + model, + outputParser, + ]).withConfig({ runName: 'Reduce' }); + + // Define the final map-reduce chain + const mapReduceChain = RunnableSequence.from([ + RunnableSequence.from([ + { doc: new RunnablePassthrough(), content: mapChain }, + (input) => + new Document({ + pageContent: input.content, + metadata: input.doc.metadata, + }), + ]) + .withConfig({ runName: 'Summarize (return doc)' }) + .map(), + collapse, + reduceChain, + ]).withConfig({ runName: 'Map reduce' }); + + return mapReduceChain; +})(); diff --git a/ui/app/components/langchain-template/ChatMessageBubble.tsx b/ui/app/components/langchain-template/ChatMessageBubble.tsx deleted file mode 100644 index 8d1beca8..00000000 --- a/ui/app/components/langchain-template/ChatMessageBubble.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import type { Message } from "ai/react"; - -export function ChatMessageBubble(props: { message: Message, aiEmoji?: string, sources: any[] }) { - const colorClassName = - props.message.role === "user" ? "bg-sky-600" : "bg-slate-50 text-black"; - const alignmentClassName = - props.message.role === "user" ? "ml-auto" : "mr-auto"; - const prefix = props.message.role === "user" ? "๐Ÿง‘" : props.aiEmoji; - return ( -
-
- {prefix} -
-
- {props.message.content} - {props.sources && props.sources.length ? <> - -

- ๐Ÿ” Sources: -

- - - {props.sources?.map((source, i) => ( -
- {i + 1}. "{source.pageContent}"{ - source.metadata?.loc?.lines !== undefined - ?

Lines {source.metadata?.loc?.lines?.from} to {source.metadata?.loc?.lines?.to}
- : "" - } -
- ))} - - : ""} -
-
- ); -} \ No newline at end of file diff --git a/ui/app/components/langchain-template/ChatWindow.tsx b/ui/app/components/langchain-template/ChatWindow.tsx deleted file mode 100644 index 34babd31..00000000 --- a/ui/app/components/langchain-template/ChatWindow.tsx +++ /dev/null @@ -1,247 +0,0 @@ -'use client'; - -import { toast, ToastContainer } from 'react-toastify'; - -import 'react-toastify/dist/ReactToastify.css'; - -import type { FormEvent, ReactElement } from 'react'; -import { useRef, useState } from 'react'; -import { useChat } from 'ai/react'; -import type { AgentStep } from 'langchain/schema'; - -import { ChatMessageBubble } from './ChatMessageBubble'; -import { IntermediateStep } from './IntermediateStep'; -import { UploadDocumentsForm } from './UploadDocumentsForm'; - -export function ChatWindow(props: { - endpoint: string; - emptyStateComponent: ReactElement; - placeholder?: string; - titleText?: string; - emoji?: string; - showIngestForm?: boolean; - showIntermediateStepsToggle?: boolean; -}) { - const messageContainerRef = useRef(null); - - const { - endpoint, - emptyStateComponent, - placeholder, - titleText = 'An LLM', - showIngestForm, - showIntermediateStepsToggle, - emoji, - } = props; - - const [showIntermediateSteps, setShowIntermediateSteps] = useState(false); - const [intermediateStepsLoading, setIntermediateStepsLoading] = - useState(false); - const ingestForm = showIngestForm && ( - - ); - const intemediateStepsToggle = showIntermediateStepsToggle && ( -
- setShowIntermediateSteps(e.target.checked)} - > - -
- ); - - const [sourcesForMessages, setSourcesForMessages] = useState< - Record - >({}); - - const { - messages, - input, - setInput, - handleInputChange, - handleSubmit, - isLoading: chatEndpointIsLoading, - setMessages, - } = useChat({ - api: endpoint, - onResponse(response) { - const sourcesHeader = response.headers.get('x-sources'); - const sources = sourcesHeader ? JSON.parse(atob(sourcesHeader)) : []; - const messageIndexHeader = response.headers.get('x-message-index'); - if (sources.length && messageIndexHeader !== null) { - setSourcesForMessages({ - ...sourcesForMessages, - [messageIndexHeader]: sources, - }); - } - }, - onError: (e) => { - toast(e.message, { - theme: 'dark', - }); - }, - }); - - async function sendMessage(e: FormEvent) { - e.preventDefault(); - if (messageContainerRef.current) { - messageContainerRef.current.classList.add('grow'); - } - if (!messages.length) { - await new Promise((resolve) => setTimeout(resolve, 300)); - } - if (chatEndpointIsLoading ?? intermediateStepsLoading) { - return; - } - if (!showIntermediateSteps) { - handleSubmit(e); - // Some extra work to show intermediate steps properly - } else { - setIntermediateStepsLoading(true); - setInput(''); - const messagesWithUserReply = messages.concat({ - id: messages.length.toString(), - content: input, - role: 'user', - }); - setMessages(messagesWithUserReply); - const response = await fetch(endpoint, { - method: 'POST', - body: JSON.stringify({ - messages: messagesWithUserReply, - show_intermediate_steps: true, - }), - }); - const json = await response.json(); - setIntermediateStepsLoading(false); - if (response.status === 200) { - // Represent intermediate steps as system messages for display purposes - const intermediateStepMessages = (json.intermediate_steps ?? []).map( - (intermediateStep: AgentStep, i: number) => { - return { - id: (messagesWithUserReply.length + i).toString(), - content: JSON.stringify(intermediateStep), - role: 'system', - }; - }, - ); - const newMessages = messagesWithUserReply; - for (const message of intermediateStepMessages) { - newMessages.push(message); - setMessages([...newMessages]); - await new Promise((resolve) => - setTimeout(resolve, 1000 + Math.random() * 1000), - ); - } - setMessages([ - ...newMessages, - { - id: ( - newMessages.length + intermediateStepMessages.length - ).toString(), - content: json.output, - role: 'assistant', - }, - ]); - } else { - if (json.error) { - toast(json.error, { - theme: 'dark', - }); - throw new Error(json.error); - } - } - } - } - - return ( -
0 ? 'border' : '' - }`} - > -

0 ? '' : 'hidden'} text-2xl`}> - {emoji} {titleText} -

- {messages.length === 0 ? emptyStateComponent : ''} -
- {messages.length > 0 - ? [...messages].reverse().map((m, i) => { - const sourceKey = (messages.length - 1 - i).toString(); - return m.role === 'system' ? ( - - ) : ( - - ); - }) - : ''} -
- - {messages.length === 0 && ingestForm} - -
-
{intemediateStepsToggle}
-
- - -
-
- -
- ); -} diff --git a/ui/app/components/langchain-template/IntermediateStep.tsx b/ui/app/components/langchain-template/IntermediateStep.tsx deleted file mode 100644 index 6bffaeba..00000000 --- a/ui/app/components/langchain-template/IntermediateStep.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { useState } from "react"; -import type { Message } from "ai/react"; -import type { AgentStep } from "langchain/schema"; - -export function IntermediateStep(props: { message: Message }) { - const parsedInput: AgentStep = JSON.parse(props.message.content); - const action = parsedInput.action; - const observation = parsedInput.observation; - const [expanded, setExpanded] = useState(false) - return ( -
-
setExpanded(!expanded)}> - - ๐Ÿ› ๏ธ {action.tool} - - ๐Ÿ”ฝ - ๐Ÿ”ผ -
-
-
- - Tool Input: -

-

- {JSON.stringify(action.toolInput)} - -
-
- {observation} -
-
-
- ); -} \ No newline at end of file diff --git a/ui/app/components/langchain-template/UploadDocumentsForm.tsx b/ui/app/components/langchain-template/UploadDocumentsForm.tsx deleted file mode 100644 index e0bf6629..00000000 --- a/ui/app/components/langchain-template/UploadDocumentsForm.tsx +++ /dev/null @@ -1,65 +0,0 @@ -'use client'; - -import type { FormEvent } from 'react'; -import { useState } from 'react'; - -export function UploadDocumentsForm() { - const [isLoading, setIsLoading] = useState(false); - const [document, setDocument] = useState(''); - const ingest = async (e: FormEvent) => { - e.preventDefault(); - setIsLoading(true); - const response = await fetch('/api/retrieval/ingest', { - method: 'POST', - body: JSON.stringify({ - text: document, - }), - }); - if (response.status === 200) { - setDocument('Uploaded!'); - } else { - const json = await response.json(); - if (json.error) { - setDocument(json.error); - } - } - setIsLoading(false); - }; - return ( -
- - -
- ); -} diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx index ac5308e4..06f53e29 100644 --- a/ui/app/components/stacks/rag-url-with-langchain.tsx +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useMemo, useRef, useState } from 'react'; import type { Message } from 'ai/react'; import { useChat } from 'ai/react'; import type { AgentStep } from 'langchain/schema'; @@ -8,6 +8,13 @@ import { toast } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; +export enum CrawlMethod { + FETCH = 'FETCH', + EDGE_BROWSER = 'EDGE_BROWSER', + PUPPETEER = 'PUPPETEER', + APIFY = 'APIFY', +} + interface ChatHistoryProps { chatHistory: Message[]; handleCancel: () => void; @@ -30,6 +37,111 @@ const isValidUrl = (urlString: string) => { return urlPattern.test(prefixedUrlString); }; +export function IntermediateStep(props: { message: Message }) { + const parsedInput: AgentStep = JSON.parse(props.message.content); + const action = parsedInput.action; + const observation = parsedInput.observation; + const [expanded, setExpanded] = useState(false); + return ( +
+
setExpanded(!expanded)} + > + + ๐Ÿ› ๏ธ {action.tool} + + ๐Ÿ”ฝ + ๐Ÿ”ผ +
+
+
+ + Tool Input: +

+

+ {JSON.stringify(action.toolInput)} + +
+
+ + {observation} + +
+
+
+ ); +} + +export function ChatMessageBubble(props: { + message: Message; + aiEmoji?: string; + sources: any[]; +}) { + const colorClassName = + props.message.role === 'user' ? 'bg-sky-600' : 'bg-slate-50 text-black'; + const alignmentClassName = + props.message.role === 'user' ? 'ml-auto' : 'mr-auto'; + const prefix = props.message.role === 'user' ? '๐Ÿง‘' : props.aiEmoji; + return ( +
+
{prefix}
+
+ {props.message.content} + {props.sources && props.sources.length ? ( + <> + +

๐Ÿ” Sources:

+ + + {props.sources?.map((source, i) => ( +
+ {i + 1}. "{source.pageContent}" + {source.metadata?.loc?.lines !== undefined ? ( +
+
+ Lines {source.metadata?.loc?.lines?.from} to{' '} + {source.metadata?.loc?.lines?.to} +
+ ) : ( + '' + )} +
+ ))} + + + ) : ( + '' + )} +
+
+ ); +} + const ChatHistory: React.FC = ({ chatHistory, handleCancel, @@ -76,6 +188,7 @@ const RAGURLWithLangchain = () => { const [showIntermediateSteps, setShowIntermediateSteps] = useState(false); const [intermediateStepsLoading, setIntermediateStepsLoading] = useState(false); + const abortControllerRef = useRef(null); const inputRef = useRef(null); const inputUrlRef = useRef(null); @@ -105,6 +218,10 @@ const RAGURLWithLangchain = () => { setMessages, } = useChat({ api: '/api/rag-url-with-langchain', + experimental_onFunctionCall(messages, functionCall) { + console.log(messages); + return Promise.resolve(); + }, onResponse(response) { const sourcesHeader = response.headers.get('x-sources'); const sources = sourcesHeader ? JSON.parse(atob(sourcesHeader)) : []; @@ -116,6 +233,9 @@ const RAGURLWithLangchain = () => { }); } }, + onFinish(message: Message) { + console.log('onFinish', message); + }, onError: (e) => { toast(e.message, { theme: 'dark', @@ -123,11 +243,11 @@ const RAGURLWithLangchain = () => { }, }); - useEffect(() => { - if (inputRef.current) { - inputRef.current.focus(); - } - }, []); + // useEffect(() => { + // if (abortControllerRef?.current) { + // abortControllerRef?.current.abort(); + // } + // }, []); const sendMessage = async (event: React.FormEvent) => { event.preventDefault(); @@ -146,45 +266,80 @@ const RAGURLWithLangchain = () => { return; } + const body = { + url: inputUrl, + chat: input, + crawlMethod: CrawlMethod.PUPPETEER.toString(), + }; if (!showIntermediateSteps) { - handleSubmit(event); + handleSubmit(event, { + options: { body }, + }); } else { setIntermediateStepsLoading(true); + setInput(''); abortControllerRef.current = new AbortController(); + const messagesWithUserReply = messages.concat({ + id: messages.length.toString(), + content: input, + role: 'user', + }); + setMessages(messagesWithUserReply); try { + console.log('inputUrl', inputUrl); + const response = await fetch('/api/rag-url-with-langchain', { method: 'POST', headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ url: inputUrl, chat: input }), + body: JSON.stringify({ + messages: messagesWithUserReply, + url: body.url, + crawlMethod: body.crawlMethod, + }), signal: abortControllerRef.current.signal, }); + const json = await response.json(); + setIntermediateStepsLoading(false); if (response.status === 200) { - const json = await response.json(); - if (showIntermediateSteps) { - const intermediateStepMessages = ( - json.intermediate_steps ?? [] - ).map((intermediateStep: AgentStep, i: number) => ({ - id: (chatHistory.length + i).toString(), - content: JSON.stringify(intermediateStep), - role: 'system', - })); - setMessages([...messages, ...intermediateStepMessages]); - } else { - setMessages([ - ...messages, - { - id: Date.now().toString(), - content: json.answer, - role: 'assistant', - }, - ]); + // Represent intermediate steps as system messages for display purposes + const intermediateStepMessages = (json.intermediate_steps ?? []).map( + (intermediateStep: AgentStep, i: number) => { + return { + id: (messagesWithUserReply.length + i).toString(), + content: JSON.stringify(intermediateStep), + role: 'system', + }; + }, + ); + const newMessages = messagesWithUserReply; + for (const message of intermediateStepMessages) { + newMessages.push(message); + setMessages([...newMessages]); + await new Promise((resolve) => + setTimeout(resolve, 1000 + Math.random() * 1000), + ); } + setMessages([ + ...newMessages, + { + id: ( + newMessages.length + intermediateStepMessages.length + ).toString(), + content: json.output, + role: 'assistant', + }, + ]); } else { - toast('An error occurred while fetching data.'); + if (json.error) { + toast(json.error, { + theme: 'dark', + }); + throw new Error(json.error); + } } } catch (error) { if (error.name !== 'AbortError') { @@ -223,10 +378,47 @@ const RAGURLWithLangchain = () => { {isLoading ? 'Loading...' : 'Send'} - { + console.log('error', e); + }} + onLoadedData={(e) => { + console.log('loaded data', e); + }} + > */} +
0 ? 'border' : '' + }`} + > + {messages.length === 0 ? <> : ''} +
+ {messages.length > 0 + ? [...messages].reverse().map((m, i) => { + const sourceKey = (messages.length - 1 - i).toString(); + return m.role === 'system' ? ( + + ) : ( + + ); + }) + : ''} +
+
+ {/* abortControllerRef.current?.abort()} - /> + /> */} {error &&

{error.message}

} ); diff --git a/ui/package-lock.json b/ui/package-lock.json index 83e60c00..6f529f4a 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -20,6 +20,7 @@ "@supabase/supabase-js": "^2.39.0", "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", + "cheerio": "^1.0.0-rc.12", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", "html-react-parser": "^5.0.7", @@ -29,6 +30,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", + "puppeteer": "^19.11.1", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", @@ -1622,6 +1624,58 @@ "resolved": "https://registry.npmjs.org/@octokit/webhooks-types/-/webhooks-types-7.1.0.tgz", "integrity": "sha512-y92CpG4kFFtBBjni8LHoV12IegJ+KFxLgKRengrVjKmGE5XMeCuGvlfRe75lTRrgXaG6XIWJlFpIDTlkoJsU8w==" }, + "node_modules/@puppeteer/browsers": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-0.5.0.tgz", + "integrity": "sha512-Uw6oB7VvmPRLE4iKsjuOh8zgDabhNX67dzo8U/BB0f9527qx+4eeUs+korU98OhG5C4ubg7ufBgVi63XYwS6TQ==", + "dependencies": { + "debug": "4.3.4", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "yargs": "17.7.1" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=14.1.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@puppeteer/browsers/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/@puppeteer/browsers/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/@stackwise-tooling/eslint-config": { "resolved": "tooling/eslint-config", "link": true @@ -1897,6 +1951,15 @@ "@types/node": "*" } }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.15.0.tgz", @@ -2693,6 +2756,21 @@ "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==" }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -2754,6 +2832,37 @@ "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", "integrity": "sha512-gvW7InbIyF8AicrqWoptdW08pUxuhq8BEgowNajy9RhiE86fmGAGl+bLKo6oB8QP0CkqHLowfN0oJdKC/J6LbA==" }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -2906,6 +3015,60 @@ "node": "*" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -2951,6 +3114,17 @@ "node": ">=10" } }, + "node_modules/chromium-bidi": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.4.7.tgz", + "integrity": "sha512-6+mJuFXwTMU6I3vYLs6IL8A1DyQTPjCfIL971X0aMPVGRbGnNfl6i6Cl0NMbxi2bRYLGESt9T2ZIMRM5PAEcIQ==", + "dependencies": { + "mitt": "3.0.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -2964,6 +3138,19 @@ "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/clsx": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz", @@ -3052,6 +3239,50 @@ "node": ">= 0.6" } }, + "node_modules/cosmiconfig": { + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.1.3.tgz", + "integrity": "sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==", + "dependencies": { + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "parse-json": "^5.0.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + } + }, + "node_modules/cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "dependencies": { + "node-fetch": "2.6.7" + } + }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -3073,6 +3304,21 @@ "node": "*" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/css-tree": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz", @@ -3086,6 +3332,17 @@ "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" } }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -3218,6 +3475,11 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1107588", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", + "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -3345,6 +3607,14 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -3356,6 +3626,14 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-abstract": { "version": "1.22.3", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.3.tgz", @@ -3999,6 +4277,25 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4067,6 +4364,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -4248,6 +4553,11 @@ "url": "https://github.com/sponsors/rawify" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs-extra": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", @@ -4412,6 +4722,14 @@ "node": ">=6.9.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", @@ -4426,6 +4744,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -4868,6 +5200,25 @@ "ms": "^2.0.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", @@ -4979,6 +5330,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, "node_modules/is-async-function": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz", @@ -5099,6 +5455,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, "node_modules/is-generator-function": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", @@ -5400,6 +5764,11 @@ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -6607,6 +6976,11 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/mitt": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.0.tgz", + "integrity": "sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ==" + }, "node_modules/mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", @@ -6618,6 +6992,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/ml-array-mean": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz", @@ -6848,6 +7227,17 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/num-sort": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz", @@ -7161,6 +7551,46 @@ "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz", "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==" }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/path": { "version": "0.12.7", "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", @@ -7227,6 +7657,11 @@ "ms": "^2.1.1" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, "node_modules/periscopic": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz", @@ -7531,6 +7966,14 @@ "node": ">= 0.6.0" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -7555,6 +7998,15 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -7563,6 +8015,116 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "19.11.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-19.11.1.tgz", + "integrity": "sha512-39olGaX2djYUdhaQQHDZ0T0GwEp+5f9UB9HmEP0qHfdQHIq0xGQZuAZ5TLnJIc/88SrPLpEflPC+xUqOTv3c5g==", + "deprecated": "< 21.3.7 is no longer supported", + "hasInstallScript": true, + "dependencies": { + "@puppeteer/browsers": "0.5.0", + "cosmiconfig": "8.1.3", + "https-proxy-agent": "5.0.1", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "puppeteer-core": "19.11.1" + } + }, + "node_modules/puppeteer-core": { + "version": "19.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", + "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", + "dependencies": { + "@puppeteer/browsers": "0.5.0", + "chromium-bidi": "0.4.7", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1107588", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "proxy-from-env": "1.1.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.13.0" + }, + "engines": { + "node": ">=14.14.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/puppeteer-core/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -7723,6 +8285,19 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -7922,6 +8497,14 @@ "yarn": ">=1.7.0" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", @@ -8241,6 +8824,32 @@ "node": ">=10.0.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "node_modules/string.prototype.matchall": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.10.tgz", @@ -8602,6 +9211,37 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -8631,6 +9271,11 @@ "node": ">=0.8" } }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -8825,6 +9470,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", @@ -9198,6 +9852,52 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -9243,6 +9943,14 @@ "node": ">=0.4" } }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", @@ -9256,6 +9964,40 @@ "node": ">= 14" } }, + "node_modules/yargs": { + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/ui/package.json b/ui/package.json index f3c1fae8..e4b8606a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -22,6 +22,7 @@ "@supabase/supabase-js": "^2.39.0", "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", + "cheerio": "^1.0.0-rc.12", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", "html-react-parser": "^5.0.7", @@ -31,6 +32,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", + "puppeteer": "^19.11.1", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", From 82bb10c08e36ea12362a241f9165c079b73b4471 Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Wed, 20 Dec 2023 11:03:50 -0800 Subject: [PATCH 03/10] wip --- .../rag-url-with-langchain/puppeteer/route.ts | 42 +++++ ui/app/api/rag-url-with-langchain/route.ts | 36 ++-- .../stacks/rag-url-with-langchain.tsx | 162 +++++++----------- 3 files changed, 124 insertions(+), 116 deletions(-) diff --git a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts index 20b3da90..ce191bfc 100644 --- a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts +++ b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts @@ -1,4 +1,5 @@ import type { NextRequest } from 'next/server'; +import type { Document } from 'langchain/document'; import type { Browser, Page, @@ -6,6 +7,12 @@ import type { PuppeteerGotoOptions, } from 'langchain/document_loaders/web/puppeteer'; import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer'; +import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; +import { ContextualCompressionRetriever } from 'langchain/retrievers/contextual_compression'; +import { DocumentCompressorPipeline } from 'langchain/retrievers/document_compressors'; +import { EmbeddingsFilter } from 'langchain/retrievers/document_compressors/embeddings_filter'; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; +import { MemoryVectorStore } from 'langchain/vectorstores/memory'; export const runtime = 'nodejs'; @@ -42,3 +49,38 @@ export const run = async ({ html, url }: { html?: string; url?: string }) => { const docs = await loaderWithOptions.loadAndSplit(); return docs; }; + +const compressedRelevantDocsRetriever = async (docs: Document[], callbacks) => { + const embeddingsFilter = new EmbeddingsFilter({ + embeddings: new OpenAIEmbeddings(), + similarityThreshold: 0.8, + k: 5, + }); + + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 400, + chunkOverlap: 40, + }); + + const compressorPipeline = new DocumentCompressorPipeline({ + transformers: [textSplitter, embeddingsFilter], + }); + + // const baseRetriever = new TavilySearchAPIRetriever({ + // includeRawContent: true, + // }); + const baseRetriever = ( + await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) + ).asRetriever({ callbacks }); + + const retriever = new ContextualCompressionRetriever({ + baseCompressor: compressorPipeline, + baseRetriever, + }); + + // const retrievedDocs = await retriever.getRelevantDocuments( + // "What did the speaker say about Justice Breyer in the 2022 State of the Union?" + // ); + // console.log({ retrievedDocs }); + return retriever; +}; diff --git a/ui/app/api/rag-url-with-langchain/route.ts b/ui/app/api/rag-url-with-langchain/route.ts index 5746a501..8e8e4c97 100644 --- a/ui/app/api/rag-url-with-langchain/route.ts +++ b/ui/app/api/rag-url-with-langchain/route.ts @@ -197,25 +197,25 @@ export async function POST(req: NextRequest) { resolveWithDocuments = resolve; }); - // const retriever = await compressedRelevantDocsRetriever(docs, [ - // { - // handleRetrieverEnd(documents) { - // resolveWithDocuments(documents); - // }, - // }, - // ]); - - const baseRetriever = ( - await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) - ).asRetriever({ - callbacks: [ - { - handleRetrieverEnd(documents) { - resolveWithDocuments(documents); - }, + const baseRetriever = await compressedRelevantDocsRetriever(docs, [ + { + handleRetrieverEnd(documents) { + resolveWithDocuments(documents); }, - ], - }); + }, + ]); + + // const baseRetriever = ( + // await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) + // ).asRetriever({ + // callbacks: [ + // { + // handleRetrieverEnd(documents) { + // resolveWithDocuments(documents); + // }, + // }, + // ], + // }); const retrievalChain = baseRetriever.pipe(combineDocumentsFn); diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx index 06f53e29..2076832c 100644 --- a/ui/app/components/stacks/rag-url-with-langchain.tsx +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -1,9 +1,10 @@ 'use client'; -import { useMemo, useRef, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import type { Message } from 'ai/react'; import { useChat } from 'ai/react'; import type { AgentStep } from 'langchain/schema'; +import { IoSend } from 'react-icons/io5'; import { toast } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; @@ -15,10 +16,6 @@ export enum CrawlMethod { APIFY = 'APIFY', } -interface ChatHistoryProps { - chatHistory: Message[]; - handleCancel: () => void; -} type SourcesForMessages = Record; const placeholderAnswering = 'A: Answeringโ€ฆ'; @@ -142,43 +139,6 @@ export function ChatMessageBubble(props: { ); } -const ChatHistory: React.FC = ({ - chatHistory, - handleCancel, -}) => ( -
0 && 'border-t' - } border-gray-200`} - > -
    - {chatHistory.map((entry, index) => ( -
  • - - {entry.content} - {entry.content === placeholderAnswering && ( - - )} - -
    • - ))} -
-
-); - const RAGURLWithLangchain = () => { const [sourcesForMessages, setSourcesForMessages] = useState({}); @@ -352,33 +312,39 @@ const RAGURLWithLangchain = () => { } }; + useEffect(() => { + toast(error?.message); + }, [error]); + return ( -
+
0 ? 'border' : '' + }`} + > + {messages.length === 0 ? <> : ''} +
+ {messages.length > 0 + ? [...messages].reverse().map((m, i) => { + const sourceKey = (messages.length - 1 - i).toString(); + return m.role === 'system' ? ( + + ) : ( + + ); + }) + : ''} +
- setInputUrl(e.target.value)} - placeholder="Enter URL" - className="w-full rounded border p-2" - /> - - -
- {/* */} -
0 ? 'border' : '' - }`} - > - {messages.length === 0 ? <> : ''} -
- {messages.length > 0 - ? [...messages].reverse().map((m, i) => { - const sourceKey = (messages.length - 1 - i).toString(); - return m.role === 'system' ? ( - - ) : ( - - ); - }) - : ''} + + {/* setInputUrl(e.target.value)} + placeholder="Enter URL" + className="w-full rounded border p-2" + /> */} + +
+ +
-
- {/* abortControllerRef.current?.abort()} - /> */} - {error &&

{error.message}

} +
); }; From 4e019aa36a8542e780a8e4d19f1cc77b3ffe8828 Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:03:36 -0800 Subject: [PATCH 04/10] config, optimization, simplification, gemini doesn't work --- .../rag-url-with-langchain/puppeteer/route.ts | 50 +- ui/app/api/rag-url-with-langchain/route.ts | 470 +++++++----------- .../stacks/rag-url-with-langchain.tsx | 457 ++++++++--------- ui/package-lock.json | 13 + ui/package.json | 1 + 5 files changed, 428 insertions(+), 563 deletions(-) diff --git a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts index ce191bfc..4e0a4c48 100644 --- a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts +++ b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts @@ -1,5 +1,4 @@ import type { NextRequest } from 'next/server'; -import type { Document } from 'langchain/document'; import type { Browser, Page, @@ -7,18 +6,11 @@ import type { PuppeteerGotoOptions, } from 'langchain/document_loaders/web/puppeteer'; import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer'; -import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; -import { ContextualCompressionRetriever } from 'langchain/retrievers/contextual_compression'; -import { DocumentCompressorPipeline } from 'langchain/retrievers/document_compressors'; -import { EmbeddingsFilter } from 'langchain/retrievers/document_compressors/embeddings_filter'; -import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; -import { MemoryVectorStore } from 'langchain/vectorstores/memory'; export const runtime = 'nodejs'; export async function POST(req: NextRequest) { const htmlOrUrl = await req.json(); - console.debug('POST rag-url-with-langchain crawl', htmlOrUrl); const docs = await run(htmlOrUrl); return new Response(JSON.stringify(docs), { headers: { @@ -29,18 +21,19 @@ export async function POST(req: NextRequest) { export const run = async ({ html, url }: { html?: string; url?: string }) => { /** Loader use evaluate function ` await page.evaluate(() => document.body.innerHTML);` as default evaluate */ - url = url ?? 'about://about'; const gotoOptions: PuppeteerGotoOptions | undefined = url - ? { waitUntil: 'domcontentloaded' } + ? { waitUntil: 'domcontentloaded', timeout: 10_000 } : undefined; + url = url ?? 'about://about'; const evaluate: PuppeteerEvaluate | undefined = html - ? async (page: Page, browser: Browser) => { + ? async (page: Page, _browser: Browser) => { return await page.evaluate(() => html); } : undefined; const loaderWithOptions = new PuppeteerWebBaseLoader(url, { launchOptions: { headless: 'new', + timeout: 10_000, }, gotoOptions, /** Pass custom evaluate , in this case you get page and browser instances */ @@ -49,38 +42,3 @@ export const run = async ({ html, url }: { html?: string; url?: string }) => { const docs = await loaderWithOptions.loadAndSplit(); return docs; }; - -const compressedRelevantDocsRetriever = async (docs: Document[], callbacks) => { - const embeddingsFilter = new EmbeddingsFilter({ - embeddings: new OpenAIEmbeddings(), - similarityThreshold: 0.8, - k: 5, - }); - - const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 400, - chunkOverlap: 40, - }); - - const compressorPipeline = new DocumentCompressorPipeline({ - transformers: [textSplitter, embeddingsFilter], - }); - - // const baseRetriever = new TavilySearchAPIRetriever({ - // includeRawContent: true, - // }); - const baseRetriever = ( - await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) - ).asRetriever({ callbacks }); - - const retriever = new ContextualCompressionRetriever({ - baseCompressor: compressorPipeline, - baseRetriever, - }); - - // const retrievedDocs = await retriever.getRelevantDocuments( - // "What did the speaker say about Justice Breyer in the 2022 State of the Union?" - // ); - // console.log({ retrievedDocs }); - return retriever; -}; diff --git a/ui/app/api/rag-url-with-langchain/route.ts b/ui/app/api/rag-url-with-langchain/route.ts index 8e8e4c97..e6968500 100644 --- a/ui/app/api/rag-url-with-langchain/route.ts +++ b/ui/app/api/rag-url-with-langchain/route.ts @@ -1,29 +1,22 @@ import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; -import type { Message as VercelChatMessage } from 'ai'; -import { StreamingTextResponse } from 'ai'; -import type { BaseCallbackConfig } from 'langchain/callbacks'; +import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; +import type { Message, Message as VercelChatMessage } from 'ai'; +import { LangChainStream, StreamingTextResponse } from 'ai'; +import { initializeAgentExecutorWithOptions } from 'langchain/agents'; import { - collapseDocs, - splitListOfDocs, -} from 'langchain/chains/combine_documents/reduce'; + createRetrieverTool, + OpenAIAgentTokenBufferMemory, +} from 'langchain/agents/toolkits'; import { ChatOpenAI } from 'langchain/chat_models/openai'; import { Document } from 'langchain/document'; import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio'; import { OpenAIEmbeddings } from 'langchain/embeddings/openai'; -import { PromptTemplate } from 'langchain/prompts'; +import { BufferMemory, ChatMessageHistory } from 'langchain/memory'; import { ContextualCompressionRetriever } from 'langchain/retrievers/contextual_compression'; import { DocumentCompressorPipeline } from 'langchain/retrievers/document_compressors'; import { EmbeddingsFilter } from 'langchain/retrievers/document_compressors/embeddings_filter'; -import { - BytesOutputParser, - StringOutputParser, -} from 'langchain/schema/output_parser'; -import { formatDocument } from 'langchain/schema/prompt_template'; -import { - RunnablePassthrough, - RunnableSequence, -} from 'langchain/schema/runnable'; +import { AIMessage, ChatMessage, HumanMessage } from 'langchain/schema'; import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; import { MemoryVectorStore } from 'langchain/vectorstores/memory'; @@ -33,59 +26,48 @@ enum CrawlMethod { FETCH = 'FETCH', EDGE_BROWSER = 'EDGE_BROWSER', PUPPETEER = 'PUPPETEER', - APIFY = 'APIFY', + // APIFY = 'APIFY', } -const combineDocumentsFn = (docs: Document[]) => { - const serializedDocs = docs.map((doc) => doc.pageContent); - return serializedDocs.join('\n\n'); -}; +const TEMPLATE = `If you don't know how to answer a question, use the available tools to look up relevant information. You should particularly do this for questions about URLs.`; -const formatVercelMessages = (chatHistory: VercelChatMessage[]) => { - const formattedDialogueTurns = chatHistory.map((message) => { - if (message.role === 'user') { - return `Human: ${message.content}`; - } else if (message.role === 'assistant') { - return `Assistant: ${message.content}`; - } else { - return `${message.role}: ${message.content}`; - } - }); - return formattedDialogueTurns.join('\n'); +const convertVercelMessageToLangChainMessage = (message: VercelChatMessage) => { + if (message.role === 'user') { + return new HumanMessage(message.content); + } else if (message.role === 'assistant') { + return new AIMessage(message.content); + } else { + return new ChatMessage(message.content, message.role); + } }; -const CONDENSE_QUESTION_TEMPLATE = `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. - - - {chat_history} - - -Follow Up Input: {question} -Standalone question:`; -const condenseQuestionPrompt = PromptTemplate.fromTemplate( - CONDENSE_QUESTION_TEMPLATE, -); - -const ANSWER_TEMPLATE = `Answer the question based only on the following context and chat history: - - {context} - - - - {chat_history} - - -Question: {question} -`; -const answerPrompt = PromptTemplate.fromTemplate(ANSWER_TEMPLATE); +const modelForName = (modelName: string) => { + switch (modelName) { + case 'gemini-pro': + return new ChatGoogleGenerativeAI({ + modelName, + temperature: 0, + }); + case 'gpt-3.5-turbo': + case 'gpt-3.5-turbo-16k': + case 'gpt-4': + case 'gpt-4-1106-preview': + default: + return new ChatOpenAI({ + modelName, + temperature: 0, + streaming: true, + }); + } +}; const docsForCrawl = async ({ origin, - url, + urls, crawlMethod, }: { origin: string | null; - url: string; + urls: string[]; crawlMethod: CrawlMethod | undefined; }): Promise => { const headers = new Headers({ @@ -96,55 +78,76 @@ const docsForCrawl = async ({ } switch (crawlMethod) { case CrawlMethod.EDGE_BROWSER: - return await (async () => { - // const browser = new WebBrowser({ - // model, - // embeddings, - // headers, - // }); - // const webBrowsePrompt = currentMessageContent; - // const response = await browser.call(`"${url}","${webBrowsePrompt}"`); - - const loader = new CheerioWebBaseLoader(url, { - selector: 'body', - }); - const docs = await loader.load(); - return docs; - })(); + return ( + await (async () => { + const loaders = urls + .map( + (url) => + new CheerioWebBaseLoader(url, { + selector: 'body', + }), + ) + .map((loader) => loader.load()); + + return await Promise.all(loaders); + })() + ).flat(); case CrawlMethod.FETCH: - return await (async () => { - const fetched = await fetch(url, { - headers, - }); - const body = await fetched.text(); - const contentType = fetched.headers.get('Content-Type'); - // TODO: handle content types differently - switch ( - contentType - // case 'text/html': - // docs = await run({ html: body, url }); - // break; - // case 'application/json': - ) { - } - return [new Document({ pageContent: body })]; - })(); + return ( + await Promise.all( + urls.map(async (url: string) => { + const fetched = await fetch(url, { + headers, + }); + const body = await fetched.text(); + const _contentType = fetched.headers.get('Content-Type'); + // TODO: handle content types differently + return [new Document({ pageContent: body })]; + }), + ) + ).flat(); case CrawlMethod.PUPPETEER: - case CrawlMethod.APIFY: - return await (async () => { - const fetched = await fetch( - `${origin}/api/rag-url-with-langchain/${crawlMethod.toLowerCase()}`, - { - method: 'POST', - body: JSON.stringify({ url, html: undefined }), - }, - ); - const body = await fetched.json(); - return body as Document[]; - })(); + return ( + await Promise.all( + urls.map(async (url: string) => { + const fetched = await fetch( + `${origin}/api/rag-url-with-langchain/${crawlMethod.toLowerCase()}`, + { + method: 'POST', + body: JSON.stringify({ url, html: undefined }), + }, + ); + return (await fetched.json()) as Document[]; + }), + ) + ).flat(); } }; + +function normalizeUrl(urlString: string): string { + // Regex pattern to match URL-like strings (with optional protocol and path/query) + const urlPattern = /(?:https?:\/\/)?[\w-]+(\.[\w-]+)+\.?(\/\S*)?/gi; + + // Check if the string matches the URL pattern + if (urlPattern.test(urlString)) { + // Check if the protocol is already present + if (!/^https?:\/\//i.test(urlString)) { + // Prepend "https://" if the protocol is missing + urlString = `https://${urlString}`; + } + } + return urlString; +} + +interface CrawlRequest { + messages: Message[]; + returnIntermediateSteps: boolean; + crawlMethod: CrawlMethod; + urls: string[]; + modelName: string; +} + /** * This handler initializes and calls a retrieval chain. It composes the chain using * LangChain Expression Language. See the docs for more information: @@ -153,100 +156,91 @@ const docsForCrawl = async ({ */ export async function POST(req: NextRequest) { try { - const body = await req.json(); + const body = (await req.json()) as Partial; const messages = body.messages ?? []; - const url = body.url; + const urls: string[] = (body.urls ?? []).map(normalizeUrl); const crawlMethod: CrawlMethod | undefined = body.crawlMethod; - console.debug('POST rag-url-with-langchain', { messages, url }); + const modelName = body.modelName ?? 'gpt-3.5-turbo'; + const returnIntermediateSteps: boolean = + body.returnIntermediateSteps ?? true; + const previousMessages = messages.slice(0, -1); const currentMessageContent = messages[messages.length - 1].content; - const html: string | undefined = undefined; - // const html = await (await fetch(url)).text(); - - // console.debug('POST rag-url-with-langchain html', { html }); - - const model = new ChatOpenAI({ - modelName: 'gpt-3.5-turbo-16k', - temperature: 0, - verbose: true, - }); - // const embeddings = new OpenAIEmbeddings(); + const model = modelForName(modelName); const origin = req.headers.get('origin'); const docs = await docsForCrawl({ origin, - url, + urls, crawlMethod, }); - /** - * We use LangChain Expression Language to compose two chains. - * To learn more, see the guide here: - * - * https://js.langchain.com/docs/guides/expression_language/cookbook - */ - const standaloneQuestionChain = RunnableSequence.from([ - condenseQuestionPrompt, - model, - new StringOutputParser(), - ]); + const baseRetriever = await compressedRelevantDocsRetriever(docs); + + const chat_history = (previousMessages ?? []).map( + convertVercelMessageToLangChainMessage, + ); + const chatHistory = new ChatMessageHistory(chat_history); + const memory = + model instanceof ChatOpenAI + ? new OpenAIAgentTokenBufferMemory({ + llm: model, + memoryKey: 'chat_history', + outputKey: 'output', + chatHistory, + }) + : new BufferMemory({ + chatHistory, + memoryKey: 'chat_history', + outputKey: 'output', + }); + + const { stream, handlers } = LangChainStream(); let resolveWithDocuments: (value: Document[]) => void; - const documentPromise = new Promise((resolve) => { + const sourcesPromise = new Promise((resolve) => { resolveWithDocuments = resolve; }); + /** + * Wrap the retriever in a tool to present it to the agent in a + * usable form. + */ + const tool = createRetrieverTool(baseRetriever, { + name: 'search_latest_knowledge', + description: 'Searches and returns up-to-date general information.', + }); - const baseRetriever = await compressedRelevantDocsRetriever(docs, [ - { - handleRetrieverEnd(documents) { - resolveWithDocuments(documents); - }, - }, - ]); - - // const baseRetriever = ( - // await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) - // ).asRetriever({ - // callbacks: [ - // { - // handleRetrieverEnd(documents) { - // resolveWithDocuments(documents); - // }, - // }, - // ], - // }); - - const retrievalChain = baseRetriever.pipe(combineDocumentsFn); - - const answerChain = RunnableSequence.from([ - { - context: RunnableSequence.from([ - (input) => input.question, - retrievalChain, - ]), - chat_history: (input) => input.chat_history, - question: (input) => input.question, + const executor = await initializeAgentExecutorWithOptions([tool], model, { + agentType: + model instanceof ChatOpenAI + ? 'openai-functions' + : 'structured-chat-zero-shot-react-description', + memory, + returnIntermediateSteps, + agentArgs: { + prefix: TEMPLATE, }, - answerPrompt, - model, - ]); + }); - const conversationalRetrievalQAChain = RunnableSequence.from([ + void executor.call( { - question: standaloneQuestionChain, - chat_history: (input) => input.chat_history, + input: currentMessageContent, }, - answerChain, - new BytesOutputParser(), - ]); - - const stream = await conversationalRetrievalQAChain.stream({ - question: currentMessageContent, - chat_history: formatVercelMessages(previousMessages), - }); + [ + handlers, + { + handleRetrieverEnd: (docs) => { + resolveWithDocuments(docs); + }, + handleChainEnd: () => { + resolveWithDocuments([]); + }, + }, + ], + ); - const documents = await documentPromise; + const documents = await sourcesPromise; const serializedSources = Buffer.from( JSON.stringify( documents.map((doc) => { @@ -264,144 +258,36 @@ export async function POST(req: NextRequest) { 'x-sources': serializedSources, }, }); - } catch (e: any) { + } catch (e) { return NextResponse.json({ error: e.message }, { status: 500 }); } } -const compressedRelevantDocsRetriever = async (docs: Document[], callbacks) => { +const compressedRelevantDocsRetriever = async (docs: Document[]) => { + const embeddings = new OpenAIEmbeddings(); const embeddingsFilter = new EmbeddingsFilter({ - embeddings: new OpenAIEmbeddings(), - similarityThreshold: 0.8, - k: 20, + embeddings, + similarityThreshold: 0.66, + k: 10, }); const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 400, - chunkOverlap: 40, + chunkSize: 200, + chunkOverlap: 20, }); const compressorPipeline = new DocumentCompressorPipeline({ transformers: [textSplitter, embeddingsFilter], }); - // const baseRetriever = new TavilySearchAPIRetriever({ - // includeRawContent: true, - // }); const baseRetriever = ( - await MemoryVectorStore.fromDocuments(docs, new OpenAIEmbeddings(), {}) - ).asRetriever({ callbacks }); + await MemoryVectorStore.fromDocuments(docs, embeddings, {}) + ).asRetriever(); const retriever = new ContextualCompressionRetriever({ baseCompressor: compressorPipeline, baseRetriever, }); - // const retrievedDocs = await retriever.getRelevantDocuments( - // "What did the speaker say about Justice Breyer in the 2022 State of the Union?" - // ); - // console.log({ retrievedDocs }); return retriever; }; - -const mapReduceDocumentsChain = (() => { - // Initialize the OpenAI model - const model = new ChatOpenAI({ - modelName: 'gpt-3.5-turbo', - temperature: 0, - verbose: true, - }); - - // Define prompt templates for document formatting, summarizing, collapsing, and combining - const documentPrompt = PromptTemplate.fromTemplate('{pageContent}'); - const summarizePrompt = PromptTemplate.fromTemplate( - 'Summarize this content:\n\n{context}', - ); - const collapsePrompt = PromptTemplate.fromTemplate( - 'Collapse this content:\n\n{context}', - ); - const combinePrompt = PromptTemplate.fromTemplate( - 'Combine these summaries:\n\n{context}', - ); - - // Wrap the `formatDocument` util so it can format a list of documents - const formatDocs = async (documents: Document[]): Promise => { - const formattedDocs = await Promise.all( - documents.map((doc) => formatDocument(doc, documentPrompt)), - ); - return formattedDocs.join('\n\n'); - }; - - // Define a function to get the number of tokens in a list of documents - const getNumTokens = async (documents: Document[]): Promise => - model.getNumTokens(await formatDocs(documents)); - - // Initialize the output parser - const outputParser = new StringOutputParser(); - - // Define the map chain to format, summarize, and parse the document - const mapChain = RunnableSequence.from([ - { context: async (i: Document) => formatDocument(i, documentPrompt) }, - summarizePrompt, - model, - outputParser, - ]); - - // Define the collapse chain to format, collapse, and parse a list of documents - const collapseChain = RunnableSequence.from([ - { context: async (documents: Document[]) => formatDocs(documents) }, - collapsePrompt, - model, - outputParser, - ]); - - // Define a function to collapse a list of documents until the total number of tokens is within the limit - const collapse = async ( - documents: Document[], - options?: { - config?: BaseCallbackConfig; - }, - tokenMax = 4000, - ) => { - const editableConfig = options?.config; - let docs = documents; - let collapseCount = 1; - while ((await getNumTokens(docs)) > tokenMax) { - if (editableConfig) { - editableConfig.runName = `Collapse ${collapseCount}`; - } - const splitDocs = splitListOfDocs(docs, getNumTokens, tokenMax); - docs = await Promise.all( - splitDocs.map((doc) => collapseDocs(doc, collapseChain.invoke)), - ); - collapseCount += 1; - } - return docs; - }; - - // Define the reduce chain to format, combine, and parse a list of documents - const reduceChain = RunnableSequence.from([ - { context: formatDocs }, - combinePrompt, - model, - outputParser, - ]).withConfig({ runName: 'Reduce' }); - - // Define the final map-reduce chain - const mapReduceChain = RunnableSequence.from([ - RunnableSequence.from([ - { doc: new RunnablePassthrough(), content: mapChain }, - (input) => - new Document({ - pageContent: input.content, - metadata: input.doc.metadata, - }), - ]) - .withConfig({ runName: 'Summarize (return doc)' }) - .map(), - collapse, - reduceChain, - ]).withConfig({ runName: 'Map reduce' }); - - return mapReduceChain; -})(); diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx index 2076832c..9b4086ee 100644 --- a/ui/app/components/stacks/rag-url-with-langchain.tsx +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -1,96 +1,178 @@ 'use client'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { Message } from 'ai/react'; import { useChat } from 'ai/react'; -import type { AgentStep } from 'langchain/schema'; +import DOMPurify from 'dompurify'; import { IoSend } from 'react-icons/io5'; import { toast } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; +import React from 'react'; + export enum CrawlMethod { FETCH = 'FETCH', EDGE_BROWSER = 'EDGE_BROWSER', PUPPETEER = 'PUPPETEER', - APIFY = 'APIFY', + // APIFY = 'APIFY', +} + +export enum ModelType { + GPT3Turbo = 'gpt-3.5-turbo', + GPT3Turbo16k = 'gpt-3.5-turbo-16k', + GPT4 = 'gpt-4', + GPT4Turbo = 'gpt-4-1106-preview', + GeminiPro = 'gemini-pro', } type SourcesForMessages = Record; -const placeholderAnswering = 'A: Answeringโ€ฆ'; +interface URLDetecting { + inputText: string; + setInputText: (inputText: string) => void; + sendMessage: () => void; +} -const ensureHttpProtocol = (urlString: string) => { - if (!/^(?:f|ht)tps?\:\/\//.test(urlString)) { - return `https://${urlString}`; - } - return urlString; -}; +const dummyEvent: React.FormEvent = { + ...(new Event('submit') as any), // Cast to 'any' to allow assignment to FormEvent + currentTarget: { + ...document.createElement('form'), + checkValidity: () => true, // Assuming the form is valid + }, + // eslint-disable-next-line @typescript-eslint/no-empty-function + preventDefault: () => {}, // No-op function + // eslint-disable-next-line @typescript-eslint/no-empty-function + stopPropagation: () => {}, // No-op function + target: document.createElement('form'), +} as React.FormEvent; -const isValidUrl = (urlString: string) => { - const urlPattern = - /^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$/; - const prefixedUrlString = ensureHttpProtocol(urlString); - return urlPattern.test(prefixedUrlString); -}; +interface URLDetectingInputProps extends URLDetecting { + placeholder?: string; +} + +// Regular expression to match URL-like patterns including path and query +const urlPattern = /(?:https?:\/\/)?[\w-]+(\.[\w-]+)+\.?(\/\S*)?/gi; + +function useURLDetector(urlDetectingContext: URLDetecting): string[] { + const { inputText } = urlDetectingContext; + + const detectedURLs = useMemo(() => { + return (inputText.match(urlPattern) ?? []) as string[]; + }, [inputText]); + + return detectedURLs; +} + +const URLDetectingInput = ({ + placeholder, + ...useDetector +}: URLDetectingInputProps) => { + const { inputText, setInputText, sendMessage } = useDetector; + const contentEditableRef = useRef(null); + const isComposing = useRef(false); + const [isFocused, setIsFocused] = useState(false); + + const handleInput = useCallback( + (event) => { + if (!isComposing.current) { + const text = event.currentTarget?.textContent ?? ''; + setInputText(text); + } + }, + [setInputText], + ); + + const handleComposition = useCallback( + (event) => { + if (event.type === 'compositionend') { + isComposing.current = false; + setInputText(event.currentTarget.textContent ?? ''); + } else { + isComposing.current = true; + } + }, + [setInputText], + ); + + const handleFocus = useCallback(() => setIsFocused(true), []); + const handleBlur = useCallback(() => setIsFocused(false), []); + + const handleKeyDown = useCallback( + (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + event.preventDefault(); // Prevent the default Enter key behavior + // Call the sendMessage function here + // You need to pass this function down from the parent component + sendMessage(); + } + }, + [sendMessage], + ); + + useEffect(() => { + if (!contentEditableRef.current) return; + contentEditableRef.current.style.whiteSpace = 'pre-wrap'; + + // Sanitize the inputText before setting it as innerHTML + const sanitizedInputText = DOMPurify.sanitize( + inputText.replace( + urlPattern, + (match) => `${match}`, + ), + ); + + contentEditableRef.current.innerHTML = sanitizedInputText; + }, [inputText]); + + useEffect(() => { + const moveCursorToEnd = () => { + const range = document.createRange(); + const selection = window.getSelection(); + + if (!contentEditableRef.current) return; + if (!selection) return; + + range.setStart( + contentEditableRef.current, + contentEditableRef.current.childNodes.length, + ); + + range.collapse(true); + selection.removeAllRanges(); + selection.addRange(range); + }; + moveCursorToEnd(); + }, [contentEditableRef.current?.textContent, inputText]); -export function IntermediateStep(props: { message: Message }) { - const parsedInput: AgentStep = JSON.parse(props.message.content); - const action = parsedInput.action; - const observation = parsedInput.observation; - const [expanded, setExpanded] = useState(false); return ( -
+
+ {/* for production applications, more care should be taken to support native affordances like undo, a11y, and also care should be taken to prevent XSS*/}
setExpanded(!expanded)} - > - - ๐Ÿ› ๏ธ {action.tool} - - ๐Ÿ”ฝ - ๐Ÿ”ผ -
-
-
- - Tool Input: -

-

- {JSON.stringify(action.toolInput)} - -
-
- - {observation} - + suppressContentEditableWarning={true} + role="textbox" + tabIndex={0} + >
+ {/* Render the placeholder text when inputText is empty and the div is not focused */} + {!inputText && !isFocused && ( +
+ {placeholder}
-
+ )}
); -} +}; export function ChatMessageBubble(props: { message: Message; @@ -142,30 +224,13 @@ export function ChatMessageBubble(props: { const RAGURLWithLangchain = () => { const [sourcesForMessages, setSourcesForMessages] = useState({}); - const [inputUrl, setInputUrl] = useState( - 'https://www.waggledance.ai', - ); - const [showIntermediateSteps, setShowIntermediateSteps] = useState(false); - const [intermediateStepsLoading, setIntermediateStepsLoading] = - useState(false); - - const abortControllerRef = useRef(null); - const inputRef = useRef(null); - const inputUrlRef = useRef(null); const messageContainerRef = useRef(null); - - const hasValidUrl = useMemo(() => isValidUrl(inputUrl), [inputUrl]); - - const chatHistory = useMemo(() => { - return Object.entries(sourcesForMessages).map( - ([index, sources]) => - ({ - id: index, - content: sources.join(' '), - role: Number.parseInt(index) % 2 === 0 ? 'user' : 'assistant', - }) as Message, - ); - }, [sourcesForMessages]); + const [selectedCrawlMethod, setSelectedCrawlMethod] = useState( + CrawlMethod.PUPPETEER, + ); + const [selectedModelType, setSelectedModelType] = useState( + ModelType.GPT3Turbo, + ); const { messages, @@ -175,14 +240,10 @@ const RAGURLWithLangchain = () => { handleInputChange, handleSubmit, isLoading, - setMessages, } = useChat({ api: '/api/rag-url-with-langchain', - experimental_onFunctionCall(messages, functionCall) { - console.log(messages); - return Promise.resolve(); - }, onResponse(response) { + console.log('onResponse', response); const sourcesHeader = response.headers.get('x-sources'); const sources = sourcesHeader ? JSON.parse(atob(sourcesHeader)) : []; const messageIndexHeader = response.headers.get('x-message-index'); @@ -197,24 +258,28 @@ const RAGURLWithLangchain = () => { console.log('onFinish', message); }, onError: (e) => { - toast(e.message, { - theme: 'dark', - }); + toast(e.message); }, + sendExtraMessageFields: true, }); - // useEffect(() => { - // if (abortControllerRef?.current) { - // abortControllerRef?.current.abort(); - // } - // }, []); + const handleInputChangeWithUrlDetection = useCallback( + (event) => { + handleInputChange(event); + setInput(event.target.value); + }, + [handleInputChange, setInput], + ); - const sendMessage = async (event: React.FormEvent) => { - event.preventDefault(); - if (!hasValidUrl) { - toast('Please enter a valid URL.'); - return; - } + const detectedURLs = useURLDetector({ + inputText: + messages.filter((m) => m.role !== 'assistant').join('\n') + input, + setInputText: handleInputChangeWithUrlDetection, + sendMessage: () => void sendMessage(), + }); + + const sendMessage = async (event?: React.FormEvent) => { + event?.preventDefault(); if (messageContainerRef.current) { messageContainerRef.current.classList.add('grow'); } @@ -222,94 +287,20 @@ const RAGURLWithLangchain = () => { await new Promise((resolve) => setTimeout(resolve, 300)); } - if (isLoading ?? intermediateStepsLoading) { + if (isLoading) { return; } const body = { - url: inputUrl, + urls: detectedURLs, chat: input, - crawlMethod: CrawlMethod.PUPPETEER.toString(), + crawlMethod: selectedCrawlMethod, + modelName: selectedModelType, + returnIntermediateSteps: true, // currently does nothing on the backend }; - if (!showIntermediateSteps) { - handleSubmit(event, { - options: { body }, - }); - } else { - setIntermediateStepsLoading(true); - setInput(''); - abortControllerRef.current = new AbortController(); - const messagesWithUserReply = messages.concat({ - id: messages.length.toString(), - content: input, - role: 'user', - }); - setMessages(messagesWithUserReply); - - try { - console.log('inputUrl', inputUrl); - - const response = await fetch('/api/rag-url-with-langchain', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - messages: messagesWithUserReply, - url: body.url, - crawlMethod: body.crawlMethod, - }), - signal: abortControllerRef.current.signal, - }); - - const json = await response.json(); - setIntermediateStepsLoading(false); - if (response.status === 200) { - // Represent intermediate steps as system messages for display purposes - const intermediateStepMessages = (json.intermediate_steps ?? []).map( - (intermediateStep: AgentStep, i: number) => { - return { - id: (messagesWithUserReply.length + i).toString(), - content: JSON.stringify(intermediateStep), - role: 'system', - }; - }, - ); - const newMessages = messagesWithUserReply; - for (const message of intermediateStepMessages) { - newMessages.push(message); - setMessages([...newMessages]); - await new Promise((resolve) => - setTimeout(resolve, 1000 + Math.random() * 1000), - ); - } - setMessages([ - ...newMessages, - { - id: ( - newMessages.length + intermediateStepMessages.length - ).toString(), - content: json.output, - role: 'assistant', - }, - ]); - } else { - if (json.error) { - toast(json.error, { - theme: 'dark', - }); - throw new Error(json.error); - } - } - } catch (error) { - if (error.name !== 'AbortError') { - toast('An error occurred while sending the message.'); - } - } finally { - setIntermediateStepsLoading(false); - abortControllerRef.current = null; - } - } + handleSubmit(event ?? dummyEvent, { + options: { body }, + }); }; useEffect(() => { @@ -330,9 +321,7 @@ const RAGURLWithLangchain = () => { {messages.length > 0 ? [...messages].reverse().map((m, i) => { const sourceKey = (messages.length - 1 - i).toString(); - return m.role === 'system' ? ( - - ) : ( + return ( { : ''}
- {/* */} - - {/* setInputUrl(e.target.value)} - placeholder="Enter URL" - className="w-full rounded border p-2" - /> */} -
-
+
+ + +
); diff --git a/ui/package-lock.json b/ui/package-lock.json index 6f529f4a..187a7948 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -21,6 +21,7 @@ "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", "cheerio": "^1.0.0-rc.12", + "dompurify": "^3.0.6", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", "html-react-parser": "^5.0.7", @@ -3559,6 +3560,11 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, + "node_modules/dompurify": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.6.tgz", + "integrity": "sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==" + }, "node_modules/domutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", @@ -6208,6 +6214,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "optional": true, + "peer": true + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", diff --git a/ui/package.json b/ui/package.json index e4b8606a..1b7b6b54 100644 --- a/ui/package.json +++ b/ui/package.json @@ -23,6 +23,7 @@ "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", "cheerio": "^1.0.0-rc.12", + "dompurify": "^3.0.6", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", "html-react-parser": "^5.0.7", From d49f746247bf4152fdeea211179ff761afb6ff9b Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:10:25 -0800 Subject: [PATCH 05/10] always show placeholder while empty --- ui/app/components/stacks/rag-url-with-langchain.tsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx index 9b4086ee..6dbee5be 100644 --- a/ui/app/components/stacks/rag-url-with-langchain.tsx +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -71,7 +71,6 @@ const URLDetectingInput = ({ const { inputText, setInputText, sendMessage } = useDetector; const contentEditableRef = useRef(null); const isComposing = useRef(false); - const [isFocused, setIsFocused] = useState(false); const handleInput = useCallback( (event) => { @@ -95,9 +94,6 @@ const URLDetectingInput = ({ [setInputText], ); - const handleFocus = useCallback(() => setIsFocused(true), []); - const handleBlur = useCallback(() => setIsFocused(false), []); - const handleKeyDown = useCallback( (event: React.KeyboardEvent) => { if (event.key === 'Enter') { @@ -152,20 +148,18 @@ const URLDetectingInput = ({ ref={contentEditableRef} contentEditable onInput={handleInput} - onFocus={handleFocus} - onBlur={handleBlur} onCompositionStart={handleComposition} onCompositionEnd={handleComposition} onKeyDown={handleKeyDown} className={`content-editable rounded border border-gray-400 p-2 ${ - !inputText && !isFocused ? 'placeholder' : '' + !inputText ? 'placeholder' : '' }`} suppressContentEditableWarning={true} role="textbox" tabIndex={0} >
{/* Render the placeholder text when inputText is empty and the div is not focused */} - {!inputText && !isFocused && ( + {!inputText && (
{placeholder}
From a8cbd49092c37eca6b6eff44ca401b564a1baf80 Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:34:32 -0800 Subject: [PATCH 06/10] do not export run puppeteer function --- ui/app/api/rag-url-with-langchain/puppeteer/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts index 4e0a4c48..a3105a44 100644 --- a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts +++ b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts @@ -19,7 +19,7 @@ export async function POST(req: NextRequest) { }); } -export const run = async ({ html, url }: { html?: string; url?: string }) => { +const run = async ({ html, url }: { html?: string; url?: string }) => { /** Loader use evaluate function ` await page.evaluate(() => document.body.innerHTML);` as default evaluate */ const gotoOptions: PuppeteerGotoOptions | undefined = url ? { waitUntil: 'domcontentloaded', timeout: 10_000 } From 82c0e1e035d1b6c6b26077e738cf8b54d685193b Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:48:55 -0800 Subject: [PATCH 07/10] try chrome-aws-lambda --- ui/package-lock.json | 487 ++++++++++++++++++++++++++++++++++++++++--- ui/package.json | 1 + 2 files changed, 456 insertions(+), 32 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 187a7948..209b13b3 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -21,6 +21,7 @@ "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", "cheerio": "^1.0.0-rc.12", + "chrome-aws-lambda": "^10.1.0", "dompurify": "^3.0.6", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", @@ -3115,6 +3116,20 @@ "node": ">=10" } }, + "node_modules/chrome-aws-lambda": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/chrome-aws-lambda/-/chrome-aws-lambda-10.1.0.tgz", + "integrity": "sha512-NZQVf+J4kqG4sVhRm3WNmOfzY0OtTSm+S8rg77pwePa9RCYHzhnzRs8YvNI6L9tALIW6RpmefWiPURt3vURXcw==", + "dependencies": { + "lambdafs": "^2.0.3" + }, + "engines": { + "node": ">= 10.16" + }, + "peerDependencies": { + "puppeteer-core": "^10.1.0" + } + }, "node_modules/chromium-bidi": { "version": "0.4.7", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.4.7.tgz", @@ -3477,9 +3492,10 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1107588", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", - "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" + "version": "0.0.901419", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz", + "integrity": "sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ==", + "peer": true }, "node_modules/didyoumean": { "version": "1.2.2", @@ -5898,6 +5914,214 @@ "json-buffer": "3.0.1" } }, + "node_modules/lambdafs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/lambdafs/-/lambdafs-2.1.1.tgz", + "integrity": "sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ==", + "bundleDependencies": [ + "tar-fs" + ], + "dependencies": { + "tar-fs": "*" + }, + "bin": { + "lambdafs": "bin/brotli.js" + }, + "engines": { + "node": ">= 10.16" + } + }, + "node_modules/lambdafs/node_modules/base64-js": { + "version": "1.5.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/bl": { + "version": "4.1.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/lambdafs/node_modules/bl/node_modules/readable-stream": { + "version": "3.6.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/lambdafs/node_modules/buffer": { + "version": "5.7.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/lambdafs/node_modules/chownr": { + "version": "1.1.4", + "inBundle": true, + "license": "ISC" + }, + "node_modules/lambdafs/node_modules/end-of-stream": { + "version": "1.4.4", + "inBundle": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/lambdafs/node_modules/fs-constants": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/ieee754": { + "version": "1.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, + "node_modules/lambdafs/node_modules/inherits": { + "version": "2.0.4", + "inBundle": true, + "license": "ISC" + }, + "node_modules/lambdafs/node_modules/mkdirp-classic": { + "version": "0.5.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/once": { + "version": "1.4.0", + "inBundle": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/lambdafs/node_modules/pump": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/lambdafs/node_modules/string_decoder": { + "version": "1.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lambdafs/node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/tar-fs": { + "version": "2.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/lambdafs/node_modules/tar-stream": { + "version": "2.2.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lambdafs/node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/lambdafs/node_modules/util-deprecate": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/wrappy": { + "version": "1.0.2", + "inBundle": true, + "license": "ISC" + }, "node_modules/langchain": { "version": "0.0.207", "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.207.tgz", @@ -7529,6 +7753,15 @@ "node": ">=8" } }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7718,6 +7951,70 @@ "node": ">= 6" } }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "peer": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "peer": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "peer": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "peer": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "peer": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/postcss": { "version": "8.4.32", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.32.tgz", @@ -8044,38 +8341,33 @@ } }, "node_modules/puppeteer-core": { - "version": "19.11.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", - "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-10.4.0.tgz", + "integrity": "sha512-KU8zyb7AIOqNjLCN3wkrFXxh+EVaG+zrs2P03ATNjc3iwSxHsu5/EvZiREpQ/IJiT9xfQbDVgKcsvRuzLCxglQ==", + "peer": true, "dependencies": { - "@puppeteer/browsers": "0.5.0", - "chromium-bidi": "0.4.7", - "cross-fetch": "3.1.5", - "debug": "4.3.4", - "devtools-protocol": "0.0.1107588", + "debug": "4.3.1", + "devtools-protocol": "0.0.901419", "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.1", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.1", + "pkg-dir": "4.2.0", + "progress": "2.0.1", "proxy-from-env": "1.1.0", - "tar-fs": "2.1.1", - "unbzip2-stream": "1.4.3", - "ws": "8.13.0" + "rimraf": "3.0.2", + "tar-fs": "2.0.0", + "unbzip2-stream": "1.3.3", + "ws": "7.4.6" }, "engines": { - "node": ">=14.14.0" - }, - "peerDependencies": { - "typescript": ">= 4.7.4" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "node": ">=10.18.1" } }, "node_modules/puppeteer-core/node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "peer": true, "dependencies": { "debug": "4" }, @@ -8083,10 +8375,34 @@ "node": ">= 6.0.0" } }, + "node_modules/puppeteer-core/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "peer": true + }, + "node_modules/puppeteer-core/node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "peer": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/puppeteer-core/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -8095,16 +8411,69 @@ "node": ">= 6" } }, + "node_modules/puppeteer-core/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "peer": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/puppeteer-core/node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "peer": true, + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/progress": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", + "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==", + "peer": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/puppeteer-core/node_modules/tar-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", + "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", + "peer": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp": "^0.5.1", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/unbzip2-stream": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", + "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", + "peer": true, + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/puppeteer-core/node_modules/ws": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", - "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "peer": true, "engines": { - "node": ">=10.0.0" + "node": ">=8.3.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" + "utf-8-validate": "^5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -8126,6 +8495,11 @@ "node": ">= 6.0.0" } }, + "node_modules/puppeteer/node_modules/devtools-protocol": { + "version": "0.0.1107588", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", + "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" + }, "node_modules/puppeteer/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -8138,6 +8512,55 @@ "node": ">= 6" } }, + "node_modules/puppeteer/node_modules/puppeteer-core": { + "version": "19.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", + "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", + "dependencies": { + "@puppeteer/browsers": "0.5.0", + "chromium-bidi": "0.4.7", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1107588", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "proxy-from-env": "1.1.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.13.0" + }, + "engines": { + "node": ">=14.14.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/ui/package.json b/ui/package.json index 1b7b6b54..19a765c4 100644 --- a/ui/package.json +++ b/ui/package.json @@ -23,6 +23,7 @@ "@vercel/analytics": "^1.1.1", "ai": "^2.2.28", "cheerio": "^1.0.0-rc.12", + "chrome-aws-lambda": "^10.1.0", "dompurify": "^3.0.6", "elevenlabs-node": "^2.0.1", "fuse.js": "^7.0.0", From dd58aa668d2ce485a4796f7bbbc1fb0d55dafd91 Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:31:48 -0800 Subject: [PATCH 08/10] use aws-lambda-chrome path and use puppeteer-core only --- .../rag-url-with-langchain/puppeteer/route.ts | 2 + ui/next.config.js | 10 + ui/package-lock.json | 293 ++++-------------- ui/package.json | 3 +- 4 files changed, 72 insertions(+), 236 deletions(-) diff --git a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts index a3105a44..8913b82d 100644 --- a/ui/app/api/rag-url-with-langchain/puppeteer/route.ts +++ b/ui/app/api/rag-url-with-langchain/puppeteer/route.ts @@ -1,4 +1,5 @@ import type { NextRequest } from 'next/server'; +import chrome from 'chrome-aws-lambda'; import type { Browser, Page, @@ -34,6 +35,7 @@ const run = async ({ html, url }: { html?: string; url?: string }) => { launchOptions: { headless: 'new', timeout: 10_000, + executablePath: await chrome.executablePath, }, gotoOptions, /** Pass custom evaluate , in this case you get page and browser instances */ diff --git a/ui/next.config.js b/ui/next.config.js index 7e7282cc..dd3429e0 100644 --- a/ui/next.config.js +++ b/ui/next.config.js @@ -51,6 +51,16 @@ const nextConfig = { }, ]; }, + webpack: (config, { isServer }) => { + // Add a rule to ignore .js.map files + config.module.rules.push({ + test: /\.js\.map$/, + use: 'ignore-loader', + }); + + // Return the modified config + return config; + }, }; module.exports = nextConfig; diff --git a/ui/package-lock.json b/ui/package-lock.json index 209b13b3..9b19d506 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -32,7 +32,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", - "puppeteer": "^19.11.1", + "puppeteer-core": "^19.11.1", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", @@ -58,6 +58,7 @@ "@types/react-dom": "^18", "@types/react-syntax-highlighter": "^15.5.11", "autoprefixer": "^10.0.1", + "ignore-loader": "^0.1.2", "postcss": "^8", "tailwindcss": "^3.3.0", "typescript": "^5" @@ -3259,6 +3260,8 @@ "version": "8.1.3", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.1.3.tgz", "integrity": "sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==", + "optional": true, + "peer": true, "dependencies": { "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -3492,10 +3495,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.901419", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz", - "integrity": "sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ==", - "peer": true + "version": "0.0.1107588", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", + "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" }, "node_modules/didyoumean": { "version": "1.2.2", @@ -3652,6 +3654,8 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "optional": true, + "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -5249,6 +5253,12 @@ "node": ">= 4" } }, + "node_modules/ignore-loader": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ignore-loader/-/ignore-loader-0.1.2.tgz", + "integrity": "sha512-yOJQEKrNwoYqrWLS4DcnzM7SEQhRKis5mB+LdKKh4cPmGYlLPR0ozRzHV5jmEk2IxptqJNQA5Cc0gw8Fj12bXA==", + "dev": true + }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -5355,7 +5365,9 @@ "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "optional": true, + "peer": true }, "node_modules/is-async-function": { "version": "2.0.0", @@ -5789,7 +5801,9 @@ "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "optional": true, + "peer": true }, "node_modules/json-schema-traverse": { "version": "0.4.1", @@ -7753,15 +7767,6 @@ "node": ">=8" } }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "peer": true, - "engines": { - "node": ">=6" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7801,6 +7806,8 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "optional": true, + "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -7951,70 +7958,6 @@ "node": ">= 6" } }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "peer": true, - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "peer": true, - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "peer": true, - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "peer": true, - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pkg-dir/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "peer": true, - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/postcss": { "version": "8.4.32", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.32.tgz", @@ -8331,6 +8274,8 @@ "integrity": "sha512-39olGaX2djYUdhaQQHDZ0T0GwEp+5f9UB9HmEP0qHfdQHIq0xGQZuAZ5TLnJIc/88SrPLpEflPC+xUqOTv3c5g==", "deprecated": "< 21.3.7 is no longer supported", "hasInstallScript": true, + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "0.5.0", "cosmiconfig": "8.1.3", @@ -8341,33 +8286,38 @@ } }, "node_modules/puppeteer-core": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-10.4.0.tgz", - "integrity": "sha512-KU8zyb7AIOqNjLCN3wkrFXxh+EVaG+zrs2P03ATNjc3iwSxHsu5/EvZiREpQ/IJiT9xfQbDVgKcsvRuzLCxglQ==", - "peer": true, + "version": "19.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", + "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", "dependencies": { - "debug": "4.3.1", - "devtools-protocol": "0.0.901419", + "@puppeteer/browsers": "0.5.0", + "chromium-bidi": "0.4.7", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1107588", "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.0", - "node-fetch": "2.6.1", - "pkg-dir": "4.2.0", - "progress": "2.0.1", + "https-proxy-agent": "5.0.1", "proxy-from-env": "1.1.0", - "rimraf": "3.0.2", - "tar-fs": "2.0.0", - "unbzip2-stream": "1.3.3", - "ws": "7.4.6" + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.13.0" }, "engines": { - "node": ">=10.18.1" + "node": ">=14.14.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } } }, "node_modules/puppeteer-core/node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "peer": true, "dependencies": { "debug": "4" }, @@ -8375,34 +8325,10 @@ "node": ">= 6.0.0" } }, - "node_modules/puppeteer-core/node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "peer": true - }, - "node_modules/puppeteer-core/node_modules/debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "peer": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/puppeteer-core/node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "peer": true, + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "dependencies": { "agent-base": "6", "debug": "4" @@ -8411,69 +8337,16 @@ "node": ">= 6" } }, - "node_modules/puppeteer-core/node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "peer": true, - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, - "node_modules/puppeteer-core/node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", - "peer": true, - "engines": { - "node": "4.x || >=6.0.0" - } - }, - "node_modules/puppeteer-core/node_modules/progress": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", - "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==", - "peer": true, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/puppeteer-core/node_modules/tar-fs": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", - "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", - "peer": true, - "dependencies": { - "chownr": "^1.1.1", - "mkdirp": "^0.5.1", - "pump": "^3.0.0", - "tar-stream": "^2.0.0" - } - }, - "node_modules/puppeteer-core/node_modules/unbzip2-stream": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", - "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", - "peer": true, - "dependencies": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, "node_modules/puppeteer-core/node_modules/ws": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", - "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", - "peer": true, + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", "engines": { - "node": ">=8.3.0" + "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -8488,6 +8361,8 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "peer": true, "dependencies": { "debug": "4" }, @@ -8495,15 +8370,12 @@ "node": ">= 6.0.0" } }, - "node_modules/puppeteer/node_modules/devtools-protocol": { - "version": "0.0.1107588", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", - "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" - }, "node_modules/puppeteer/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -8512,55 +8384,6 @@ "node": ">= 6" } }, - "node_modules/puppeteer/node_modules/puppeteer-core": { - "version": "19.11.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", - "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", - "dependencies": { - "@puppeteer/browsers": "0.5.0", - "chromium-bidi": "0.4.7", - "cross-fetch": "3.1.5", - "debug": "4.3.4", - "devtools-protocol": "0.0.1107588", - "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.1", - "proxy-from-env": "1.1.0", - "tar-fs": "2.1.1", - "unbzip2-stream": "1.4.3", - "ws": "8.13.0" - }, - "engines": { - "node": ">=14.14.0" - }, - "peerDependencies": { - "typescript": ">= 4.7.4" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/puppeteer/node_modules/ws": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", - "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/ui/package.json b/ui/package.json index 19a765c4..e0cd0263 100644 --- a/ui/package.json +++ b/ui/package.json @@ -34,7 +34,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", - "puppeteer": "^19.11.1", + "puppeteer-core": "^19.11.1", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", @@ -60,6 +60,7 @@ "@types/react-dom": "^18", "@types/react-syntax-highlighter": "^15.5.11", "autoprefixer": "^10.0.1", + "ignore-loader": "^0.1.2", "postcss": "^8", "tailwindcss": "^3.3.0", "typescript": "^5" From ea56717cad8187587933161612da399de218b93d Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:36:56 -0800 Subject: [PATCH 09/10] downgrading puppeteer-core to fix version conflict --- ui/package-lock.json | 314 +++++++++++++++++++++++++++++++++++++------ ui/package.json | 2 +- 2 files changed, 277 insertions(+), 39 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 9b19d506..e15139d2 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -32,7 +32,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", - "puppeteer-core": "^19.11.1", + "puppeteer-core": "^10.4.0", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", @@ -1631,6 +1631,8 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-0.5.0.tgz", "integrity": "sha512-Uw6oB7VvmPRLE4iKsjuOh8zgDabhNX67dzo8U/BB0f9527qx+4eeUs+korU98OhG5C4ubg7ufBgVi63XYwS6TQ==", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -1660,6 +1662,8 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "peer": true, "dependencies": { "debug": "4" }, @@ -1671,6 +1675,8 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -3135,6 +3141,8 @@ "version": "0.4.7", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.4.7.tgz", "integrity": "sha512-6+mJuFXwTMU6I3vYLs6IL8A1DyQTPjCfIL971X0aMPVGRbGnNfl6i6Cl0NMbxi2bRYLGESt9T2ZIMRM5PAEcIQ==", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -3159,6 +3167,8 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "optional": true, + "peer": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -3279,6 +3289,8 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "2.6.7" } @@ -3287,6 +3299,8 @@ "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -3495,9 +3509,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1107588", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", - "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==" + "version": "0.0.901419", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz", + "integrity": "sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ==" }, "node_modules/didyoumean": { "version": "1.2.2", @@ -4752,6 +4766,8 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "optional": true, + "peer": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -5493,6 +5509,8 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -7230,7 +7248,9 @@ "node_modules/mitt": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.0.tgz", - "integrity": "sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ==" + "integrity": "sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ==", + "optional": true, + "peer": true }, "node_modules/mkdirp": { "version": "1.0.4", @@ -7246,7 +7266,9 @@ "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "optional": true, + "peer": true }, "node_modules/ml-array-mean": { "version": "1.1.6", @@ -7767,6 +7789,14 @@ "node": ">=8" } }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7958,6 +7988,65 @@ "node": ">= 6" } }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/postcss": { "version": "8.4.32", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.32.tgz", @@ -8223,6 +8312,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -8286,32 +8377,25 @@ } }, "node_modules/puppeteer-core": { - "version": "19.11.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", - "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-10.4.0.tgz", + "integrity": "sha512-KU8zyb7AIOqNjLCN3wkrFXxh+EVaG+zrs2P03ATNjc3iwSxHsu5/EvZiREpQ/IJiT9xfQbDVgKcsvRuzLCxglQ==", "dependencies": { - "@puppeteer/browsers": "0.5.0", - "chromium-bidi": "0.4.7", - "cross-fetch": "3.1.5", - "debug": "4.3.4", - "devtools-protocol": "0.0.1107588", + "debug": "4.3.1", + "devtools-protocol": "0.0.901419", "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.1", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.1", + "pkg-dir": "4.2.0", + "progress": "2.0.1", "proxy-from-env": "1.1.0", - "tar-fs": "2.1.1", - "unbzip2-stream": "1.4.3", - "ws": "8.13.0" + "rimraf": "3.0.2", + "tar-fs": "2.0.0", + "unbzip2-stream": "1.3.3", + "ws": "7.4.6" }, "engines": { - "node": ">=14.14.0" - }, - "peerDependencies": { - "typescript": ">= 4.7.4" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "node": ">=10.18.1" } }, "node_modules/puppeteer-core/node_modules/agent-base": { @@ -8325,10 +8409,31 @@ "node": ">= 6.0.0" } }, + "node_modules/puppeteer-core/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/puppeteer-core/node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/puppeteer-core/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", "dependencies": { "agent-base": "6", "debug": "4" @@ -8337,16 +8442,63 @@ "node": ">= 6" } }, + "node_modules/puppeteer-core/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/puppeteer-core/node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/progress": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", + "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/puppeteer-core/node_modules/tar-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", + "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp": "^0.5.1", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/unbzip2-stream": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", + "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/puppeteer-core/node_modules/ws": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", - "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", "engines": { - "node": ">=10.0.0" + "node": ">=8.3.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" + "utf-8-validate": "^5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -8370,6 +8522,13 @@ "node": ">= 6.0.0" } }, + "node_modules/puppeteer/node_modules/devtools-protocol": { + "version": "0.0.1107588", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1107588.tgz", + "integrity": "sha512-yIR+pG9x65Xko7bErCUSQaDLrO/P1p3JUzEk7JCU4DowPcGHkTGUGQapcfcLc4qj0UaALwZ+cr0riFgiqpixcg==", + "optional": true, + "peer": true + }, "node_modules/puppeteer/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -8384,6 +8543,59 @@ "node": ">= 6" } }, + "node_modules/puppeteer/node_modules/puppeteer-core": { + "version": "19.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-19.11.1.tgz", + "integrity": "sha512-qcuC2Uf0Fwdj9wNtaTZ2OvYRraXpAK+puwwVW8ofOhOgLPZyz1c68tsorfIZyCUOpyBisjr+xByu7BMbEYMepA==", + "optional": true, + "peer": true, + "dependencies": { + "@puppeteer/browsers": "0.5.0", + "chromium-bidi": "0.4.7", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1107588", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "proxy-from-env": "1.1.0", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.13.0" + }, + "engines": { + "node": ">=14.14.0" + }, + "peerDependencies": { + "typescript": ">= 4.7.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "optional": true, + "peer": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -8760,6 +8972,8 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -9095,6 +9309,8 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "optional": true, + "peer": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -9107,7 +9323,9 @@ "node_modules/string-width/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "optional": true, + "peer": true }, "node_modules/string.prototype.matchall": { "version": "4.0.10", @@ -9474,6 +9692,8 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "optional": true, + "peer": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -9484,7 +9704,9 @@ "node_modules/tar-fs/node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "optional": true, + "peer": true }, "node_modules/tar-stream": { "version": "2.2.0", @@ -9733,6 +9955,8 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -10115,6 +10339,8 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "optional": true, + "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -10131,6 +10357,8 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "optional": true, + "peer": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -10145,6 +10373,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "optional": true, + "peer": true, "dependencies": { "color-name": "~1.1.4" }, @@ -10155,7 +10385,9 @@ "node_modules/wrap-ansi/node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "optional": true, + "peer": true }, "node_modules/wrappy": { "version": "1.0.2", @@ -10206,6 +10438,8 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -10227,6 +10461,8 @@ "version": "17.7.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", + "optional": true, + "peer": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -10244,6 +10480,8 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "optional": true, + "peer": true, "engines": { "node": ">=12" } diff --git a/ui/package.json b/ui/package.json index e0cd0263..31fab61a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -34,7 +34,7 @@ "openai": "^4.20.1", "path": "^0.12.7", "pdf-parse": "^1.1.1", - "puppeteer-core": "^19.11.1", + "puppeteer-core": "^10.4.0", "react": "^18", "react-compare-slider": "^3.0.1", "react-dom": "^18", From 02c99ba17c4cd2a2a75eb61b1aae58bf51c71b0a Mon Sep 17 00:00:00 2001 From: jon <906671+jondwillis@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:38:46 -0800 Subject: [PATCH 10/10] remove some extra padding --- ui/app/components/stacks/rag-url-with-langchain.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/app/components/stacks/rag-url-with-langchain.tsx b/ui/app/components/stacks/rag-url-with-langchain.tsx index 6dbee5be..ac5c331d 100644 --- a/ui/app/components/stacks/rag-url-with-langchain.tsx +++ b/ui/app/components/stacks/rag-url-with-langchain.tsx @@ -303,7 +303,7 @@ const RAGURLWithLangchain = () => { return (
0 ? 'border' : '' }`} >