diff --git a/web/package-lock.json b/web/package-lock.json index d81aaa14..f67664e9 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -28,6 +28,9 @@ "react-dom": "^18", "react-markdown": "^9.0.1", "reactstrap": "^9.2.2", + "rehype-autolink-headings": "^7.1.0", + "rehype-raw": "^7.0.0", + "rehype-slug": "^6.0.0", "remark-gfm": "^4.0.0" }, "devDependencies": { @@ -3816,6 +3819,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/github-slugger": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", + "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", + "license": "ISC" + }, "node_modules/glob": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", @@ -4028,6 +4037,100 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz", + "integrity": "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^7.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-parse5/node_modules/property-information": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", + "integrity": "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-heading-rank": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-heading-rank/-/hast-util-heading-rank-3.0.0.tgz", + "integrity": "sha512-EJKb8oMUXVHcWZTDepnr+WNbfnXKFNf9duMesmr4S8SXTJBJ9M4Yok08pu9vxdJwdlGRhVumk9mEhkEvKGifwA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", + "integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^7.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.0.tgz", @@ -4054,6 +4157,38 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.0.tgz", + "integrity": "sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-string": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz", + "integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-whitespace": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", @@ -4066,6 +4201,33 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hastscript": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", + "integrity": "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript/node_modules/property-information": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", + "integrity": "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/hpagent": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", @@ -4083,6 +4245,16 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/humanize-ms": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", @@ -6136,6 +6308,30 @@ "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz", "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==" }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -6576,6 +6772,56 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/rehype-autolink-headings": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/rehype-autolink-headings/-/rehype-autolink-headings-7.1.0.tgz", + "integrity": "sha512-rItO/pSdvnvsP4QRB1pmPiNHUskikqtPojZKJPPPAVx9Hj8i8TwMBhofrrAYRhYOOBZH9tgmG5lPqDLuIWPWmw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-heading-rank": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-slug": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/rehype-slug/-/rehype-slug-6.0.0.tgz", + "integrity": "sha512-lWyvf/jwu+oS5+hL5eClVd3hNdmwM1kAC0BUvEGD19pajQMIzcNUd/k9GsfQ+FfECvX+JE+e9/btsKH0EjJT6A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "github-slugger": "^2.0.0", + "hast-util-heading-rank": "^3.0.0", + "hast-util-to-string": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-gfm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz", @@ -7687,6 +7933,20 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/vfile-location": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", + "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/vfile-message": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", @@ -7729,6 +7989,16 @@ "loose-envify": "^1.0.0" } }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", diff --git a/web/package.json b/web/package.json index 67ca7ada..df4510f0 100644 --- a/web/package.json +++ b/web/package.json @@ -33,6 +33,9 @@ "react-dom": "^18", "react-markdown": "^9.0.1", "reactstrap": "^9.2.2", + "rehype-autolink-headings": "^7.1.0", + "rehype-raw": "^7.0.0", + "rehype-slug": "^6.0.0", "remark-gfm": "^4.0.0" }, "devDependencies": { diff --git a/web/src/app/styles/_wonksidebar.scss b/web/src/app/styles/_wonksidebar.scss index ba99bf28..283799f4 100644 --- a/web/src/app/styles/_wonksidebar.scss +++ b/web/src/app/styles/_wonksidebar.scss @@ -107,4 +107,142 @@ ul.footer-links { li { display: inline; } -} \ No newline at end of file +} + +// source sidebar section +.source-sidebar-wrapper { + position: fixed; + top: 0; + right: 0; + height: 100dvh; + min-width: 600px; + background-color: $primary-bg; + transform: translateX(100%); + transition: transform 0.3s ease-in-out; + z-index: 50; + &.open { + transform: translateX(0); + } + .source-sidebar-header { + display: grid; + grid-template-columns: 1fr auto; + grid-template-areas: "title close"; + align-items: center; + padding: 8px 16px; + border-bottom: 1px solid $borders; + background: $ucop-blue-light; + column-gap: 12px; + &::before { content: ""; grid-column: 1 / 2; } + .sidebar-title { + grid-area: title; + min-width: 0; + margin: 0; + text-align: center; + } + .sidebar-title-link { + display: -webkit-box; + -webkit-box-orient: vertical; + overflow: hidden; + white-space: normal; + line-height: 1.2; + } + + .sidebar-close { + grid-area: close; + justify-self: end; + background: none; + border: none; + cursor: pointer; + flex: 0 0 auto; + white-space: nowrap; + } + } + + .source-sidebar-main { + padding: 16px; + overflow-y: auto; + height: calc(100vh - 60px); + } +} +.source-sidebar-main { + overflow-y: auto; + overflow-x: hidden; + + .markdown { + overflow-wrap: anywhere; + word-break: break-word; + + img { + max-width: 100%; + height: auto; + display: block; + } + + table { + width: 100%; + table-layout: fixed; + border-collapse: collapse; + border: 1px solid $borders; + margin: 0 0 1rem 0; + } + + thead th { + background: $ucop-blue-light; + } + + th, + td { + border: 1px solid $borders; + padding: 8px; + vertical-align: top; + overflow-wrap: anywhere; + word-break: break-word; + } + + pre, + code { + white-space: pre-wrap; + word-break: break-word; + } + + a { + word-break: break-all; + } + } +} + +.source-sidebar-main .markdown h2#table-of-contents + table { + border: 0; + table-layout: auto; + border-collapse: separate; + margin: 0 0 0.75rem 0; +} + +.source-sidebar-main .markdown h2#table-of-contents + table thead { + display: none; +} + +.source-sidebar-main .markdown h2#table-of-contents + table tr { + display: block; + padding: 6px 0; + position: relative; + padding-left: 1.1rem; +} + +.source-sidebar-main .markdown h2#table-of-contents + table tr::before { + content: "•"; + position: absolute; + left: 0; + top: 0; +} + +.source-sidebar-main .markdown h2#table-of-contents + table th, +.source-sidebar-main .markdown h2#table-of-contents + table td { + display: block; + border: 0; + padding: 0; + line-height: 1.3; + overflow-wrap: anywhere; + word-break: break-word; +} + diff --git a/web/src/components/chat/answer/wonkAnswer.tsx b/web/src/components/chat/answer/wonkAnswer.tsx index eecb692d..f4b73d0a 100644 --- a/web/src/components/chat/answer/wonkAnswer.tsx +++ b/web/src/components/chat/answer/wonkAnswer.tsx @@ -9,9 +9,10 @@ import { GTagEvents } from '../../../models/gtag'; interface WonkAnswerProps { text: string; + onCitationClick?: (citationHref: string) => void; } -const WonkAnswer: React.FC = ({ text }) => { +const WonkAnswer: React.FC = ({ text, onCitationClick }) => { const gtagEvent = useGtagEvent(); const sanitizedText = sanitizeMarkdown(text); @@ -25,10 +26,12 @@ const WonkAnswer: React.FC = ({ text }) => { if (props.href?.startsWith('http')) { return ( { - gtagEvent({ - event: GTagEvents.CITATION_EXTERNAL, - }); + className='citation-link text-blue-600 hover:underline' + onClick={(e) => { + e.preventDefault(); + if (onCitationClick && props.href) { + onCitationClick(props.href || ''); + } }} {...props} target='_blank' @@ -70,7 +73,19 @@ const WonkAnswer: React.FC = ({ text }) => { ); } else { // regular links (like mailto:) - return ; + return ( + { + e.preventDefault(); // prevent actual navigation + if (onCitationClick && props.href) { + onCitationClick(props.href); + } + }} + > + {props.children} + + ); } }, }} diff --git a/web/src/components/chat/answer/wonkMessage.tsx b/web/src/components/chat/answer/wonkMessage.tsx index 848fd6a9..7f3b05a0 100644 --- a/web/src/components/chat/answer/wonkMessage.tsx +++ b/web/src/components/chat/answer/wonkMessage.tsx @@ -1,11 +1,12 @@ 'use client'; -import React from 'react'; +import React, { useState, useEffect } from 'react'; import { StreamableValue } from 'ai/rsc'; import WonkyClientError from '../../../lib/error/wonkyClientError'; import WonkyErrorBoundary from '../../../lib/error/wonkyErrorBoundary'; import { useStreamableText } from '../../../lib/hooks/useStreamableText'; +import SourceSidebar from '../../layout/sourceSidebar'; import ChatActions from './chatActions'; import WonkAnswer from './wonkAnswer'; @@ -14,35 +15,74 @@ export const WonkMessage = ({ content, isLoading, wonkThoughts, + citationDocs, }: { content: string | StreamableValue; isLoading: boolean; wonkThoughts: StreamableValue | string; + citationDocs?: { + title: string; + content: string; + url: string; + }[]; }) => { const text = useStreamableText(content); const wonkText = useStreamableText(wonkThoughts, { shouldAppend: false }); + const [isSidebarOpen, setIsSidebarOpen] = useState(false); + useEffect(() => {}, [isSidebarOpen]); + const [selectedDoc, setSelectedDoc] = useState<{ + title: string; + content: string; + url: string; + } | null>(null); + + const onCitationClick = (citationHref: string) => { + if (!citationDocs) { + return; + } + + const doc = citationDocs.find((d) => d.url === citationHref); + if (doc) { + setSelectedDoc(doc); + setIsSidebarOpen(true); + } else { + } + }; + return ( -
-
-
- {text ? ( - - } - > - - - ) : ( - wonkText - )} +
+
+
+
+ {text ? ( + + } + > + + + ) : ( + wonkText + )} +
+ {!isLoading && }
- {!isLoading && } + { + setIsSidebarOpen(false); + setSelectedDoc(null); + }} + selectedDoc={isSidebarOpen ? selectedDoc : null} + />
); }; diff --git a/web/src/components/layout/sourceSidebar.tsx b/web/src/components/layout/sourceSidebar.tsx new file mode 100644 index 00000000..69f3ecc6 --- /dev/null +++ b/web/src/components/layout/sourceSidebar.tsx @@ -0,0 +1,58 @@ +'use client'; +import React from 'react'; + +import { faClose } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import rehypeAutolinkHeadings from 'rehype-autolink-headings'; +import rehypeRaw from 'rehype-raw'; +import rehypeSlug from 'rehype-slug'; +import remarkGfm from 'remark-gfm'; + +import { MemoizedReactMarkdown } from '../../lib/markdown'; + +interface SourceSidebarProps { + isOpen: boolean; + onClose: () => void; + selectedDoc: { title: string; content: string; url: string } | null; +} + +const SourceSidebar: React.FC = ({ + isOpen, + onClose, + selectedDoc, +}) => { + return ( + <> +
+ +
+ {selectedDoc && ( + <> + + {selectedDoc.content} + + + )} +
+
+ + ); +}; + +export default React.memo(SourceSidebar); diff --git a/web/src/lib/actions.tsx b/web/src/lib/actions.tsx index 2b3c8248..d3266d74 100644 --- a/web/src/lib/actions.tsx +++ b/web/src/lib/actions.tsx @@ -20,6 +20,7 @@ import { llmModel, transformContentWithCitations, getSearchResultsElastic, + getDocumentContents, } from '../services/chatService'; import { removeChat, @@ -119,21 +120,27 @@ export const submitUserMessage = async (userInput: string) => { // `text` is called when an AI returns a text response (as opposed to a tool call). // Its content is streamed from the LLM, so this function will be called // multiple times with `content` being incremental. `delta` is the new text to append. - text: ({ content, done, delta }) => { + text: async ({ content, done, delta }) => { if (done) { textStream.done(); - // once we are finished, we need to modify the content to transform the citations - const finalContent = transformContentWithCitations( + const { transformedText, citations } = transformContentWithCitations( content, searchResults ); - + const citationDocs = await Promise.all( + citations.map(async ({ title }) => { + const doc = await getDocumentContents(title); + return doc ? { ...doc, title } : null; + }) + ).then((docs) => docs.filter((doc) => doc !== null)); + console.log('citationDocs', citationDocs); const finalNode = ( ); @@ -145,7 +152,7 @@ export const submitUserMessage = async (userInput: string) => { { id: nanoid(), // new id for the message role: 'assistant', - content: finalContent, + content: transformedText, }, ]; (async () => { diff --git a/web/src/lib/markdown.tsx b/web/src/lib/markdown.tsx index f64fcd1e..15bb098d 100644 --- a/web/src/lib/markdown.tsx +++ b/web/src/lib/markdown.tsx @@ -1,10 +1,34 @@ import { FC, memo } from 'react'; import ReactMarkdown, { Options } from 'react-markdown'; +import rehypeAutolinkHeadings from 'rehype-autolink-headings'; +import rehypeRaw from 'rehype-raw'; +import rehypeSlug from 'rehype-slug'; +import remarkGfm from 'remark-gfm'; export const MemoizedReactMarkdown: FC = memo( - ReactMarkdown, - (prevProps, nextProps) => - prevProps.children === nextProps.children && - prevProps.className === nextProps.className + (props: Options) => ( +
+ ( + + ), + thead: ({ node, ...p }) => , + th: ({ node, ...p }) =>
, + td: ({ node, ...p }) => , + a: ({ node, ...p }) => , + code: ({ node, inline, ...p }: { node?: any; inline?: boolean }) => + inline ? : , + pre: ({ node, ...p }) => 
,
+          img: ({ node, ...p }) => ,
+        }}
+        {...props}
+      />
+    
+  ),
+  (prev, next) =>
+    prev.children === next.children && prev.className === next.className
 );
diff --git a/web/src/services/chatService.ts b/web/src/services/chatService.ts
index b130a8e0..a95cc3f3 100644
--- a/web/src/services/chatService.ts
+++ b/web/src/services/chatService.ts
@@ -250,7 +250,8 @@ export const expandedTransformSearchResults = (
 export const transformContentWithCitations = (
   docText: string,
   policies: PolicyIndex[]
-) => {
+  // modified to return url and title of source to pass to getDocumentContents
+): { transformedText: string; citations: { url: string; title: string }[] } => {
   // our content contains citations in the form 
   // we need to replace those w/ markdown citations
   // markdown citations replace inline in the form of [^1]
@@ -261,7 +262,7 @@ export const transformContentWithCitations = (
 
   // if there are no citations, we don't need to do anything
   if (citations.length === 0) {
-    return docText;
+    return { transformedText: docText, citations: [] };
   }
 
   // 2. replace the citations in the text w/ markdown citations and keep track of the citations
@@ -286,6 +287,10 @@ export const transformContentWithCitations = (
   const usedPolicies = policies.filter((p) =>
     usedCitationDocNums.has(p.docNumber)
   );
+  const citationMetadata = usedPolicies.map((p) => ({
+    url: p.metadata.url,
+    title: p.metadata.title,
+  }));
 
   const citationFootnoteMarkdown = usedPolicies
     .map((p) => {
@@ -296,9 +301,32 @@ export const transformContentWithCitations = (
   // 4. add the citations to the end of the document
   transformedText += `\n\n## Citations\n${citationFootnoteMarkdown}\n`;
 
-  return transformedText;
+  return { transformedText, citations: citationMetadata };
 };
 
+export async function getDocumentContents(title: string): Promise<{
+  url: string;
+  content: string;
+} | null> {
+  const documentContents = await prisma.documentContents.findFirst({
+    where: {
+      AND: [{ document: { title: title } }],
+    },
+    include: {
+      document: true,
+    },
+  });
+
+  if (!documentContents || !documentContents.content) {
+    return null;
+  }
+
+  return {
+    url: documentContents.document.url ?? '',
+    content: documentContents.content,
+  };
+}
+
 export const getSystemMessage = (docText: string) => {
   if (!docText) {
     // if we don't have any documents, we can't do anything, but still use the llm to respond so the pipeline is consistent