diff --git a/apps/roam/src/components/VectorDuplicateMatches.tsx b/apps/roam/src/components/VectorDuplicateMatches.tsx new file mode 100644 index 000000000..8cd398b37 --- /dev/null +++ b/apps/roam/src/components/VectorDuplicateMatches.tsx @@ -0,0 +1,177 @@ +import React, { useEffect, useState, useMemo } from "react"; +import { Collapse, Spinner, Icon } from "@blueprintjs/core"; +import { findSimilarNodesVectorOnly, type VectorMatch } from "~/utils/hyde"; +import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; +import { DiscourseNode } from "~/utils/getDiscourseNodes"; +import extractContentFromTitle from "~/utils/extractContentFromTitle"; +import { handleTitleAdditions } from "~/utils/handleTitleAdditions"; + +type VectorSearchParams = { + text: string; + threshold?: number; + limit?: number; +}; + +const vectorSearch = (params: VectorSearchParams) => + findSimilarNodesVectorOnly(params); + +export const VectorDuplicateMatches = ({ + pageTitle, + text, + limit = 15, + node, +}: { + pageTitle?: string; + text?: string; + limit?: number; + node: DiscourseNode; +}) => { + const [debouncedText, setDebouncedText] = useState(text); + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedText(text); + }, 500); + return () => { + clearTimeout(handler); + }; + }, [text]); + + const [isOpen, setIsOpen] = useState(false); + const [suggestionsLoading, setSuggestionsLoading] = useState(false); + const [hasSearched, setHasSearched] = useState(false); + const [suggestions, setSuggestions] = useState([]); + + const searchText = extractContentFromTitle(pageTitle || "", node); + const pageUid = getPageUidByPageTitle(searchText); + const activeContext = useMemo( + () => + text !== undefined + ? { searchText: debouncedText || "", pageUid: null } + : { searchText, pageUid }, + [text, debouncedText, searchText, pageUid], + ); + + useEffect(() => { + setHasSearched(false); + }, [activeContext?.searchText]); + + useEffect(() => { + let isCancelled = false; + const fetchSuggestions = async () => { + if (!isOpen || hasSearched) return; + if (!activeContext || !activeContext.searchText.trim()) return; + + const { searchText, pageUid } = activeContext; + + setSuggestionsLoading(true); + try { + const raw = await vectorSearch({ + text: searchText, + threshold: 0.3, + limit, + }); + const results: VectorMatch[] = raw.filter((candidate) => { + const sameUid = !!pageUid && candidate.node.uid === pageUid; + return !sameUid; + }); + if (!isCancelled) { + setSuggestions(results); + setSuggestionsLoading(false); + setHasSearched(true); + } + } catch (error: unknown) { + console.error("Error fetching vector duplicates:", error); + if (!isCancelled) { + setSuggestionsLoading(false); + } + } + }; + void fetchSuggestions(); + return () => { + isCancelled = true; + }; + }, [isOpen, hasSearched, activeContext, pageTitle, limit]); + + const handleSuggestionClick = async (node: VectorMatch["node"]) => { + await window.roamAlphaAPI.ui.rightSidebar.addWindow({ + window: { + type: "outline", + // @ts-expect-error - type definition mismatch + // eslint-disable-next-line @typescript-eslint/naming-convention + "block-uid": node.uid, + }, + }); + }; + + if (!activeContext) { + return null; + } + + const hasSuggestions = suggestions.length > 0; + + return ( +
+
{ + setIsOpen(!isOpen); + }} + > +
+ +
Possible Duplicates
+
+ {hasSearched && !suggestionsLoading && hasSuggestions && ( + + {suggestions.length} + + )} +
+ + +
+ {suggestionsLoading && ( +
+ + + Searching for duplicates... + +
+ )} + + {!suggestionsLoading && hasSearched && !hasSuggestions && ( +

No matches found.

+ )} + + {!suggestionsLoading && hasSearched && hasSuggestions && ( + + )} +
+ +
+ ); +}; + +export const renderPossibleDuplicates = ( + h1: HTMLHeadingElement, + title: string, + node: DiscourseNode, +) => { + handleTitleAdditions( + h1, + , + ); +}; diff --git a/apps/roam/src/utils/extractContentFromTitle.ts b/apps/roam/src/utils/extractContentFromTitle.ts new file mode 100644 index 000000000..d88002776 --- /dev/null +++ b/apps/roam/src/utils/extractContentFromTitle.ts @@ -0,0 +1,28 @@ +import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression"; + +const extractContentFromTitle = ( + title: string, + node: { format: string }, +): string => { + if (!node.format) return title; + const placeholderRegex = /{([\w\d-]+)}/g; + const placeholders: string[] = []; + let placeholderMatch: RegExpExecArray | null = null; + while ((placeholderMatch = placeholderRegex.exec(node.format))) { + placeholders.push(placeholderMatch[1]); + } + const expression = getDiscourseNodeFormatExpression(node.format); + const expressionMatch = expression.exec(title); + if (!expressionMatch || expressionMatch.length <= 1) { + return title; + } + const contentIndex = placeholders.findIndex( + (name) => name.toLowerCase() === "content", + ); + if (contentIndex >= 0) { + return expressionMatch[contentIndex + 1]?.trim() || title; + } + return expressionMatch[1]?.trim() || title; +}; + +export default extractContentFromTitle; diff --git a/apps/roam/src/utils/hyde.ts b/apps/roam/src/utils/hyde.ts index 361d38725..730c07974 100644 --- a/apps/roam/src/utils/hyde.ts +++ b/apps/roam/src/utils/hyde.ts @@ -1,6 +1,7 @@ import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; import { Result } from "./types"; import normalizePageTitle from "roamjs-components/queries/normalizePageTitle"; +import { render as renderToast } from "roamjs-components/components/Toast"; import findDiscourseNode from "./findDiscourseNode"; import { nextApiRoot } from "@repo/utils/execContext"; import { DiscourseNode } from "./getDiscourseNodes"; @@ -58,7 +59,7 @@ type EmbeddingFunc = (text: string) => Promise; type SearchFunc = (params: { queryEmbedding: EmbeddingVectorType; - indexData: CandidateNodeWithEmbedding[]; + indexData: Result[]; }) => Promise; const API_CONFIG = { @@ -530,3 +531,62 @@ export const performHydeSearch = async ({ } return []; }; + +export type VectorMatch = { + node: Result; + score: number; +}; + +export const findSimilarNodesVectorOnly = async ({ + text, + threshold = 0.4, + limit = 15, +}: { + text: string; + threshold?: number; + limit?: number; +}): Promise => { + if (!text.trim()) { + return []; + } + + try { + const supabase = await getLoggedInClient(); + if (!supabase) return []; + + const queryEmbedding = await createEmbedding(text); + + const { data, error } = await supabase.rpc("match_content_embeddings", { + query_embedding: JSON.stringify(queryEmbedding), + match_threshold: threshold, + match_count: limit, + }); + + if (error) { + console.error("Vector search failed:", error); + throw error; + } + + if (!data || !Array.isArray(data)) return []; + + const results: VectorMatch[] = data.map((item) => ({ + node: { + uid: item.roam_uid, + text: item.text_content, + }, + score: item.similarity, + })); + + return results; + } catch (error) { + console.error("Error in vector-only similar nodes search:", error); + renderToast({ + content: `Error in vector-only similar nodes search: ${ + error instanceof Error ? error.message : String(error) + }`, + intent: "danger", + id: "vector-search-error", + }); + return []; + } +}; diff --git a/apps/roam/src/utils/initializeObserversAndListeners.ts b/apps/roam/src/utils/initializeObserversAndListeners.ts index e8ac5378a..4fc7fc9f1 100644 --- a/apps/roam/src/utils/initializeObserversAndListeners.ts +++ b/apps/roam/src/utils/initializeObserversAndListeners.ts @@ -4,7 +4,7 @@ import { getPageTitleValueByHtmlElement, } from "roamjs-components/dom"; import { createBlock } from "roamjs-components/writes"; -import { renderLinkedReferenceAdditions } from "~/utils/renderLinkedReferenceAdditions"; +import { renderDiscourseContextAndCanvasReferences } from "~/utils/renderLinkedReferenceAdditions"; import { createConfigObserver } from "roamjs-components/components/ConfigPage"; import { renderTldrawCanvas, @@ -54,6 +54,9 @@ import { getUidAndBooleanSetting } from "./getExportSettings"; import { getCleanTagText } from "~/components/settings/NodeConfig"; import getPleasingColors from "@repo/utils/getPleasingColors"; import { colord } from "colord"; +import { renderPossibleDuplicates } from "~/components/VectorDuplicateMatches"; +import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; +import findDiscourseNode from "./findDiscourseNode"; const debounce = (fn: () => void, delay = 250) => { let timeout: number; @@ -85,6 +88,33 @@ export const initObservers = async ({ const title = getPageTitleValueByHtmlElement(h1); const props = { title, h1, onloadArgs }; + const isSuggestiveModeEnabled = getUidAndBooleanSetting({ + tree: getBasicTreeByParentUid( + getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE), + ), + text: "(BETA) Suggestive Mode Enabled", + }).value; + + const uid = getPageUidByPageTitle(title); + const nodes = getDiscourseNodes(); + const node = findDiscourseNode(uid, nodes); + const isDiscourseNode = node && node.backedBy !== "default"; + if (isDiscourseNode) { + if (isSuggestiveModeEnabled) { + renderPossibleDuplicates(h1, title, node); + } + const linkedReferencesDiv = document.querySelector( + ".rm-reference-main", + ) as HTMLDivElement; + if (linkedReferencesDiv) { + renderDiscourseContextAndCanvasReferences( + linkedReferencesDiv, + uid, + onloadArgs, + ); + } + } + if (isNodeConfigPage(title)) renderNodeConfigPage(props); else if (isQueryPage(props)) renderQueryPage(props); else if (isCurrentPageCanvas(props)) renderTldrawCanvas(props); @@ -92,17 +122,6 @@ export const initObservers = async ({ }, }); - // TODO: contains roam query: https://github.com/DiscourseGraphs/discourse-graph/issues/39 - const linkedReferencesObserver = createHTMLObserver({ - tag: "DIV", - useBody: true, - className: "rm-reference-main", - callback: async (el) => { - const div = el as HTMLDivElement; - await renderLinkedReferenceAdditions(div, onloadArgs); - }, - }); - const queryBlockObserver = createButtonObserver({ attribute: "query-block", render: (b) => renderQueryBlock(b, onloadArgs), @@ -391,7 +410,6 @@ export const initObservers = async ({ pageTitleObserver, queryBlockObserver, configPageObserver, - linkedReferencesObserver, graphOverviewExportObserver, nodeTagPopupButtonObserver, leftSidebarObserver, diff --git a/apps/roam/src/utils/renderLinkedReferenceAdditions.ts b/apps/roam/src/utils/renderLinkedReferenceAdditions.ts index d7caa1629..2cac6f08f 100644 --- a/apps/roam/src/utils/renderLinkedReferenceAdditions.ts +++ b/apps/roam/src/utils/renderLinkedReferenceAdditions.ts @@ -1,50 +1,40 @@ import { createElement } from "react"; -import { getPageTitleValueByHtmlElement } from "roamjs-components/dom"; -import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; import renderWithUnmount from "roamjs-components/util/renderWithUnmount"; import { DiscourseContext } from "~/components"; import CanvasReferences from "~/components/canvas/CanvasReferences"; -import isDiscourseNode from "./isDiscourseNode"; import { OnloadArgs } from "roamjs-components/types"; -export const renderLinkedReferenceAdditions = async ( +export const renderDiscourseContextAndCanvasReferences = ( div: HTMLDivElement, + uid: string, onloadArgs: OnloadArgs, -) => { - const isMainWindow = !!div.closest(".roam-article"); - const uid = isMainWindow - ? await window.roamAlphaAPI.ui.mainWindow.getOpenPageOrBlockUid() - : getPageUidByPageTitle(getPageTitleValueByHtmlElement(div)); - if ( - uid && - isDiscourseNode(uid) && - !div.getAttribute("data-roamjs-discourse-context") - ) { - div.setAttribute("data-roamjs-discourse-context", "true"); - const parent = div.firstElementChild; - if (parent) { - const insertBefore = parent.firstElementChild; +): void => { + if (div.getAttribute("data-roamjs-discourse-context")) return; - const p = document.createElement("div"); - parent.insertBefore(p, insertBefore); - renderWithUnmount( - createElement(DiscourseContext, { - uid, - results: [], - }), - p, - onloadArgs, - ); + div.setAttribute("data-roamjs-discourse-context", "true"); + const parent = div.firstElementChild; + if (!parent) return; - const canvasP = document.createElement("div"); - parent.insertBefore(canvasP, insertBefore); - renderWithUnmount( - createElement(CanvasReferences, { - uid, - }), - canvasP, - onloadArgs, - ); - } - } + const insertBefore = parent.firstElementChild; + + const p = document.createElement("div"); + parent.insertBefore(p, insertBefore); + renderWithUnmount( + createElement(DiscourseContext, { + uid, + results: [], + }), + p, + onloadArgs, + ); + + const canvasP = document.createElement("div"); + parent.insertBefore(canvasP, insertBefore); + renderWithUnmount( + createElement(CanvasReferences, { + uid, + }), + canvasP, + onloadArgs, + ); };