ENG-1074 Prod duplicate node alert on page

apps/roam/src/components/VectorDuplicateMatches.tsx

@@ -0,0 +1,177 @@
+import React, { useEffect, useState, useMemo } from "react";
+import { Collapse, Spinner, Icon } from "@blueprintjs/core";
+import { findSimilarNodesVectorOnly, type VectorMatch } from "~/utils/hyde";
+import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
+import { DiscourseNode } from "~/utils/getDiscourseNodes";
+import extractContentFromTitle from "~/utils/extractContentFromTitle";
+import { handleTitleAdditions } from "~/utils/handleTitleAdditions";
+
+type VectorSearchParams = {
+  text: string;
+  threshold?: number;
+  limit?: number;
+};
+
+const vectorSearch = (params: VectorSearchParams) =>
+  findSimilarNodesVectorOnly(params);
+
+export const VectorDuplicateMatches = ({
+  pageTitle,
+  text,
+  limit = 15,
+  node,
+}: {
+  pageTitle?: string;
+  text?: string;
+  limit?: number;
+  node: DiscourseNode;
+}) => {
+  const [debouncedText, setDebouncedText] = useState(text);
+  useEffect(() => {
+    const handler = setTimeout(() => {
+      setDebouncedText(text);
+    }, 500);
+    return () => {
+      clearTimeout(handler);
+    };
+  }, [text]);
+
+  const [isOpen, setIsOpen] = useState(false);
+  const [suggestionsLoading, setSuggestionsLoading] = useState(false);
+  const [hasSearched, setHasSearched] = useState(false);
+  const [suggestions, setSuggestions] = useState<VectorMatch[]>([]);
+
+  const searchText = extractContentFromTitle(pageTitle || "", node);
+  const pageUid = getPageUidByPageTitle(searchText);
+  const activeContext = useMemo(
+    () =>
+      text !== undefined
+        ? { searchText: debouncedText || "", pageUid: null }
+        : { searchText, pageUid },
+    [text, debouncedText, searchText, pageUid],
+  );
+
+  useEffect(() => {
+    setHasSearched(false);
+  }, [activeContext?.searchText]);
+
+  useEffect(() => {
+    let isCancelled = false;
+    const fetchSuggestions = async () => {
+      if (!isOpen || hasSearched) return;
+      if (!activeContext || !activeContext.searchText.trim()) return;
+
+      const { searchText, pageUid } = activeContext;
+
+      setSuggestionsLoading(true);
+      try {
+        const raw = await vectorSearch({
+          text: searchText,
+          threshold: 0.3,
+          limit,
+        });
+        const results: VectorMatch[] = raw.filter((candidate) => {
+          const sameUid = !!pageUid && candidate.node.uid === pageUid;
+          return !sameUid;
+        });
+        if (!isCancelled) {
+          setSuggestions(results);
+          setSuggestionsLoading(false);
+          setHasSearched(true);
+        }
+      } catch (error: unknown) {
+        console.error("Error fetching vector duplicates:", error);
+        if (!isCancelled) {
+          setSuggestionsLoading(false);
+        }
+      }
+    };
+    void fetchSuggestions();
+    return () => {
+      isCancelled = true;
+    };
+  }, [isOpen, hasSearched, activeContext, pageTitle, limit]);
+
+  const handleSuggestionClick = async (node: VectorMatch["node"]) => {
+    await window.roamAlphaAPI.ui.rightSidebar.addWindow({
+      window: {
+        type: "outline",
+        // @ts-expect-error - type definition mismatch
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        "block-uid": node.uid,
+      },
+    });
+  };
+
+  if (!activeContext) {
+    return null;
+  }
+
+  const hasSuggestions = suggestions.length > 0;
+
+  return (
+    <div className="my-2 rounded border border-gray-200">
+      <div
+        className="flex cursor-pointer items-center justify-between p-2"
+        onClick={() => {
+          setIsOpen(!isOpen);
+        }}
+      >
+        <div className="flex items-center gap-2">
+          <Icon icon={isOpen ? "chevron-down" : "chevron-right"} />
+          <h5 className="m-0 font-semibold">Possible Duplicates</h5>
+        </div>
+        {hasSearched && !suggestionsLoading && hasSuggestions && (
+          <span className="rounded-full bg-orange-500 px-2 py-0.5 text-xs text-white">
+            {suggestions.length}
+          </span>
+        )}
+      </div>
+
+      <Collapse isOpen={isOpen}>
+        <div className="border-t border-gray-200 p-2">
+          {suggestionsLoading && (
+            <div className="ml-2 flex items-center gap-2 py-4">
+              <Spinner size={20} />
+              <span className="text-sm text-gray-600">
+                Searching for duplicates...
+              </span>
+            </div>
+          )}
+
+          {!suggestionsLoading && hasSearched && !hasSuggestions && (
+            <p className="py-2 text-sm text-gray-600">No matches found.</p>
+          )}
+
+          {!suggestionsLoading && hasSearched && hasSuggestions && (
+            <ul className="flex flex-col gap-1">
+              {suggestions.map((match) => (
+                <li key={match.node.uid} className="flex items-start gap-2">
+                  <a
+                    onClick={() => {
+                      void handleSuggestionClick(match.node);
+                    }}
+                    className="min-w-0 flex-1 cursor-pointer break-words text-blue-600 opacity-70 hover:underline"
+                  >
+                    {match.node.text}
+                  </a>
+                </li>
+              ))}
+            </ul>
+          )}
+        </div>
+      </Collapse>
+    </div>
+  );
+};
+
+export const renderPossibleDuplicates = (
+  h1: HTMLHeadingElement,
+  title: string,
+  node: DiscourseNode,
+) => {
+  handleTitleAdditions(
+    h1,
+    <VectorDuplicateMatches pageTitle={title} node={node} />,
+  );
+};

apps/roam/src/utils/extractContentFromTitle.ts

@@ -0,0 +1,28 @@
+import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression";
+
+const extractContentFromTitle = (
+  title: string,
+  node: { format: string },
+): string => {
+  if (!node.format) return title;
+  const placeholderRegex = /{([\w\d-]+)}/g;
+  const placeholders: string[] = [];
+  let placeholderMatch: RegExpExecArray | null = null;
+  while ((placeholderMatch = placeholderRegex.exec(node.format))) {
+    placeholders.push(placeholderMatch[1]);
+  }
+  const expression = getDiscourseNodeFormatExpression(node.format);
+  const expressionMatch = expression.exec(title);
+  if (!expressionMatch || expressionMatch.length <= 1) {
+    return title;
+  }
+  const contentIndex = placeholders.findIndex(
+    (name) => name.toLowerCase() === "content",
+  );
+  if (contentIndex >= 0) {
+    return expressionMatch[contentIndex + 1]?.trim() || title;
+  }
+  return expressionMatch[1]?.trim() || title;
+};
+
+export default extractContentFromTitle;

apps/roam/src/utils/hyde.ts

@@ -1,6 +1,7 @@
 import { getLoggedInClient, getSupabaseContext } from "./supabaseContext";
 import { Result } from "./types";
 import normalizePageTitle from "roamjs-components/queries/normalizePageTitle";
+import { render as renderToast } from "roamjs-components/components/Toast";
 import findDiscourseNode from "./findDiscourseNode";
 import { nextApiRoot } from "@repo/utils/execContext";
 import { DiscourseNode } from "./getDiscourseNodes";
@@ -58,7 +59,7 @@ type EmbeddingFunc = (text: string) => Promise<EmbeddingVectorType>;
 
 type SearchFunc = (params: {
   queryEmbedding: EmbeddingVectorType;
-  indexData: CandidateNodeWithEmbedding[];
+  indexData: Result[];
 }) => Promise<NodeSearchResult[]>;
 
 const API_CONFIG = {
@@ -530,3 +531,62 @@ export const performHydeSearch = async ({
   }
   return [];
 };
+
+export type VectorMatch = {
+  node: Result;
+  score: number;
+};
+
+export const findSimilarNodesVectorOnly = async ({
+  text,
+  threshold = 0.4,
+  limit = 15,
+}: {
+  text: string;
+  threshold?: number;
+  limit?: number;
+}): Promise<VectorMatch[]> => {
+  if (!text.trim()) {
+    return [];
+  }
+
+  try {
+    const supabase = await getLoggedInClient();
+    if (!supabase) return [];
+
+    const queryEmbedding = await createEmbedding(text);
+
+    const { data, error } = await supabase.rpc("match_content_embeddings", {
+      query_embedding: JSON.stringify(queryEmbedding),
+      match_threshold: threshold,
+      match_count: limit,
+    });
+
+    if (error) {
+      console.error("Vector search failed:", error);
+      throw error;
+    }
+
+    if (!data || !Array.isArray(data)) return [];
+
+    const results: VectorMatch[] = data.map((item) => ({
+      node: {
+        uid: item.roam_uid,
+        text: item.text_content,
+      },
+      score: item.similarity,
+    }));
+
+    return results;
+  } catch (error) {
+    console.error("Error in vector-only similar nodes search:", error);
+    renderToast({
+      content: `Error in vector-only similar nodes search: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+      intent: "danger",
+      id: "vector-search-error",
+    });
+    return [];
+  }
+};

apps/roam/src/utils/initializeObserversAndListeners.ts

@@ -4,7 +4,7 @@ import {
   getPageTitleValueByHtmlElement,
 } from "roamjs-components/dom";
 import { createBlock } from "roamjs-components/writes";
-import { renderLinkedReferenceAdditions } from "~/utils/renderLinkedReferenceAdditions";
+import { renderDiscourseContextAndCanvasReferences } from "~/utils/renderLinkedReferenceAdditions";
 import { createConfigObserver } from "roamjs-components/components/ConfigPage";
 import {
   renderTldrawCanvas,
@@ -54,6 +54,9 @@ import { getUidAndBooleanSetting } from "./getExportSettings";
 import { getCleanTagText } from "~/components/settings/NodeConfig";
 import getPleasingColors from "@repo/utils/getPleasingColors";
 import { colord } from "colord";
+import { renderPossibleDuplicates } from "~/components/VectorDuplicateMatches";
+import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
+import findDiscourseNode from "./findDiscourseNode";
 
 const debounce = (fn: () => void, delay = 250) => {
   let timeout: number;
@@ -85,24 +88,40 @@ export const initObservers = async ({
       const title = getPageTitleValueByHtmlElement(h1);
       const props = { title, h1, onloadArgs };
 
+      const isSuggestiveModeEnabled = getUidAndBooleanSetting({
+        tree: getBasicTreeByParentUid(
+          getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE),
+        ),
+        text: "(BETA) Suggestive Mode Enabled",
+      }).value;
+
+      const uid = getPageUidByPageTitle(title);
+      const nodes = getDiscourseNodes();
+      const node = findDiscourseNode(uid, nodes);
+      const isDiscourseNode = node && node.backedBy !== "default";
+      if (isDiscourseNode) {
+        if (isSuggestiveModeEnabled) {
+          renderPossibleDuplicates(h1, title, node);
+        }
+        const linkedReferencesDiv = document.querySelector(
+          ".rm-reference-main",
+        ) as HTMLDivElement;
+        if (linkedReferencesDiv) {
+          renderDiscourseContextAndCanvasReferences(
+            linkedReferencesDiv,
+            uid,
+            onloadArgs,
+          );
+        }
+      }
+
       if (isNodeConfigPage(title)) renderNodeConfigPage(props);
       else if (isQueryPage(props)) renderQueryPage(props);
       else if (isCurrentPageCanvas(props)) renderTldrawCanvas(props);
       else if (isSidebarCanvas(props)) renderTldrawCanvasInSidebar(props);
     },
   });
 
-  // TODO: contains roam query: https://github.com/DiscourseGraphs/discourse-graph/issues/39
-  const linkedReferencesObserver = createHTMLObserver({
-    tag: "DIV",
-    useBody: true,
-    className: "rm-reference-main",
-    callback: async (el) => {
-      const div = el as HTMLDivElement;
-      await renderLinkedReferenceAdditions(div, onloadArgs);
-    },
-  });
-
   const queryBlockObserver = createButtonObserver({
     attribute: "query-block",
     render: (b) => renderQueryBlock(b, onloadArgs),
@@ -391,7 +410,6 @@ export const initObservers = async ({
       pageTitleObserver,
       queryBlockObserver,
       configPageObserver,
-      linkedReferencesObserver,
       graphOverviewExportObserver,
       nodeTagPopupButtonObserver,
       leftSidebarObserver,