Refine highlight pagination and enforce explicit confirmations

Author: softpudding

AGENTS.md

@@ -169,8 +169,7 @@ OpenBrowser uses Jinja2 templates for agent prompts, enabling dynamic content in
 - Model profile is resolved from session metadata and exposed to prompt rendering as `model_profile` / `small_model`; see `server/agent/manager.py` and `server/agent/tools/prompt_context.py`
 - Tool prompt variants are split by model profile under `server/agent/prompts/small_model/` and `server/agent/prompts/big_model/`
 - Small-model browser guidance intentionally avoids `keywords` fallback and leans harder on same-mode highlight pagination when dense UI may be split across collision-aware pages
-- Observation rendering also differs by model profile: large models keep clickable highlights compact (`... and N clickable elements`), while small models include clickable element HTML in the LLM-visible observation text for extra semantic grounding
-- The small-model clickable-observation branch is implemented in `server/agent/tools/base.py`; the per-conversation `small_model` flag is attached in `server/agent/tools/browser_executor.py`
+- Observation rendering now includes clickable element HTML for all model profiles; prompt differences still vary by model profile
 
 ### Keyword Discipline
 - Highlight pagination remains the default discovery flow for controls and dense UI

extension/src/background/index.ts

@@ -45,8 +45,7 @@ import {
 } from '../commands/label-constants';
 import { getOrCreateUUID } from '../uuid/uuidGenerator';
 import {
-  selectCollisionFreePage,
-  calculateTotalPages,
+  paginateCollisionFreeElements,
   sortElementsByVisualOrder,
 } from '../utils/collision-detection';
 import {
@@ -161,35 +160,25 @@ async function runRawScreenshotPrime(options: {
 
 function buildStoredHighlightPages(options: {
   filteredElements: InteractiveElement[];
-  totalPages: number;
   viewportWidth: number;
   viewportHeight: number;
   keywordMode: boolean;
 }): InteractiveElement[][] {
-  const {
-    filteredElements,
-    totalPages,
-    viewportWidth,
-    viewportHeight,
-    keywordMode,
-  } = options;
+  const { filteredElements, viewportWidth, viewportHeight, keywordMode } =
+    options;
 
   if (keywordMode) {
     return [sortElementsByVisualOrder(filteredElements)];
   }
 
-  const pages: InteractiveElement[][] = [];
-  for (let page = 1; page <= totalPages; page++) {
-    const pageElements = selectCollisionFreePage(
-      filteredElements,
-      page,
-      viewportWidth,
-      viewportHeight,
-    );
-    pages.push(sortElementsByVisualOrder(pageElements));
-  }
-
-  return pages;
+  const collisionFreePages = paginateCollisionFreeElements(
+    filteredElements,
+    viewportWidth,
+    viewportHeight,
+  );
+  return collisionFreePages.map((collisionFreePage) =>
+    sortElementsByVisualOrder(collisionFreePage),
+  );
 }
 
 function buildHighlightConsistencyScript(
@@ -389,38 +378,39 @@ async function captureHighlightedPageState(
       `⏱️ [HighlightTrace] background keyword-filter ${Date.now() - keywordFilterStart}ms (keywords=${keywordList.length}, kept=${filteredElements.length}/${allElements.length})`,
     );
 
+    let storedPages: InteractiveElement[][];
     let paginatedElements: InteractiveElement[];
     let totalPages: number;
     let currentPage = page;
 
     if (keywordList.length > 0) {
-      paginatedElements = filteredElements;
-      totalPages = 1;
+      storedPages = buildStoredHighlightPages({
+        filteredElements,
+        viewportWidth: detectedViewportWidth,
+        viewportHeight: detectedViewportHeight,
+        keywordMode: true,
+      });
+      paginatedElements = storedPages[0] ?? [];
+      totalPages = storedPages.length || 1;
       currentPage = 1;
       console.log(
         `🔍 [${logLabel}] Keywords [${keywordList.join(', ')}] matched ${paginatedElements.length} elements (no pagination)`,
       );
     } else {
-      const paginationSelectionStart = Date.now();
-      paginatedElements = selectCollisionFreePage(
+      const paginationBuildStart = Date.now();
+      storedPages = buildStoredHighlightPages({
         filteredElements,
-        page,
-        detectedViewportWidth,
-        detectedViewportHeight,
-      );
-      const paginationSelectionMs = Date.now() - paginationSelectionStart;
-      const totalPagesStart = Date.now();
-      totalPages = calculateTotalPages(
-        filteredElements,
-        detectedViewportWidth,
-        detectedViewportHeight,
-      );
-      const totalPagesMs = Date.now() - totalPagesStart;
+        viewportWidth: detectedViewportWidth,
+        viewportHeight: detectedViewportHeight,
+        keywordMode: false,
+      });
+      paginatedElements = storedPages[Math.max(0, page - 1)] ?? [];
+      totalPages = storedPages.length;
       console.log(
         `📄 [${logLabel}] Page ${page}/${totalPages}, showing ${paginatedElements.length} of ${filteredElements.length} elements`,
       );
       console.log(
-        `⏱️ [HighlightTrace] background pagination select=${paginationSelectionMs}ms totalPages=${totalPagesMs}ms (page=${page}, viewport=${detectedViewportWidth}x${detectedViewportHeight})`,
+        `⏱️ [HighlightTrace] background pagination build-pages=${Date.now() - paginationBuildStart}ms (page=${page}, viewport=${detectedViewportWidth}x${detectedViewportHeight})`,
       );
     }
 
@@ -515,13 +505,6 @@ async function captureHighlightedPageState(
       );
     }
 
-    const storedPages = buildStoredHighlightPages({
-      filteredElements,
-      totalPages,
-      viewportWidth: detectedViewportWidth,
-      viewportHeight: detectedViewportHeight,
-      keywordMode: keywordList.length > 0,
-    });
     const displayOrderedElements = storedPages[currentPage - 1] ?? [];
 
     const cacheStoreStart = Date.now();