Merge pull request #59 from softpudding/feat/recording-dnd-slider

Bundle keyframes into event_detail; recover stalled compiler agent

Author: softpudding

.gitignore

@@ -76,3 +76,7 @@ reference/
 eval/.server_port
 eval/.locks/
 eval/output
+
+# Claude Code local state
+.claude/
+CLAUDE.local.md

extension/src/__tests__/recording-keyframe-policy.test.ts

@@ -46,8 +46,8 @@ describe('recording keyframe policy', () => {
     const options = getRecordingKeyframeCaptureOptions();
 
     expect(options.preferredFormat).toBe('jpeg');
-    expect(options.maxOutputWidth).toBe(960);
-    expect(options.maxOutputHeight).toBe(540);
+    expect(options.maxOutputWidth).toBe(1920);
+    expect(options.maxOutputHeight).toBe(1080);
     expect(options.warmupBeforeCapture).toBe(true);
     expect(options.settleBeforeCapture).toBe(true);
   });
@@ -57,8 +57,8 @@ describe('recording keyframe policy', () => {
 
     expect(getRecordingPreActionWaitForRender()).toBe(0);
     expect(options.preferredFormat).toBe('jpeg');
-    expect(options.maxOutputWidth).toBe(960);
-    expect(options.maxOutputHeight).toBe(540);
+    expect(options.maxOutputWidth).toBe(1920);
+    expect(options.maxOutputHeight).toBe(1080);
     expect(options.warmupBeforeCapture).toBe(false);
     expect(options.settleBeforeCapture).toBe(false);
   });

extension/src/content/index.ts

@@ -19,6 +19,16 @@ let scrollTimeoutId: number | null = null;
 const CONTAINER_TEXT_MAX_LENGTH = 280;
 const ELEMENT_HTML_MAX_LENGTH = 1500;
 
+// Drag-and-drop recording state
+const DRAG_DISTANCE_THRESHOLD = 30;
+let pendingDrag: {
+  sourceElement: Element;
+  startX: number;
+  startY: number;
+} | null = null;
+let dragConfirmedByHtml5 = false;
+let suppressNextClickRecording = false;
+
 function isOpenBrowserUiPage(): boolean {
   if (
     window.location.protocol !== 'http:' &&
@@ -537,6 +547,14 @@ function findOwningForm(element: Element): HTMLFormElement | null {
   return element instanceof HTMLFormElement ? element : element.closest('form');
 }
 
+function isSliderLikeElement(el: Element): boolean {
+  if (el instanceof HTMLInputElement && el.type === 'range') {
+    return true;
+  }
+  const role = el.getAttribute('role');
+  return role === 'slider' && el.hasAttribute('aria-valuemin');
+}
+
 function installRecordingListeners(): void {
   document.addEventListener(
     'pointerdown',
@@ -553,6 +571,13 @@ function installRecordingListeners(): void {
         return;
       }
 
+      pendingDrag = {
+        sourceElement: event.target,
+        startX: pointerEvent.clientX,
+        startY: pointerEvent.clientY,
+      };
+      dragConfirmedByHtml5 = false;
+
       const form = findOwningForm(event.target);
       sendRecordingPreAction('click', {
         element: serializeElement(event.target),
@@ -564,9 +589,97 @@ function installRecordingListeners(): void {
     true,
   );
 
+  // HTML5 drag-and-drop detection
+  document.addEventListener(
+    'dragstart',
+    (event) => {
+      if (!shouldRecordTrustedEvent(event) || !pendingDrag) {
+        return;
+      }
+      dragConfirmedByHtml5 = true;
+    },
+    true,
+  );
+
+  document.addEventListener(
+    'drop',
+    (event) => {
+      if (
+        !shouldRecordTrustedEvent(event) ||
+        !isElement(event.target) ||
+        !pendingDrag ||
+        !dragConfirmedByHtml5
+      ) {
+        return;
+      }
+
+      sendRecordingEvent('drag_and_drop', {
+        sourceElement: serializeElement(pendingDrag.sourceElement),
+        targetElement: serializeElement(event.target),
+        startX: pendingDrag.startX,
+        startY: pendingDrag.startY,
+        endX: (event as DragEvent).clientX,
+        endY: (event as DragEvent).clientY,
+      });
+      suppressNextClickRecording = true;
+      pendingDrag = null;
+      dragConfirmedByHtml5 = false;
+    },
+    true,
+  );
+
+  document.addEventListener(
+    'dragend',
+    () => {
+      // Cleanup for cancelled HTML5 drags where drop never fired
+      pendingDrag = null;
+      dragConfirmedByHtml5 = false;
+    },
+    true,
+  );
+
+  // Pointer-based drag detection (for non-HTML5 DnD libraries)
+  document.addEventListener(
+    'pointerup',
+    (event) => {
+      if (!pendingDrag || dragConfirmedByHtml5) {
+        pendingDrag = null;
+        return;
+      }
+
+      if (!shouldRecordTrustedEvent(event) || !isElement(event.target)) {
+        pendingDrag = null;
+        return;
+      }
+
+      const dx = (event as PointerEvent).clientX - pendingDrag.startX;
+      const dy = (event as PointerEvent).clientY - pendingDrag.startY;
+      const distance = Math.sqrt(dx * dx + dy * dy);
+
+      if (distance > DRAG_DISTANCE_THRESHOLD) {
+        sendRecordingEvent('drag_and_drop', {
+          sourceElement: serializeElement(pendingDrag.sourceElement),
+          targetElement: serializeElement(event.target),
+          startX: pendingDrag.startX,
+          startY: pendingDrag.startY,
+          endX: (event as PointerEvent).clientX,
+          endY: (event as PointerEvent).clientY,
+        });
+        suppressNextClickRecording = true;
+      }
+
+      pendingDrag = null;
+    },
+    true,
+  );
+
   document.addEventListener(
     'click',
     (event) => {
+      if (suppressNextClickRecording) {
+        suppressNextClickRecording = false;
+        return;
+      }
       if (!shouldRecordTrustedEvent(event) || !isElement(event.target)) {
         return;
       }
@@ -598,6 +711,15 @@ function installRecordingListeners(): void {
         return;
       }
 
+      // Suppress per-frame input events from range sliders — the final
+      // value is captured by the change handler as a set_slider event.
+      if (
+        event.target instanceof HTMLInputElement &&
+        event.target.type === 'range'
+      ) {
+        return;
+      }
+
       sendRecordingEvent('input', {
         element: serializeElement(event.target),
       });
@@ -612,6 +734,26 @@ function installRecordingListeners(): void {
         return;
       }
 
+      if (isSliderLikeElement(event.target)) {
+        const el = event.target;
+        sendRecordingEvent('set_slider', {
+          element: serializeElement(el),
+          value:
+            el instanceof HTMLInputElement
+              ? el.value
+              : el.getAttribute('aria-valuenow'),
+          min:
+            el instanceof HTMLInputElement
+              ? el.min
+              : el.getAttribute('aria-valuemin'),
+          max:
+            el instanceof HTMLInputElement
+              ? el.max
+              : el.getAttribute('aria-valuemax'),
+        });
+        return;
+      }
+
       sendRecordingEvent('change', {
         element: serializeElement(event.target),
       });

extension/src/recording/keyframe-annotation.ts

@@ -1,6 +1,12 @@
 import { calculateConfirmationBannerLayout } from '../commands/single-highlight';
 
-type AnnotatableRecordingEventType = 'click' | 'focus' | 'change' | 'submit';
+type AnnotatableRecordingEventType =
+  | 'click'
+  | 'focus'
+  | 'change'
+  | 'submit'
+  | 'drag_and_drop'
+  | 'set_slider';
 
 interface RecordingEventBBox {
   x: number;
@@ -11,7 +17,7 @@ interface RecordingEventBBox {
 
 interface RecordingKeyframeAnnotationTarget {
   bbox: RecordingEventBBox;
-  intendedAction: 'click' | 'keyboard_input';
+  intendedAction: 'click' | 'keyboard_input' | 'drag_and_drop' | 'set_slider';
   message: string;
 }
 
@@ -57,7 +63,14 @@ function getEventTargetRecord(
   eventType: AnnotatableRecordingEventType,
   eventData: Record<string, unknown>,
 ): Record<string, unknown> | null {
-  const candidate = eventType === 'submit' ? eventData.form : eventData.element;
+  let candidate: unknown;
+  if (eventType === 'submit') {
+    candidate = eventData.form;
+  } else if (eventType === 'drag_and_drop') {
+    candidate = eventData.targetElement;
+  } else {
+    candidate = eventData.element;
+  }
 
   return isObjectRecord(candidate) ? candidate : null;
 }
@@ -77,6 +90,14 @@ export function getRecordingAnnotationMessage(
     return 'This is the input the user just focused.';
   }
 
+  if (eventType === 'drag_and_drop') {
+    return 'This is where the user dropped the dragged element.';
+  }
+
+  if (eventType === 'set_slider') {
+    return 'This is the slider the user adjusted.';
+  }
+
   return 'This is the element the user just typed into.';
 }
 
@@ -88,7 +109,9 @@ export function resolveRecordingKeyframeAnnotationTarget(
     eventType !== 'click' &&
     eventType !== 'focus' &&
     eventType !== 'change' &&
-    eventType !== 'submit'
+    eventType !== 'submit' &&
+    eventType !== 'drag_and_drop' &&
+    eventType !== 'set_slider'
   ) {
     return null;
   }
@@ -103,9 +126,20 @@ export function resolveRecordingKeyframeAnnotationTarget(
     return null;
   }
 
+  let intendedAction: RecordingKeyframeAnnotationTarget['intendedAction'];
+  if (eventType === 'click') {
+    intendedAction = 'click';
+  } else if (eventType === 'drag_and_drop') {
+    intendedAction = 'drag_and_drop';
+  } else if (eventType === 'set_slider') {
+    intendedAction = 'set_slider';
+  } else {
+    intendedAction = 'keyboard_input';
+  }
+
   return {
     bbox,
-    intendedAction: eventType === 'click' ? 'click' : 'keyboard_input',
+    intendedAction,
     message: getRecordingAnnotationMessage(eventType),
   };
 }

extension/src/recording/keyframe-policy.ts

@@ -1,13 +1,20 @@
 import type { ScreenshotCaptureOptions } from '../utils/highlight-screenshot';
 
-const ACTION_KEYFRAME_EVENT_TYPES = new Set(['click', 'change', 'submit']);
+const ACTION_KEYFRAME_EVENT_TYPES = new Set([
+  'click',
+  'change',
+  'submit',
+  'drag_and_drop',
+  'set_slider',
+]);
 const DEFAULT_RECORDING_KEYFRAME_WAIT_MS = 180;
 const ACTION_RECORDING_KEYFRAME_WAIT_MS = 60;
 const PRE_ACTION_RECORDING_KEYFRAME_WAIT_MS = 0;
+const AFTER_ACTION_RECORDING_KEYFRAME_WAIT_MS = 400;
 const RECORDING_KEYFRAME_CAPTURE_OPTIONS: ScreenshotCaptureOptions = {
   preferredFormat: 'jpeg',
-  maxOutputWidth: 960,
-  maxOutputHeight: 540,
+  maxOutputWidth: 1920,
+  maxOutputHeight: 1080,
   warmupBeforeCapture: true,
   warmupMaxAttempts: 1,
   settleBeforeCapture: true,
@@ -17,8 +24,8 @@ const RECORDING_KEYFRAME_CAPTURE_OPTIONS: ScreenshotCaptureOptions = {
 const PRE_ACTION_RECORDING_KEYFRAME_CAPTURE_OPTIONS: ScreenshotCaptureOptions =
   {
     preferredFormat: 'jpeg',
-    maxOutputWidth: 960,
-    maxOutputHeight: 540,
+    maxOutputWidth: 1920,
+    maxOutputHeight: 1080,
     warmupBeforeCapture: false,
     settleBeforeCapture: false,
   };
@@ -106,6 +113,14 @@ export function getRecordingPreActionCaptureOptions(): ScreenshotCaptureOptions
   return { ...PRE_ACTION_RECORDING_KEYFRAME_CAPTURE_OPTIONS };
 }
 
+export function shouldCaptureAfterKeyframe(eventType: string): boolean {
+  return ACTION_KEYFRAME_EVENT_TYPES.has(eventType);
+}
+
+export function getRecordingAfterKeyframeWaitForRender(): number {
+  return AFTER_ACTION_RECORDING_KEYFRAME_WAIT_MS;
+}
+
 export function shouldDiscardPostCaptureRecordingKeyframe(
   eventType: string,
   eventData: Record<string, unknown>,