Add WASD and mouse controller input support

frontend/src/components/VideoOutput.tsx

@@ -14,6 +14,12 @@ interface VideoOutputProps {
   onPlayPauseToggle?: () => void;
   onStartStream?: () => void;
   onVideoPlaying?: () => void;
+  // Controller input props
+  supportsControllerInput?: boolean;
+  isPointerLocked?: boolean;
+  onRequestPointerLock?: () => void;
+  /** Ref to expose the video container element for pointer lock */
+  videoContainerRef?: React.RefObject<HTMLDivElement | null>;
 }
 
 export function VideoOutput({
@@ -27,12 +33,20 @@ export function VideoOutput({
   onPlayPauseToggle,
   onStartStream,
   onVideoPlaying,
+  supportsControllerInput = false,
+  isPointerLocked = false,
+  onRequestPointerLock,
+  videoContainerRef,
 }: VideoOutputProps) {
   const videoRef = useRef<HTMLVideoElement>(null);
+  const internalContainerRef = useRef<HTMLDivElement>(null);
   const [showOverlay, setShowOverlay] = useState(false);
   const [isFadingOut, setIsFadingOut] = useState(false);
   const overlayTimeoutRef = useRef<number | null>(null);
 
+  // Use external ref if provided, otherwise use internal
+  const containerRef = videoContainerRef || internalContainerRef;
+
   useEffect(() => {
     if (videoRef.current && remoteStream) {
       videoRef.current.srcObject = remoteStream;
@@ -87,7 +101,16 @@ export function VideoOutput({
   }, [onPlayPauseToggle, remoteStream]);
 
   const handleVideoClick = () => {
-    triggerPlayPause();
+    // If controller input is supported and not locked, request pointer lock
+    if (supportsControllerInput && !isPointerLocked && onRequestPointerLock) {
+      onRequestPointerLock();
+      return;
+    }
+
+    // Otherwise toggle play/pause
+    if (!isPointerLocked) {
+      triggerPlayPause();
+    }
   };
 
   // Handle spacebar press for play/pause
@@ -134,6 +157,7 @@ export function VideoOutput({
       <CardContent className="flex-1 flex items-center justify-center min-h-0 p-4">
         {remoteStream ? (
           <div
+            ref={containerRef}
             className="relative w-full h-full cursor-pointer flex items-center justify-center"
             onClick={handleVideoClick}
           >
@@ -158,6 +182,12 @@ export function VideoOutput({
                 </div>
               </div>
             )}
+            {/* Controller Input Overlay - only show before pointer lock (browser shows ESC hint) */}
+            {supportsControllerInput && !isPointerLocked && (
+              <div className="absolute bottom-4 left-1/2 -translate-x-1/2 bg-black/70 text-white px-4 py-2 rounded-lg text-sm pointer-events-none">
+                Click to enable controller input
+              </div>
+            )}
           </div>
         ) : isDownloading ? (
           <div className="text-center text-muted-foreground text-lg">

frontend/src/hooks/useControllerInput.ts

@@ -0,0 +1,252 @@
+import { useEffect, useRef, useCallback, useState } from "react";
+
+/**
+ * Controller input state matching the backend CtrlInput format.
+ * Uses W3C event.code strings for key identification.
+ */
+export interface ControllerInputState {
+  /** Set of currently pressed keys (W3C event.code values) */
+  button: string[];
+  /** Mouse velocity/delta as [dx, dy] tuple */
+  mouse: [number, number];
+}
+
+/**
+ * Configuration for the controller input hook.
+ */
+export interface ControllerInputConfig {
+  /** Target send rate in Hz (default: 60) */
+  sendRateHz?: number;
+  /** Mouse sensitivity multiplier (default: 0.002) */
+  mouseSensitivity?: number;
+  /** Keys to capture (default: WASD, arrows, space, shift) */
+  capturedKeys?: Set<string>;
+}
+
+/** Default keys to capture */
+const DEFAULT_CAPTURED_KEYS = new Set([
+  "KeyW",
+  "KeyA",
+  "KeyS",
+  "KeyD",
+  "ArrowUp",
+  "ArrowDown",
+  "ArrowLeft",
+  "ArrowRight",
+  "Space",
+  "ShiftLeft",
+  "ShiftRight",
+  "KeyQ",
+  "KeyE",
+  "KeyR",
+  "KeyF",
+  "KeyC",
+  "KeyX",
+  "KeyZ",
+]);
+
+/**
+ * Hook for capturing WASD keyboard and mouse input for streaming to backend.
+ *
+ * Uses a pygame-inspired state dictionary pattern:
+ * - Tracks which keys are currently held down (not just press events)
+ * - Accumulates mouse deltas between send intervals
+ * - Sends state snapshots at a fixed rate (default 60Hz)
+ *
+ * @param sendFn Function to send controller input to backend
+ * @param enabled Whether controller input capture is enabled
+ * @param targetRef Ref to the element that should capture input (for pointer lock)
+ * @param config Optional configuration
+ */
+export function useControllerInput(
+  sendFn: (params: { ctrl_input: ControllerInputState }) => void,
+  enabled: boolean,
+  targetRef: React.RefObject<HTMLElement | null>,
+  config?: ControllerInputConfig
+) {
+  const {
+    sendRateHz = 60,
+    mouseSensitivity = 1.5,
+    capturedKeys = DEFAULT_CAPTURED_KEYS,
+  } = config || {};
+
+  // State for UI feedback
+  const [isPointerLocked, setIsPointerLocked] = useState(false);
+  const [pressedKeys, setPressedKeys] = useState<Set<string>>(new Set());
+
+  // Refs for tracking input state (mutable for performance)
+  const pressedKeysRef = useRef<Set<string>>(new Set());
+  const mouseDeltaRef = useRef<[number, number]>([0, 0]);
+  const lastSentStateRef = useRef<string>("");
+  const sendIntervalRef = useRef<number | null>(null);
+
+  // Handle keyboard events
+  const handleKeyDown = useCallback(
+    (e: KeyboardEvent) => {
+      if (!enabled || !isPointerLocked) return;
+
+      // Ignore if typing in an input field
+      const target = e.target as HTMLElement;
+      if (
+        target.tagName === "INPUT" ||
+        target.tagName === "TEXTAREA" ||
+        target.tagName === "SELECT" ||
+        target.isContentEditable
+      ) {
+        return;
+      }
+
+      if (capturedKeys.has(e.code)) {
+        e.preventDefault();
+        pressedKeysRef.current.add(e.code);
+        setPressedKeys(new Set(pressedKeysRef.current));
+      }
+    },
+    [enabled, isPointerLocked, capturedKeys]
+  );
+
+  const handleKeyUp = useCallback(
+    (e: KeyboardEvent) => {
+      if (!enabled) return;
+
+      if (capturedKeys.has(e.code)) {
+        e.preventDefault();
+        pressedKeysRef.current.delete(e.code);
+        setPressedKeys(new Set(pressedKeysRef.current));
+      }
+    },
+    [enabled, capturedKeys]
+  );
+
+  // Handle mouse movement (only when pointer is locked)
+  const handleMouseMove = useCallback(
+    (e: MouseEvent) => {
+      if (!enabled || !isPointerLocked) return;
+
+      // Accumulate mouse deltas
+      mouseDeltaRef.current[0] += e.movementX * mouseSensitivity;
+      mouseDeltaRef.current[1] += e.movementY * mouseSensitivity;
+    },
+    [enabled, isPointerLocked, mouseSensitivity]
+  );
+
+  // Handle pointer lock changes
+  const handlePointerLockChange = useCallback(() => {
+    const isLocked = document.pointerLockElement === targetRef.current;
+    setIsPointerLocked(isLocked);
+
+    if (!isLocked) {
+      // Clear pressed keys when pointer lock is released
+      pressedKeysRef.current.clear();
+      setPressedKeys(new Set());
+      mouseDeltaRef.current = [0, 0];
+    }
+  }, [targetRef]);
+
+  // Request pointer lock
+  const requestPointerLock = useCallback(() => {
+    if (targetRef.current && enabled) {
+      targetRef.current.requestPointerLock();
+    }
+  }, [targetRef, enabled]);
+
+  // Release pointer lock
+  const releasePointerLock = useCallback(() => {
+    if (document.pointerLockElement) {
+      document.exitPointerLock();
+    }
+  }, []);
+
+  // Send controller input at fixed interval
+  const sendControllerInput = useCallback(() => {
+    if (!enabled || !isPointerLocked) return;
+
+    const state: ControllerInputState = {
+      button: Array.from(pressedKeysRef.current),
+      mouse: [...mouseDeltaRef.current] as [number, number],
+    };
+
+    // Only send if state has changed (optimization)
+    const stateStr = JSON.stringify(state);
+    if (stateStr !== lastSentStateRef.current) {
+      sendFn({ ctrl_input: state });
+      lastSentStateRef.current = stateStr;
+    }
+
+    // Reset mouse delta after sending (it's accumulated between sends)
+    mouseDeltaRef.current = [0, 0];
+  }, [enabled, isPointerLocked, sendFn]);
+
+  // Set up event listeners
+  useEffect(() => {
+    if (!enabled) return;
+
+    window.addEventListener("keydown", handleKeyDown);
+    window.addEventListener("keyup", handleKeyUp);
+    window.addEventListener("mousemove", handleMouseMove);
+    document.addEventListener("pointerlockchange", handlePointerLockChange);
+
+    return () => {
+      window.removeEventListener("keydown", handleKeyDown);
+      window.removeEventListener("keyup", handleKeyUp);
+      window.removeEventListener("mousemove", handleMouseMove);
+      document.removeEventListener(
+        "pointerlockchange",
+        handlePointerLockChange
+      );
+    };
+  }, [
+    enabled,
+    handleKeyDown,
+    handleKeyUp,
+    handleMouseMove,
+    handlePointerLockChange,
+  ]);
+
+  // Set up send interval
+  useEffect(() => {
+    if (!enabled || !isPointerLocked) {
+      if (sendIntervalRef.current) {
+        clearInterval(sendIntervalRef.current);
+        sendIntervalRef.current = null;
+      }
+      return;
+    }
+
+    const intervalMs = 1000 / sendRateHz;
+    sendIntervalRef.current = window.setInterval(
+      sendControllerInput,
+      intervalMs
+    );
+
+    return () => {
+      if (sendIntervalRef.current) {
+        clearInterval(sendIntervalRef.current);
+        sendIntervalRef.current = null;
+      }
+    };
+  }, [enabled, isPointerLocked, sendRateHz, sendControllerInput]);
+
+  // Clean up on unmount
+  useEffect(() => {
+    return () => {
+      if (sendIntervalRef.current) {
+        clearInterval(sendIntervalRef.current);
+      }
+      if (document.pointerLockElement) {
+        document.exitPointerLock();
+      }
+    };
+  }, []);
+
+  return {
+    /** Whether pointer lock is currently active */
+    isPointerLocked,
+    /** Set of currently pressed keys (for UI display) */
+    pressedKeys,
+    /** Request pointer lock on the target element */
+    requestPointerLock,
+    /** Release pointer lock */
+    releasePointerLock,
+  };
+}

frontend/src/hooks/usePipelines.ts

@@ -36,6 +36,10 @@ export function usePipelines() {
             }
           }
 
+          // Check if pipeline supports controller input (has ctrl_input field in schema)
+          const supportsControllerInput =
+            schema.config_schema?.properties?.ctrl_input !== undefined;
+
           transformed[id] = {
             name: schema.name,
             about: schema.description,
@@ -60,6 +64,7 @@ export function usePipelines() {
               schema.recommended_quantization_vram_threshold ?? undefined,
             modified: schema.modified,
             vaeTypes,
+            supportsControllerInput,
           };
         }
 

frontend/src/hooks/useWebRTC.ts

@@ -329,6 +329,7 @@ export function useWebRTC(options?: UseWebRTCOptions) {
       vace_ref_images?: string[];
       vace_use_input_video?: boolean;
       vace_context_scale?: number;
+      ctrl_input?: { button: string[]; mouse: [number, number] };
     }) => {
       if (
         dataChannelRef.current &&