Camera streaming feature

examples/28_camera_streaming.py

@@ -0,0 +1,63 @@
+"""Camera on-demand capture
+
+Demonstrates how to request camera frames from the client on-demand.
+"""
+
+import time
+
+import numpy as np
+
+import viser
+
+
+def main():
+    server = viser.ViserServer()
+
+    # Attach camera capture handlers to each client.
+    @server.on_client_connect
+    def _(client: viser.ClientHandle):
+        # Camera configuration controls
+        facing_mode_dropdown = client.gui.add_dropdown(
+            "Camera", options=("user", "environment"), initial_value="user"
+        )
+
+        client_id = client.client_id
+
+        # Create placeholder image displays
+        dummy_image = np.zeros((480, 640, 3), dtype=np.uint8)
+        client_image_handle = client.gui.add_image(dummy_image)
+        server.scene.add_transform_controls(
+            name=f"/camera_frame_{client_id}",
+            scale=0.2,
+            position=(client_id, 0, 0),
+            active_axes=(True, True, False),
+        )
+        server_image_handle = server.scene.add_image(
+            name=f"/camera_frame_{client_id}/img",
+            image=dummy_image,
+            render_width=0.5,
+            render_height=0.5,
+            position=(0.25, 0.25, -0.001),
+        )
+
+        # Configure camera with facing mode
+        client.configure_camera_access(enabled=True, facing_mode=facing_mode_dropdown.value)
+
+        # Update camera configuration when facing mode changes
+        @facing_mode_dropdown.on_update
+        def _(_):
+            client.configure_camera_access(enabled=True, facing_mode=facing_mode_dropdown.value)
+
+        while True:
+            image = client.capture_frame(timeout=2.0)
+            if image is not None:
+                client_image_handle.image = np.array(image)
+                server_image_handle.image = np.array(image)
+
+            time.sleep(1 / 20)
+
+    server.sleep_forever()
+
+
+if __name__ == "__main__":
+    main()

src/viser/__init__.py

@@ -1,4 +1,5 @@
 from ._gui_api import GuiApi as GuiApi
+from ._gui_handles import CameraStreamFrameEvent as CameraStreamFrameEvent
 from ._gui_handles import GuiButtonGroupHandle as GuiButtonGroupHandle
 from ._gui_handles import GuiButtonHandle as GuiButtonHandle
 from ._gui_handles import GuiCheckboxHandle as GuiCheckboxHandle

src/viser/_gui_handles.py

@@ -21,6 +21,7 @@
 
 import imageio.v3 as iio
 import numpy as np
+from PIL.Image import Image
 from typing_extensions import Protocol, override
 
 from ._assignable_props_api import AssignablePropsBase
@@ -809,3 +810,18 @@ def image(self, image: np.ndarray) -> None:
         )
         self._data = data
         del media_type
+
+
+@dataclasses.dataclass(frozen=True)
+class CameraStreamFrameEvent:
+    """Event passed to camera stream frame callbacks."""
+
+    client: ClientHandle
+    """Client that sent this frame."""
+    client_id: int
+    """ID of client that sent this frame."""
+    image: Image
+    """Frame as PIL image."""
+    timestamp: float
+    """Timestamp when the frame was captured."""
+

src/viser/_messages.py

@@ -1600,3 +1600,36 @@ class SetGuiPanelLabelMessage(Message):
     """Message from server->client to set the label of the GUI panel."""
 
     label: Optional[str]
+
+
+@dataclasses.dataclass
+class CameraAccessConfigMessage(Message):
+    """Message from server->client to configure camera access."""
+
+    enabled: bool
+    facing_mode: Optional[Literal["user", "environment"]] = None
+
+
+@dataclasses.dataclass
+class CameraFrameRequestMessage(Message):
+    """Message from server->client requesting a camera frame."""
+
+    request_id: str
+
+    @override
+    def redundancy_key(self) -> str:
+        return type(self).__name__ + "-" + self.request_id
+
+
+@dataclasses.dataclass
+class CameraFrameResponseMessage(Message):
+    """Message from client->server responding with a camera frame."""
+
+    request_id: str
+    frame_data: Optional[bytes]  # None if capture failed
+    timestamp: float
+    error: Optional[str]  # Error message if capture failed
+
+    @override
+    def redundancy_key(self) -> str:
+        return type(self).__name__ + "-" + self.request_id

src/viser/_viser.py

@@ -16,6 +16,7 @@
 import numpy as np
 import numpy.typing as npt
 import rich
+from PIL import Image
 from rich import box, style
 from rich.panel import Panel
 from rich.table import Table
@@ -445,6 +446,66 @@ def send_file_download(
             )
             self.flush()
 
+    def capture_frame(
+        self,
+        timeout: float = 2.0,
+    ) -> Image.Image | None:
+        """Request a camera frame from this client.
+
+        Args:
+            timeout: Maximum time to wait for frame capture in seconds.
+
+        Returns:
+            PIL Image when frame is captured.
+
+        Raises:
+            TimeoutError: If frame capture takes longer than timeout.
+            RuntimeError: If camera capture fails.
+        """
+        frame_ready_event = threading.Event()
+        frame: Image.Image | None = None
+
+        connection = self._websock_connection
+
+        def got_frame_cb(
+            client_id: int, message: _messages.CameraFrameResponseMessage
+        ) -> None:
+            del client_id
+            connection.unregister_handler(_messages.CameraFrameResponseMessage, got_frame_cb)
+            nonlocal frame
+            if message.frame_data is None:
+                frame = None
+            else:
+                frame = Image.open(io.BytesIO(message.frame_data))
+            frame_ready_event.set()
+
+        connection.register_handler(_messages.CameraFrameResponseMessage, got_frame_cb)
+
+        self._websock_connection.queue_message(
+            _messages.CameraFrameRequestMessage(
+                request_id=_make_uuid(),
+            )
+        )
+        frame_ready_event.wait(timeout=timeout)
+        return frame
+
+    def configure_camera_access(
+        self, 
+        enabled: bool, 
+        facing_mode: Literal["user", "environment"] | None = None
+    ) -> None:
+        """Configure camera access for this client.
+
+        Args:
+            enabled: Whether to enable camera access. When True, the client will
+                    request camera permissions and make the camera available for
+                    frame capture. When False, camera access is disabled.
+            facing_mode: Camera facing mode ("user" for front camera, "environment" for back camera).
+        """
+        self._websock_connection.queue_message(
+            _messages.CameraAccessConfigMessage(enabled=enabled, facing_mode=facing_mode)
+        )
+
     def add_notification(
         self,
         title: str,
@@ -675,6 +736,7 @@ async def handle_camera_message(
                     first = False
                     with self._client_lock:
                         self._connected_clients[conn.client_id] = client
+
                         for cb in self._client_connect_cb:
                             if asyncio.iscoroutinefunction(cb):
                                 await cb(client)

src/viser/client/package.json

@@ -36,6 +36,7 @@
     "react-error-boundary": "^4.0.10",
     "react-intersection-observer": "^9.13.1",
     "react-qr-code": "^2.0.12",
+    "react-webcam": "^7.2.0",
     "rehype-color-chips": "^0.1.3",
     "remark-gfm": "^4.0.0",
     "three": "^0.174.0",

src/viser/client/src/App.tsx

@@ -45,6 +45,7 @@ import { BrowserWarning } from "./BrowserWarning";
 import { MacWindowWrapper } from "./MacWindowWrapper";
 import { CsmDirectionalLight } from "./CsmDirectionalLight";
 import { VISER_VERSION } from "./VersionInfo";
+import { CameraStream } from "./CameraStream";
 
 // ======= Utility functions =======
 
@@ -211,6 +212,11 @@ function ViewerRoot() {
 
     // Global hover state tracking.
     hoveredElementsCount: 0,
+
+    // Camera stream state.
+    cameraStreamConfig: {
+      enabled: false,
+    },
   });
 
   // Create the context value with hooks and single ref.
@@ -246,6 +252,8 @@ function ViewerContents({ children }: { children: React.ReactNode }) {
   const colors = viewer.useGui((state) => state.theme.colors);
   const controlLayout = viewer.useGui((state) => state.theme.control_layout);
   const showLogo = viewer.useGui((state) => state.theme.show_logo);
+  const connected = viewer.useGui((state) => state.websocketConnected);
+  const cameraEnabled = viewer.useGui((state) => state.cameraEnabled);
   const { messageSource } = viewer;
 
   // Create Mantine theme with custom colors if provided.
@@ -319,6 +327,7 @@ function ViewerContents({ children }: { children: React.ReactNode }) {
             )}
           </Box>
         </Box>
+        {connected && cameraEnabled && <CameraStream />}
       </MantineProvider>
     </>
   );

src/viser/client/src/CameraStream.tsx

@@ -0,0 +1,127 @@
+import { useEffect, useRef, useContext, useCallback } from "react";
+import { Box } from "@mantine/core";
+import { ViewerContext } from "./ViewerContext";
+import Webcam from "react-webcam";
+
+export function CameraStream() {
+  const viewer = useContext(ViewerContext)!;
+  const viewerMutable = viewer.mutable.current; // Get mutable once.
+  const connected = viewer.useGui((state) => state.websocketConnected);
+  const cameraEnabled = viewer.useGui((state) => state.cameraEnabled);
+  const cameraReady = viewer.useGui((state) => state.cameraReady);
+  const activeCameraRequest = viewer.useGui((state) => state.activeCameraRequest);
+  const cameraFacingMode = viewer.useGui((state) => state.cameraFacingMode);
+  const setCameraReady = viewer.useGui((state) => state.setCameraReady);
+  const setCameraRequest = viewer.useGui((state) => state.setCameraRequest);
+  const webcamRef = useRef<Webcam>(null);
+
+  // Handle camera frame capture requests.
+  useEffect(() => {
+    if (!activeCameraRequest) return;
+
+    const request = activeCameraRequest;
+    const timestamp = Date.now() / 1000;
+
+    // Camera not enabled.
+    if (!cameraEnabled) {
+      viewerMutable.sendMessage({
+        type: "CameraFrameResponseMessage",
+        request_id: request.request_id,
+        frame_data: null,
+        timestamp: timestamp,
+        error: "Camera access disabled",
+      });
+      setCameraRequest(null);
+      return;
+    }
+
+    // Camera not found, or not ready.
+    if (!webcamRef.current || !cameraReady) {
+      viewerMutable.sendMessage({
+        type: "CameraFrameResponseMessage",
+        request_id: request.request_id,
+        frame_data: null,
+        timestamp: timestamp,
+        error: "Camera not ready",
+      });
+      setCameraRequest(null);
+      return;
+    }
+
+    const imageSrc = webcamRef.current.getScreenshot();
+
+    // Tried to capture frame, but failed.
+    if (!imageSrc) {
+      viewerMutable.sendMessage({
+        type: "CameraFrameResponseMessage",
+        request_id: request.request_id,
+        frame_data: null,
+        timestamp: timestamp,
+        error: "Failed to capture frame",
+      });
+      setCameraRequest(null);
+      return;
+    }
+
+    // Convert base64 to Uint8Array.
+    const byteString = atob(imageSrc.split(',')[1]);
+    const ab = new ArrayBuffer(byteString.length);
+    const ia = new Uint8Array(ab);
+    for (let i = 0; i < byteString.length; i++) {
+      ia[i] = byteString.charCodeAt(i);
+    }
+
+    const response = {
+      type: "CameraFrameResponseMessage" as const,
+      request_id: request.request_id,
+      frame_data: ia,
+      timestamp: timestamp,
+      error: null,
+    };
+    viewerMutable.sendMessage(response);
+
+    // Clear the request after processing.
+    console.log("Camera frame captured");
+    setCameraRequest(null);
+  }, [activeCameraRequest]);
+
+  // Set camera to "ready" when enabled, by default.
+  useEffect(() => {
+    if (cameraEnabled) {
+      setCameraReady(true);
+    } else {
+      setCameraReady(false);
+    }
+  }, [cameraEnabled]);
+
+  // Let the error trigger the webcam to create a "enabled-but-not-ready" state.
+  const handleUserMediaError = useCallback(() => { setCameraReady(false); }, []);
+
+  // Reset camera ready state when disconnected.
+  useEffect(() => {
+    if (!connected) { setCameraReady(false); }
+    else { setCameraReady(true); }
+  }, [connected]);
+
+  // Only render webcam if connected and enabled.
+  if (!connected || !cameraEnabled) {
+    return null;
+  }
+
+  return (
+    // This is a hack -- {display: none} doesn't work.
+    // It seems to fetch the current webcam render.
+    <Box style={{ position: "absolute", zIndex: -1000 }}>
+      <Webcam
+        ref={webcamRef}
+        audio={false}
+        screenshotFormat="image/jpeg"
+        onUserMediaError={handleUserMediaError}
+        mirrored={false}
+        videoConstraints={{
+          facingMode: cameraFacingMode == "environment" ? {exact: "environment"} : "user",
+        }}
+      />
+    </Box>
+  );
+}