Skip to content

Click events for GUI images #383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
brentyi wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Click events for GUI images #383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5d9ec3f
(WIP) clickable image prototype
brentyi Jan 31, 2025
76a5d12
image is button
brentyi Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/viser/_gui_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@
GuiVector3Handle,
SupportsRemoveProtocol,
UploadedFile,
_GuiHandle,
_GuiHandleState,
_GuiInputHandle,
_make_uuid,
)
from ._icons import svg_from_icon
Expand Down Expand Up @@ -203,7 +203,7 @@ def __init__(
)
"""Interface for sending and listening to messages."""

self._gui_input_handle_from_uuid: dict[str, _GuiInputHandle[Any]] = {}
self._gui_handle_from_uuid: dict[str, _GuiHandle[Any]] = {}
self._container_handle_from_uuid: dict[str, GuiContainerProtocol] = {
"root": _RootGuiContainer({})
}
Expand All @@ -228,7 +228,7 @@ async def _handle_gui_updates(
self, client_id: ClientId, message: _messages.GuiUpdateMessage
) -> None:
"""Callback for handling GUI messages."""
handle = self._gui_input_handle_from_uuid.get(message.uuid, None)
handle = self._gui_handle_from_uuid.get(message.uuid, None)
if handle is None:
return
handle_state = handle._impl
Expand Down Expand Up @@ -293,7 +293,7 @@ async def _handle_gui_updates(
def _handle_file_transfer_start(
self, client_id: ClientId, message: _messages.FileTransferStart
) -> None:
if message.source_component_uuid not in self._gui_input_handle_from_uuid:
if message.source_component_uuid not in self._gui_handle_from_uuid:
return
self._current_file_upload_states[message.transfer_uuid] = {
"filename": message.filename,
Expand All @@ -310,7 +310,7 @@ def _handle_file_transfer_part(
) -> None:
if message.transfer_uuid not in self._current_file_upload_states:
return
assert message.source_component_uuid in self._gui_input_handle_from_uuid
assert message.source_component_uuid in self._gui_handle_from_uuid

state = self._current_file_upload_states[message.transfer_uuid]
state["parts"][message.part] = message.content
Expand All @@ -336,9 +336,7 @@ def _handle_file_transfer_part(
assert state["transferred_bytes"] == total_bytes
state = self._current_file_upload_states.pop(message.transfer_uuid)

handle = self._gui_input_handle_from_uuid.get(
message.source_component_uuid, None
)
handle = self._gui_handle_from_uuid.get(message.source_component_uuid, None)
if handle is None:
return

Expand Down Expand Up @@ -649,6 +647,7 @@ def add_image(
media_type="image/png" if format == "png" else "image/jpeg",
order=_apply_default_order(order),
visible=visible,
_clickable=False,
),
)
self._websock_interface.queue_message(message)
Expand All @@ -657,9 +656,10 @@ def add_image(
_GuiHandleState(
message.uuid,
self,
None,
[0.0, 0.0],
props=message.props,
parent_container_id=message.container_uuid,
is_button=True,
),
_image=image,
_jpeg_quality=jpeg_quality,
Expand Down
43 changes: 36 additions & 7 deletions src/viser/_gui_handles.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from ._messages import (
GuiBaseProps,
GuiButtonGroupProps,
GuiButtonProps,
GuiCheckboxProps,
GuiCloseModalMessage,
GuiDropdownProps,
Expand Down Expand Up @@ -64,7 +65,7 @@


T = TypeVar("T")
TGuiHandle = TypeVar("TGuiHandle", bound="_GuiInputHandle")
TGuiHandle = TypeVar("TGuiHandle", bound="_GuiHandle")
NoneOrCoroutine = TypeVar("NoneOrCoroutine", None, Coroutine)


Expand Down Expand Up @@ -165,8 +166,7 @@ def __init__(self, _impl: _GuiHandleState[T]) -> None:
]
parent._children[self._impl.uuid] = self

if isinstance(self, _GuiInputHandle):
self._impl.gui_api._gui_input_handle_from_uuid[self._impl.uuid] = self
self._impl.gui_api._gui_handle_from_uuid[self._impl.uuid] = self

def remove(self) -> None:
"""Permanently remove this GUI element from the visualizer."""
Expand All @@ -191,8 +191,7 @@ def remove(self) -> None:
parent = gui_api._container_handle_from_uuid[self._impl.parent_container_id]
parent._children.pop(self._impl.uuid)

if isinstance(self, _GuiInputHandle):
gui_api._gui_input_handle_from_uuid.pop(self._impl.uuid)
gui_api._gui_handle_from_uuid.pop(self._impl.uuid)


class _GuiInputHandle(
Expand Down Expand Up @@ -397,7 +396,7 @@ class GuiEvent(Generic[TGuiHandle]):
"""GUI element that was affected."""


class GuiButtonHandle(_GuiInputHandle[bool]):
class GuiButtonHandle(_GuiInputHandle[bool], GuiButtonProps):
"""Handle for a button input in our visualizer.

.. attribute:: value
Expand Down Expand Up @@ -818,7 +817,7 @@ def figure(self, figure: go.Figure) -> None:
self._plotly_json_str = json_str


class GuiImageHandle(_GuiHandle[None], GuiImageProps):
class GuiImageHandle(_GuiHandle[Tuple[float, float]], GuiImageProps):
"""Handle for updating and removing images."""

def __init__(
Expand All @@ -831,6 +830,36 @@ def __init__(
self._image = _image
self._jpeg_quality = _jpeg_quality

@property
def clicked_xy(self) -> Tuple[float, float]:
"""Last-clicked XY coordinate of the image. Normalized [0, 1]."""
return self._impl.value

def on_click(
self, func: Callable[[GuiEvent[TGuiHandle]], NoneOrCoroutine]
) -> Callable[[GuiEvent[TGuiHandle]], NoneOrCoroutine]:
"""Attach a function to call when an image is clicked."""
self._impl.update_cb.append(func)
self._clickable = True
return func

def remove_click_callback(
self, callback: Literal["all"] | Callable = "all"
) -> None:
"""Remove click callbacks from the GUI input.

Args:
callback: Either "all" to remove all callbacks, or a specific callback function to remove.
"""
if callback == "all":
self._impl.update_cb.clear()
else:
self._impl.update_cb = [cb for cb in self._impl.update_cb if cb != callback]

# Set clickable to False if not more callbacks.
if len(self._impl.update_cb) == 0:
self._clickable = False

@property
def image(self) -> np.ndarray:
"""Current content of this image element. Synchronized automatically when assigned."""
Expand Down
2 changes: 2 additions & 0 deletions src/viser/_messages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -915,6 +915,8 @@ class GuiImageProps:
"""Format of the provided image ('image/jpeg' or 'image/png'). Synchronized automatically when assigned."""
visible: bool
"""Visibility state of the image. Synchronized automatically when assigned."""
_clickable: bool
"""(Private) Whether the image is clickable. Synchronized automatically when assigned."""


@dataclasses.dataclass
Expand Down
3 changes: 2 additions & 1 deletion src/viser/client/src/WebsocketMessages.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@ export interface GuiImageMessage {
_data: Uint8Array | null;
media_type: "image/jpeg" | "image/png";
visible: boolean;
_clickable: boolean;
};
}
/** GuiTabGroupMessage(uuid: 'str', container_uuid: 'str', props: 'GuiTabGroupProps')
Expand Down Expand Up @@ -522,7 +523,7 @@ export interface GuiSliderMessage {
_marks: { value: number; label: string | null }[] | null;
};
}
/** GuiMultiSliderMessage(uuid: 'str', value: 'tuple[float, ...]', container_uuid: 'str', props: 'GuiMultiSliderProps')
/** GuiMultiSliderMessage(uuid: 'str', value: 'Tuple[float, ...]', container_uuid: 'str', props: 'GuiMultiSliderProps')
*
* (automatically generated)
*/
Expand Down
18 changes: 17 additions & 1 deletion src/viser/client/src/components/Image.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import React from "react";
import { useEffect, useState } from "react";
import { GuiImageMessage } from "../WebsocketMessages";
import { Box, Text } from "@mantine/core";
import { ViewerContext } from "../ViewerContext";

function ImageComponent({ props }: GuiImageMessage) {
function ImageComponent({ uuid, props }: GuiImageMessage) {
if (!props.visible) return <></>;

const [imageUrl, setImageUrl] = useState<string | null>(null);
const viewer = React.useContext(ViewerContext)!;

useEffect(() => {
if (props._data === null) {
Expand All @@ -28,11 +31,24 @@ function ImageComponent({ props }: GuiImageMessage) {
{props.label}
</Text>
)}

<img
src={imageUrl}
style={{
maxWidth: "100%",
height: "auto",
cursor: props._clickable === false ? "default" : "pointer",
}}
onClick={(e) => {
if (props._clickable === false) return;
const rect = e.currentTarget.getBoundingClientRect();
const x = (e.clientX - rect.left) / rect.width;
const y = (e.clientY - rect.top) / rect.height;
viewer.sendMessageRef.current({
type: "GuiUpdateMessage",
uuid: uuid,
updates: { value: [x, y] },
});
}}
/>
</Box>
Expand Down