From 0e786279e52e064c392e9c77e69710cccf77895c Mon Sep 17 00:00:00 2001 From: ruthwikdasyam Date: Sat, 14 Mar 2026 01:01:30 -0700 Subject: [PATCH 1/6] feat: deag, click on enable --- dimos/src/interaction/keyboard.rs | 88 ++++++++++++++++++------ dimos/src/viewer.rs | 107 +----------------------------- 2 files changed, 72 insertions(+), 123 deletions(-) diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index b6cdcd809c37..9e7d2b79776a 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -19,7 +19,6 @@ const BASE_ANGULAR_SPEED: f64 = 0.8; // rad/s const FAST_MULTIPLIER: f64 = 2.0; // Shift modifier /// Overlay styling -const OVERLAY_MARGIN: f32 = 12.0; const OVERLAY_PADDING: f32 = 10.0; const OVERLAY_ROUNDING: f32 = 8.0; const OVERLAY_BG: egui::Color32 = egui::Color32::from_rgba_premultiplied(20, 20, 30, 220); @@ -66,11 +65,13 @@ impl KeyState { } /// Handles keyboard input and publishes Twist via LCM. +/// Must be activated by clicking the overlay before keys are captured. pub struct KeyboardHandler { publisher: LcmPublisher, state: KeyState, was_active: bool, estop_flash: bool, // true briefly after space pressed + engaged: bool, // true when user has clicked the overlay to activate } impl KeyboardHandler { @@ -82,29 +83,30 @@ impl KeyboardHandler { state: KeyState::new(), was_active: false, estop_flash: false, + engaged: false, }) } /// Process keyboard input from egui and publish Twist if keys are held. /// Called once per frame from DimosApp.ui(). + /// Only captures keys when the overlay has been clicked (engaged). /// /// Returns true if any movement key is active (for UI overlay). pub fn process(&mut self, ctx: &egui::Context) -> bool { self.estop_flash = false; - // Check if any text widget has focus - if so, skip keyboard capture - let text_has_focus = ctx.memory(|m| m.focused().is_some()); - if text_has_focus { + // If not engaged, don't capture any keys + if !self.engaged { if self.was_active { if let Err(e) = self.publish_stop() { - re_log::warn!("Failed to send stop command on focus change: {e:?}"); + re_log::warn!("Failed to send stop on disengage: {e:?}"); } self.was_active = false; } return false; } - // Update key state from egui input + // Update key state from egui input (engaged flag is the only gate) self.update_key_state(ctx); // Check for emergency stop (Space key pressed - one-shot action) @@ -134,33 +136,71 @@ impl KeyboardHandler { self.state.any_active() } - /// Draw keyboard overlay HUD. Always shown (dim when idle, bright when active). - pub fn draw_overlay(&self, ctx: &egui::Context) { + /// Draw keyboard overlay HUD at bottom-right of the 3D viewport area. + /// Clickable: clicking the overlay toggles engaged state. + pub fn draw_overlay(&mut self, ctx: &egui::Context) { + let screen_rect = ctx.content_rect(); + // Default position: bottom-right of the 3D viewport area + let overlay_width = 140.0; + let overlay_height = 160.0; + let right_panel_offset = 320.0; + let bottom_timeline_offset = 120.0; + let default_pos = egui::pos2( + screen_rect.max.x - overlay_width - right_panel_offset, + screen_rect.max.y - overlay_height - bottom_timeline_offset, + ); + egui::Area::new("keyboard_hud".into()) - .fixed_pos(egui::pos2(OVERLAY_MARGIN, OVERLAY_MARGIN)) + .default_pos(default_pos) + .movable(true) .order(egui::Order::Foreground) - .interactable(false) + .interactable(true) .show(ctx, |ui| { - egui::Frame::new() + let border_color = if self.engaged { + egui::Color32::from_rgb(60, 180, 75) // green border when active + } else { + egui::Color32::from_rgb(80, 80, 100) // dim border when inactive + }; + + let response = egui::Frame::new() .fill(OVERLAY_BG) .corner_radius(egui::CornerRadius::same(OVERLAY_ROUNDING as u8)) .inner_margin(egui::Margin::same(OVERLAY_PADDING as i8)) + .stroke(egui::Stroke::new(2.0, border_color)) .show(ui, |ui| { self.draw_hud_content(ui); }); + + // Make the frame rect clickable (Frame doesn't have click sense by default) + let click_response = ui.interact( + response.response.rect, + ui.id().with("wasd_click"), + egui::Sense::click(), + ).on_hover_cursor(egui::CursorIcon::Default); + + // Toggle engaged state on click + if click_response.clicked() { + self.engaged = !self.engaged; + if !self.engaged { + // Send stop when disengaging + if let Err(e) = self.publish_stop() { + re_log::warn!("Failed to send stop on disengage: {e:?}"); + } + self.state.reset(); + self.was_active = false; + } + re_log::info!( + "Keyboard teleop {}", + if self.engaged { "ENGAGED" } else { "DISENGAGED" } + ); + } }); + } fn draw_hud_content(&self, ui: &mut egui::Ui) { - let active = self.state.any_active() || self.estop_flash; - // Title - let title_color = if active { - egui::Color32::WHITE - } else { - egui::Color32::from_rgb(120, 120, 140) - }; - ui.label(egui::RichText::new("🎮 Keyboard Teleop").color(title_color).size(13.0)); + ui.label(egui::RichText::new("Keyboard Teleop").color(LABEL_COLOR).size(13.0)); ui.add_space(4.0); // Key grid: [Q] [W] [E] @@ -352,6 +392,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED); @@ -368,6 +409,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -381,6 +423,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -397,6 +440,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -410,6 +454,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -427,6 +472,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED * FAST_MULTIPLIER); @@ -444,6 +490,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED); @@ -471,6 +518,7 @@ mod tests { assert!(handler.is_ok()); let handler = handler.unwrap(); assert!(!handler.was_active); + assert!(!handler.engaged); assert!(!handler.state.any_active()); } @@ -484,6 +532,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -498,6 +547,7 @@ mod tests { state: KeyState::new(), was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, lin_z, ang_x, ang_y, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); diff --git a/dimos/src/viewer.rs b/dimos/src/viewer.rs index 7af7282ef188..86a7532458b1 100644 --- a/dimos/src/viewer.rs +++ b/dimos/src/viewer.rs @@ -1,25 +1,15 @@ -use std::cell::RefCell; -use std::rc::Rc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - use clap::Parser; -use dimos_viewer::interaction::{LcmPublisher, KeyboardHandler, click_event_from_ms}; +use dimos_viewer::interaction::KeyboardHandler; use rerun::external::{eframe, egui, re_crash_handler, re_grpc_server, re_log, re_memory, re_viewer}; #[global_allocator] static GLOBAL: re_memory::AccountingAllocator = re_memory::AccountingAllocator::new(mimalloc::MiMalloc); -/// LCM channel for click events (follows RViz convention) -const LCM_CHANNEL: &str = "/clicked_point#geometry_msgs.PointStamped"; -/// Minimum time between click events (debouncing) -const CLICK_DEBOUNCE_MS: u64 = 100; -/// Maximum rapid clicks to log as warning -const RAPID_CLICK_THRESHOLD: usize = 5; /// Default gRPC listen port (9877 to avoid conflict with stock Rerun on 9876) const DEFAULT_PORT: u16 = 9877; -/// DimOS Interactive Viewer — a custom Rerun viewer with LCM click-to-navigate. +/// DimOS Interactive Viewer — a custom Rerun viewer with WASD keyboard teleop. /// /// Accepts the same CLI flags as the stock `rerun` binary so it can be spawned /// seamlessly via `rerun_bindings.spawn(executable_name="dimos-viewer")`. @@ -73,7 +63,7 @@ impl eframe::App for DimosApp { // Process keyboard input before delegating to Rerun self.keyboard.process(ui.ctx()); - // Always draw the keyboard HUD overlay (dims when inactive) + // Draw the keyboard HUD overlay (click to engage/disengage) self.keyboard.draw_overlay(ui.ctx()); // Delegate to Rerun's main ui method @@ -124,20 +114,11 @@ async fn main() -> Result<(), Box> { re_grpc_server::shutdown::never(), ); - // Create LCM publisher for click events - let lcm_publisher = LcmPublisher::new(LCM_CHANNEL.to_string()) - .expect("Failed to create LCM publisher"); - re_log::info!("LCM publisher created for channel: {LCM_CHANNEL}"); - // Create keyboard handler let keyboard_handler = KeyboardHandler::new() .expect("Failed to create keyboard handler"); re_log::info!("Keyboard handler initialized for WASD controls on /cmd_vel"); - // State for debouncing and rapid click detection - let last_click_time = Rc::new(RefCell::new(Instant::now())); - let rapid_click_count = Rc::new(RefCell::new(0usize)); - let mut native_options = re_viewer::native::eframe_options(None); native_options.viewport = native_options .viewport @@ -151,88 +132,6 @@ async fn main() -> Result<(), Box> { let startup_options = re_viewer::StartupOptions { memory_limit, - on_event: Some(Rc::new({ - let last_click_time = last_click_time.clone(); - let rapid_click_count = rapid_click_count.clone(); - - move |event: re_viewer::ViewerEvent| { - if let re_viewer::ViewerEventKind::SelectionChange { items } = event.kind { - let mut has_position = false; - let mut no_position_count = 0; - - for item in items { - match item { - re_viewer::SelectionChangeItem::Entity { - entity_path, - view_name: _, - position: Some(pos), - .. - } => { - has_position = true; - - // Debouncing - let now = Instant::now(); - let elapsed = now.duration_since(*last_click_time.borrow()); - - if elapsed < Duration::from_millis(CLICK_DEBOUNCE_MS) { - let mut count = rapid_click_count.borrow_mut(); - *count += 1; - if *count == RAPID_CLICK_THRESHOLD { - re_log::warn!( - "Rapid click detected ({} clicks within {}ms)", - RAPID_CLICK_THRESHOLD, - CLICK_DEBOUNCE_MS - ); - } - continue; - } else { - *rapid_click_count.borrow_mut() = 0; - } - *last_click_time.borrow_mut() = now; - - let timestamp_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - - // Build click event and publish via LCM - let click = click_event_from_ms( - [pos.x, pos.y, pos.z], - &entity_path.to_string(), - timestamp_ms, - ); - - match lcm_publisher.publish(&click) { - Ok(_) => { - re_log::debug!( - "LCM click event published: entity={}, pos=({:.2}, {:.2}, {:.2})", - entity_path, - pos.x, - pos.y, - pos.z - ); - } - Err(err) => { - re_log::error!("Failed to publish LCM click event: {err:?}"); - } - } - } - re_viewer::SelectionChangeItem::Entity { position: None, .. } => { - no_position_count += 1; - } - _ => {} - } - } - - if !has_position && no_position_count > 0 { - re_log::trace!( - "Selection change without position data ({no_position_count} items). \ - This is normal for hover/keyboard navigation." - ); - } - } - } - })), ..Default::default() }; From 117df644935d9972a82aa44b716f480a8c9128fa Mon Sep 17 00:00:00 2001 From: ruthwikdasyam Date: Sat, 14 Mar 2026 01:31:29 -0700 Subject: [PATCH 2/6] feat: click anywhere else to disengage --- dimos/src/interaction/keyboard.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index 9e7d2b79776a..88a556c9c042 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -150,7 +150,7 @@ impl KeyboardHandler { screen_rect.max.y - overlay_height - bottom_timeline_offset, ); - egui::Area::new("keyboard_hud".into()) + let area_response = egui::Area::new("keyboard_hud".into()) .default_pos(default_pos) .movable(true) .order(egui::Order::Foreground) @@ -176,7 +176,12 @@ impl KeyboardHandler { response.response.rect, ui.id().with("wasd_click"), egui::Sense::click(), - ).on_hover_cursor(egui::CursorIcon::Default); + ); + + // Force arrow cursor over the entire overlay (overrides label I-beam) + if click_response.hovered() { + ctx.set_cursor_icon(egui::CursorIcon::Default); + } // Toggle engaged state on click if click_response.clicked() { @@ -194,8 +199,22 @@ impl KeyboardHandler { if self.engaged { "ENGAGED" } else { "DISENGAGED" } ); } - }); - + }) + .response; + + // Disengage when clicking anywhere outside the overlay + if self.engaged + && !ctx.rect_contains_pointer(area_response.layer_id, area_response.interact_rect) + && ctx.input(|i| i.pointer.primary_clicked()) + { + self.engaged = false; + if let Err(e) = self.publish_stop() { + re_log::warn!("Failed to send stop on outside click: {e:?}"); + } + self.state.reset(); + self.was_active = false; + re_log::info!("Keyboard teleop DISENGAGED (clicked outside)"); + } } fn draw_hud_content(&self, ui: &mut egui::Ui) { From d57c63de81056958acf45382ef8d2e2eab6a9c52 Mon Sep 17 00:00:00 2001 From: ruthwikdasyam Date: Sat, 14 Mar 2026 01:44:21 -0700 Subject: [PATCH 3/6] feat: click to navigate --- dimos/src/viewer.rs | 105 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/dimos/src/viewer.rs b/dimos/src/viewer.rs index 86a7532458b1..f2c5f98ca31c 100644 --- a/dimos/src/viewer.rs +++ b/dimos/src/viewer.rs @@ -1,15 +1,25 @@ +use std::cell::RefCell; +use std::rc::Rc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + use clap::Parser; -use dimos_viewer::interaction::KeyboardHandler; +use dimos_viewer::interaction::{LcmPublisher, KeyboardHandler, click_event_from_ms}; use rerun::external::{eframe, egui, re_crash_handler, re_grpc_server, re_log, re_memory, re_viewer}; #[global_allocator] static GLOBAL: re_memory::AccountingAllocator = re_memory::AccountingAllocator::new(mimalloc::MiMalloc); +/// LCM channel for click events (follows RViz convention) +const LCM_CHANNEL: &str = "/clicked_point#geometry_msgs.PointStamped"; +/// Minimum time between click events (debouncing) +const CLICK_DEBOUNCE_MS: u64 = 100; +/// Maximum rapid clicks to log as warning +const RAPID_CLICK_THRESHOLD: usize = 5; /// Default gRPC listen port (9877 to avoid conflict with stock Rerun on 9876) const DEFAULT_PORT: u16 = 9877; -/// DimOS Interactive Viewer — a custom Rerun viewer with WASD keyboard teleop. +/// DimOS Interactive Viewer — a custom Rerun viewer with LCM click-to-navigate. /// /// Accepts the same CLI flags as the stock `rerun` binary so it can be spawned /// seamlessly via `rerun_bindings.spawn(executable_name="dimos-viewer")`. @@ -114,11 +124,20 @@ async fn main() -> Result<(), Box> { re_grpc_server::shutdown::never(), ); + // Create LCM publisher for click events + let lcm_publisher = LcmPublisher::new(LCM_CHANNEL.to_string()) + .expect("Failed to create LCM publisher"); + re_log::info!("LCM publisher created for channel: {LCM_CHANNEL}"); + // Create keyboard handler let keyboard_handler = KeyboardHandler::new() .expect("Failed to create keyboard handler"); re_log::info!("Keyboard handler initialized for WASD controls on /cmd_vel"); + // State for debouncing and rapid click detection + let last_click_time = Rc::new(RefCell::new(Instant::now())); + let rapid_click_count = Rc::new(RefCell::new(0usize)); + let mut native_options = re_viewer::native::eframe_options(None); native_options.viewport = native_options .viewport @@ -132,6 +151,88 @@ async fn main() -> Result<(), Box> { let startup_options = re_viewer::StartupOptions { memory_limit, + on_event: Some(Rc::new({ + let last_click_time = last_click_time.clone(); + let rapid_click_count = rapid_click_count.clone(); + + move |event: re_viewer::ViewerEvent| { + if let re_viewer::ViewerEventKind::SelectionChange { items } = event.kind { + let mut has_position = false; + let mut no_position_count = 0; + + for item in items { + match item { + re_viewer::SelectionChangeItem::Entity { + entity_path, + view_name: _, + position: Some(pos), + .. + } => { + has_position = true; + + // Debouncing + let now = Instant::now(); + let elapsed = now.duration_since(*last_click_time.borrow()); + + if elapsed < Duration::from_millis(CLICK_DEBOUNCE_MS) { + let mut count = rapid_click_count.borrow_mut(); + *count += 1; + if *count == RAPID_CLICK_THRESHOLD { + re_log::warn!( + "Rapid click detected ({} clicks within {}ms)", + RAPID_CLICK_THRESHOLD, + CLICK_DEBOUNCE_MS + ); + } + continue; + } else { + *rapid_click_count.borrow_mut() = 0; + } + *last_click_time.borrow_mut() = now; + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + + // Build click event and publish via LCM + let click = click_event_from_ms( + [pos.x, pos.y, pos.z], + &entity_path.to_string(), + timestamp_ms, + ); + + match lcm_publisher.publish(&click) { + Ok(_) => { + re_log::debug!( + "LCM click event published: entity={}, pos=({:.2}, {:.2}, {:.2})", + entity_path, + pos.x, + pos.y, + pos.z + ); + } + Err(err) => { + re_log::error!("Failed to publish LCM click event: {err:?}"); + } + } + } + re_viewer::SelectionChangeItem::Entity { position: None, .. } => { + no_position_count += 1; + } + _ => {} + } + } + + if !has_position && no_position_count > 0 { + re_log::trace!( + "Selection change without position data ({no_position_count} items). \ + This is normal for hover/keyboard navigation." + ); + } + } + } + })), ..Default::default() }; From c1baf07b6777668430c6b05b37e8d9e215e3ec0d Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Sat, 14 Mar 2026 12:51:26 -0700 Subject: [PATCH 4/6] fix: remove logging --- dimos/src/interaction/keyboard.rs | 5 ----- dimos/src/viewer.rs | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index 88a556c9c042..b3f631b711ec 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -194,10 +194,6 @@ impl KeyboardHandler { self.state.reset(); self.was_active = false; } - re_log::info!( - "Keyboard teleop {}", - if self.engaged { "ENGAGED" } else { "DISENGAGED" } - ); } }) .response; @@ -213,7 +209,6 @@ impl KeyboardHandler { } self.state.reset(); self.was_active = false; - re_log::info!("Keyboard teleop DISENGAGED (clicked outside)"); } } diff --git a/dimos/src/viewer.rs b/dimos/src/viewer.rs index f2c5f98ca31c..0b71b71f8484 100644 --- a/dimos/src/viewer.rs +++ b/dimos/src/viewer.rs @@ -132,7 +132,7 @@ async fn main() -> Result<(), Box> { // Create keyboard handler let keyboard_handler = KeyboardHandler::new() .expect("Failed to create keyboard handler"); - re_log::info!("Keyboard handler initialized for WASD controls on /cmd_vel"); + // State for debouncing and rapid click detection let last_click_time = Rc::new(RefCell::new(Instant::now())); From a952495af4abfab8cf3f8312d92fe6a7b10cbb85 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Sat, 14 Mar 2026 12:57:40 -0700 Subject: [PATCH 5/6] fix: reposition default --- dimos/src/interaction/keyboard.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index b3f631b711ec..67afe4e39c31 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -140,13 +140,12 @@ impl KeyboardHandler { /// Clickable: clicking the overlay toggles engaged state. pub fn draw_overlay(&mut self, ctx: &egui::Context) { let screen_rect = ctx.content_rect(); - // Default position: bottom-right of the 3D viewport area - let overlay_width = 140.0; + // Default position: bottom-left, just above the timeline bar let overlay_height = 160.0; - let right_panel_offset = 320.0; + let left_margin = 12.0; let bottom_timeline_offset = 120.0; let default_pos = egui::pos2( - screen_rect.max.x - overlay_width - right_panel_offset, + screen_rect.min.x + left_margin, screen_rect.max.y - overlay_height - bottom_timeline_offset, ); From d22140933a37967573cfb8275e6a21ec397bc4b8 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Sat, 14 Mar 2026 13:16:51 -0700 Subject: [PATCH 6/6] fix: default position --- dimos/src/interaction/keyboard.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index 67afe4e39c31..1e7d51f3c71f 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -150,6 +150,7 @@ impl KeyboardHandler { ); let area_response = egui::Area::new("keyboard_hud".into()) + .pivot(egui::Align2::LEFT_BOTTOM) .default_pos(default_pos) .movable(true) .order(egui::Order::Foreground)