From 18a4bb27662f959c10ddb3b829934ffb0ebbe96c Mon Sep 17 00:00:00 2001 From: Iason Paraskevopoulos Date: Wed, 28 Jan 2026 22:42:23 +0000 Subject: [PATCH 1/3] fix: wrong display issue Fixes an issue where the window was placed in the wrong display in a system with 3 displays. --- core/src/graphics/graphics_context.rs | 5 +- core/src/lib.rs | 184 ++++------------- core/src/window_manager.rs | 282 ++++++++++++++++++++++++++ core/tests/src/main.rs | 6 + core/tests/src/screenshare_client.rs | 49 +++++ 5 files changed, 382 insertions(+), 144 deletions(-) create mode 100644 core/src/window_manager.rs diff --git a/core/src/graphics/graphics_context.rs b/core/src/graphics/graphics_context.rs index 7728795f..2211bebb 100644 --- a/core/src/graphics/graphics_context.rs +++ b/core/src/graphics/graphics_context.rs @@ -171,10 +171,9 @@ impl<'a> GraphicsContext<'a> { /// # Platform-Specific Behavior /// /// - **Windows**: Initializes DirectComposition for transparent overlay rendering - pub fn new(window: Window, texture_path: String, scale: f64) -> OverlayResult { + pub fn new(window_arc: Arc, texture_path: String, scale: f64) -> OverlayResult { log::info!("GraphicsContext::new"); - let size = window.inner_size(); - let window_arc = Arc::new(window); + let size = window_arc.inner_size(); let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { backends: wgpu::Backends::PRIMARY, ..Default::default() diff --git a/core/src/lib.rs b/core/src/lib.rs index 986c0443..96e957e9 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -23,6 +23,7 @@ pub mod utils { } pub(crate) mod overlay_window; +pub(crate) mod window_manager; use capture::capturer::{poll_stream, Capturer}; use graphics::graphics_context::GraphicsContext; @@ -42,28 +43,21 @@ use std::thread::JoinHandle; use thiserror::Error; use utils::geometry::{Extent, Frame}; use winit::application::ApplicationHandler; -use winit::dpi::{LogicalSize, PhysicalPosition}; use winit::error::EventLoopError; use winit::event::WindowEvent; use winit::event_loop::{ActiveEventLoop, EventLoop, EventLoopProxy}; use winit::monitor::MonitorHandle; #[cfg(target_os = "macos")] -use winit::platform::macos::{EventLoopBuilderExtMacOS, WindowExtMacOS}; +use winit::platform::macos::EventLoopBuilderExtMacOS; #[cfg(target_os = "windows")] use winit::platform::windows::WindowExtWindows; -use winit::window::{WindowAttributes, WindowLevel}; - use crate::overlay_window::DisplayInfo; use crate::room_service::DrawingMode; use crate::utils::geometry::Position; -// Constants for magic numbers -/// Initial size for the overlay window (width and height in logical pixels) -const OVERLAY_WINDOW_INITIAL_SIZE: f64 = 1.0; - /// Timeout in seconds for socket message reception const SOCKET_MESSAGE_TIMEOUT_SECONDS: u64 = 30; @@ -95,19 +89,6 @@ pub enum ServerError { CursorControllerCreationError, } -pub fn get_window_attributes() -> WindowAttributes { - WindowAttributes::default() - .with_title("Overlay window") - .with_window_level(WindowLevel::AlwaysOnTop) - .with_decorations(false) - .with_transparent(true) - .with_inner_size(LogicalSize::new( - OVERLAY_WINDOW_INITIAL_SIZE, - OVERLAY_WINDOW_INITIAL_SIZE, - )) - .with_content_protected(true) -} - /// Encapsulates the active remote control session components. /// /// This struct manages all the components needed for an active remote control session, @@ -186,6 +167,7 @@ pub struct Application<'a> { socket: CursorSocket, room_service: Option, event_loop_proxy: EventLoopProxy, + window_manager: Option, } #[derive(Error, Debug)] @@ -233,10 +215,11 @@ impl<'a> Application<'a> { socket, room_service: None, event_loop_proxy, + window_manager: None, }) } - fn get_available_content(&mut self) -> Vec { + fn get_available_content(&mut self, event_loop: &ActiveEventLoop) -> Vec { let mut screen_capturer = self.screen_capturer.lock().unwrap(); let res = screen_capturer.get_available_content(); @@ -245,6 +228,10 @@ impl<'a> Application<'a> { return vec![]; } + if let Some(wm) = self.window_manager.as_mut() { + let _ = wm.update(event_loop); + } + res.unwrap() } @@ -277,7 +264,6 @@ impl<'a> Application<'a> { &mut self, screenshare_input: ScreenShareMessage, monitors: Vec, - event_loop: &ActiveEventLoop, ) -> Result<(), ServerError> { log::info!( "screenshare: resolution: {:?} content: {} accessibility_permission: {} use_av1: {}", @@ -352,11 +338,7 @@ impl<'a> Application<'a> { let monitor = screen_capturer.get_selected_monitor(&monitors, screenshare_input.content.id); drop(screen_capturer); - let res = self.create_overlay_window( - monitor, - event_loop, - screenshare_input.accessibility_permission, - ); + let res = self.create_overlay_window(monitor, screenshare_input.accessibility_permission); if let Err(e) = res { self.stop_screenshare(); log::error!("screenshare: error creating overlay window: {e:?}"); @@ -388,58 +370,21 @@ impl<'a> Application<'a> { fn create_overlay_window( &mut self, selected_monitor: MonitorHandle, - event_loop: &ActiveEventLoop, accessibility_permission: bool, ) -> Result<(), ServerError> { log::info!("create_overlay_window: selected_monitor: {selected_monitor:?} {accessibility_permission}",); - let attributes = get_window_attributes(); - let window = match event_loop.create_window(attributes) { - Ok(window) => window, - Err(_error) => { - return Err(ServerError::WindowCreationError); - } - }; - - #[cfg(target_os = "linux")] - { - /* This is needed for getting the system picker for screen sharing. */ - let _ = window.request_inner_size(selected_monitor.size().clone()); - } - - let res = window.set_cursor_hittest(false); - if let Err(_error) = res { - return Err(ServerError::CursorHittestError); - } - - #[cfg(target_os = "windows")] - { - window.set_skip_taskbar(true); - } - - #[cfg(target_os = "macos")] - { - window.set_has_shadow(false); - } - window.set_visible(true); + let window = self + .window_manager + .as_mut() + .ok_or(ServerError::WindowCreationError)? + .show_window(&selected_monitor) + .map_err(|e| { + log::error!("create_overlay_window: Error showing window: {:?}", e); + ServerError::from(e) + })?; let monitor_position = selected_monitor.position(); - let mut monitor_scale = 1.0; - if let Some(active_monitor) = window.current_monitor() { - monitor_scale = active_monitor.scale_factor(); - } - let monitor_position_x = (monitor_position.x as f64) * monitor_scale; - let monitor_position_y = (monitor_position.y as f64) * monitor_scale; - window.set_outer_position(PhysicalPosition::new( - monitor_position_x, - monitor_position_y, - )); - - let res = set_fullscreen(&window, selected_monitor.clone()); - if let Err(error) = res { - log::error!("create_overlay_window: Error setting fullscreen {error:?}"); - return Err(ServerError::FullscreenError); - } let window_position = match window.outer_position() { Ok(position) => position, @@ -531,6 +476,9 @@ impl<'a> Application<'a> { fn destroy_overlay_window(&mut self) { log::info!("destroy_overlay_window"); + if let Some(wm) = self.window_manager.as_mut() { + wm.hide_active_window(); + } self.remote_control = None; } @@ -701,7 +649,7 @@ impl<'a> ApplicationHandler for Application<'a> { } UserEvent::GetAvailableContent => { log::debug!("user_event: Get available content"); - let content = self.get_available_content(); + let content = self.get_available_content(event_loop); if content.is_empty() { log::error!("user_event: No available content"); sentry_utils::upload_logs_event("No available content".to_string()); @@ -724,7 +672,7 @@ impl<'a> ApplicationHandler for Application<'a> { .available_monitors() .collect::>(); - let result_message = match self.screenshare(data, monitors, event_loop) { + let result_message = match self.screenshare(data, monitors) { Ok(_) => Ok(()), Err(e) => { log::error!("user_event: Screen share failed: {e:?}"); @@ -1013,7 +961,18 @@ impl<'a> ApplicationHandler for Application<'a> { } } - fn resumed(&mut self, _event_loop: &ActiveEventLoop) {} + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window_manager.is_none() { + log::info!("Application::resumed: initializing WindowManager"); + match window_manager::WindowManager::new(event_loop) { + Ok(wm) => self.window_manager = Some(wm), + Err(e) => log::error!( + "Application::resumed: failed to initialize WindowManager: {:?}", + e + ), + } + } + } // Once we get movement input from guest, we will call Window::request_redraw fn window_event( @@ -1037,6 +996,13 @@ impl<'a> ApplicationHandler for Application<'a> { let cursor_controller = &mut remote_control.cursor_controller; remote_control.gfx.draw(cursor_controller); } + WindowEvent::ScaleFactorChanged { .. } + | WindowEvent::Resized(_) + | WindowEvent::Moved(_) => { + if let Some(wm) = self.window_manager.as_mut() { + let _ = wm.update(event_loop); + } + } _ => {} } } @@ -1229,67 +1195,3 @@ impl RenderEventLoop { }) } } - -#[derive(Error, Debug)] -enum FullscreenError { - #[error("Failed to get raw window handle")] - #[cfg(target_os = "macos")] - GetRawWindowHandleError, - #[error("Failed to get NSView")] - #[cfg(target_os = "macos")] - GetNSViewError, - #[error("Failed to get NSWindow")] - #[cfg(target_os = "macos")] - GetNSWindowError, - #[error("Failed to get raw window handle")] - #[cfg(target_os = "macos")] - FailedToGetRawWindowHandle, -} - -fn set_fullscreen( - window: &winit::window::Window, - selected_monitor: MonitorHandle, -) -> Result<(), FullscreenError> { - log::info!("set_fullscreen: {selected_monitor:?}"); - #[cfg(target_os = "macos")] - { - /* WA for putting the window in the right place. */ - window.set_maximized(true); - window.set_simple_fullscreen(true); - - use objc2::rc::Retained; - use objc2_app_kit::NSMainMenuWindowLevel; - use objc2_app_kit::NSView; - use raw_window_handle::HasWindowHandle; - use raw_window_handle::RawWindowHandle; - - let raw_handle = window - .window_handle() - .map_err(|_| FullscreenError::GetRawWindowHandleError)?; - if let RawWindowHandle::AppKit(handle) = raw_handle.as_raw() { - let view = handle.ns_view.as_ptr(); - let ns_view: Option> = unsafe { Retained::retain(view.cast()) }; - if ns_view.is_none() { - return Err(FullscreenError::GetNSViewError); - } - let ns_view = ns_view.unwrap(); - let ns_window = ns_view.window(); - if ns_window.is_none() { - return Err(FullscreenError::GetNSWindowError); - } - let ns_window = ns_window.unwrap(); - /* This is a hack to make the overlay window to appear above the main menu. */ - ns_window.setLevel(NSMainMenuWindowLevel + 1); - return Ok(()); - } - Err(FullscreenError::FailedToGetRawWindowHandle) - } - #[cfg(any(target_os = "windows", target_os = "linux"))] - { - use winit::window::Fullscreen; - - window.set_fullscreen(Some(Fullscreen::Borderless(Some(selected_monitor)))); - - Ok(()) - } -} diff --git a/core/src/window_manager.rs b/core/src/window_manager.rs new file mode 100644 index 00000000..f03c2935 --- /dev/null +++ b/core/src/window_manager.rs @@ -0,0 +1,282 @@ +use std::sync::Arc; +use thiserror::Error; +use winit::dpi::{LogicalPosition, LogicalSize, PhysicalPosition}; +use winit::event_loop::ActiveEventLoop; +use winit::monitor::MonitorHandle; +use winit::window::{Window, WindowAttributes, WindowLevel}; + +#[cfg(target_os = "macos")] +use winit::platform::macos::WindowExtMacOS; + +use crate::ServerError; + +// Constants for magic numbers +/// Initial size for the overlay window (width and height in logical pixels) +const OVERLAY_WINDOW_INITIAL_SIZE: f64 = 1.0; + +fn get_window_attributes() -> WindowAttributes { + WindowAttributes::default() + .with_title("Overlay window") + .with_window_level(WindowLevel::AlwaysOnTop) + .with_decorations(false) + .with_transparent(true) + .with_inner_size(LogicalSize::new( + OVERLAY_WINDOW_INITIAL_SIZE, + OVERLAY_WINDOW_INITIAL_SIZE, + )) + .with_content_protected(true) +} + +#[derive(Error, Debug)] +pub enum WindowManagerError { + #[error("Failed to create window")] + WindowCreationError, + #[error("Monitor not found")] + MonitorNotFound, + #[error("Fullscreen error: {0}")] + FullscreenError(String), +} + +impl From for ServerError { + fn from(err: WindowManagerError) -> Self { + match err { + WindowManagerError::WindowCreationError => ServerError::WindowCreationError, + WindowManagerError::MonitorNotFound => ServerError::WindowCreationError, // Map to WindowCreationError for now + WindowManagerError::FullscreenError(_) => ServerError::FullscreenError, + } + } +} + +struct WindowEntry { + window: Arc, + monitor_position: PhysicalPosition, +} + +pub struct WindowManager { + windows: Vec, + active_window_index: Option, +} + +impl WindowManager { + pub fn new(event_loop: &ActiveEventLoop) -> Result { + log::info!("WindowManager::new: creating windows for available monitors"); + let mut windows = Vec::new(); + + for monitor in event_loop.available_monitors() { + let window_entry = Self::create_window_entry(event_loop, &monitor)?; + windows.push(window_entry); + } + + Ok(Self { + windows, + active_window_index: None, + }) + } + + fn create_window_entry( + event_loop: &ActiveEventLoop, + monitor: &MonitorHandle, + ) -> Result { + let attributes = get_window_attributes(); + let window = event_loop + .create_window(attributes) + .map_err(|_| WindowManagerError::WindowCreationError)?; + + let window = Arc::new(window); + + #[cfg(target_os = "linux")] + { + /* This is needed for getting the system picker for screen sharing. */ + let _ = window.request_inner_size(monitor.size().clone()); + } + + let _ = window.set_cursor_hittest(false); + + #[cfg(target_os = "windows")] + { + use winit::platform::windows::WindowExtWindows; + window.set_skip_taskbar(true); + } + + #[cfg(target_os = "macos")] + { + window.set_has_shadow(false); + } + + let monitor_position = monitor.position(); + let logical_position = monitor_position.to_logical::(monitor.scale_factor()); + + let final_position = + LogicalPosition::new(logical_position.x + 30., logical_position.y + 30.); + + window.set_outer_position(final_position); + window.set_visible(false); + + Ok(WindowEntry { + window, + monitor_position, + }) + } + + pub fn show_window( + &mut self, + monitor: &MonitorHandle, + ) -> Result, WindowManagerError> { + let monitor_position = monitor.position(); + log::info!( + "WindowManager::show_window: looking for window at {:?}", + monitor_position + ); + + for entry in &self.windows { + log::info!( + "WindowManager::show_window: display {:?} window {:?}", + entry.monitor_position, + entry.window.outer_position() + ); + } + let index = self + .windows + .iter() + .position(|entry| entry.monitor_position == monitor_position) + .ok_or(WindowManagerError::MonitorNotFound)?; + + let window = &self.windows[index].window; + + log::info!("window fullscreen {:?}", window.fullscreen()); + if let Err(e) = set_fullscreen(window, monitor.clone()) { + log::error!( + "WindowManager::show_window: error setting fullscreen: {:?}", + e + ); + return Err(WindowManagerError::FullscreenError(e.to_string())); + } + + window.set_visible(true); + self.active_window_index = Some(index); + + Ok(window.clone()) + } + + pub fn hide_active_window(&mut self) { + if let Some(index) = self.active_window_index.take() { + log::info!( + "WindowManager::hide_active_window: hiding window at index {}", + index + ); + self.windows[index].window.set_visible(false); + } + } + + pub fn update(&mut self, event_loop: &ActiveEventLoop) -> Result<(), WindowManagerError> { + let monitors: Vec = event_loop.available_monitors().collect(); + let mut monitor_positions: Vec> = + monitors.iter().map(|m| m.position()).collect(); + + log::info!( + "WindowManager::update: checking {} monitors", + monitors.len() + ); + + self.windows.retain(|entry| { + if let Some(pos_index) = monitor_positions + .iter() + .position(|&pos| pos == entry.monitor_position) + { + monitor_positions.remove(pos_index); + true + } else { + log::info!( + "WindowManager::update: removing window at outdated position {:?}", + entry.monitor_position + ); + false + } + }); + + for position in monitor_positions { + log::info!( + "WindowManager::update: adding new window for position {:?}", + position + ); + + let monitor = monitors + .iter() + .find(|m| m.position() == position) + .ok_or(WindowManagerError::MonitorNotFound)?; + + let window_entry = Self::create_window_entry(event_loop, monitor)?; + self.windows.push(window_entry); + } + + log::info!( + "WindowManager::update: windows list length: {:?}", + self.windows.len() + ); + + Ok(()) + } +} + +#[derive(Error, Debug)] +enum FullscreenError { + #[error("Failed to get raw window handle")] + #[cfg(target_os = "macos")] + GetRawWindowHandleError, + #[error("Failed to get NSView")] + #[cfg(target_os = "macos")] + GetNSViewError, + #[error("Failed to get NSWindow")] + #[cfg(target_os = "macos")] + GetNSWindowError, + #[error("Failed to get raw window handle")] + #[cfg(target_os = "macos")] + FailedToGetRawWindowHandle, +} + +fn set_fullscreen( + window: &winit::window::Window, + selected_monitor: MonitorHandle, +) -> Result<(), FullscreenError> { + log::info!("set_fullscreen: {selected_monitor:?}"); + #[cfg(target_os = "macos")] + { + /* WA for putting the window in the right place. */ + window.set_simple_fullscreen(true); + + use objc2::rc::Retained; + use objc2_app_kit::NSMainMenuWindowLevel; + use objc2_app_kit::NSView; + use raw_window_handle::HasWindowHandle; + use raw_window_handle::RawWindowHandle; + + let raw_handle = window + .window_handle() + .map_err(|_| FullscreenError::GetRawWindowHandleError)?; + if let RawWindowHandle::AppKit(handle) = raw_handle.as_raw() { + let view = handle.ns_view.as_ptr(); + let ns_view: Option> = unsafe { Retained::retain(view.cast()) }; + if ns_view.is_none() { + return Err(FullscreenError::GetNSViewError); + } + let ns_view = ns_view.unwrap(); + let ns_window = ns_view.window(); + if ns_window.is_none() { + return Err(FullscreenError::GetNSWindowError); + } + let ns_window = ns_window.unwrap(); + /* This is a hack to make the overlay window to appear above the main menu. */ + ns_window.setLevel(NSMainMenuWindowLevel + 1); + return Ok(()); + } + Err(FullscreenError::FailedToGetRawWindowHandle) + } + #[cfg(any(target_os = "windows", target_os = "linux"))] + { + use winit::window::Fullscreen; + + window.set_fullscreen(Some(Fullscreen::Borderless(Some(selected_monitor)))); + + Ok(()) + } +} diff --git a/core/tests/src/main.rs b/core/tests/src/main.rs index 9e52e09a..93cb4ef6 100644 --- a/core/tests/src/main.rs +++ b/core/tests/src/main.rs @@ -96,6 +96,8 @@ enum ScreenshareTest { Basic, /// Test available content consistency across multiple requests AvailableContent, + /// Screen share every available monitor for 10 seconds each + EveryMonitor, } #[derive(Clone, ValueEnum, Debug)] @@ -214,6 +216,10 @@ async fn main() -> io::Result<()> { println!("Running available content test..."); screenshare_client::test_available_content_consistency()?; } + ScreenshareTest::EveryMonitor => { + println!("Running every monitor screenshare test..."); + screenshare_client::test_every_monitor()?; + } } println!("Screenshare test finished."); } diff --git a/core/tests/src/screenshare_client.rs b/core/tests/src/screenshare_client.rs index 792dd25f..ab4b8552 100644 --- a/core/tests/src/screenshare_client.rs +++ b/core/tests/src/screenshare_client.rs @@ -166,3 +166,52 @@ pub fn test_available_content_consistency() -> io::Result<()> { ))) } } + +pub fn test_every_monitor() -> io::Result<()> { + let mut socket = connect_socket()?; + println!("Connected to socket."); + + let livekit_server_url = + env::var("LIVEKIT_URL").expect("LIVEKIT_URL environment variable not set"); + socket.send_message(Message::LivekitServerUrl(livekit_server_url))?; + + let available_content = match get_available_content(&mut socket)? { + Message::AvailableContent(available_content) => available_content, + _ => return Err(io::Error::other("Failed to get available content")), + }; + + let monitors: Vec<_> = available_content + .content + .into_iter() + .filter(|c| matches!(c.content.content_type, ContentType::Display)) + .collect(); + + println!("Found {} monitors to test.", monitors.len()); + + for (i, monitor) in monitors.iter().enumerate() { + println!( + "Testing monitor {}/{} (ID: {})", + i + 1, + monitors.len(), + monitor.content.id + ); + + // Start screen share + let width = 1920.0; + let height = 1080.0; + request_screenshare(&mut socket, monitor.content.id, width, height)?; + println!("Screen share started for monitor {}.", monitor.content.id); + + std::thread::sleep(std::time::Duration::from_secs(10)); + + // Stop screen share + stop_screenshare(&mut socket)?; + println!("Screen share stopped for monitor {}.", monitor.content.id); + + // Small delay between monitors + std::thread::sleep(std::time::Duration::from_secs(1)); + } + + println!("✓ Success: All monitors tested."); + Ok(()) +} From 2584d901875b8558bf73f551a4223e6c3e13fa96 Mon Sep 17 00:00:00 2001 From: Iason Paraskevopoulos Date: Wed, 28 Jan 2026 23:08:26 +0000 Subject: [PATCH 2/3] chore: clippy --- core/src/lib.rs | 10 ---------- core/src/window_manager.rs | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/core/src/lib.rs b/core/src/lib.rs index 96e957e9..00932b61 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -51,9 +51,6 @@ use winit::monitor::MonitorHandle; #[cfg(target_os = "macos")] use winit::platform::macos::EventLoopBuilderExtMacOS; -#[cfg(target_os = "windows")] -use winit::platform::windows::WindowExtWindows; - use crate::overlay_window::DisplayInfo; use crate::room_service::DrawingMode; use crate::utils::geometry::Position; @@ -996,13 +993,6 @@ impl<'a> ApplicationHandler for Application<'a> { let cursor_controller = &mut remote_control.cursor_controller; remote_control.gfx.draw(cursor_controller); } - WindowEvent::ScaleFactorChanged { .. } - | WindowEvent::Resized(_) - | WindowEvent::Moved(_) => { - if let Some(wm) = self.window_manager.as_mut() { - let _ = wm.update(event_loop); - } - } _ => {} } } diff --git a/core/src/window_manager.rs b/core/src/window_manager.rs index f03c2935..a7275475 100644 --- a/core/src/window_manager.rs +++ b/core/src/window_manager.rs @@ -8,6 +8,9 @@ use winit::window::{Window, WindowAttributes, WindowLevel}; #[cfg(target_os = "macos")] use winit::platform::macos::WindowExtMacOS; +#[cfg(target_os = "windows")] +use winit::platform::windows::WindowExtWindows; + use crate::ServerError; // Constants for magic numbers From d230b2109df12ee74e49dade1d661d02aa207e25 Mon Sep 17 00:00:00 2001 From: Iason Paraskevopoulos Date: Thu, 29 Jan 2026 00:28:54 +0000 Subject: [PATCH 3/3] chore: clippy --- core/src/window_manager.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/window_manager.rs b/core/src/window_manager.rs index a7275475..51feb3d4 100644 --- a/core/src/window_manager.rs +++ b/core/src/window_manager.rs @@ -97,7 +97,6 @@ impl WindowManager { #[cfg(target_os = "windows")] { - use winit::platform::windows::WindowExtWindows; window.set_skip_taskbar(true); }