Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/RemoteViewer.Client/NativeMethods.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ KEYBDINPUT
MOUSEINPUT
KEYBD_EVENT_FLAGS
MOUSE_EVENT_FLAGS
VIRTUAL_KEY
MapVirtualKey
GetSystemMetrics
SYSTEM_METRICS_INDEX

Expand Down
1 change: 0 additions & 1 deletion src/RemoteViewer.Client/RemoteViewer.Client.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
</ItemGroup>

<ItemGroup Condition="$(TargetFramework.Contains('windows'))">
<PackageReference Include="H.InputSimulator" Version="1.5.0" />
<PackageReference Include="Microsoft.Windows.CsWin32" Version="0.3.269">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
Expand Down
152 changes: 152 additions & 0 deletions src/RemoteViewer.Client/Services/InputInjection/InputHelpers.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
#if WINDOWS
using Windows.Win32;
using Windows.Win32.UI.Input.KeyboardAndMouse;

namespace RemoteViewer.Client.Services.InputInjection;

internal static class InputHelpers
{
public static void SendMouseMove(int absX, int absY)
{
var input = CreateMouseInput(
absX,
absY,
MOUSE_EVENT_FLAGS.MOUSEEVENTF_MOVE |
MOUSE_EVENT_FLAGS.MOUSEEVENTF_ABSOLUTE |
MOUSE_EVENT_FLAGS.MOUSEEVENTF_VIRTUALDESK);

SendSingleInput(input);
}

public static void SendMouseButton(MOUSE_EVENT_FLAGS flags, uint mouseData = 0)
{
var input = CreateMouseInput(0, 0, flags, mouseData);
SendSingleInput(input);
}

public static void SendMouseWheel(int scrollAmount, bool horizontal)
{
var flags = horizontal
? MOUSE_EVENT_FLAGS.MOUSEEVENTF_HWHEEL
: MOUSE_EVENT_FLAGS.MOUSEEVENTF_WHEEL;

var input = CreateMouseInput(0, 0, flags, (uint)scrollAmount);
SendSingleInput(input);
}

public static void SendKeyDown(ushort vk)
{
var flags = IsExtendedKey(vk) ? KEYBD_EVENT_FLAGS.KEYEVENTF_EXTENDEDKEY : 0;
var input = CreateKeyboardInput(vk, flags);
SendSingleInput(input);
}

public static void SendKeyUp(ushort vk)
{
var flags = KEYBD_EVENT_FLAGS.KEYEVENTF_KEYUP;
if (IsExtendedKey(vk))
flags |= KEYBD_EVENT_FLAGS.KEYEVENTF_EXTENDEDKEY;

var input = CreateKeyboardInput(vk, flags);
SendSingleInput(input);
}

public static void SendUnicodeText(string text)
{
if (text.Length == 0)
return;

var inputs = new INPUT[text.Length * 2];
var index = 0;

foreach (var c in text)
{
inputs[index++] = CreateUnicodeInput(c, 0);
inputs[index++] = CreateUnicodeInput(c, KEYBD_EVENT_FLAGS.KEYEVENTF_KEYUP);
}

SendInputs(inputs);
}

private static INPUT CreateMouseInput(int absX, int absY, MOUSE_EVENT_FLAGS flags, uint mouseData = 0)
{
var input = new INPUT { type = INPUT_TYPE.INPUT_MOUSE };
input.Anonymous.mi = new MOUSEINPUT
{
dx = absX,
dy = absY,
mouseData = mouseData,
dwFlags = flags,
time = 0,
dwExtraInfo = nuint.Zero
};
return input;
}

private static INPUT CreateKeyboardInput(ushort vk, KEYBD_EVENT_FLAGS flags)
{
var input = new INPUT { type = INPUT_TYPE.INPUT_KEYBOARD };
input.Anonymous.ki = new KEYBDINPUT
{
wVk = (VIRTUAL_KEY)vk,
wScan = (ushort)(PInvoke.MapVirtualKey(vk, 0) & 0xFFu),
dwFlags = flags,
time = 0,
dwExtraInfo = nuint.Zero
};
return input;
}

private static INPUT CreateUnicodeInput(char c, KEYBD_EVENT_FLAGS flags)
{
ushort scanCode = c;

// Handle extended keys: if scan code has 0xE0 prefix
if ((scanCode & 0xFF00) == 0xE000)
{
flags |= KEYBD_EVENT_FLAGS.KEYEVENTF_EXTENDEDKEY;
}

var input = new INPUT { type = INPUT_TYPE.INPUT_KEYBOARD };
input.Anonymous.ki = new KEYBDINPUT
{
wVk = 0,
wScan = scanCode,
dwFlags = flags | KEYBD_EVENT_FLAGS.KEYEVENTF_UNICODE,
time = 0,
dwExtraInfo = nuint.Zero
};
return input;
}

private static unsafe void SendSingleInput(INPUT input)
{
PInvoke.SendInput(new ReadOnlySpan<INPUT>(ref input), sizeof(INPUT));
}

private static unsafe void SendInputs(INPUT[] inputs)
{
PInvoke.SendInput(inputs, sizeof(INPUT));
}

private static bool IsExtendedKey(ushort vk) => vk is
0x03 or // VK_CANCEL (Break)
0x11 or // VK_CONTROL (generic)
0x12 or // VK_MENU (generic Alt)
0x21 or // VK_PRIOR (Page Up)
0x22 or // VK_NEXT (Page Down)
0x23 or // VK_END
0x24 or // VK_HOME
0x25 or // VK_LEFT
0x26 or // VK_UP
0x27 or // VK_RIGHT
0x28 or // VK_DOWN
0x2C or // VK_SNAPSHOT (Print Screen)
0x2D or // VK_INSERT
0x2E or // VK_DELETE
0x6F or // VK_DIVIDE (Numpad /)
0x90 or // VK_NUMLOCK
0xA3 or // VK_RCONTROL
0xA5; // VK_RMENU (Right Alt)
}
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using RemoteViewer.Shared;
using RemoteViewer.Shared.Protocol;
using Windows.Win32;
using WindowsInput;
using Windows.Win32.UI.Input.KeyboardAndMouse;

using ProtocolMouseButton = RemoteViewer.Shared.Protocol.MouseButton;

Expand All @@ -19,9 +19,8 @@ public class WindowsInputInjectionService : IInputInjectionService, IDisposable
private readonly IWin32SessionService? _sessionService;
private readonly SessionRecorderRpcClient? _rpcClient;
private readonly ILogger<WindowsInputInjectionService> _logger;
private readonly InputSimulator _simulator = new();

private readonly ConcurrentDictionary<VirtualKeyCode, DateTime> _pressedModifiers = new();
private readonly ConcurrentDictionary<ushort, DateTime> _pressedModifiers = new();
private DateTime _lastInputTime = DateTime.UtcNow;

private float _verticalScrollAccumulator;
Expand Down Expand Up @@ -122,7 +121,7 @@ private Task ActualInjectMouseMove(DisplayInfo display, float normalizedX, float
{
this.CheckAndReleaseStuckModifiers();
var (absX, absY) = NormalizedToAbsolute(display, normalizedX, normalizedY);
this._simulator.Mouse.MoveMouseToPositionOnVirtualDesktop(absX, absY);
InputHelpers.SendMouseMove(absX, absY);
}, ct);
return Task.CompletedTask;
}
Expand All @@ -135,44 +134,25 @@ private Task ActualInjectMouseButton(DisplayInfo display, ProtocolMouseButton bu
this._lastInputTime = DateTime.UtcNow;

var (absX, absY) = NormalizedToAbsolute(display, normalizedX, normalizedY);
this._simulator.Mouse.MoveMouseToPositionOnVirtualDesktop(absX, absY);
InputHelpers.SendMouseMove(absX, absY);

switch (button)
var (flags, mouseData) = button switch
{
case ProtocolMouseButton.Left:
if (isDown)
this._simulator.Mouse.LeftButtonDown();
else
this._simulator.Mouse.LeftButtonUp();
break;
case ProtocolMouseButton.Right:
if (isDown)
this._simulator.Mouse.RightButtonDown();
else
this._simulator.Mouse.RightButtonUp();
break;
case ProtocolMouseButton.Middle:
if (isDown)
this._simulator.Mouse.MiddleButtonDown();
else
this._simulator.Mouse.MiddleButtonUp();
break;
case ProtocolMouseButton.XButton1:
if (isDown)
this._simulator.Mouse.XButtonDown(1);
else
this._simulator.Mouse.XButtonUp(1);
break;
case ProtocolMouseButton.XButton2:
if (isDown)
this._simulator.Mouse.XButtonDown(2);
else
this._simulator.Mouse.XButtonUp(2);
break;
default:
this._logger.LogWarning("Unknown mouse button: {Button}", button);
break;
ProtocolMouseButton.Left => (isDown ? MOUSE_EVENT_FLAGS.MOUSEEVENTF_LEFTDOWN : MOUSE_EVENT_FLAGS.MOUSEEVENTF_LEFTUP, 0u),
ProtocolMouseButton.Right => (isDown ? MOUSE_EVENT_FLAGS.MOUSEEVENTF_RIGHTDOWN : MOUSE_EVENT_FLAGS.MOUSEEVENTF_RIGHTUP, 0u),
ProtocolMouseButton.Middle => (isDown ? MOUSE_EVENT_FLAGS.MOUSEEVENTF_MIDDLEDOWN : MOUSE_EVENT_FLAGS.MOUSEEVENTF_MIDDLEUP, 0u),
ProtocolMouseButton.XButton1 => (isDown ? MOUSE_EVENT_FLAGS.MOUSEEVENTF_XDOWN : MOUSE_EVENT_FLAGS.MOUSEEVENTF_XUP, 1u),
ProtocolMouseButton.XButton2 => (isDown ? MOUSE_EVENT_FLAGS.MOUSEEVENTF_XDOWN : MOUSE_EVENT_FLAGS.MOUSEEVENTF_XUP, 2u),
_ => (default(MOUSE_EVENT_FLAGS), 0u)
};

if (flags == default)
{
this._logger.LogWarning("Unknown mouse button: {Button}", button);
return;
}

InputHelpers.SendMouseButton(flags, mouseData);
}, ct);
return Task.CompletedTask;
}
Expand All @@ -185,21 +165,21 @@ private Task ActualInjectMouseWheel(DisplayInfo display, float deltaX, float del
this._lastInputTime = DateTime.UtcNow;

var (absX, absY) = NormalizedToAbsolute(display, normalizedX, normalizedY);
this._simulator.Mouse.MoveMouseToPositionOnVirtualDesktop(absX, absY);
InputHelpers.SendMouseMove(absX, absY);

this._verticalScrollAccumulator += deltaY;
var verticalClicks = (int)this._verticalScrollAccumulator;
if (verticalClicks != 0)
{
this._simulator.Mouse.VerticalScroll(verticalClicks);
InputHelpers.SendMouseWheel(verticalClicks * 120, horizontal: false);
this._verticalScrollAccumulator -= verticalClicks;
}

this._horizontalScrollAccumulator += deltaX;
var horizontalClicks = (int)this._horizontalScrollAccumulator;
if (horizontalClicks != 0)
{
this._simulator.Mouse.HorizontalScroll(horizontalClicks);
InputHelpers.SendMouseWheel(horizontalClicks * 120, horizontal: true);
this._horizontalScrollAccumulator -= horizontalClicks;
}
}, ct);
Expand All @@ -213,27 +193,25 @@ private Task ActualInjectKey(ushort keyCode, bool isDown, CancellationToken ct)
this.CheckAndReleaseStuckModifiers();
this._lastInputTime = DateTime.UtcNow;

var vk = (VirtualKeyCode)keyCode;

if (IsModifierKey(vk))
if (IsModifierKey(keyCode))
{
if (isDown)
{
this._pressedModifiers[vk] = DateTime.UtcNow;
this._pressedModifiers[keyCode] = DateTime.UtcNow;
}
else
{
this._pressedModifiers.TryRemove(vk, out _);
this._pressedModifiers.TryRemove(keyCode, out _);
}
}

if (isDown)
{
this._simulator.Keyboard.KeyDown(vk);
InputHelpers.SendKeyDown(keyCode);
}
else
{
this._simulator.Keyboard.KeyUp(vk);
InputHelpers.SendKeyUp(keyCode);
}
}, ct);
return Task.CompletedTask;
Expand All @@ -245,7 +223,7 @@ private Task ActualInjectText(string text, CancellationToken ct)
{
this.CheckAndReleaseStuckModifiers();
this._lastInputTime = DateTime.UtcNow;
this._simulator.Keyboard.TextEntry(text);
InputHelpers.SendUnicodeText(text);
}, ct);
return Task.CompletedTask;
}
Expand All @@ -257,7 +235,7 @@ private Task ActualReleaseAllModifiers(CancellationToken ct)
foreach (var vk in this._pressedModifiers.Keys)
{
this._logger.LogInformation("Releasing modifier key on cleanup: {Key}", vk);
this._simulator.Keyboard.KeyUp(vk);
InputHelpers.SendKeyUp(vk);
}
this._pressedModifiers.Clear();
}, ct);
Expand All @@ -282,11 +260,11 @@ private async Task ExecuteWithFallbackAsync(string? connectionId, Func<string, T
await localAction();
}

private static bool IsModifierKey(VirtualKeyCode vk) => vk is
VirtualKeyCode.LSHIFT or VirtualKeyCode.RSHIFT or
VirtualKeyCode.LCONTROL or VirtualKeyCode.RCONTROL or
VirtualKeyCode.LMENU or VirtualKeyCode.RMENU or
VirtualKeyCode.LWIN or VirtualKeyCode.RWIN;
private static bool IsModifierKey(ushort vk) => vk is
0xA0 or 0xA1 or // VK_LSHIFT, VK_RSHIFT
0xA2 or 0xA3 or // VK_LCONTROL, VK_RCONTROL
0xA4 or 0xA5 or // VK_LMENU, VK_RMENU
0x5B or 0x5C; // VK_LWIN, VK_RWIN

private void CheckAndReleaseStuckModifiers()
{
Expand All @@ -304,7 +282,7 @@ private void CheckAndReleaseStuckModifiers()
if (now - pressedTime >= s_modifierTimeout)
{
this._logger.LogWarning("Auto-releasing stuck modifier key: {Key} (held for {Duration:F1}s)", vk, (now - pressedTime).TotalSeconds);
this._simulator.Keyboard.KeyUp(vk);
InputHelpers.SendKeyUp(vk);
this._pressedModifiers.TryRemove(vk, out _);
}
}
Expand Down
1 change: 0 additions & 1 deletion src/RemoteViewer.Client/Views/About/AboutViewModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ public partial class AboutViewModel : ObservableObject
new("Serilog", "https://serilog.net/", "Apache-2.0", "https://github.com/serilog/serilog/blob/dev/LICENSE"),
new("Quamotion.TurboJpegWrapper", "https://github.com/quamotion/AS.TurboJpegWrapper", "MIT", "https://github.com/quamotion/AS.TurboJpegWrapper/blob/master/LICENSE"),
new("ZiggyCreatures.FusionCache", "https://github.com/ZiggyCreatures/FusionCache", "MIT", "https://github.com/ZiggyCreatures/FusionCache/blob/main/LICENSE"),
new("H.InputSimulator", "https://github.com/HavenDV/H.InputSimulator", "MIT", "https://github.com/HavenDV/H.InputSimulator/blob/master/LICENSE.md"),
new("PolyType", "https://github.com/eiriktsarpalis/PolyType", "MIT", "https://github.com/eiriktsarpalis/PolyType/blob/main/LICENSE"),
];

Expand Down