trigger mouse events manually for WebXR controllers

[fmarinofiuba]

Hi, I'm not sure if this is there right place to post this message.
I found this library quite useful for creating UIs for VR app in three.js.

The only problem is that if I want to use the VR controllers (like Meta Quest 2 controllers) to trigger the pointerup and pointerdown events, I couldn´'t find any public method to do that.

I read the source code but I could understand exactly what to modify. But it seems it should be relatively simple.
This modification would enable interesting uses cases for VR

I figured out the the method setMouse(x,y) is used to set the local coordinates of the pointer on the ui, but I could not trigger the mousedown or mouseup from the controller's event handler

[lo-th]

Hi all mouse event start on core/Roots.js
and is dispatch on handleEvent function

[fmarinofiuba]

Thank you, I was able to extend it so that the mouse events could be triggered from Gui.js class

[sylwesterdigital]

Hi @fmarinofiuba - I am super curious how you managed that to work with WebXR. I'm not sure how to add this to setAnimationLoop? Let say I have this prototype code:

let ui, ui_screen, ui_plane
let useEncoding = true;

function addUIL() {

    ui_plane = new THREE.Mesh(
        new THREE.PlaneGeometry(2, 0.2), 
        new THREE.MeshBasicMaterial({ transparent: true })
    );
    ui_plane.position.set(0, 1.7, 0); // Adjust position as needed

    playground.add(ui_plane);


    ui = new UIL.Gui({ w: 256, h: 32, css: 'right:10px; top:10px;', center: true });

    // Add a slider to the UI
    ui.add('slide', {
        name: 'Blur',
        min: 0,
        max: 1,
        value: playground.backgroundBlurriness,
        precision: 2,
        step: 0.01,
        callback: (value) => playground.backgroundBlurriness = value
    });

    ui_screen = new THREE.Texture(ui.canvas);
    ui_screen.minFilter = THREE.LinearFilter;
    if (useEncoding) ui_screen.encoding = THREE.sRGBEncoding;
    ui_plane.material.map = ui_screen;


}

and I try to update ui_screen in render loop:

   if(ui_screen) {
        ui_screen.needsUpdate = true;        
    }

but it does no work, any clue?

[fmarinofiuba]

Hi @fmarinofiuba - I am super curious how you managed that to work with WebXR. I'm not sure how to add this to setAnimationLoop? Let say I have this prototype code:

let ui, ui_screen, ui_plane
let useEncoding = true;

function addUIL() {

    ui_plane = new THREE.Mesh(
        new THREE.PlaneGeometry(2, 0.2), 
        new THREE.MeshBasicMaterial({ transparent: true })
    );
    ui_plane.position.set(0, 1.7, 0); // Adjust position as needed

    playground.add(ui_plane);


    ui = new UIL.Gui({ w: 256, h: 32, css: 'right:10px; top:10px;', center: true });

    // Add a slider to the UI
    ui.add('slide', {
        name: 'Blur',
        min: 0,
        max: 1,
        value: playground.backgroundBlurriness,
        precision: 2,
        step: 0.01,
        callback: (value) => playground.backgroundBlurriness = value
    });

    ui_screen = new THREE.Texture(ui.canvas);
    ui_screen.minFilter = THREE.LinearFilter;
    if (useEncoding) ui_screen.encoding = THREE.sRGBEncoding;
    ui_plane.material.map = ui_screen;


}

and I try to update ui_screen in render loop:

   if(ui_screen) {
        ui_screen.needsUpdate = true;        
    }

but it does no work, any clue?

Hi, if you check the examples section, there is one called "uil_3d".
In order to update the texture used in the 3D scene for the UI you have to define ui.onDraw function.
Here is an example

        ui.onDraw = function () {

            if( screen === null ){

                screen = new THREE.Texture( this.canvas )
                screen.minFilter = THREE.LinearFilter
                if( useEncoding ) screen.encoding = THREE.sRGBEncoding
                plane.material.map = screen;
                plane.material.needsUpdate = true;
                plane.visible = true;                
            }

            screen.needsUpdate = true
        }

You don't need to put anything in the render loop.

As for WebXR, in order to use it you need to modify the source code, since all interactions are tied to the mouse.
And the library adds event listeners for mouse or pointer by default.
What i did is to remove those listeners and add methods to explicity call the mouse events from outside.

So when I use the Oculus touch controllers, I can use its buttons and ray direction instead of the mouse

Here is my fork of UIL with the modifications https://github.com/fedemarino31/uil/

And this is a test I built
https://github.com/fedemarino31/uil/blob/gh-pages/examples/uil_vr_test.html