Feature Request: Coordinate-based mouse control for canvas/drawing apps (working prototype included)

[mauricekindermann]

I'm building a big canvas-based design tool and needed to test drag interactions on a FabricJS canvas. The existing drag() method works great for element-to-element dragging, but canvas tools need coordinate-based mouse control.

I haven't found any discussions or examples showing what I need to do, so I ended up building a small Mouse class that talks directly to Playwright's mouse API via the Client. I also wrapped the Pest browser in a CavasTestHelpers class so I can call canvas()-> and chain domain-specific methods like assertCanvasObjectCount(2) while still proxying through to Pest's methods via __call. Saves me writing lots of repetitive raw ->script() tags and needing to escape the visit constantly.

My tests are passing and --debug shows objects are moving as you'd expect. I'd love to see the mouse functionality built into the plugin natively.

Example:

it('can scale objects by dragging control handles', function () {
    canvas()->visit('/editor')
        ->click('@add-rectangle')
        ->selectFirstObject()
        ->dragControlWithMouse('br', 100, 100)
        ->assertScaleXGreaterThan(1.0);
});

it('can manipulate multiple objects', function () {
    canvas()->visit('/editor')
        ->click('@add-rectangle')
        ->click('@add-rectangle')
        ->assertCanvasObjectCount(2)
        ->selectFirstObject()
        ->dragObject(80, 40)
        ->selectLastObject()
        ->dragControlWithMouse('br', 60, 60)
        ->assertScaleXGreaterThan(1.0);
});

it('can perform precise multi-step drag with withMouse', function () {
    canvas()->visit('/editor')
        ->click('@add-rectangle')
        ->selectFirstObject()
        ->withMouse(function ($mouse, $canvas) {
            $pos = $canvas->controlScreenPosition('br');
            $mouse
                ->move($pos['x'], $pos['y'])
                ->down()
                ->move($pos['x'] + 50, $pos['y'] + 25)
                ->move($pos['x'] + 100, $pos['y'] + 50)
                ->up();
        })
        ->assertScaleXGreaterThan(1.0);
});

Proof of concept:

// tests/Browser/Concerns/Mouse.php
<?php

declare(strict_types=1);

namespace Tests\Browser\Concerns;

use Pest\Browser\Playwright\Client;

/**
 * @internal
 */
final readonly class Mouse
{
    public function __construct(
        private string $pageGuid,
    ) {
        //
    }

    public function move(int $x, int $y, int $steps = 1): self
    {
        $this->send('mouseMove', ['x' => $x, 'y' => $y, 'steps' => $steps]);

        return $this;
    }

    public function down(string $button = 'left', int $clickCount = 1): self
    {
        $this->send('mouseDown', ['button' => $button, 'clickCount' => $clickCount]);

        return $this;
    }

    public function up(string $button = 'left', int $clickCount = 1): self
    {
        $this->send('mouseUp', ['button' => $button, 'clickCount' => $clickCount]);

        return $this;
    }

    public function click(int $x, int $y, string $button = 'left'): self
    {
        $this->send('mouseClick', ['x' => $x, 'y' => $y, 'button' => $button, 'clickCount' => 1]);

        return $this;
    }

    public function dblclick(int $x, int $y, string $button = 'left'): self
    {
        $this->send('mouseClick', ['x' => $x, 'y' => $y, 'button' => $button, 'clickCount' => 2]);

        return $this;
    }

    public function wheel(int $deltaX, int $deltaY): self
    {
        $this->send('mouseWheel', ['deltaX' => $deltaX, 'deltaY' => $deltaY]);

        return $this;
    }

    public function drag(int $fromX, int $fromY, int $toX, int $toY, int $steps = 10): self
    {
        $this->move($fromX, $fromY)->down();

        for ($i = 1; $i <= $steps; $i++) {
            $progress = $i / $steps;
            $this->move(
                (int) ($fromX + ($toX - $fromX) * $progress),
                (int) ($fromY + ($toY - $fromY) * $progress),
            );
            usleep(10_000);
        }

        return $this->up();
    }

    /**
     * @param  array<string, mixed>  $params
     */
    private function send(string $method, array $params = []): void
    {
        iterator_to_array(Client::instance()->execute($this->pageGuid, $method, $params));
    }
}

// tests/Browser/Concerns/InteractsWithMouse.php
<?php

declare(strict_types=1);

namespace Tests\Browser\Concerns;

use Pest\Browser\Playwright\Page;
use ReflectionClass;

trait InteractsWithMouse
{
    public function mouse(): Mouse
    {
        return new Mouse($this->resolvePageGuid());
    }

    public function withMouse(callable $callback): self
    {
        $callback($this->mouse(), $this);

        return $this;
    }

    public function mouseMove(int $x, int $y, int $steps = 1): self
    {
        $this->mouse()->move($x, $y, $steps);

        return $this;
    }

    public function mouseClick(int $x, int $y): self
    {
        $this->mouse()->click($x, $y);

        return $this;
    }

    public function mouseDrag(int $fromX, int $fromY, int $toX, int $toY, int $steps = 10): self
    {
        $this->mouse()->drag($fromX, $fromY, $toX, $toY, $steps);

        return $this;
    }

    private function resolvePageGuid(): string
    {
        $page = $this->resolvePage();

        return (new ReflectionClass($page))->getProperty('guid')->getValue($page);
    }

    private function resolvePage(): Page
    {
        if (method_exists($this, 'page')) {
            return $this->page();
        }

        return $this->browser->page();
    }
}

The only hack is using reflection to get the page GUID since it's private. If this were built into Pest it wouldn't be needed.