ci(playwright): use playwright for tests instead of selenium

.github/workflows/test_and_release.yml

@@ -56,20 +56,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Chrome
-      run: |
-        sudo apt install google-chrome-stable
-
-    - name: Check the console scripts interface
-      run: |
-        pip install seleniumbase
-        seleniumbase
-        sbase
-
-    - name: Install chromedriver
-      run: |
-        seleniumbase install chromedriver
-
     - name: Set Up Node
       uses: actions/setup-node@v4
       with:
@@ -91,11 +77,13 @@ jobs:
       run: |
         pip install .
         pip install -r tests/requirements.txt
+        # Install playwright requirements
+        playwright install
         # Run the tests with coverage so we get a coverage report too
         pip install coverage
-        # coverage run --source . -m pytest -s .
+        coverage run --source . -m pytest -s .
         # Print the coverage report
-        # coverage report -m
+        coverage report -m
 
     - name: Upload Coverage to Codecov
       uses: codecov/codecov-action@v3

examples/vue3/js_call.py

@@ -1,5 +1,5 @@
 from trame.app import get_server
-from trame_client.widgets import client, html
+from trame.widgets import client, html
 from trame_client.ui.html import DivLayout
 from trame_client.utils.testing import enable_testing
 

tests/conftest.py

@@ -6,11 +6,9 @@
 HELPER = FixtureHelper(ROOT_PATH)
 
 
-@pytest.fixture()
-def baseline_image():
-    HELPER.remove_page_urls()
-    yield
-    HELPER.remove_page_urls()
+@pytest.fixture
+def ref_dir() -> Path:
+    return Path(__file__).parent / "refs"
 
 
 @pytest.fixture

tests/refs/simple_count_1.yml

@@ -0,0 +1,3 @@
+- document:
+  - text: "1"
+  - button "Add to count"

tests/refs/simple_count_5.yml

@@ -0,0 +1,3 @@
+- document:
+  - text: "5"
+  - button "Add to count"

tests/refs/test_dynamic_template_final.yml

@@ -0,0 +1,4 @@
+- document:
+  - text: Static text 6 count = 6 tts = 2
+  - button "Update template"
+  - button "count++"

tests/refs/test_dynamic_template_initial.yml

@@ -0,0 +1,4 @@
+- document:
+  - text: Static text 2 count = 2 tts = 0
+  - button "Update template"
+  - button "count++"

tests/requirements.txt

@@ -1,8 +1,8 @@
 pytest
 pytest-asyncio
-seleniumbase
+pytest-playwright
 pixelmatch
 Pillow
 pytest-xprocess
 trame>=3.6
-trame-server>=3
+trame-server>=3

tests/test_reactivity.py

@@ -1,23 +1,30 @@
+from playwright.sync_api import expect
 import pytest
-from seleniumbase import SB
+
+from trame_client.utils.testing import assert_snapshot_matches
 
 
 @pytest.mark.parametrize("server_path", ["examples/test/reactivity.py"])
-def test_reactivity(server, baseline_image):
-    with SB() as sb:
-        url = f"http://127.0.0.1:{server.port}/"
-        sb.open(url)
-        sb.check_window(name="simple_count_1", level=3)
-        sb.assert_exact_text("1", ".countValue")
-        assert server.get("count") == 1
-        sb.click(".plusButton")
-        sb.assert_exact_text("2", ".countValue")
-        assert server.get("count") == 2
-        sb.click(".plusButton")
-        sb.click(".plusButton")
-        sb.assert_exact_text("4", ".countValue")
-        assert server.get("count") == 4
-        sb.click(".plusButton")
-        sb.assert_exact_text("5", ".countValue")
-        assert server.get("count") == 5
-        sb.check_window(name="simple_count_5", level=3)
+def test_reactivity(server, page, ref_dir):
+    url = f"http://127.0.0.1:{server.port}/"
+    page.goto(url)
+
+    assert_snapshot_matches(page, ref_dir, "simple_count_1")
+
+    plus_button = page.locator(".plusButton")
+    count_value = page.locator(".countValue")
+
+    expect(count_value).to_have_text("1")
+    assert server.get("count") == 1
+    plus_button.click()
+    expect(count_value).to_have_text("2")
+    assert server.get("count") == 2
+    plus_button.click()
+    plus_button.click()
+    expect(count_value).to_have_text("4")
+    assert server.get("count") == 4
+    plus_button.click()
+    expect(count_value).to_have_text("5")
+    assert server.get("count") == 5
+
+    assert_snapshot_matches(page, ref_dir, "simple_count_5")

tests/test_vue23.py

@@ -1,53 +1,55 @@
 import pytest
-from seleniumbase import SB
-from trame_client.utils.testing import wait_for_ready
+
+from playwright.sync_api import expect
+from trame_client.utils.testing import assert_snapshot_matches
 
 
 @pytest.mark.parametrize(
     "server_path",
     ["examples/vue2/dynamic_template.py"],
 )
-def test_dynamic_template(server, baseline_image):
-    with SB() as sb:
-        url = f"http://127.0.0.1:{server.port}/"
-        sb.open(url)
-        wait_for_ready(sb, 60)
-        sb.check_window(name="init", level=3)
-        sb.assert_exact_text("Static text 2", ".staticDiv")
-        sb.assert_exact_text("count = 2", ".countDiv")
-        sb.assert_exact_text("tts = 0", ".ttsDiv")
-        assert server.get("count") == 2
-        sb.click(".plusBtn")
-        sb.click(".plusBtn")
-        sb.assert_exact_text("Static text 2", ".staticDiv")
-        assert server.get("count") == 4
-        sb.assert_exact_text("count = 4", ".countDiv")
-        sb.assert_exact_text("tts = 0", ".ttsDiv")
-        sb.click(".updateBtn")
-        sb.click(".updateBtn")
-        sb.assert_exact_text("Static text 6", ".staticDiv")
-        assert server.get("count") == 6
-        sb.assert_exact_text("count = 6", ".countDiv")
-        sb.assert_exact_text("tts = 2", ".ttsDiv")
-        sb.check_window(name="final", level=3)
+def test_dynamic_template(server, page, ref_dir):
+    url = f"http://127.0.0.1:{server.port}/"
+    page.goto(url)
+
+    assert_snapshot_matches(page, ref_dir, "test_dynamic_template_initial")
+
+    expect(page.locator(".staticDiv")).to_have_text("Static text 2")
+    expect(page.locator(".countDiv")).to_have_text("count = 2")
+    expect(page.locator(".ttsDiv")).to_have_text("tts = 0")
+    assert server.get("count") == 2
+
+    page.locator(".plusBtn").click()
+    page.locator(".plusBtn").click()
+    expect(page.locator(".staticDiv")).to_have_text("Static text 2")
+    assert server.get("count") == 4
+    expect(page.locator(".countDiv")).to_have_text("count = 4")
+    expect(page.locator(".ttsDiv")).to_have_text("tts = 0")
+    page.locator(".updateBtn").click()
+    page.locator(".updateBtn").click()
+    expect(page.locator(".staticDiv")).to_have_text("Static text 6")
+    assert server.get("count") == 6
+    expect(page.locator(".countDiv")).to_have_text("count = 6")
+    expect(page.locator(".ttsDiv")).to_have_text("tts = 2")
+
+    assert_snapshot_matches(page, ref_dir, "test_dynamic_template_final")
 
 
 @pytest.mark.parametrize(
     "server_path",
     ["examples/vue2/js_call.py", "examples/vue3/js_call.py"],
 )
-def test_js_call(server, baseline_image):
-    with SB() as sb:
-        url = f"http://127.0.0.1:{server.port}/"
-        sb.open(url)
-        wait_for_ready(sb, 60)
-        sb.assert_exact_text("Alert", ".jsAlert")
-        assert server.get("message") == "hello world"
-        sb.click(".alertMsg")
-        sb.assert_exact_text("hello world", ".jsAlert")
-        sb.click(".alertMe")
-        sb.assert_exact_text("Yes me", ".jsAlert")
-        sb.click(".swapMsg")
-        assert server.get("message") == "dlrow olleh"
-        sb.click(".alertMsg")
-        sb.assert_exact_text("dlrow olleh", ".jsAlert")
+def test_js_call(server, page):
+    url = f"http://127.0.0.1:{server.port}/"
+    page.goto(url)
+
+    expect(page.locator(".jsAlert")).to_have_text("Alert")
+    assert server.get("message") == "hello world"
+    page.locator(".alertMsg").click()
+    expect(page.locator(".jsAlert")).to_have_text("hello world")
+    page.locator(".alertMe").click()
+    expect(page.locator(".jsAlert")).to_have_text("Yes me")
+    page.locator(".swapMsg").click()
+    assert server.get("message") == "dlrow olleh"
+    page.locator(".alertMsg").click()
+    expect(page.locator(".jsAlert")).to_have_text("dlrow olleh")

trame_client/utils/testing.py

@@ -5,6 +5,8 @@
 from PIL import Image
 from pixelmatch.contrib.PIL import pixelmatch
 
+from playwright.sync_api import expect, Page
+
 
 # ---------------------------------------------------------
 # Pytest helpers
@@ -84,56 +86,66 @@ class Starter(ProcessStarter):
         return Path(server_path).name, Starter, TrameServerMonitor
 
 
-# ---------------------------------------------------------
-# Seleniumbase helper functions
-# ---------------------------------------------------------
-
-
-def set_browser_size(sb, width=300, height=300):
-    delta_width = 0
-    delta_height = 0
-    agent = sb.get_user_agent()
-    if "Firefox" in agent:
-        delta_height = 85
-    elif "Chrome" in agent:
-        delta_height = 0
-    elif "Safari" in agent:
-        delta_height = 0
+def assert_images_match(img_test: Image, ref_path: Path, threshold=0.1):
+    img_ref = Image.open(ref_path)
+    img_diff = Image.new("RGBA", img_ref.size)
+    diff_path = ref_path.parent / f"diff_{ref_path.with_suffix('.png').name}"
 
-    sb.set_window_size(width + delta_width, height + delta_height)
-    wait_for_ready(sb)
+    mismatch = pixelmatch(img_ref, img_test, img_diff, threshold=threshold)
+    img_diff.save(diff_path)
+    assert mismatch < threshold
 
 
-def baseline_comparison(sb, check_baseline_path, threshold=0.1):
-    baseline_test = Path(check_baseline_path)
-    baseline_refs = baseline_test.parent.glob("baseline_ref*.png")
-    baseline_diff = baseline_test.with_name("baseline_diff.png")
+# ---------------------------------------------------------
+# Playwright helpers
+# ---------------------------------------------------------
 
-    img_test = Image.open(baseline_test)
-    min_mismatch = 1000000
 
-    for baseline_ref in baseline_refs:
-        img_ref = Image.open(baseline_ref)
-        img_diff = Image.new("RGBA", img_ref.size)
-        baseline_diff = (
-            baseline_ref.parent / f"baseline_diff{baseline_ref.name[12:-4]}.png"
-        )
+def assert_screenshot_matches(
+    page: Page, ref_dir: Path, name: str, threshold: float = 0.1
+):
+    img_dir = ref_dir / name
+    ref_images = list(img_dir.glob("ref*.png"))
+    if not ref_images:
+        print(f"No reference images exist in {img_dir}. Creating one...")
+        img_dir.mkdir(parents=True, exist_ok=True)
+        ref_img_path = img_dir / "ref1.png"
+        page.screenshot(path=ref_img_path)
+        return
+
+    # Save the test image
+    test_img_path = img_dir / "test_image.png"
+    page.screenshot(path=test_img_path)
+    img_test = Image.open(test_img_path)
+
+    # If there are reference images, find one that matches
+    # Only write out image diffs if none succeed
+    for ref_img_path in ref_images:
+        try:
+            # `expect(page).to_have_screenshot()` is not yet supported
+            # in Python playwright, even though it is available in
+            # JavaScript... :-\
+            # Use our own comparison for now.
+            # expect(page).to_have_screenshot(threshold=threshold)
+            assert_images_match(img_test, ref_img_path, threshold)
+        except AssertionError:
+            # We'll just try the next one
+            continue
+        else:
+            # It matched. Return!
+            return
 
-        mismatch = pixelmatch(img_ref, img_test, img_diff, threshold=threshold)
-        img_diff.save(baseline_diff)
-        min_mismatch = min(min_mismatch, mismatch)
+    raise AssertionError(f"No reference images matched in {img_dir}")
 
-    sb.assert_true(
-        min_mismatch < threshold,
-        f"Baseline threshold {min_mismatch} < {threshold}",
-    )
 
+def assert_snapshot_matches(page: Page, ref_dir: Path, name: str):
+    html = page.locator("html")
+    ref_path = ref_dir / f"{name}.yml"
+    if not ref_path.exists():
+        print(f"'{ref_path}' does not exist. Creating...")
+        with open(ref_path, "w") as wf:
+            wf.write(html.aria_snapshot())
+        return
 
-def wait_for_ready(sb, timeout=60):
-    for i in range(timeout):
-        print(f"wait_for_ready {i}")
-        if sb.is_element_present(".trame__loader"):
-            sb.sleep(1)
-        else:
-            print("Ready")
-            return
+    with open(ref_path, "r") as rf:
+        expect(html).to_match_aria_snapshot(rf.read())