Generate dynamic protocol headers

.github/workflows/ci.yml

@@ -192,7 +192,7 @@ jobs:
         with:
           python-version: ${{ env.python-version }}
       - name: Run ruff check
-        uses: chartboost/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
   mypy-test:
     name: mypy tests
     runs-on: ubuntu-24.04
@@ -227,7 +227,6 @@ jobs:
         run: |
           mypy -p wlroots
           mypy -p tiny
-          mypy check_headers.py
   black-test:
     name: black tests
     runs-on: ubuntu-24.04
@@ -244,7 +243,7 @@ jobs:
         run: |
           pip install black
       - name: Run black test
-        run: black --check wlroots tiny check_headers.py
+        run: black --check wlroots tiny
   packaging-test:
     name: packaging tests
     runs-on: ubuntu-24.04
@@ -279,7 +278,7 @@ jobs:
         run: |
           check-manifest
           python setup.py check -m -s
-          python setup.py sdist
+          python -m build --sdist
           twine check dist/*
   protocol-headers-test:
     name: protocol headers test
@@ -312,4 +311,6 @@ jobs:
           sudo apt update
           sudo apt-get install -y --no-install-recommends pkgconf
       - name: Run protocol test
-        run: python check_headers.py --wlroots-dir wlroots-${{ env.wlroots-version }}
+        run: |
+          python wlroots/include/check_headers.py --generate
+          python wlroots/include/check_headers.py --wlroots-dir wlroots-${{ env.wlroots-version }}

.gitignore

@@ -21,3 +21,6 @@ pywlroots.egg-info/
 .eggs
 
 *.swp
+
+# Protocol headers
+wlroots/include/*.h

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.1.0
+    rev: 25.1.0
     hooks:
       - id: black
         # Use the latest supported version here
         language_version: python3.13
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.13.0
     hooks:
       - id: ruff

MANIFEST.in

@@ -4,5 +4,5 @@ include *.yaml
 include *.rst
 include tiny/*.py
 recursive-include tests *.py
-recursive-include wlroots *.h
 recursive-include wlroots *.rst
+recursive-include wlroots *.xml

README.rst

@@ -24,6 +24,12 @@ To build pywlroots from source, the Python requirements will need to be
 installed manually.  These are available in ``requirements.txt``.  The cffi
 bindings are built by running ``python wlroots/ffi_build.py``.
 
+Included Protocols
+------------------
+
+XML files containing the protocol specification that are needed for the
+pywlroots ffi build. Read more `here <wlroots/include/README.rst>`_.
+
 Versioning and Releases
 -----------------------
 

pyproject.toml

@@ -3,6 +3,8 @@ requires = [
     "setuptools>=77.0.0",
     "wheel",
     "cffi>=1.12.0; platform_python_implementation != 'PyPy'",
+    "pywayland>=0.4.14",
+    "xkbcommon",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -31,7 +33,7 @@ classifiers = [
 dependencies = [
     "cffi >= 1.12.0; platform_python_implementation != 'PyPy'",
     "pywayland >= 0.4.14",
-    "xkbcommon >= 0.2",
+    "xkbcommon",
 ]
 dynamic = ["version"]
 

setup.py

@@ -1,34 +1,33 @@
 # Copyright (c) Sean Vig 2018
 
+import glob
+import os
+import subprocess
 import sys
-from distutils.command.build import build
 
 from setuptools import setup
-from setuptools.command.install import install
+from setuptools.command.sdist import sdist
 
-NO_SETUP_REQUIRES_ARGUMENTS = ["--help", "-h", "sdist"]
 
-
-class DummyBuild(build):
-    def run(self):
-        raise RuntimeError("Requested running build")
-
-
-class DummyInstall(install):
+class SdistClean(sdist):
     def run(self):
-        raise RuntimeError("Requested running install")
-
-
-def keywords_with_side_effects(argv):
-    if any(arg in NO_SETUP_REQUIRES_ARGUMENTS for arg in argv):
-        return {"cmdclass": {"build": DummyBuild, "install": DummyInstall}}
-    else:
-        return {
-            "setup_requires": ["cffi>=1.12.0", "pywayland>=0.4.14", "xkbcommon>=0.2"],
-            "cffi_modules": ["wlroots/ffi_build.py:ffi_builder"],
-        }
+        # These files are not tracked by git and as a result check-manifest complains
+        # when present, '_build.py' needs to be removed manually since it keeps being
+        # created by other setup commands and '.h' files are needen only during build
+        files = glob.glob("wlroots/include/*.h")
+        files.append("wlroots/_build.py")
+        for file in files:
+            try:
+                os.remove(file)
+            except Exception:
+                pass
+        sdist.run(self)
 
 
 sys.path.insert(0, "wlroots")
 
-setup(**keywords_with_side_effects(sys.argv))
+subprocess.run(["python", "wlroots/include/check_headers.py", "--generate"])
+setup(
+    cmdclass={"sdist": SdistClean},
+    cffi_modules=["wlroots/ffi_build.py:ffi_builder"],
+)

tests/test_helper.py

@@ -6,5 +6,7 @@
 
 def test_build_compositor():
     with Display() as display:
-        _, _, _, backend, _ = build_compositor(display, backend_type=BackendType.HEADLESS)
+        _, _, _, backend, _ = build_compositor(
+            display, backend_type=BackendType.HEADLESS
+        )
         backend.destroy()

tiny/__main__.py

@@ -28,9 +28,7 @@
 
 def main(argv) -> None:
     with Display() as display:
-        compositor, allocator, renderer, backend, subcompositor = build_compositor(
-            display
-        )
+        _, allocator, renderer, backend, _ = build_compositor(display)
         device_manager = DataDeviceManager(display)  # noqa: F841
         xdg_shell = XdgShell(display)
         with (

wlroots/ffi_build.py

@@ -2,16 +2,17 @@
 # Copyright (c) 2022 Aakash Sen Sharma
 
 import importlib.util
-import sys
 import os
+import subprocess
+import sys
 from pathlib import Path
 
 from cffi import FFI, VerificationError
 from pywayland.ffi_build import ffi_builder as pywayland_ffi
 from xkbcommon.ffi_build import ffibuilder as xkb_ffi
 
-include_dir = (Path(__file__).parent / "include").resolve()
-assert include_dir.is_dir(), f"missing {include_dir}"
+INCLUDE_PATH = (Path(__file__).parent / "include").resolve()
+assert INCLUDE_PATH.is_dir(), f"missing {INCLUDE_PATH}"
 
 
 def load_version():
@@ -40,7 +41,7 @@ def load_wlroots_version():
             lib = importlib.import_module("wlroots").lib
         return f"{lib.WLR_VERSION_MAJOR}.{lib.WLR_VERSION_MINOR}.{lib.WLR_VERSION_MICRO}"  # type: ignore[attr-defined]
     else:
-        return os.getenv("PYTHON_CROSSENV_WLROOTS_VERSION")  # type: ignore[attr-defined]
+        return os.getenv("PYTHON_CROSSENV_WLROOTS_VERSION")
 
 
 def check_version():
@@ -80,7 +81,7 @@ def has_xwayland() -> bool:
         FFI().verify(
             "#include <wlr/xwayland.h>",
             define_macros=[("WLR_USE_UNSTABLE", 1)],
-            include_dirs=["/usr/include/pixman-1", include_dir.as_posix()],
+            include_dirs=["/usr/include/pixman-1", INCLUDE_PATH.as_posix()],
         )
         has_xwayland = True
     except VerificationError:
@@ -3390,11 +3391,12 @@ def has_xwayland() -> bool:
     SOURCE,
     libraries=["wlroots"],
     define_macros=[("WLR_USE_UNSTABLE", None)],
-    include_dirs=["/usr/include/pixman-1", include_dir],
+    include_dirs=["/usr/include/pixman-1", INCLUDE_PATH],
 )
 ffi_builder.include(pywayland_ffi)
 ffi_builder.include(xkb_ffi)
 ffi_builder.cdef(CDEF)
 
 if __name__ == "__main__":
+    subprocess.run(["python", f"{INCLUDE_PATH}/check_headers.py", "--generate"])
     ffi_builder.compile()

wlroots/include/README.rst

@@ -1,27 +1,19 @@
-Included Headers
+Protocol Headers
 ----------------
 
-Generated Wayland protocol headers that are included with the pywlroots ffi
-build, and are shipped with the library for downstream libraries to be able to
-include these headers.
+Dynamically generated Wayland and wlroots protocol headers that are included
+with the pywlroots ffi build, and are automatically generated when running
+``python wlroots/ffi_build.py``. You can generate them manually by running:
 
-Updating Headers
-----------------
+.. code-block::
+
+    python check_headers.py --generate
 
-When protocol versions are updated or the wayland scanner generation changes,
-the included headers should be updated as well.  The match between the latest
-protocol and the included files is run in the CI.  When that starts to fail,
-run:
-```
-wayland-scanner server-header /usr/share/wayland-protocols/path/to/file.xml wlroots/include/file.h
-```
-This can also be used to add new headers to pywlroots.
+Updating Protocols
+------------------
 
-Alternatively, if you download a copy of the upstream wlroots source code, you
-can run:
-```
-python check_headers.py --wlroots-dir /path/to/wlroots/source --generate
-```
-Which will regenerate all of theh current set of protocol headers.  Leaving off
-the `--generate` option will instread run a check on the currently generated
-protocols.
+When protocol versions are updated, the included protocols XML files should
+be updated as well. The match between the latest protocol and the generated
+headers is run in the CI.  When that starts to fail, update the corresponding
+XML file in ``wlroots/protocol`` with the one in the `wlroots repository
+<https://gitlab.freedesktop.org/wlroots/wlroots>`_.

check_headers.py → wlroots/include/check_headers.py

@@ -6,7 +6,7 @@
 import sys
 import tempfile
 
-INCLUDE_PATH = pathlib.Path(__file__).parent / "wlroots" / "include"
+INCLUDE_PATH = pathlib.Path(__file__).parent
 WAYLAND_PROCOTOLS = [
     "stable/xdg-shell/xdg-shell.xml",
     "unstable/idle-inhibit/idle-inhibit-unstable-v1.xml",
@@ -18,6 +18,10 @@
 ]
 
 
+def get_wlroots_protocols_dir() -> pathlib.Path:
+    return pathlib.Path(__file__).parent.parent
+
+
 def get_wayland_protocols_dir() -> pathlib.Path | None:
     args = ["pkgconf", "--variable=pkgdatadir", "wayland-protocols"]
     try:
@@ -49,7 +53,7 @@ def check(protocols: list[pathlib.Path]) -> None:
         header_filename(protocol_xml) for protocol_xml in protocols
     }
     protocol_files = {
-        path.name for path in INCLUDE_PATH.iterdir() if path.name != "README.rst"
+        path.name for path in INCLUDE_PATH.iterdir() if path.suffix == ".h"
     }
 
     if expected_protocol_files != protocol_files:
@@ -95,12 +99,14 @@ def generate(protocols: list[pathlib.Path]) -> None:
         generate_protocol_header(protocol_xml, INCLUDE_PATH)
 
 
-def parse_args(argv):
+def parse_args(argv) -> tuple[list[pathlib.Path], bool]:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--wayland-dir", default=get_wayland_protocols_dir(), type=pathlib.Path
     )
-    parser.add_argument("--wlroots-dir", required=True, type=pathlib.Path)
+    parser.add_argument(
+        "--wlroots-dir", default=get_wlroots_protocols_dir(), type=pathlib.Path
+    )
     parser.add_argument("--generate", action="store_true")
     args = parser.parse_args(argv)