Migrate build to meson-python from setuptools

.github/workflows/ci.yml

@@ -11,8 +11,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        numpy-version: ["oldest-supported-numpy", "numpy"]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        numpy-version: ["numpy"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
@@ -21,8 +21,8 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python dependencies
-      run: python -m pip install --prefer-binary setuptools "cython>=0.28,<3.0" "matplotlib>=3,<4" ${{ matrix.numpy-version }}
+      run: python -m pip install --prefer-binary meson-python meson ninja setuptools "cython>=0.28,<3.0" "matplotlib>=3,<4" ${{ matrix.numpy-version }}
     - name: Build and install Raysect
-      run: dev/build.sh
+      run: dev/install_editable.sh
     - name: Run tests
       run: dev/test.sh

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+venv/
 build/
 develop-eggs/
 dist/

CHANGELOG.txt

@@ -1,6 +1,19 @@
 Raysect Changelog
 =================
 
+Release 0.9.0 (TBD)
+-------------------
+
+Build changes:
+* The legacy setuptools build system has been replaced with meson-python.
+  - The dev/build.sh and dev/clean.sh scripts have been updated to work with the new build system.
+  - A meson.build generator script is provided to automate the discovery of pyx, pxd,py and data files. Please see dev/generate_meson_files.py.
+  - When installed in editable mode, any modified pyx files will automatically trigger a rebuild when python attempts to import from the package. Please be aware that (as with the previous build system) changes to pxd files will require a clean rebuild of the project.
+  - A dev/install_editable.sh script is provided to simplify the installation of the package in editable mode with verbose build output enabled.
+  - Meson-python performs the build out of the project folder, so the project source folders will no longer be polluted with build artefacts.
+  - Cython build annotations are now always enabled, the annotation files can be found in the build folder under the build artefacts folder associated with each so file (*.so.p folder).
+
+
 Release 0.8.1 (12 Feb 2023)
 ---------------------------
 

dev/abi.py

@@ -0,0 +1,50 @@
+#!/bin/env python
+
+import sys
+import sysconfig
+from typing import Union
+
+"""
+Derived from the meson-python codebase. Original license terms:
+
+Copyright © 2022 the meson-python contributors
+Copyright © 2021 Quansight Labs and Filipe Laíns
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next
+paragraph) shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+"""
+
+
+def get_cpython_abi() -> str:
+    version = sys.version_info
+    debug = pymalloc = ''
+    if version < (3, 8) and _get_config_var('WITH_PYMALLOC', True):
+        pymalloc = 'm'
+    return f'cp{version[0]}{version[1]}{debug}{pymalloc}'
+
+
+def _get_config_var(name: str, default: Union[str, int, None] = None) -> Union[str, int, None]:
+    value: Union[str, int, None] = sysconfig.get_config_var(name)
+    if value is None:
+        return default
+    return value
+
+
+if __name__ == '__main__':
+    print(get_cpython_abi())

dev/build.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
+set -e  # exit if an error occurs
 
-CORES=`nproc --all`
+BUILD_PATH="build/`dev/abi.py`"
 
-echo "Rebuilding Raysect extension modules (in place)..."
-python setup.py build_ext -j$CORES --inplace $1 $2 $3 $4 $5
+echo Rebuilding $BUILD_PATH...
+meson compile -C $BUILD_PATH

dev/clean.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
+set -e  # exit if an error occurs
 
-echo Removing all .c, .so and .html files...
+BUILD_PATH="build/`dev/abi.py`"
+
+echo Cleaning $BUILD_PATH...
+meson compile -C $BUILD_PATH --clean
 
-find raysect -type f -name '*.c' -exec rm {} +
-find raysect -type f -name '*.so' -exec rm {} +
-find raysect -type f -name '*.html' -exec rm {} +
-rm build -rf

dev/generate_meson_files.py

@@ -0,0 +1,238 @@
+#!/bin/env python
+
+# Copyright (c) 2014-2025, Dr Alex Meakins, Raysect Project
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     1. Redistributions of source code must retain the above copyright notice,
+#        this list of conditions and the following disclaimer.
+#
+#     2. Redistributions in binary form must reproduce the above copyright
+#        notice, this list of conditions and the following disclaimer in the
+#        documentation and/or other materials provided with the distribution.
+#
+#     3. Neither the name of the Raysect Project nor the names of its
+#        contributors may be used to endorse or promote products derived from
+#        this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from pathlib import Path
+
+PACKAGES = ['raysect']
+
+EXCLUDE_DIR_FILE = '.meson-exclude'
+CUSTOM_MESON_BUILD = '.meson-custom'
+EXCLUDED_DIRS = ['__pycache__']
+
+
+def generate_meson_files(packages):
+    """
+    Generates the meson.build files for the project from a set of template files.
+
+    A root build.meson will be placed in the project root and meson.build files will be generated for the specified
+    package folders. This script must be executed from the root folder of the project.
+
+    If substantive changes are needed to the meson.build files throughout the project, it will be easier to modify the
+    templates and trigger the regeneration process.
+
+    This script will remove any existing meson.build files, so be sure any changes are captured before re-running this
+    script. There are two template files:
+
+        * root-meson.build
+        * subdir-meson.build
+
+    The root-meson.build file is used to generate the root meson.build for the project. The subdir-meson.build is used
+    to generate the meson.build files in the project sub-directory. The templates are read and handled as a python
+    f-strings. See the script implementation for the variables available to the template. The meson.build files will
+    consist of the template followed by a set of subdir() entries for each descendant of the current sub-directory
+    (if not excluded).
+
+    A sub-directory may be excluded from the generation process by placing a file in the subfolder called
+    ".meson-exclude". If the exclusion file is found, the sub-directory and its descendants will be ignored during
+    the generation process.
+
+    Occasionally a meson.build file may need to be customised in the source tree. Placing a file called ".meson-custom"
+    in the same directory as the meson.build file will protect the customised file from deletion or replacement by this
+    script.
+
+    :param packages: A list of package names.
+    """
+
+    root_path = Path('.')
+    package_paths = [Path(package) for package in packages]
+
+    # Walk the project folder and specified packages to remove all (non-custom) meson.build files.
+    # Any stale meson.build files found in excluded directories are also removed.
+    _remove_meson_files(root_path, subdirs=package_paths)
+
+    # Add root meson.build file.
+    _install_root_meson_file(root_path, subdirs=package_paths)
+
+    # Walk the specified packages and add the sub-directory meson.build files.
+    for path in package_paths:
+        _install_subdir_meson_files(path)
+
+
+def _remove_meson_files(path, subdirs=None):
+    """
+    Removes any meson.build files found under the directory tree referenced by path.
+
+    By default, this function recurses through the entire directory tree under the supplied path. If sub-dirs is
+    provided, then only the specified sub-directories will be explored.
+    """
+
+    # validate
+    if not path.is_dir():
+        raise ValueError('The supplied path is not a directory.')
+
+    if subdirs and any([not subdir.is_dir() for subdir in subdirs]):
+        raise ValueError('The list of sub-directories must only contain paths to valid directories.')
+
+    # remove meson.build in this directory if it is not flagged as customised
+    if not _custom_meson_file(path):
+        meson_file = path / 'meson.build'
+        meson_file.unlink(missing_ok=True)
+
+    # generate a list of subdirectories if none supplied
+    if not subdirs:
+        subdirs = [child for child in path.iterdir() if child.is_dir()]
+
+    # recurse into sub-directories
+    for subdir in subdirs:
+        _remove_meson_files(subdir)
+
+
+def _install_root_meson_file(path, subdirs):
+
+    # validate
+    if not path.is_dir():
+        raise ValueError('The supplied path is not a directory.')
+
+    if any([not subdir.is_dir() for subdir in subdirs]):
+        raise ValueError('The list of sub-directories must only contain paths to valid directories.')
+
+    # write meson file
+    _write_meson_file(path, _generate_root_meson_file(subdirs))
+
+
+def _install_subdir_meson_files(path):
+
+    # validate
+    if not path.is_dir():
+        raise ValueError('The supplied path is not a directory.')
+
+    # generate a list of subdirectories, filtering excluded
+    subdirs = [child for child in path.iterdir() if child.is_dir() and not _excluded_dir(child)]
+
+    # write meson file
+    _write_meson_file(path, _generate_subdir_meson_file(path, subdirs))
+
+    # recurse into sub-directories
+    for subdir in subdirs:
+        _install_subdir_meson_files(subdir)
+
+
+def _write_meson_file(path, contents):
+    if not _custom_meson_file(path):
+        meson_file = path / 'meson.build'
+        meson_file.write_text(contents)
+
+
+def _custom_meson_file(path):
+    return (path / CUSTOM_MESON_BUILD).exists()
+
+
+def _excluded_dir(path):
+    foo = (path / EXCLUDE_DIR_FILE).exists() or path.name in EXCLUDED_DIRS
+    return foo
+
+
+def _generate_root_meson_file(subdirs):
+
+    # read template
+    template_path = Path(__file__).parent / 'root-meson.build'
+    template = template_path.read_text()
+
+    # start contents with a warning
+    contents = (
+        "# WARNING: This file is automatically generated by dev/generate_meson_files.py.\n"
+        "# The template file used to generate this file is dev/root-meson.build.\n\n"
+    )
+
+    # add template
+    contents += template
+
+    # add subdir entries
+    contents += '\n'
+    for subdir in subdirs:
+        contents += f'subdir(\'{subdir.name}\')\n'
+
+    return contents
+
+
+def _generate_subdir_meson_file(path, subdirs):
+
+    # read template
+    template_path = Path(__file__).parent / 'subdir-meson.build'
+    template = template_path.read_text()
+
+    # build file lists
+    pyx = []
+    pxd = []
+    py = []
+    data = []
+    for child in path.iterdir():
+
+        if child.is_dir():
+            continue
+
+        elif child.suffix == '.pyx':
+            pyx.append(child.name)
+
+        elif child.suffix == '.pxd':
+            pxd.append(child.name)
+
+        elif child.suffix == '.py':
+            py.append(child.name)
+
+        else:
+            data.append(child.name)
+
+    # start contents with a warning
+    contents = (
+        "# WARNING: This file is automatically generated by dev/generate_meson_files.py.\n"
+        "# The template file used to generate this file is dev/subdir-meson.build.\n\n"
+    )
+
+    # add template, filling in the variables
+    contents += template.format(
+        target=f'\'{str(path)}\'',
+        pyx_files=str(pyx),
+        pxd_files=str(pxd),
+        py_files=str(py),
+        data_files=str(data)
+    )
+
+    # add subdir entries
+    contents += '\n'
+    for subdir in subdirs:
+        contents += f'subdir(\'{subdir.name}\')\n'
+
+    return contents
+
+
+if __name__ == '__main__':
+    generate_meson_files(PACKAGES)

dev/install_editable.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e  # exit if an error occurs
+
+echo Installing package as editable...
+python -m pip install --config-settings=editable-verbose=true --no-build-isolation --upgrade --editable .

dev/root-meson.build

@@ -0,0 +1,8 @@
+project('raysect', 'cython', default_options: ['python.install-env=auto'])
+
+py = import('python').find_installation(pure: false)
+numpy = dependency('numpy', method: 'config-tool')
+fs = import('fs')
+
+cython_args = ['--annotate']
+cython_dependencies = [numpy]