Add type annotations, modernise build, publish sdist, add agent harness

.github/scripts/build-linux.sh

@@ -63,23 +63,6 @@ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release \
 make install -j$(nproc)
 popd
 
-# Install libraw
-libraw_dir=$(pwd)/external/LibRaw
-pushd external/LibRaw-cmake
-mkdir build
-cd build
-cmake .. \
-    -DCMAKE_INSTALL_PREFIX=/usr \
-    -DLIBRAW_PATH=$libraw_dir \
-    -DENABLE_X3FTOOLS=ON \
-    -DENABLE_6BY9RPI=ON \
-    -DENABLE_EXAMPLES=OFF \
-    -DENABLE_RAWSPEED=OFF \
-    -DCMAKE_BUILD_TYPE=Release
-make
-make install -j$(nproc)
-popd
-
 # Install matplotlib (a scikit-image dependency) dependencies
 retry dnf install -y libpng-devel freetype-devel
 
@@ -90,15 +73,13 @@ retry dnf install -y lapack-devel blas-devel
 ${PYBIN}/python -m pip install --upgrade pip
 export PIP_PREFER_BINARY=1
 
-# install compile-time dependencies
-retry ${PYBIN}/pip install numpy==${NUMPY_VERSION} cython setuptools
-
 # List installed packages
 ${PYBIN}/pip freeze
 
 # Build rawpy wheel
+# Relies on pyproject.toml to create an isolated build env with the correct numpy version
 export LDFLAGS="-Wl,--strip-debug"
-${PYBIN}/python setup.py bdist_wheel --dist-dir dist-tmp
+${PYBIN}/pip wheel . --wheel-dir dist-tmp --no-deps
 
 # Bundle external shared libraries into wheel and fix the wheel tags
 mkdir dist

.github/scripts/build-macos.sh

@@ -34,8 +34,8 @@ popd
 python -m pip install --upgrade pip
 export PIP_PREFER_BINARY=1
 
-# Install dependencies
-pip install numpy==$NUMPY_VERSION cython wheel delocate setuptools
+# Install delocate for bundling shared libraries into the wheel
+pip install delocate
 
 # List installed packages
 pip freeze
@@ -110,7 +110,7 @@ export LDFLAGS=$CFLAGS
 export ARCHFLAGS=$CFLAGS
 
 # Build wheel
-python setup.py bdist_wheel
+pip wheel . --wheel-dir dist --no-deps
 
 DYLD_LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib delocate-listdeps --all --depending dist/*.whl # lists library dependencies
 DYLD_LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib delocate-wheel --verbose --require-archs=${PYTHON_ARCH} dist/*.whl # copies library dependencies into wheel

.github/scripts/build-windows.ps1

@@ -13,52 +13,6 @@ function exec {
     }
 }
 
-function Initialize-Python {
-    if ($env:USE_CONDA -eq 1) {
-        $env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH"
-        & .\.github\scripts\install-miniconda.ps1
-        & $env:CONDA_ROOT\shell\condabin\conda-hook.ps1
-        exec { conda update --yes -n base -c defaults conda }
-    }
-    # Check Python version
-    exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" }
-}
-
-function Create-VEnv {
-    [CmdletBinding()]
-    param([Parameter(Position=0,Mandatory=1)][string]$name)
-    if ($env:USE_CONDA -eq 1) {
-        exec { conda create --yes --name $name -c defaults --strict-channel-priority python=$env:PYTHON_VERSION --force }
-    } else {
-        exec { python -m venv env\$name }
-    }
-}
-
-function Enter-VEnv {
-    [CmdletBinding()]
-    param([Parameter(Position=0,Mandatory=1)][string]$name)
-    if ($env:USE_CONDA -eq 1) {
-        conda activate $name
-    } else {
-        & .\env\$name\scripts\activate
-    }
-}
-
-function Create-And-Enter-VEnv {
-    [CmdletBinding()]
-    param([Parameter(Position=0,Mandatory=1)][string]$name)
-    Create-VEnv $name
-    Enter-VEnv $name
-}
-
-function Exit-VEnv {
-    if ($env:USE_CONDA -eq 1) {
-        conda deactivate
-    } else {
-        deactivate
-    }
-}
-
 function Initialize-VS {
     # https://wiki.python.org/moin/WindowsCompilers
     # setuptools automatically selects the right compiler for building
@@ -113,12 +67,11 @@ if (!$env:PYTHON_VERSION) {
 if ($env:PYTHON_ARCH -ne 'x86' -and $env:PYTHON_ARCH -ne 'x86_64') {
     throw "PYTHON_ARCH env var must be x86 or x86_64"
 }
-if (!$env:NUMPY_VERSION) {
-    throw "NUMPY_VERSION env var missing"
-}
 
 Initialize-VS
-Initialize-Python
+
+# Check Python version
+exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" }
 
 # Prefer binary packages over building from source
 $env:PIP_PREFER_BINARY = 1
@@ -133,10 +86,9 @@ if (!(Test-Path ./vcpkg)) {
 exec { ./vcpkg/vcpkg install zlib libjpeg-turbo[jpeg8] jasper lcms --triplet=x64-windows-static --recurse }
 $env:CMAKE_PREFIX_PATH = $pwd.Path + "\vcpkg\installed\x64-windows-static"
 
-
-# Build the wheel.
-Create-And-Enter-VEnv build
-exec { python -m pip install --upgrade pip wheel setuptools }
-exec { python -m pip install --only-binary :all: numpy==$env:NUMPY_VERSION cython }
-exec { python -u setup.py bdist_wheel }
-Exit-VEnv
+# Build the wheel in a virtual environment
+exec { python -m venv env\build }
+& .\env\build\scripts\activate
+exec { python -m pip install --upgrade pip }
+exec { python -m pip wheel . --wheel-dir dist --no-deps }
+deactivate

.github/scripts/test-sdist-linux.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Test sdist: install from source in a clean venv and run the test suite.
+# This validates that the sdist contains everything needed to build.
+#
+# When RAWPY_USE_SYSTEM_LIBRAW=1 is set, the sdist is built against the
+# system libraw and linkage is verified (no bundled libraw_r.so, ldd
+# points to system library).
+set -e -x
+
+PYTHON_BIN="python${PYTHON_VERSION}"
+
+# Install system build dependencies
+sudo apt-get update -q
+sudo apt-get install -y -q \
+    liblcms2-dev \
+    libjpeg-dev
+
+# Create a clean venv
+${PYTHON_BIN} -m venv sdist-test-env
+source sdist-test-env/bin/activate
+python -m pip install --upgrade pip
+
+# Install the sdist (pip will build from source with build isolation)
+# RAWPY_USE_SYSTEM_LIBRAW is inherited from the environment if set.
+SDIST=$(ls dist/rawpy-*.tar.gz | head -1)
+pip install "${SDIST}[test]"
+
+# Run tests from a temp directory to avoid importing from the source tree
+mkdir tmp_for_test
+pushd tmp_for_test
+
+# Verify system libraw linkage when applicable
+if [ "$RAWPY_USE_SYSTEM_LIBRAW" = "1" ]; then
+    python -c "
+import rawpy._rawpy as _rawpy
+import os, subprocess, sys
+
+ext_path = _rawpy.__file__
+pkg_dir = os.path.dirname(ext_path)
+print(f'Extension: {ext_path}')
+
+# No bundled libraw_r.so in the package directory
+bundled = [f for f in os.listdir(pkg_dir) if f.startswith('libraw_r.so')]
+if bundled:
+    print(f'FAIL: Found bundled libraw files: {bundled}')
+    sys.exit(1)
+print('OK: No bundled libraw_r.so in package directory')
+
+# ldd shows system libraw, not a local path
+result = subprocess.run(['ldd', ext_path], capture_output=True, text=True)
+libraw_lines = [l.strip() for l in result.stdout.splitlines() if 'libraw_r' in l]
+if not libraw_lines:
+    print('FAIL: libraw_r.so not found in ldd output')
+    sys.exit(1)
+for line in libraw_lines:
+    print(f'ldd: {line}')
+    if pkg_dir in line:
+        print('FAIL: libraw_r.so resolves to the package directory')
+        sys.exit(1)
+print('OK: libraw_r.so links to system library')
+"
+fi
+
+pytest --verbosity=3 -s ../test
+popd
+
+deactivate

.github/scripts/test-sdist-macos.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Test sdist: install from source in a clean venv and run the test suite.
+# This validates that the sdist contains everything needed to build on macOS.
+set -e -x
+
+# Install build dependencies
+brew install jasper
+
+# Create a clean venv
+python${PYTHON_VERSION} -m venv sdist-test-env
+source sdist-test-env/bin/activate
+python -m pip install --upgrade pip
+
+# Install the sdist (pip will build from source with build isolation)
+SDIST=$(ls dist/rawpy-*.tar.gz | head -1)
+pip install "${SDIST}[test]"
+
+# Run tests from a temp directory to avoid importing from the source tree
+mkdir tmp_for_test
+pushd tmp_for_test
+pytest --verbosity=3 -s ../test
+popd
+
+deactivate

.github/scripts/test-sdist-windows.ps1

@@ -0,0 +1,89 @@
+$ErrorActionPreference = 'Stop'
+
+function exec {
+    [CmdletBinding()]
+    param([Parameter(Position=0,Mandatory=1)][scriptblock]$cmd)
+    Write-Host "$cmd"
+    $ErrorActionPreference = 'Continue'
+    & $cmd
+    $ErrorActionPreference = 'Stop'
+    if ($lastexitcode -ne 0) {
+        throw ("ERROR exit code $lastexitcode")
+    }
+}
+
+function Initialize-VS {
+    $VS_ROOTS = @(
+        "C:\Program Files\Microsoft Visual Studio",
+        "C:\Program Files (x86)\Microsoft Visual Studio"
+    )
+    $VS_VERSIONS = @("2017", "2019", "2022")
+    $VS_EDITIONS = @("Enterprise", "Professional", "Community")
+    $VS_INIT_CMD_SUFFIX = "Common7\Tools\vsdevcmd.bat"
+
+    $VS_ARCH = if ($env:PYTHON_ARCH -eq 'x86') { 'x86' } else { 'x64' }
+    $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo"
+
+    $found = $false
+    :outer foreach ($VS_ROOT in $VS_ROOTS) {
+        foreach ($version in $VS_VERSIONS) {
+            foreach ($edition in $VS_EDITIONS) {
+                $VS_INIT_CMD = "$VS_ROOT\$version\$edition\$VS_INIT_CMD_SUFFIX"
+                if (Test-Path $VS_INIT_CMD) {
+                    $found = $true
+                    break outer
+                }
+            }
+        }
+    }
+
+    if (!$found) {
+        throw ("No suitable Visual Studio installation found")
+    }
+
+    Write-Host "Executing: $VS_INIT_CMD $VS_INIT_ARGS"
+
+    & "${env:COMSPEC}" /s /c "`"$VS_INIT_CMD`" $VS_INIT_ARGS && set" | foreach-object {
+        $name, $value = $_ -split '=', 2
+        try {
+            set-content env:\"$name" $value
+        } catch {
+        }
+    }
+}
+
+if (!$env:PYTHON_VERSION) {
+    throw "PYTHON_VERSION env var missing, must be x.y"
+}
+if ($env:PYTHON_ARCH -ne 'x86' -and $env:PYTHON_ARCH -ne 'x86_64') {
+    throw "PYTHON_ARCH env var must be x86 or x86_64"
+}
+
+Initialize-VS
+
+# Check Python version
+exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" }
+
+# Install vcpkg dependencies (needed for building from source)
+if (!(Test-Path ./vcpkg)) {
+    exec { git clone https://github.com/microsoft/vcpkg -b 2025.01.13 --depth 1 }
+    exec { ./vcpkg/bootstrap-vcpkg }
+}
+exec { ./vcpkg/vcpkg install zlib libjpeg-turbo[jpeg8] jasper lcms --triplet=x64-windows-static --recurse }
+$env:CMAKE_PREFIX_PATH = $pwd.Path + "\vcpkg\installed\x64-windows-static"
+
+# Create a clean venv and install the sdist
+exec { python -m venv sdist-test-env }
+& .\sdist-test-env\scripts\activate
+exec { python -m pip install --upgrade pip }
+
+$sdist = Get-ChildItem dist\rawpy-*.tar.gz | Select-Object -First 1
+exec { pip install "$($sdist.FullName)[test]" }
+
+# Run tests from a temp directory to avoid importing from the source tree
+mkdir -f tmp_for_test | out-null
+pushd tmp_for_test
+exec { pytest --verbosity=3 -s ../test }
+popd
+
+deactivate