diff --git a/.github/scripts/build-linux.sh b/.github/scripts/build-linux.sh index 9c518571..46179fb1 100755 --- a/.github/scripts/build-linux.sh +++ b/.github/scripts/build-linux.sh @@ -63,23 +63,6 @@ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release \ make install -j$(nproc) popd -# Install libraw -libraw_dir=$(pwd)/external/LibRaw -pushd external/LibRaw-cmake -mkdir build -cd build -cmake .. \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DLIBRAW_PATH=$libraw_dir \ - -DENABLE_X3FTOOLS=ON \ - -DENABLE_6BY9RPI=ON \ - -DENABLE_EXAMPLES=OFF \ - -DENABLE_RAWSPEED=OFF \ - -DCMAKE_BUILD_TYPE=Release -make -make install -j$(nproc) -popd - # Install matplotlib (a scikit-image dependency) dependencies retry dnf install -y libpng-devel freetype-devel @@ -90,15 +73,13 @@ retry dnf install -y lapack-devel blas-devel ${PYBIN}/python -m pip install --upgrade pip export PIP_PREFER_BINARY=1 -# install compile-time dependencies -retry ${PYBIN}/pip install numpy==${NUMPY_VERSION} cython setuptools - # List installed packages ${PYBIN}/pip freeze # Build rawpy wheel +# Relies on pyproject.toml to create an isolated build env with the correct numpy version export LDFLAGS="-Wl,--strip-debug" -${PYBIN}/python setup.py bdist_wheel --dist-dir dist-tmp +${PYBIN}/pip wheel . --wheel-dir dist-tmp --no-deps # Bundle external shared libraries into wheel and fix the wheel tags mkdir dist diff --git a/.github/scripts/build-macos.sh b/.github/scripts/build-macos.sh index 90700921..90c0aa79 100755 --- a/.github/scripts/build-macos.sh +++ b/.github/scripts/build-macos.sh @@ -34,8 +34,8 @@ popd python -m pip install --upgrade pip export PIP_PREFER_BINARY=1 -# Install dependencies -pip install numpy==$NUMPY_VERSION cython wheel delocate setuptools +# Install delocate for bundling shared libraries into the wheel +pip install delocate # List installed packages pip freeze @@ -110,7 +110,7 @@ export LDFLAGS=$CFLAGS export ARCHFLAGS=$CFLAGS # Build wheel -python setup.py bdist_wheel +pip wheel . --wheel-dir dist --no-deps DYLD_LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib delocate-listdeps --all --depending dist/*.whl # lists library dependencies DYLD_LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib delocate-wheel --verbose --require-archs=${PYTHON_ARCH} dist/*.whl # copies library dependencies into wheel diff --git a/.github/scripts/build-windows.ps1 b/.github/scripts/build-windows.ps1 index 9156d253..f438cf90 100644 --- a/.github/scripts/build-windows.ps1 +++ b/.github/scripts/build-windows.ps1 @@ -13,52 +13,6 @@ function exec { } } -function Initialize-Python { - if ($env:USE_CONDA -eq 1) { - $env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH" - & .\.github\scripts\install-miniconda.ps1 - & $env:CONDA_ROOT\shell\condabin\conda-hook.ps1 - exec { conda update --yes -n base -c defaults conda } - } - # Check Python version - exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } -} - -function Create-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - if ($env:USE_CONDA -eq 1) { - exec { conda create --yes --name $name -c defaults --strict-channel-priority python=$env:PYTHON_VERSION --force } - } else { - exec { python -m venv env\$name } - } -} - -function Enter-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - if ($env:USE_CONDA -eq 1) { - conda activate $name - } else { - & .\env\$name\scripts\activate - } -} - -function Create-And-Enter-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - Create-VEnv $name - Enter-VEnv $name -} - -function Exit-VEnv { - if ($env:USE_CONDA -eq 1) { - conda deactivate - } else { - deactivate - } -} - function Initialize-VS { # https://wiki.python.org/moin/WindowsCompilers # setuptools automatically selects the right compiler for building @@ -113,12 +67,11 @@ if (!$env:PYTHON_VERSION) { if ($env:PYTHON_ARCH -ne 'x86' -and $env:PYTHON_ARCH -ne 'x86_64') { throw "PYTHON_ARCH env var must be x86 or x86_64" } -if (!$env:NUMPY_VERSION) { - throw "NUMPY_VERSION env var missing" -} Initialize-VS -Initialize-Python + +# Check Python version +exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } # Prefer binary packages over building from source $env:PIP_PREFER_BINARY = 1 @@ -133,10 +86,9 @@ if (!(Test-Path ./vcpkg)) { exec { ./vcpkg/vcpkg install zlib libjpeg-turbo[jpeg8] jasper lcms --triplet=x64-windows-static --recurse } $env:CMAKE_PREFIX_PATH = $pwd.Path + "\vcpkg\installed\x64-windows-static" - -# Build the wheel. -Create-And-Enter-VEnv build -exec { python -m pip install --upgrade pip wheel setuptools } -exec { python -m pip install --only-binary :all: numpy==$env:NUMPY_VERSION cython } -exec { python -u setup.py bdist_wheel } -Exit-VEnv +# Build the wheel in a virtual environment +exec { python -m venv env\build } +& .\env\build\scripts\activate +exec { python -m pip install --upgrade pip } +exec { python -m pip wheel . --wheel-dir dist --no-deps } +deactivate diff --git a/.github/scripts/install-miniconda.ps1 b/.github/scripts/install-miniconda.ps1 deleted file mode 100644 index 6ec40281..00000000 --- a/.github/scripts/install-miniconda.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$MINICONDA_URL = "http://repo.continuum.io/miniconda/" - -function DownloadMiniconda ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - - $filename = "Miniconda3-latest-Windows-" + $platform_suffix + ".exe" - - $url = $MINICONDA_URL + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - if ($i + 1 -eq $retry_attempts) { - throw - } else { - Start-Sleep 1 - } - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function InstallMiniconda ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - $filepath = DownloadMiniconda $python_version $architecture - Write-Host "Installing" $filepath "to" $python_home - $install_log = $python_home + ".log" - $args = "/RegisterPython=0 /AddToPath=0 /S /D=$python_home" - Write-Host $filepath $args - Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - -InstallMiniconda $env:PYTHON_VERSION $env:PYTHON_ARCH $env:CONDA_ROOT diff --git a/.github/scripts/test-sdist-linux.sh b/.github/scripts/test-sdist-linux.sh new file mode 100755 index 00000000..d470aa02 --- /dev/null +++ b/.github/scripts/test-sdist-linux.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Test sdist: install from source in a clean venv and run the test suite. +# This validates that the sdist contains everything needed to build. +# +# When RAWPY_USE_SYSTEM_LIBRAW=1 is set, the sdist is built against the +# system libraw and linkage is verified (no bundled libraw_r.so, ldd +# points to system library). +set -e -x + +PYTHON_BIN="python${PYTHON_VERSION}" + +# Install system build dependencies +sudo apt-get update -q +sudo apt-get install -y -q \ + liblcms2-dev \ + libjpeg-dev + +# Create a clean venv +${PYTHON_BIN} -m venv sdist-test-env +source sdist-test-env/bin/activate +python -m pip install --upgrade pip + +# Install the sdist (pip will build from source with build isolation) +# RAWPY_USE_SYSTEM_LIBRAW is inherited from the environment if set. +SDIST=$(ls dist/rawpy-*.tar.gz | head -1) +pip install "${SDIST}[test]" + +# Run tests from a temp directory to avoid importing from the source tree +mkdir tmp_for_test +pushd tmp_for_test + +# Verify system libraw linkage when applicable +if [ "$RAWPY_USE_SYSTEM_LIBRAW" = "1" ]; then + python -c " +import rawpy._rawpy as _rawpy +import os, subprocess, sys + +ext_path = _rawpy.__file__ +pkg_dir = os.path.dirname(ext_path) +print(f'Extension: {ext_path}') + +# No bundled libraw_r.so in the package directory +bundled = [f for f in os.listdir(pkg_dir) if f.startswith('libraw_r.so')] +if bundled: + print(f'FAIL: Found bundled libraw files: {bundled}') + sys.exit(1) +print('OK: No bundled libraw_r.so in package directory') + +# ldd shows system libraw, not a local path +result = subprocess.run(['ldd', ext_path], capture_output=True, text=True) +libraw_lines = [l.strip() for l in result.stdout.splitlines() if 'libraw_r' in l] +if not libraw_lines: + print('FAIL: libraw_r.so not found in ldd output') + sys.exit(1) +for line in libraw_lines: + print(f'ldd: {line}') + if pkg_dir in line: + print('FAIL: libraw_r.so resolves to the package directory') + sys.exit(1) +print('OK: libraw_r.so links to system library') +" +fi + +pytest --verbosity=3 -s ../test +popd + +deactivate diff --git a/.github/scripts/test-sdist-macos.sh b/.github/scripts/test-sdist-macos.sh new file mode 100755 index 00000000..dcc673ce --- /dev/null +++ b/.github/scripts/test-sdist-macos.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Test sdist: install from source in a clean venv and run the test suite. +# This validates that the sdist contains everything needed to build on macOS. +set -e -x + +# Install build dependencies +brew install jasper + +# Create a clean venv +python${PYTHON_VERSION} -m venv sdist-test-env +source sdist-test-env/bin/activate +python -m pip install --upgrade pip + +# Install the sdist (pip will build from source with build isolation) +SDIST=$(ls dist/rawpy-*.tar.gz | head -1) +pip install "${SDIST}[test]" + +# Run tests from a temp directory to avoid importing from the source tree +mkdir tmp_for_test +pushd tmp_for_test +pytest --verbosity=3 -s ../test +popd + +deactivate diff --git a/.github/scripts/test-sdist-windows.ps1 b/.github/scripts/test-sdist-windows.ps1 new file mode 100644 index 00000000..33e28d8a --- /dev/null +++ b/.github/scripts/test-sdist-windows.ps1 @@ -0,0 +1,89 @@ +$ErrorActionPreference = 'Stop' + +function exec { + [CmdletBinding()] + param([Parameter(Position=0,Mandatory=1)][scriptblock]$cmd) + Write-Host "$cmd" + $ErrorActionPreference = 'Continue' + & $cmd + $ErrorActionPreference = 'Stop' + if ($lastexitcode -ne 0) { + throw ("ERROR exit code $lastexitcode") + } +} + +function Initialize-VS { + $VS_ROOTS = @( + "C:\Program Files\Microsoft Visual Studio", + "C:\Program Files (x86)\Microsoft Visual Studio" + ) + $VS_VERSIONS = @("2017", "2019", "2022") + $VS_EDITIONS = @("Enterprise", "Professional", "Community") + $VS_INIT_CMD_SUFFIX = "Common7\Tools\vsdevcmd.bat" + + $VS_ARCH = if ($env:PYTHON_ARCH -eq 'x86') { 'x86' } else { 'x64' } + $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo" + + $found = $false + :outer foreach ($VS_ROOT in $VS_ROOTS) { + foreach ($version in $VS_VERSIONS) { + foreach ($edition in $VS_EDITIONS) { + $VS_INIT_CMD = "$VS_ROOT\$version\$edition\$VS_INIT_CMD_SUFFIX" + if (Test-Path $VS_INIT_CMD) { + $found = $true + break outer + } + } + } + } + + if (!$found) { + throw ("No suitable Visual Studio installation found") + } + + Write-Host "Executing: $VS_INIT_CMD $VS_INIT_ARGS" + + & "${env:COMSPEC}" /s /c "`"$VS_INIT_CMD`" $VS_INIT_ARGS && set" | foreach-object { + $name, $value = $_ -split '=', 2 + try { + set-content env:\"$name" $value + } catch { + } + } +} + +if (!$env:PYTHON_VERSION) { + throw "PYTHON_VERSION env var missing, must be x.y" +} +if ($env:PYTHON_ARCH -ne 'x86' -and $env:PYTHON_ARCH -ne 'x86_64') { + throw "PYTHON_ARCH env var must be x86 or x86_64" +} + +Initialize-VS + +# Check Python version +exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } + +# Install vcpkg dependencies (needed for building from source) +if (!(Test-Path ./vcpkg)) { + exec { git clone https://github.com/microsoft/vcpkg -b 2025.01.13 --depth 1 } + exec { ./vcpkg/bootstrap-vcpkg } +} +exec { ./vcpkg/vcpkg install zlib libjpeg-turbo[jpeg8] jasper lcms --triplet=x64-windows-static --recurse } +$env:CMAKE_PREFIX_PATH = $pwd.Path + "\vcpkg\installed\x64-windows-static" + +# Create a clean venv and install the sdist +exec { python -m venv sdist-test-env } +& .\sdist-test-env\scripts\activate +exec { python -m pip install --upgrade pip } + +$sdist = Get-ChildItem dist\rawpy-*.tar.gz | Select-Object -First 1 +exec { pip install "$($sdist.FullName)[test]" } + +# Run tests from a temp directory to avoid importing from the source tree +mkdir -f tmp_for_test | out-null +pushd tmp_for_test +exec { pytest --verbosity=3 -s ../test } +popd + +deactivate diff --git a/.github/scripts/test-windows.ps1 b/.github/scripts/test-windows.ps1 index 7cbf0f45..66a94372 100644 --- a/.github/scripts/test-windows.ps1 +++ b/.github/scripts/test-windows.ps1 @@ -13,52 +13,6 @@ function exec { } } -function Initialize-Python { - if ($env:USE_CONDA -eq 1) { - $env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH" - & .\.github\scripts\install-miniconda.ps1 - & $env:CONDA_ROOT\shell\condabin\conda-hook.ps1 - exec { conda update --yes -n base -c defaults conda } - } - # Check Python version/arch - exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } -} - -function Create-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - if ($env:USE_CONDA -eq 1) { - exec { conda create --yes --name $name -c defaults --strict-channel-priority python=$env:PYTHON_VERSION --force } - } else { - exec { python -m venv env\$name } - } -} - -function Enter-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - if ($env:USE_CONDA -eq 1) { - conda activate $name - } else { - & .\env\$name\scripts\activate - } -} - -function Create-And-Enter-VEnv { - [CmdletBinding()] - param([Parameter(Position=0,Mandatory=1)][string]$name) - Create-VEnv $name - Enter-VEnv $name -} - -function Exit-VEnv { - if ($env:USE_CONDA -eq 1) { - conda deactivate - } else { - deactivate - } -} - if (!$env:PYTHON_VERSION) { throw "PYTHON_VERSION env var missing, must be x.y" } @@ -71,7 +25,8 @@ if (!$env:NUMPY_VERSION) { $PYVER = ($env:PYTHON_VERSION).Replace('.', '') -Initialize-Python +# Check Python version/arch +exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } # Upgrade pip and prefer binary packages exec { python -m pip install --upgrade pip } @@ -81,7 +36,8 @@ Get-ChildItem env: # Install and import in an empty environment. # This is to catch DLL issues that may be hidden with dependencies. -Create-And-Enter-VEnv import-test +exec { python -m venv env\import-test } +& .\env\import-test\scripts\activate python -m pip uninstall -y rawpy ls dist\*cp${PYVER}*win*.whl | % { exec { python -m pip install $_ } } @@ -91,10 +47,11 @@ pushd tmp_for_test exec { python -c "import rawpy" } popd -Exit-VEnv +deactivate # Run test suite with all required and optional dependencies -Create-And-Enter-VEnv testsuite +exec { python -m venv env\testsuite } +& .\env\testsuite\scripts\activate python -m pip uninstall -y rawpy ls dist\*cp${PYVER}*win*.whl | % { exec { python -m pip install $_ } } exec { python -m pip install -r dev-requirements.txt numpy==$env:NUMPY_VERSION } @@ -105,4 +62,4 @@ pushd tmp_for_test exec { pytest --verbosity=3 -s ../test } popd -Exit-VEnv +deactivate diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f9c8f90..ad981218 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,142 +46,118 @@ jobs: docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.9' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.10' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.11' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.12' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.13' - numpy-version: '2.1.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_x86_64 python-arch: 'x86_64' python-version: '3.14' - numpy-version: '2.4.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.9' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.10' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.11' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.12' - numpy-version: '2.0.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.13' - numpy-version: '2.1.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux_2_28_aarch64 python-arch: 'aarch64' python-version: '3.14' - numpy-version: '2.4.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.9' - numpy-version: '2.0.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.10' - numpy-version: '2.0.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.11' - numpy-version: '2.0.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.12' - numpy-version: '2.0.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.13' - numpy-version: '2.1.*' - os-image: macos-15 # Apple Silicon os-name: mac macos-min-version: '11.0' python-arch: 'arm64' python-version: '3.14' - numpy-version: '2.4.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.9' - numpy-version: '2.0.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.10' - numpy-version: '2.0.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.11' - numpy-version: '2.0.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.12' - numpy-version: '2.0.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.13' - numpy-version: '2.1.*' - os-image: windows-2022 os-name: windows python-arch: 'x86_64' python-version: '3.14' - numpy-version: '2.4.*' runs-on: ${{ matrix.config.os-image }} @@ -196,11 +172,10 @@ jobs: - name: Build wheels (Linux) if: matrix.config.os-name == 'linux' - run: docker run --rm -e PYTHON_ARCH -e PYTHON_VERSION -e NUMPY_VERSION -v `pwd`:/io ${{ matrix.config.docker-image }} /io/.github/scripts/build-linux.sh + run: docker run --rm -e PYTHON_ARCH -e PYTHON_VERSION -v `pwd`:/io ${{ matrix.config.docker-image }} /io/.github/scripts/build-linux.sh env: PYTHON_ARCH: ${{ matrix.config.python-arch }} PYTHON_VERSION: ${{ matrix.config.python-version }} - NUMPY_VERSION: ${{ matrix.config.numpy-version }} - name: Build wheels (macOS) if: matrix.config.os-name == 'mac' @@ -209,7 +184,6 @@ jobs: MACOS_MIN_VERSION: ${{ matrix.config.macos-min-version }} PYTHON_ARCH: ${{ matrix.config.python-arch }} PYTHON_VERSION: ${{ matrix.config.python-version }} - NUMPY_VERSION: ${{ matrix.config.numpy-version }} - name: Setup Python (Windows) if: matrix.config.os-name == 'windows' @@ -224,7 +198,6 @@ jobs: env: PYTHON_VERSION: ${{ matrix.config.python-version }} PYTHON_ARCH: ${{ matrix.config.python-arch }} - NUMPY_VERSION: ${{ matrix.config.numpy-version }} - name: Store wheels as artifacts uses: actions/upload-artifact@v4 @@ -426,11 +399,12 @@ jobs: - name: Test wheel (Linux) if: matrix.config.os-name == 'linux' - run: docker run --rm -e PYTHON_ARCH -e PYTHON_VERSION -e NUMPY_VERSION -v `pwd`:/io ${{ matrix.config.docker-image }} /io/.github/scripts/test-linux.sh + run: docker run --rm -e PYTHON_ARCH -e PYTHON_VERSION -e NUMPY_VERSION -e RAWPY_SKIP_EXAMPLES -v `pwd`:/io ${{ matrix.config.docker-image }} /io/.github/scripts/test-linux.sh env: PYTHON_ARCH: ${{ matrix.config.python-arch }} PYTHON_VERSION: ${{ matrix.config.python-version }} NUMPY_VERSION: ${{ matrix.config.numpy-version }} + RAWPY_SKIP_EXAMPLES: ${{ matrix.config.python-arch == 'aarch64' && '1' || '' }} - name: Setup Python (Windows) if: matrix.config.os-name == 'windows' @@ -461,6 +435,118 @@ jobs: PYTHON_VERSION: ${{ matrix.config.python-version }} NUMPY_VERSION: ${{ matrix.config.numpy-version }} + build-sdist: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build sdist + run: | + pip install build + python -m build --sdist + + - name: Store sdist as artifact + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + test-sdist: + strategy: + fail-fast: false + matrix: + config: + - os-image: ubuntu-latest + os-name: linux + python-version: '3.12' + - os-image: macos-15 + os-name: mac + python-version: '3.12' + - os-image: windows-2022 + os-name: windows + python-arch: 'x86_64' + python-version: '3.12' + + runs-on: ${{ matrix.config.os-image }} + + needs: build-sdist + + steps: + - uses: actions/checkout@v4 + + - name: Download sdist from artifact storage + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.config.python-version }} + + - name: Test sdist (Linux) + if: matrix.config.os-name == 'linux' + run: .github/scripts/test-sdist-linux.sh + env: + PYTHON_VERSION: ${{ matrix.config.python-version }} + RAWPY_CI_NO_JASPER: '1' + + - name: Test sdist (macOS) + if: matrix.config.os-name == 'mac' + run: .github/scripts/test-sdist-macos.sh + env: + PYTHON_VERSION: ${{ matrix.config.python-version }} + + - name: Test sdist (Windows) + if: matrix.config.os-name == 'windows' + run: .github/scripts/test-sdist-windows.ps1 + shell: pwsh + env: + PYTHON_VERSION: ${{ matrix.config.python-version }} + PYTHON_ARCH: ${{ matrix.config.python-arch }} + + test-system-libraw: + runs-on: ubuntu-latest + + needs: build-sdist + + steps: + - uses: actions/checkout@v4 + with: + submodules: false # not needed — using system libraw + + - name: Download sdist from artifact storage + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Install system libraw + run: | + sudo apt-get update -q + sudo apt-get install -y -q libraw-dev pkg-config + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Test sdist with system libraw + run: .github/scripts/test-sdist-linux.sh + env: + PYTHON_VERSION: '3.12' + RAWPY_USE_SYSTEM_LIBRAW: '1' + RAWPY_CI_NO_JASPER: '1' + docs: runs-on: ubuntu-latest @@ -494,10 +580,10 @@ jobs: with: path: dist-docs/ - publish-wheels: + publish-dist: runs-on: ubuntu-latest - needs: [test, docs] + needs: [test, test-sdist, test-system-libraw, docs] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') @@ -516,6 +602,12 @@ jobs: merge-multiple: true path: dist + - name: Download sdist from artifact storage + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + - name: Setup Python uses: actions/setup-python@v5 @@ -525,7 +617,7 @@ jobs: publish-docs: runs-on: ubuntu-latest - needs: [publish-wheels] + needs: [publish-dist] permissions: pages: write # to deploy to Pages diff --git a/.gitignore b/.gitignore index 42445ff8..2614e493 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ # C extensions *.so +*.so.* # Distribution / packaging .Python @@ -74,3 +75,6 @@ Miniconda*.exe /vcpkg !/logo/logo.png +.venv/ +.venv-test/ +tmp/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..775fe91b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,328 @@ +# Agent Development Guide + +This repository wraps the C++ `LibRaw` library using Cython. + +## Prerequisites + +Before starting, ensure you have: +- **Python 3.9+** +- **C++ compiler** - `apt install g++` (Ubuntu) / Xcode Command Line Tools (macOS) + +## Critical: Compilation Required + +**You are working with Cython (`.pyx`) files.** +Changes to `rawpy/_rawpy.pyx` or C++ files **will not take effect** until you recompile. + +| File type | After editing... | +|-----------|------------------| +| `.py` files | Changes apply immediately (editable install) | +| `.pyx` files | Must run `bash scripts/rebuild.sh` | +| C++ files in `external/` | Must run `bash scripts/rebuild.sh` | +| `MANIFEST.in` | Rebuild: `bash scripts/build_dist.sh` | + +## Quick Commands + +| Task | Command | +|------|---------| +| First-time setup | `bash scripts/setup_dev_env.sh` | +| Setup with specific Python | `bash scripts/setup_dev_env.sh 3.12` | +| Activate environment | `source .venv/bin/activate` | +| Rebuild after .pyx/C++ changes | `bash scripts/rebuild.sh` | +| Quick sanity check | `bash scripts/dev_check.sh` | +| Build sdist + wheel | `bash scripts/build_dist.sh` | +| Test built sdist | `bash scripts/test_dist.sh sdist` | +| Test built wheel | `bash scripts/test_dist.sh wheel` | +| Test with numpy version | `bash scripts/test_dist.sh wheel 2.0.2` | +| Test sdist with system libraw | `bash scripts/build_dist.sh && RAWPY_USE_SYSTEM_LIBRAW=1 bash scripts/test_dist.sh sdist` | +| Test wheel with system libraw | `RAWPY_USE_SYSTEM_LIBRAW=1 bash scripts/build_dist.sh && RAWPY_USE_SYSTEM_LIBRAW=1 bash scripts/test_dist.sh wheel` | +| Run single test | `pytest test/test_basic.py::testName -v` | +| Run all tests | `pytest test/` | +| Type check | `mypy rawpy` | +| Switch numpy version | `bash scripts/setup_numpy.sh 2.0.2` | +| Build docs | `cd docs && sphinx-build -b html . _build/html` | +| Serve & view docs | `cd docs/_build/html && python -m http.server 8765` then open `http://localhost:8765` | + +> **System libraw requires LibRaw ≥ 0.21.** Ubuntu 22.04's `libraw-dev` (0.20.2) is +> too old. Use Ubuntu 24.04+ or build without `RAWPY_USE_SYSTEM_LIBRAW`. +> +> Note: The sdist build command does **not** use `RAWPY_USE_SYSTEM_LIBRAW=1` +> because sdist just packages source files — it doesn't compile anything. The +> env var is only needed at install/test time, when pip builds the sdist from +> source. For wheel, the env var is needed at both build **and** test time. + +## Environment Setup + +**First time only:** +```bash +bash scripts/setup_dev_env.sh +``` + +This will: +1. Create a `.venv` virtual environment +2. Check for required system dependencies (cmake, C++ compiler) +3. Initialize git submodules (LibRaw source) +4. Install Python dependencies +5. Build and install rawpy in editable mode + +**With a specific Python version (Ubuntu only):** +```bash +bash scripts/setup_dev_env.sh 3.12 +``` + +This installs the requested Python via the deadsnakes PPA, creates a `.venv` +with it, then runs the full setup. You can also use `scripts/setup_python.sh` +directly if you only need to switch the Python version without rebuilding. + +**For subsequent sessions:** +```bash +source .venv/bin/activate +``` + +## Architecture + +| Path | Purpose | +|------|---------| +| `rawpy/_rawpy.pyx` | Main Cython implementation (RawPy class, C++ bindings) | +| `rawpy/_rawpy.cpp` | **Generated** C++ from `.pyx` — do not edit manually. `setup.py` calls `cythonize()` which regenerates this, but only when the `.pyx` has a newer timestamp than the `.cpp`. A stale `.cpp` from a previous build can cause failures if the NumPy ABI has changed. `scripts/rebuild.sh` deletes it to force regeneration. | +| `rawpy/_rawpy.pyi` | Type stubs (update when changing API) | +| `rawpy/__init__.py` | Python entry point | +| `rawpy/enhance.py` | Pure Python utilities (bad pixel repair, etc.) | +| `external/LibRaw/` | LibRaw C++ library (git submodule) | +| `external/LibRaw/libraw/*.h` | LibRaw headers (check these for C++ signatures) | +| `external/LibRaw-cmake/` | CMake build system for LibRaw (git submodule) | +| `setup.py` | Build configuration (compiles LibRaw from source, links Cython extension) | +| `tmp/` | Scratch directory for build logs etc. (git-ignored) | +| `.github/workflows/ci.yml` | CI workflow (build matrix for Linux/macOS/Windows × Python versions) | +| `.github/scripts/` | Platform-specific CI build/test scripts | + +## Common Tasks + +### Building and viewing documentation + +The docs use Sphinx with the Read the Docs theme. Both are already installed +in the dev venv (via `dev-requirements.txt`). + +1. Build: `cd docs && sphinx-build -b html . _build/html` +2. Serve: `cd docs/_build/html && python -m http.server 8765` (run as background process) +3. Open `http://localhost:8765` in the Simple Browser + +- Source files: `docs/index.rst`, `docs/api/*.rst` +- Config: `docs/conf.py` +- Output: `docs/_build/html/` (git-ignored) +- The docs use `autodoc` to pull docstrings from the built Cython extension, + so `rawpy._rawpy` must be importable (i.e., the extension must be compiled). + Run `bash scripts/rebuild.sh` first if needed. + +### Adding a new LibRaw method + +1. Find the C++ signature in `external/LibRaw/libraw/libraw.h` +2. Add the `cdef extern` declaration in `rawpy/_rawpy.pyx` +3. Add a Python method in the `RawPy` class in `rawpy/_rawpy.pyx` +4. Add type stub in `rawpy/_rawpy.pyi` +5. Rebuild: `bash scripts/rebuild.sh` +6. Add a test in `test/` + +### Testing sdist and wheel artifacts + +The editable install (`pip install -e .`) is convenient for development but +doesn't catch packaging problems (missing files in `MANIFEST.in`, broken +build isolation, etc.). To test what end-users will get: + +```bash +# Build sdist and wheel (output in dist/) +bash scripts/build_dist.sh + +# Test the sdist — builds from source in a clean venv, then runs pytest +bash scripts/test_dist.sh sdist + +# Test the wheel +bash scripts/test_dist.sh wheel + +# Test with a specific numpy version +bash scripts/test_dist.sh sdist 2.0.2 + +# Test with a specific Python version (Ubuntu, via deadsnakes) +bash scripts/setup_python.sh 3.12 +bash scripts/build_dist.sh +bash scripts/test_dist.sh sdist +``` + +The test script creates an isolated `.venv-test` (separate from the dev +`.venv`), installs the artifact, runs the test suite from a temp directory +(so the source tree's `rawpy/` isn't accidentally imported), and cleans up +automatically. + +**Tip:** Building from source (sdist install, `pip install .`, etc.) compiles +LibRaw and the Cython extension, which can take several minutes. Use `tee` to +save output to `tmp/` (git-ignored) while still seeing progress: + +```bash +mkdir -p tmp +bash scripts/build_dist.sh 2>&1 | tee tmp/build.log +# Then inspect: +grep -i error tmp/build.log # just errors +tail -30 tmp/build.log # last 30 lines +``` + +> `tee` overwrites by default (like `>`), so re-running always gives a fresh log. + +### Running specific tests + +```bash +# Run a single test +pytest test/test_basic.py::testFileOpenAndPostProcess -v + +# Run tests matching a pattern +pytest -k "thumbnail" -v + +# Run with print output visible +pytest -s test/test_basic.py +``` + +## Troubleshooting + +### "No module named rawpy._rawpy" +The Cython extension isn't built. Run: +```bash +bash scripts/rebuild.sh +``` + +### "PyArray_Descr has no member named 'subarray'" or similar NumPy ABI errors +The generated `_rawpy.cpp` is stale (compiled against a different NumPy version). +`scripts/rebuild.sh` already handles this by deleting the `.cpp` so `cythonize()` +regenerates it. Just re-run `bash scripts/rebuild.sh`. To fix manually: +```bash +rm rawpy/_rawpy.cpp +pip install --no-build-isolation -e . +``` + +### "cmake: command not found" +cmake is installed automatically as a build dependency via `pyproject.toml`. +If you see this error during an editable install (`--no-build-isolation`), +install it into your venv: +```bash +pip install cmake +``` + +### "fatal error: libraw/libraw.h: No such file or directory" +Git submodules aren't initialized: +```bash +git submodule update --init --recursive +``` + +### Build fails with compiler errors +Ensure you have a C++ compiler: +```bash +# Ubuntu/Debian +sudo apt install g++ + +# macOS (installs clang) +xcode-select --install +``` + +### Mypy errors about missing stubs +If you added new API, update `rawpy/_rawpy.pyi` to match. + +### System libraw build fails with missing struct members +The system `libraw-dev` is too old. rawpy requires LibRaw ≥ 0.21. +Ubuntu 22.04 ships LibRaw 0.20.2, which is incompatible. Errors look like: +``` +error: 'libraw_raw_unpack_params_t' was not declared in this scope +error: 'struct libraw_image_sizes_t' has no member named 'raw_inset_crops' +``` +Use Ubuntu 24.04+ (ships 0.22) or build without `RAWPY_USE_SYSTEM_LIBRAW=1` +(the default), which compiles LibRaw 0.22 from the bundled submodule. + +## CI Architecture + +The CI workflow is in `.github/workflows/ci.yml`. It builds wheels across a matrix: + +| Platform | Runner | Container | Architectures | +|----------|--------|-----------|---------------| +| Linux | `ubuntu-latest` | `manylinux_2_28` (RHEL-based) | x86_64, aarch64 (via QEMU) | +| macOS | `macos-15` | native | arm64 | +| Windows | `windows-2022` | native | x86_64 | + +Build scripts in `.github/scripts/`: +- `build-linux.sh` — runs inside Docker; installs deps, builds wheel, runs `auditwheel` +- `build-macos.sh` — installs deps from source (respects `MACOSX_DEPLOYMENT_TARGET`), uses `delocate` +- `build-windows.ps1` — uses vcpkg for deps, VS build tools + +### Reproducing CI builds locally + +```bash +# Build sdist + wheel with build isolation (recommended): +bash scripts/build_dist.sh + +# Low-level alternative (what CI does for wheels): +pip wheel . --wheel-dir dist --no-deps + +# Build without isolation (faster, for local dev): +pip install --no-build-isolation -e . +``` + +Note: `python -m build` (used by `build_dist.sh`) and `pip wheel .` both use +build isolation and create a fresh environment from `pyproject.toml`'s +`build-system.requires`. This is different from the local dev workflow +(`--no-build-isolation`) which reuses the current venv. + +### Reproducing CI test failures locally + +CI tests run across multiple Python and NumPy versions. Type checking (mypy) +is particularly sensitive to the NumPy stubs version bundled with each NumPy +release. + +**Test artifacts against a specific Python + NumPy (closest to CI):** +```bash +# Install Python 3.12 via deadsnakes, build artifacts, and test sdist +bash scripts/setup_python.sh 3.12 +bash scripts/build_dist.sh +bash scripts/test_dist.sh sdist 2.0.2 +``` + +**Test with a specific NumPy version (editable install):** +```bash +# Switch to numpy 2.0.x, then use normal commands +bash scripts/setup_numpy.sh 2.0.2 +source .venv/bin/activate +pytest test/test_mypy.py -v + +# Switch back when done +bash scripts/setup_numpy.sh 2.2.6 +``` + +**Test with a specific Python version (editable install):** +```bash +# Install Python 3.12 and rebuild everything +bash scripts/setup_dev_env.sh 3.12 + +# Then run tests +source .venv/bin/activate +pytest test/ -v +``` + +**Reference CI NumPy versions** (check `.github/workflows/ci.yml` test matrix): + +| Python | NumPy | +|--------|-------| +| 3.9–3.12 | 2.0.* | +| 3.13 | 2.1.* | +| 3.14 | 2.4.* | + +## Platform-Specific Notes + +- **Linux (RHEL/manylinux):** CMake's `GNUInstallDirs` installs libraries to + `lib64/` instead of `lib/` on 64-bit RHEL-based systems. The `setup.py` + handles this by passing `-DCMAKE_INSTALL_LIBDIR=lib` to cmake. If you modify + cmake arguments in `setup.py`, always keep this flag. +- **macOS:** `-DCMAKE_INSTALL_NAME_DIR` is required for dylib resolution. + Build scripts install dependencies from source to control `MACOSX_DEPLOYMENT_TARGET`. +- **Windows:** Uses a separate `windows_libraw_compile()` code path with + NMake Makefiles generator and vcpkg for native dependencies. + +## Examples + +See `examples/` for API usage: +- `basic_process.py` - Load RAW, postprocess to RGB, save +- `thumbnail_extract.py` - Extract embedded JPEG thumbnail +- `bad_pixel_repair.py` - Detect and repair bad pixels diff --git a/MANIFEST.in b/MANIFEST.in index 2b76cc7a..b4b8a91a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,31 @@ -include README.rst -include rawpy/def_helper.h \ No newline at end of file +# Include documentation and license files +include README.md +include LICENSE +include LICENSE.LibRaw +# Note: AGENTS.md is excluded - it's for development only, not end users + +# Include Cython source and helper headers +include rawpy/_rawpy.pyx +include rawpy/def_helper.h +include rawpy/data_helper.h + +# Include type stub and marker +include rawpy/py.typed +include rawpy/_rawpy.pyi + +# Include external LibRaw source code (required for building from source) +recursive-include external/LibRaw *.h *.cpp +include external/LibRaw/COPYRIGHT +include external/LibRaw/LICENSE.CDDL +include external/LibRaw/LICENSE.LGPL +include external/LibRaw/Changelog.txt +recursive-include external/LibRaw-cmake *.cmake *.cmake.in CMakeLists.txt + +# Exclude build artifacts +prune external/LibRaw-cmake/build + +# Exclude development-only directories +prune test + +# Exclude generated files (regenerated from .pyx during build) +exclude rawpy/_rawpy.cpp diff --git a/README.md b/README.md index c9e9e64a..f600ac7b 100644 --- a/README.md +++ b/README.md @@ -150,21 +150,17 @@ for libraries by default in some Linux distributions. These instructions are experimental and support is not provided for them. Typically, there should be no need to build manually since wheels are hosted on PyPI. -You need to have Visual Studio installed to build rawpy. +You need to have Visual Studio and Python installed to build rawpy. In a PowerShell window: ```sh -$env:USE_CONDA = '1' -$env:PYTHON_VERSION = '3.7' -$env:PYTHON_ARCH = '64' -$env:NUMPY_VERSION = '1.14.*' +$env:PYTHON_VERSION = '3.12' +$env:PYTHON_ARCH = 'x86_64' git clone https://github.com/letmaik/rawpy cd rawpy +git submodule update --init .github/scripts/build-windows.ps1 ``` -The above will download all build dependencies (including a Python installation) -and is fully configured through the four environment variables. -Set `USE_CONDA = '0'` to build within an existing Python environment. ## FAQ diff --git a/dev-requirements.txt b/dev-requirements.txt index b4dae9ce..1a3f9ea7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,7 @@ # build dependencies wheel>=0.31.0 +setuptools>=69 +build delocate;sys.platform == 'darwin' cython @@ -12,7 +14,7 @@ scikit-image # test dependencies pytest imageio>=2.21 # for imageio.v3 / iio support -setuptools +mypy # documentation dependencies sphinx_rtd_theme diff --git a/docs/conf.py b/docs/conf.py index ec8947f4..2cd0c478 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,38 +14,38 @@ import sys import os -import sphinx_rtd_theme import rawpy # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'rawpy' -copyright = u'2014, Maik Riechert' +project = "rawpy" +copyright = "2014, Maik Riechert" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -58,66 +58,66 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'logo_only': True, - 'display_version': True, + "logo_only": True, + "display_version": True, } # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# html_theme_path = [] # Not needed for sphinx_rtd_theme >= 1.0 # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -126,116 +126,110 @@ # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -html_extra_path = ['gh-pages'] +html_extra_path = ["gh-pages"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'rawpydoc' +htmlhelp_basename = "rawpydoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'rawpy.tex', u'rawpy Documentation', - u'Maik Riechert', 'manual'), + ("index", "rawpy.tex", "rawpy Documentation", "Maik Riechert", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'rawpy', u'rawpy Documentation', - [u'Maik Riechert'], 1) -] +man_pages = [("index", "rawpy", "rawpy Documentation", ["Maik Riechert"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -244,21 +238,32 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'rawpy', u'rawpy Documentation', - u'Maik Riechert', 'rawpy', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "rawpy", + "rawpy Documentation", + "Maik Riechert", + "rawpy", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -autoclass_content = 'both' \ No newline at end of file +autoclass_content = "both" + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable", None), +} diff --git a/examples/bad_pixel_repair.py b/examples/bad_pixel_repair.py new file mode 100644 index 00000000..c154d64e --- /dev/null +++ b/examples/bad_pixel_repair.py @@ -0,0 +1,63 @@ +""" +Bad Pixel Repair Example + +Demonstrates: +- Using rawpy.enhance module for bad pixel detection/repair +- Loading pre-computed bad pixel coordinates +- Repairing bad pixels using median interpolation + +Note: In practice, you would first detect bad pixels using: + bad_pixels = rawpy.enhance.find_bad_pixels(['image1.NEF', 'image2.NEF', ...]) + +Usage: + python examples/bad_pixel_repair.py +""" + +import numpy as np +import rawpy +import rawpy.enhance +import imageio.v3 as iio +import os +import sys +import tempfile + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) +TEST_IMAGE = os.path.join(REPO_ROOT, "test", "iss030e122639.NEF") +BAD_PIXELS_FILE = os.path.join(REPO_ROOT, "test", "bad_pixels.gz") + + +def main(): + if not os.path.exists(TEST_IMAGE): + print(f"Error: Test image not found at {TEST_IMAGE}") + return 1 + + # Load bad pixel coordinates (if available) + if not os.path.exists(BAD_PIXELS_FILE): + print(f"Bad pixel file not found: {BAD_PIXELS_FILE}") + print("Skipping repair demo. In practice, you would run:") + print(" bad_pixels = rawpy.enhance.find_bad_pixels([...image paths...])") + return 0 + + bad_pixels = np.loadtxt(BAD_PIXELS_FILE, dtype=int) + + print(f"Loaded {len(bad_pixels)} bad pixel coordinates") + print(f"Processing: {TEST_IMAGE}") + + with rawpy.imread(TEST_IMAGE) as raw: + # Repair bad pixels in-place before postprocessing + rawpy.enhance.repair_bad_pixels(raw, bad_pixels, method="median") + + # Now postprocess the repaired data + rgb = raw.postprocess() + + output_path = os.path.join(tempfile.gettempdir(), "rawpy_repaired.tiff") + iio.imwrite(output_path, rgb) + print(f"Saved repaired image to: {output_path}") + + print("Done!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/basic_process.py b/examples/basic_process.py new file mode 100644 index 00000000..13f0f453 --- /dev/null +++ b/examples/basic_process.py @@ -0,0 +1,51 @@ +""" +Basic RAW Processing Example + +Demonstrates: +- Loading a RAW file with rawpy.imread() +- Converting to RGB with postprocess() +- Saving the result + +Usage: + python examples/basic_process.py +""" + +import rawpy +import imageio.v3 as iio +import os +import sys +import tempfile + +# Locate test image (works from repo root or examples/ directory) +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) +TEST_IMAGE = os.path.join(REPO_ROOT, "test", "iss030e122639.NEF") + + +def main(): + if not os.path.exists(TEST_IMAGE): + print(f"Error: Test image not found at {TEST_IMAGE}") + print("This example requires the test data from the repository.") + return 1 + + print(f"Loading: {TEST_IMAGE}") + + with rawpy.imread(TEST_IMAGE) as raw: + print(f" Raw type: {raw.raw_type}") + print(f" Image size: {raw.sizes.width}x{raw.sizes.height}") + + # Convert RAW to RGB using default parameters + rgb = raw.postprocess() + print(f" Output shape: {rgb.shape}") + + # Save to temp directory (avoids polluting repo) + output_path = os.path.join(tempfile.gettempdir(), "rawpy_basic_output.tiff") + iio.imwrite(output_path, rgb) + print(f" Saved to: {output_path}") + + print("Done!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/thumbnail_extract.py b/examples/thumbnail_extract.py new file mode 100644 index 00000000..351fc603 --- /dev/null +++ b/examples/thumbnail_extract.py @@ -0,0 +1,61 @@ +""" +Thumbnail Extraction Example + +Demonstrates: +- Extracting embedded JPEG thumbnails from RAW files +- Handling different thumbnail formats (JPEG vs BITMAP) +- Error handling for missing/unsupported thumbnails + +Usage: + python examples/thumbnail_extract.py +""" + +import rawpy +import imageio.v3 as iio +import os +import sys +import tempfile + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) +TEST_IMAGE = os.path.join(REPO_ROOT, "test", "iss030e122639.NEF") + + +def main(): + if not os.path.exists(TEST_IMAGE): + print(f"Error: Test image not found at {TEST_IMAGE}") + return 1 + + print(f"Extracting thumbnail from: {TEST_IMAGE}") + + with rawpy.imread(TEST_IMAGE) as raw: + try: + thumb = raw.extract_thumb() + except rawpy.LibRawNoThumbnailError: + print("No thumbnail embedded in this file.") + return 0 + except rawpy.LibRawUnsupportedThumbnailError: + print("Thumbnail format not supported.") + return 0 + + print(f" Thumbnail format: {thumb.format}") + + output_dir = tempfile.gettempdir() + + if thumb.format == rawpy.ThumbFormat.JPEG: + output_path = os.path.join(output_dir, "rawpy_thumb.jpg") + with open(output_path, "wb") as f: + f.write(thumb.data) + print(f" Saved JPEG to: {output_path}") + + elif thumb.format == rawpy.ThumbFormat.BITMAP: + output_path = os.path.join(output_dir, "rawpy_thumb.tiff") + iio.imwrite(output_path, thumb.data) + print(f" Saved TIFF to: {output_path}") + + print("Done!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..d817e5c9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,83 @@ +[build-system] +requires = [ + "setuptools>=69.0.0", + "wheel", + "Cython>=0.29.32", + "cmake", + # Build against NumPy 2.x headers. Extensions compiled with NumPy 2.0+ + # are backward-compatible with NumPy >= 1.19 at runtime. + "numpy>=2.0.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "rawpy" +dynamic = ["version"] +description = "RAW image processing for Python, a wrapper for libraw" +readme = "README.md" +authors = [ + {name = "Maik Riechert"} +] +license = "MIT" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Cython", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: Unix", + "Topic :: Multimedia :: Graphics", + "Topic :: Software Development :: Libraries", +] +requires-python = ">=3.9" +dependencies = [ + "numpy>=1.26.0" +] + +[project.urls] +Homepage = "https://github.com/letmaik/rawpy" +Documentation = "https://letmaik.github.io/rawpy/" +Repository = "https://github.com/letmaik/rawpy.git" +Issues = "https://github.com/letmaik/rawpy/issues" +Changelog = "https://github.com/letmaik/rawpy/releases" + +[project.optional-dependencies] +test = [ + "pytest", + "imageio>=2.21", + "mypy", + "scikit-image", +] + +[tool.setuptools.packages.find] +include = ["rawpy*"] + +[tool.mypy] +# Global mypy configuration for rawpy project +warn_unused_configs = true +check_untyped_defs = true + +# Selectively ignore missing imports for optional dependencies +[[tool.mypy.overrides]] +module = [ + "skimage.*", + "cv2", + "scipy.*", + "imageio", + "imageio.*", + "imageio.v3" +] +ignore_missing_imports = true + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = ["test"] diff --git a/rawpy/__init__.py b/rawpy/__init__.py index 93c92424..b15c53bb 100644 --- a/rawpy/__init__.py +++ b/rawpy/__init__.py @@ -1,4 +1,47 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations + +from typing import Union, TYPE_CHECKING + +if TYPE_CHECKING: + from typing import BinaryIO + from rawpy._rawpy import ( + # Module-level attributes + flags, + libraw_version, + # Main classes + RawPy, + Params, + # Named tuples + ImageSizes, + Thumbnail, + # Enums + RawType, + ThumbFormat, + DemosaicAlgorithm, + FBDDNoiseReductionMode, + ColorSpace, + HighlightMode, + # Exceptions + LibRawError, + LibRawFatalError, + LibRawNonFatalError, + LibRawUnspecifiedError, + LibRawFileUnsupportedError, + LibRawRequestForNonexistentImageError, + LibRawOutOfOrderCallError, + LibRawNoThumbnailError, + LibRawUnsupportedThumbnailError, + LibRawInputClosedError, + LibRawNotImplementedError, + LibRawUnsufficientMemoryError, + LibRawDataError, + LibRawIOError, + LibRawCancelledByCallbackError, + LibRawBadCropError, + LibRawTooBigError, + LibRawMemPoolOverflowError, + NotSupportedError, + ) from ._version import __version__ @@ -59,21 +102,26 @@ def _check_multiprocessing_fork(): # multiprocessing not available pass -def imread(pathOrFile, shot_select=0): +def imread(pathOrFile: Union[str, BinaryIO], shot_select: int = 0) -> RawPy: """ Convenience function that creates a :class:`rawpy.RawPy` instance, opens the given file, and returns the :class:`rawpy.RawPy` instance for further processing. - :param str|file pathOrFile: path or file object of RAW image that will be read - :param int shot_select: select which image to extract from RAW files that contain multiple images - (e.g., Dual Pixel RAW). Default is 0 for the first/main image. - :rtype: :class:`rawpy.RawPy` + :param pathOrFile: path or file object of RAW image that will be read + :type pathOrFile: str or file-like object + :param shot_select: select which image to extract from RAW files that contain multiple images + (e.g., Dual Pixel RAW). Default is 0 for the first/main image. + :type shot_select: int + :return: RawPy instance with the opened RAW image + :rtype: rawpy.RawPy """ _check_multiprocessing_fork() d = RawPy() - if hasattr(pathOrFile, 'read'): - d.open_buffer(pathOrFile) - else: + if isinstance(pathOrFile, str): + # pathOrFile is a string file path d.open_file(pathOrFile) + else: + # pathOrFile is a file-like object with read() method + d.open_buffer(pathOrFile) d.set_unpack_params(shot_select=shot_select) return d \ No newline at end of file diff --git a/rawpy/_rawpy.pyi b/rawpy/_rawpy.pyi new file mode 100644 index 00000000..a669c897 --- /dev/null +++ b/rawpy/_rawpy.pyi @@ -0,0 +1,614 @@ +"""Type stubs for rawpy._rawpy Cython module""" +from __future__ import annotations + +from enum import Enum +from typing import Any, Optional, Tuple, List, Union, NamedTuple, BinaryIO +import numpy as np +from numpy.typing import NDArray + +# Module-level version +libraw_version: Tuple[int, int, int] +flags: Optional[dict[str, bool]] + +# Named tuples +class ImageSizes(NamedTuple): + raw_height: int + raw_width: int + height: int + width: int + top_margin: int + left_margin: int + iheight: int + iwidth: int + pixel_aspect: float + flip: int + crop_left_margin: int + crop_top_margin: int + crop_width: int + crop_height: int + +class Thumbnail(NamedTuple): + format: ThumbFormat + data: Union[bytes, NDArray[np.uint8]] + +# Enums +class RawType(Enum): + """ + RAW image type. + """ + Flat = 0 + """ Bayer type or black and white """ + Stack = 1 + """ Foveon type or sRAW/mRAW files or RawSpeed decoding """ + +class ThumbFormat(Enum): + """ + Thumbnail/preview image type. + """ + JPEG = 1 + """ JPEG image as bytes object. """ + BITMAP = 2 + """ RGB image as ndarray object. """ + +class DemosaicAlgorithm(Enum): + """ + Identifiers for demosaic algorithms. + """ + LINEAR = 0 + VNG = 1 + PPG = 2 + AHD = 3 + DCB = 4 + # 5-9 only usable if demosaic pack GPL2 available + MODIFIED_AHD = 5 + AFD = 6 + VCD = 7 + VCD_MODIFIED_AHD = 8 + LMMSE = 9 + # 10 only usable if demosaic pack GPL3 available + AMAZE = 10 + # 11-12 only usable for LibRaw >= 0.16 + DHT = 11 + AAHD = 12 + + @property + def isSupported(self) -> Optional[bool]: + """ + Return True if the demosaic algorithm is supported, False if it is not, + and None if the support status is unknown. The latter is returned if + LibRaw < 0.15.4 is used or if it was compiled without cmake. + + The necessary information is read from the libraw_config.h header which + is only written with cmake builds >= 0.15.4. + """ + ... + + def checkSupported(self) -> Optional[bool]: + """ + Like :attr:`isSupported` but raises an exception for the `False` case. + """ + ... + +class FBDDNoiseReductionMode(Enum): + """ + FBDD noise reduction modes. + """ + Off = 0 + Light = 1 + Full = 2 + +class ColorSpace(Enum): + """ + Color spaces. + """ + raw = 0 + sRGB = 1 + Adobe = 2 + Wide = 3 + ProPhoto = 4 + XYZ = 5 + ACES = 6 + P3D65 = 7 + Rec2020 = 8 + +class HighlightMode(Enum): + """ + Highlight modes. + """ + Clip = 0 + Ignore = 1 + Blend = 2 + ReconstructDefault = 5 + + @classmethod + def Reconstruct(cls, level: int) -> int: + """ + :param int level: 3 to 9, low numbers favor whites, high numbers favor colors + """ + ... + +# Exceptions +class LibRawError(Exception): ... +class LibRawFatalError(LibRawError): ... +class LibRawNonFatalError(LibRawError): ... +class LibRawUnspecifiedError(LibRawNonFatalError): ... +class LibRawFileUnsupportedError(LibRawNonFatalError): ... +class LibRawRequestForNonexistentImageError(LibRawNonFatalError): ... +class LibRawOutOfOrderCallError(LibRawNonFatalError): ... +class LibRawNoThumbnailError(LibRawNonFatalError): ... +class LibRawUnsupportedThumbnailError(LibRawNonFatalError): ... +class LibRawInputClosedError(LibRawNonFatalError): ... +class LibRawNotImplementedError(LibRawNonFatalError): ... +class LibRawUnsufficientMemoryError(LibRawFatalError): ... +class LibRawDataError(LibRawFatalError): ... +class LibRawIOError(LibRawFatalError): ... +class LibRawCancelledByCallbackError(LibRawFatalError): ... +class LibRawBadCropError(LibRawFatalError): ... +class LibRawTooBigError(LibRawFatalError): ... +class LibRawMemPoolOverflowError(LibRawFatalError): ... + +class NotSupportedError(Exception): + def __init__(self, message: str, min_version: Optional[Tuple[int, int, int]] = None) -> None: ... + +# Params class +class Params: + """ + A class that handles postprocessing parameters. + """ + + def __init__( + self, + demosaic_algorithm: Optional[DemosaicAlgorithm] = None, + half_size: bool = False, + four_color_rgb: bool = False, + dcb_iterations: int = 0, + dcb_enhance: bool = False, + fbdd_noise_reduction: FBDDNoiseReductionMode = FBDDNoiseReductionMode.Off, + noise_thr: Optional[float] = None, + median_filter_passes: int = 0, + use_camera_wb: bool = False, + use_auto_wb: bool = False, + user_wb: Optional[List[float]] = None, + output_color: ColorSpace = ColorSpace.sRGB, + output_bps: int = 8, + user_flip: Optional[int] = None, + user_black: Optional[int] = None, + user_cblack: Optional[List[int]] = None, + user_sat: Optional[int] = None, + no_auto_bright: bool = False, + auto_bright_thr: Optional[float] = None, + adjust_maximum_thr: float = 0.75, + bright: float = 1.0, + highlight_mode: Union[HighlightMode, int] = HighlightMode.Clip, + exp_shift: Optional[float] = None, + exp_preserve_highlights: float = 0.0, + no_auto_scale: bool = False, + gamma: Optional[Tuple[float, float]] = None, + chromatic_aberration: Optional[Tuple[float, float]] = None, + bad_pixels_path: Optional[str] = None, + ) -> None: + """ + If use_camera_wb and use_auto_wb are False and user_wb is None, then + daylight white balance correction is used. + If both use_camera_wb and use_auto_wb are True, then use_auto_wb has priority. + + :param rawpy.DemosaicAlgorithm demosaic_algorithm: default is AHD + :param bool half_size: outputs image in half size by reducing each 2x2 block to one pixel + instead of interpolating + :param bool four_color_rgb: whether to use separate interpolations for two green channels + :param int dcb_iterations: number of DCB correction passes, requires DCB demosaicing algorithm + :param bool dcb_enhance: DCB interpolation with enhanced interpolated colors + :param rawpy.FBDDNoiseReductionMode fbdd_noise_reduction: controls FBDD noise reduction before demosaicing + :param float noise_thr: threshold for wavelet denoising (default disabled) + :param int median_filter_passes: number of median filter passes after demosaicing to reduce color artifacts + :param bool use_camera_wb: whether to use the as-shot white balance values + :param bool use_auto_wb: whether to try automatically calculating the white balance + :param list user_wb: list of length 4 with white balance multipliers for each color + :param rawpy.ColorSpace output_color: output color space + :param int output_bps: 8 or 16 + :param int user_flip: 0=none, 3=180, 5=90CCW, 6=90CW, + default is to use image orientation from the RAW image if available + :param int user_black: custom black level + :param list user_cblack: list of length 4 with per-channel corrections to user_black. + These are offsets applied on top of user_black for [R, G, B, G2] channels. + :param int user_sat: saturation adjustment (custom white level) + :param bool no_auto_scale: Whether to disable pixel value scaling + :param bool no_auto_bright: whether to disable automatic increase of brightness + :param float auto_bright_thr: ratio of clipped pixels when automatic brighness increase is used + (see `no_auto_bright`). Default is 0.01 (1%). + :param float adjust_maximum_thr: see libraw docs + :param float bright: brightness scaling + :param highlight_mode: highlight mode + :type highlight_mode: :class:`rawpy.HighlightMode` | int + :param float exp_shift: exposure shift in linear scale. + Usable range from 0.25 (2-stop darken) to 8.0 (3-stop lighter). + :param float exp_preserve_highlights: preserve highlights when lightening the image with `exp_shift`. + From 0.0 to 1.0 (full preservation). + :param tuple gamma: pair (power,slope), default is (2.222, 4.5) for rec. BT.709 + :param tuple chromatic_aberration: pair (red_scale, blue_scale), default is (1,1), + corrects chromatic aberration by scaling the red and blue channels + :param str bad_pixels_path: path to dcraw bad pixels file. Each bad pixel will be corrected using + the mean of the neighbor pixels. See the :mod:`rawpy.enhance` module + for alternative repair algorithms, e.g. using the median. + """ + ... + + # Instance attributes (accessible after __init__) + user_qual: int + half_size: bool + four_color_rgb: bool + dcb_iterations: int + dcb_enhance: bool + fbdd_noise_reduction: int + noise_thr: float + median_filter_passes: int + use_camera_wb: bool + use_auto_wb: bool + user_mul: List[float] + output_color: int + output_bps: int + user_flip: int + user_black: int + user_cblack: List[int] + user_sat: int + no_auto_bright: bool + auto_bright_thr: float + adjust_maximum_thr: float + bright: float + highlight_mode: int + exp_shift: float + exp_preserve_highlights: float + no_auto_scale: bool + gamm: Tuple[float, float] + aber: Tuple[float, float] + bad_pixels_path: Optional[str] + + +# Main RawPy class +class RawPy: + """ + Load RAW images, work on their data, and create a postprocessed (demosaiced) image. + + All operations are implemented using numpy arrays. + """ + + def __enter__(self) -> RawPy: ... + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + + def close(self) -> None: + """ + Release all resources and close the RAW image. + + Consider using context managers for the same effect: + + .. code-block:: python + + with rawpy.imread('image.nef') as raw: + # work with raw object + """ + ... + + def open_file(self, path: str) -> None: + """ + Opens the given RAW image file. Should be followed by a call to :meth:`~rawpy.RawPy.unpack`. + + .. NOTE:: This is a low-level method, consider using :func:`rawpy.imread` instead. + + :param str path: The path to the RAW image. + """ + ... + + def open_buffer(self, fileobj: BinaryIO) -> None: + """ + Opens the given RAW image file-like object. Should be followed by a call to :meth:`~rawpy.RawPy.unpack`. + + .. NOTE:: This is a low-level method, consider using :func:`rawpy.imread` instead. + + :param file fileobj: The file-like object. + """ + ... + + def set_unpack_params(self, shot_select: int = 0) -> None: + """ + Set parameters that affect RAW image unpacking. + + This should be called after opening a file and before unpacking. + + .. NOTE:: This is a low-level method. When using :func:`rawpy.imread`, + unpack parameters can be provided directly. + + :param int shot_select: select which image to extract from RAW files that contain multiple images + (e.g., Dual Pixel RAW). Default is 0 for the first/main image. + """ + ... + + def unpack(self) -> None: + """ + Unpacks/decodes the opened RAW image. + + .. NOTE:: This is a low-level method, consider using :func:`rawpy.imread` instead. + """ + ... + + def unpack_thumb(self) -> None: + """ + Unpacks/decodes the thumbnail/preview image, whichever is bigger. + + .. NOTE:: This is a low-level method, consider using :meth:`~rawpy.RawPy.extract_thumb` instead. + """ + ... + + @property + def raw_type(self) -> RawType: + """ + Return the RAW type. + + :rtype: :class:`rawpy.RawType` + """ + ... + + @property + def raw_image(self) -> NDArray[np.uint16]: + """ + View of RAW image. Includes margin. + + For Bayer images, a 2D ndarray is returned. + For Foveon and other RGB-type images, a 3D ndarray is returned. + Note that there may be 4 color channels, where the 4th channel can be blank (zeros). + + Modifying the returned array directly influences the result of + calling :meth:`~rawpy.RawPy.postprocess`. + + .. WARNING:: The returned numpy array can only be accessed while this RawPy instance + is not closed yet, that is, within a :code:`with` block or before calling :meth:`~rawpy.RawPy.close`. + If you need to work on the array after closing the RawPy instance, + make sure to create a copy of it with :code:`raw_image = raw.raw_image.copy()`. + + :rtype: ndarray of shape (h,w[,c]) + """ + ... + + @property + def raw_image_visible(self) -> NDArray[np.uint16]: + """ + Like raw_image but without margin. + + :rtype: ndarray of shape (hv,wv[,c]) + """ + ... + + def raw_value(self, row: int, column: int) -> int: + """ + Return RAW value at given position relative to the full RAW image. + Only usable for flat RAW images (see :attr:`~rawpy.RawPy.raw_type` property). + """ + ... + + def raw_value_visible(self, row: int, column: int) -> int: + """ + Return RAW value at given position relative to visible area of image. + Only usable for flat RAW images (see :attr:`~rawpy.RawPy.raw_type` property). + """ + ... + + @property + def sizes(self) -> ImageSizes: + """ + Return a :class:`rawpy.ImageSizes` instance with size information of + the RAW image and postprocessed image. + """ + ... + + @property + def num_colors(self) -> int: + """ + Number of colors. + Note that e.g. for RGBG this can be 3 or 4, depending on the camera model, + as some use two different greens. + """ + ... + + @property + def color_desc(self) -> bytes: + """ + String description of colors numbered from 0 to 3 (RGBG,RGBE,GMCY, or GBTG). + Note that same letters may not refer strictly to the same color. + There are cameras with two different greens for example. + """ + ... + + def raw_color(self, row: int, column: int) -> int: + """ + Return color index for the given coordinates relative to the full RAW size. + + :rtype: 0 to 3 (sometimes 4) + """ + ... + + @property + def raw_colors(self) -> NDArray[np.uint8]: + """ + An array of color indices for each pixel in the RAW image. + Equivalent to calling raw_color(y,x) for each pixel. + Only usable for flat RAW images (see raw_type property). + + :rtype: ndarray of shape (h,w) + """ + ... + + @property + def raw_colors_visible(self) -> NDArray[np.uint8]: + """ + Like raw_colors but without margin. + + :rtype: ndarray of shape (hv,wv) + """ + ... + + @property + def raw_pattern(self) -> Optional[NDArray[np.uint8]]: + """ + The smallest possible Bayer pattern of this image. + + :rtype: ndarray, or None if not a flat RAW image + """ + ... + + @property + def camera_whitebalance(self) -> List[float]: + """ + White balance coefficients (as shot). Either read from file or calculated. + + :rtype: list of length 4 + """ + ... + + @property + def daylight_whitebalance(self) -> List[float]: + """ + White balance coefficients for daylight (daylight balance). + Either read from file, or calculated on the basis of file data, + or taken from hardcoded constants. + + :rtype: list of length 4 + """ + ... + + @property + def auto_whitebalance(self) -> Optional[List[float]]: + """ + White balance coefficients used during postprocessing. + + This property returns the actual white balance multipliers that were used + during postprocessing, regardless of the white balance mode: + whether from camera settings, auto white balance calculation, user-specified + values, or daylight balance. + + This property must be accessed after calling :meth:`~rawpy.RawPy.postprocess` + or :meth:`~rawpy.RawPy.dcraw_process` to get the coefficients that were + actually applied. If accessed before postprocessing, it returns None. + + This corresponds to LibRaw's ``imgdata.color.pre_mul[]`` array after processing, + which contains the white balance multipliers applied to the raw sensor data. + + :rtype: list of length 4, or None if postprocessing hasn't been called yet + """ + ... + + @property + def black_level_per_channel(self) -> List[int]: + """ + Per-channel black level correction. + + :rtype: list of length 4 + """ + ... + + @property + def white_level(self) -> int: + """ + Level at which the raw pixel value is considered to be saturated. + """ + ... + + @property + def camera_white_level_per_channel(self) -> Optional[List[int]]: + """ + Per-channel saturation levels read from raw file metadata, if it exists. Otherwise None. + + :rtype: list of length 4, or None if metadata missing + """ + ... + + @property + def color_matrix(self) -> NDArray[np.float32]: + """ + Color matrix, read from file for some cameras, calculated for others. + + :rtype: ndarray of shape (3,4) + """ + ... + + @property + def rgb_xyz_matrix(self) -> NDArray[np.float32]: + """ + Camera RGB - XYZ conversion matrix. + This matrix is constant (different for different models). + Last row is zero for RGB cameras and non-zero for different color models (CMYG and so on). + + :rtype: ndarray of shape (4,3) + """ + ... + + @property + def tone_curve(self) -> NDArray[np.uint16]: + """ + Camera tone curve, read from file for Nikon, Sony and some other cameras. + + :rtype: ndarray of length 65536 + """ + ... + + def dcraw_process(self, params: Optional[Params] = None, **kw: Any) -> None: + """ + Postprocess the currently loaded RAW image. + + .. NOTE:: This is a low-level method, consider using :meth:`~rawpy.RawPy.postprocess` instead. + + :param rawpy.Params params: + The parameters to use for postprocessing. + :param **kw: + Alternative way to provide postprocessing parameters. + The keywords are used to construct a :class:`rawpy.Params` instance. + If keywords are given, then `params` must be omitted. + """ + ... + + def dcraw_make_mem_image(self) -> NDArray[np.uint8]: + """ + Return the postprocessed image (see :meth:`~rawpy.RawPy.dcraw_process`) as numpy array. + + .. NOTE:: This is a low-level method, consider using :meth:`~rawpy.RawPy.postprocess` instead. + + :rtype: ndarray of shape (h,w,c) + """ + ... + + def dcraw_make_mem_thumb(self) -> Thumbnail: + """ + Return the thumbnail/preview image (see :meth:`~rawpy.RawPy.unpack_thumb`) + as :class:`rawpy.Thumbnail` object. + For JPEG thumbnails, data is a bytes object and can be written as-is to file. + For bitmap thumbnails, data is an ndarray of shape (h,w,c). + If no image exists or the format is unsupported, an exception is raised. + + .. NOTE:: This is a low-level method, consider using :meth:`~rawpy.RawPy.extract_thumb` instead. + + :rtype: :class:`rawpy.Thumbnail` + """ + ... + + def postprocess(self, params: Optional[Params] = None, **kw: Any) -> NDArray[np.uint8]: + """ + Postprocess the currently loaded RAW image and return the + new resulting image as numpy array. + + :param rawpy.Params params: + The parameters to use for postprocessing. + :param **kw: + Alternative way to provide postprocessing parameters. + The keywords are used to construct a :class:`rawpy.Params` instance. + If keywords are given, then `params` must be omitted. + :rtype: ndarray of shape (h,w,c) + """ + ... + + def extract_thumb(self) -> Thumbnail: + """ + Extracts and returns the thumbnail/preview image (whichever is bigger) + of the opened RAW image as :class:`rawpy.Thumbnail` object. + + :rtype: :class:`rawpy.Thumbnail` + """ + ... diff --git a/rawpy/_rawpy.pyx b/rawpy/_rawpy.pyx index 10f6c48d..20b5edda 100644 --- a/rawpy/_rawpy.pyx +++ b/rawpy/_rawpy.pyx @@ -4,10 +4,12 @@ from __future__ import print_function -from cpython.ref cimport PyObject, Py_INCREF +from typing import Optional, Union, Tuple, List, Any, BinaryIO +from numpy.typing import NDArray + +from cpython.ref cimport Py_INCREF from cpython.bytes cimport PyBytes_FromStringAndSize from cpython.mem cimport PyMem_Free -from cython.operator cimport dereference as deref from libc.stddef cimport wchar_t import numpy as np @@ -16,8 +18,6 @@ cimport numpy as np np.import_array() import os -import sys -import warnings from enum import Enum cdef extern from "limits.h": @@ -392,13 +392,13 @@ cdef class RawPy: def __dealloc__(self): del self.p - def __enter__(self): + def __enter__(self) -> RawPy: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: self.close() - def close(self): + def close(self) -> None: """ Release all resources and close the RAW image. @@ -413,7 +413,7 @@ cdef class RawPy: with nogil: self.p.recycle() - def open_file(self, path): + def open_file(self, path: str) -> None: """ Opens the given RAW image file. Should be followed by a call to :meth:`~rawpy.RawPy.unpack`. @@ -437,7 +437,7 @@ cdef class RawPy: res = self.p.open_file(path.encode('UTF-8')) self.handle_error(res) - def open_buffer(self, fileobj): + def open_buffer(self, fileobj: BinaryIO) -> None: """ Opens the given RAW image file-like object. Should be followed by a call to :meth:`~rawpy.RawPy.unpack`. @@ -456,7 +456,7 @@ cdef class RawPy: e = self.p.open_buffer(buf, buf_len) self.handle_error(e) - def set_unpack_params(self, shot_select=0): + def set_unpack_params(self, shot_select: int = 0) -> None: """ Set parameters that affect RAW image unpacking. @@ -471,7 +471,7 @@ cdef class RawPy: cdef libraw_raw_unpack_params_t* rp = &self.p.imgdata.rawparams rp.shot_select = shot_select - def unpack(self): + def unpack(self) -> None: """ Unpacks/decodes the opened RAW image. @@ -487,7 +487,7 @@ cdef class RawPy: if not self.unpack_called: self.unpack() - def unpack_thumb(self): + def unpack_thumb(self) -> None: """ Unpacks/decodes the thumbnail/preview image, whichever is bigger. @@ -502,20 +502,21 @@ cdef class RawPy: if not self.unpack_thumb_called: self.unpack_thumb() - property raw_type: + @property + def raw_type(self) -> RawType: """ Return the RAW type. :rtype: :class:`rawpy.RawType` """ - def __get__(self): - self.ensure_unpack() - if self.p.imgdata.rawdata.raw_image != NULL: - return RawType.Flat - else: - return RawType.Stack + self.ensure_unpack() + if self.p.imgdata.rawdata.raw_image != NULL: + return RawType.Flat + else: + return RawType.Stack - property raw_image: + @property + def raw_image(self) -> NDArray[np.uint16]: """ View of RAW image. Includes margin. @@ -533,47 +534,46 @@ cdef class RawPy: :rtype: ndarray of shape (h,w[,c]) """ - def __get__(self): - self.ensure_unpack() - cdef np.npy_intp shape_bayer[2] - cdef np.npy_intp shape_rgb[3] - cdef np.ndarray ndarr - if self.p.imgdata.rawdata.raw_image != NULL: - shape_bayer[0] = self.p.imgdata.sizes.raw_height - shape_bayer[1] = self.p.imgdata.sizes.raw_width - ndarr = np.PyArray_SimpleNewFromData(2, shape_bayer, np.NPY_USHORT, self.p.imgdata.rawdata.raw_image) - elif self.p.imgdata.rawdata.color3_image != NULL: - shape_rgb[0] = self.p.imgdata.sizes.raw_height - shape_rgb[1] = self.p.imgdata.sizes.raw_width - shape_rgb[2] = 3 - ndarr = np.PyArray_SimpleNewFromData(3, shape_rgb, np.NPY_USHORT, self.p.imgdata.rawdata.color3_image) - elif self.p.imgdata.rawdata.color4_image != NULL: - shape_rgb[0] = self.p.imgdata.sizes.raw_height - shape_rgb[1] = self.p.imgdata.sizes.raw_width - shape_rgb[2] = 4 - ndarr = np.PyArray_SimpleNewFromData(3, shape_rgb, np.NPY_USHORT, self.p.imgdata.rawdata.color4_image) - else: - raise RuntimeError('unsupported raw data') - - # ndarr must hold a reference to this object, - # otherwise the underlying data gets lost when the RawPy instance gets out of scope - # (which would trigger __dealloc__) - np.PyArray_SetBaseObject(ndarr, self) - # Python doesn't know about above assignment as it's in C-level - Py_INCREF(self) - return ndarr + self.ensure_unpack() + cdef np.npy_intp shape_bayer[2] + cdef np.npy_intp shape_rgb[3] + cdef np.ndarray ndarr + if self.p.imgdata.rawdata.raw_image != NULL: + shape_bayer[0] = self.p.imgdata.sizes.raw_height + shape_bayer[1] = self.p.imgdata.sizes.raw_width + ndarr = np.PyArray_SimpleNewFromData(2, shape_bayer, np.NPY_USHORT, self.p.imgdata.rawdata.raw_image) + elif self.p.imgdata.rawdata.color3_image != NULL: + shape_rgb[0] = self.p.imgdata.sizes.raw_height + shape_rgb[1] = self.p.imgdata.sizes.raw_width + shape_rgb[2] = 3 + ndarr = np.PyArray_SimpleNewFromData(3, shape_rgb, np.NPY_USHORT, self.p.imgdata.rawdata.color3_image) + elif self.p.imgdata.rawdata.color4_image != NULL: + shape_rgb[0] = self.p.imgdata.sizes.raw_height + shape_rgb[1] = self.p.imgdata.sizes.raw_width + shape_rgb[2] = 4 + ndarr = np.PyArray_SimpleNewFromData(3, shape_rgb, np.NPY_USHORT, self.p.imgdata.rawdata.color4_image) + else: + raise RuntimeError('unsupported raw data') + + # ndarr must hold a reference to this object, + # otherwise the underlying data gets lost when the RawPy instance gets out of scope + # (which would trigger __dealloc__) + np.PyArray_SetBaseObject(ndarr, self) + # Python doesn't know about above assignment as it's in C-level + Py_INCREF(self) + return ndarr - property raw_image_visible: + @property + def raw_image_visible(self) -> NDArray[np.uint16]: """ Like raw_image but without margin. :rtype: ndarray of shape (hv,wv[,c]) """ - def __get__(self): - self.ensure_unpack() - s = self.sizes - return self.raw_image[s.top_margin:s.top_margin+s.height, - s.left_margin:s.left_margin+s.width] + self.ensure_unpack() + s = self.sizes + return self.raw_image[s.top_margin:s.top_margin+s.height, + s.left_margin:s.left_margin+s.width] cpdef ushort raw_value(self, int row, int column): """ @@ -601,44 +601,44 @@ cdef class RawPy: cdef ushort raw_width = self.p.imgdata.sizes.raw_width return raw[(row+top_margin)*raw_width + column + left_margin] - property sizes: + @property + def sizes(self) -> ImageSizes: """ Return a :class:`rawpy.ImageSizes` instance with size information of the RAW image and postprocessed image. """ - def __get__(self): - cdef libraw_image_sizes_t* s = &self.p.imgdata.sizes - - # LibRaw returns 65535 for cleft and ctop in some files - probably those that do not specify them - cdef bint has_cleft = s.raw_inset_crops[0].cleft != USHRT_MAX - cdef bint has_ctop = s.raw_inset_crops[0].ctop != USHRT_MAX - - return ImageSizes(raw_height=s.raw_height, raw_width=s.raw_width, - height=s.height, width=s.width, - top_margin=s.top_margin, left_margin=s.left_margin, - iheight=s.iheight, iwidth=s.iwidth, - pixel_aspect=s.pixel_aspect, flip=s.flip, - crop_left_margin=s.raw_inset_crops[0].cleft if has_cleft else 0, - crop_top_margin=s.raw_inset_crops[0].ctop if has_ctop else 0, - crop_width=s.raw_inset_crops[0].cwidth, crop_height=s.raw_inset_crops[0].cheight) + cdef libraw_image_sizes_t* s = &self.p.imgdata.sizes + + # LibRaw returns 65535 for cleft and ctop in some files - probably those that do not specify them + cdef bint has_cleft = s.raw_inset_crops[0].cleft != USHRT_MAX + cdef bint has_ctop = s.raw_inset_crops[0].ctop != USHRT_MAX + + return ImageSizes(raw_height=s.raw_height, raw_width=s.raw_width, + height=s.height, width=s.width, + top_margin=s.top_margin, left_margin=s.left_margin, + iheight=s.iheight, iwidth=s.iwidth, + pixel_aspect=s.pixel_aspect, flip=s.flip, + crop_left_margin=s.raw_inset_crops[0].cleft if has_cleft else 0, + crop_top_margin=s.raw_inset_crops[0].ctop if has_ctop else 0, + crop_width=s.raw_inset_crops[0].cwidth, crop_height=s.raw_inset_crops[0].cheight) - property num_colors: + @property + def num_colors(self) -> int: """ Number of colors. Note that e.g. for RGBG this can be 3 or 4, depending on the camera model, as some use two different greens. """ - def __get__(self): - return self.p.imgdata.idata.colors + return self.p.imgdata.idata.colors - property color_desc: + @property + def color_desc(self) -> bytes: """ String description of colors numbered from 0 to 3 (RGBG,RGBE,GMCY, or GBTG). Note that same letters may not refer strictly to the same color. There are cameras with two different greens for example. """ - def __get__(self): - return self.p.imgdata.idata.cdesc + return self.p.imgdata.idata.cdesc cpdef int raw_color(self, int row, int column): """ @@ -653,7 +653,8 @@ cdef class RawPy: # COLOR's coordinates are relative to visible image size. return self.p.COLOR(row - top_margin, column - left_margin) - property raw_colors: + @property + def raw_colors(self) -> NDArray[np.uint8]: """ An array of color indices for each pixel in the RAW image. Equivalent to calling raw_color(y,x) for each pixel. @@ -661,79 +662,79 @@ cdef class RawPy: :rtype: ndarray of shape (h,w) """ - def __get__(self): - self.ensure_unpack() - if self.p.imgdata.rawdata.raw_image == NULL: - raise RuntimeError('RAW image is not flat') - cdef np.ndarray pattern = self.raw_pattern - cdef int n = pattern.shape[0] - cdef int height = self.p.imgdata.sizes.raw_height - cdef int width = self.p.imgdata.sizes.raw_width - return np.pad(pattern, ((0, height - n), (0, width - n)), mode='wrap') + self.ensure_unpack() + if self.p.imgdata.rawdata.raw_image == NULL: + raise RuntimeError('RAW image is not flat') + cdef np.ndarray pattern = self.raw_pattern + cdef int n = pattern.shape[0] + cdef int height = self.p.imgdata.sizes.raw_height + cdef int width = self.p.imgdata.sizes.raw_width + return np.pad(pattern, ((0, height - n), (0, width - n)), mode='wrap') - property raw_colors_visible: + @property + def raw_colors_visible(self) -> NDArray[np.uint8]: """ Like raw_colors but without margin. :rtype: ndarray of shape (hv,wv) """ - def __get__(self): - s = self.sizes - return self.raw_colors[s.top_margin:s.top_margin+s.height, - s.left_margin:s.left_margin+s.width] + s = self.sizes + return self.raw_colors[s.top_margin:s.top_margin+s.height, + s.left_margin:s.left_margin+s.width] - property raw_pattern: + @property + def raw_pattern(self) -> Optional[NDArray[np.uint8]]: """ The smallest possible Bayer pattern of this image. :rtype: ndarray, or None if not a flat RAW image """ - def __get__(self): - self.ensure_unpack() - if self.p.imgdata.rawdata.raw_image == NULL: - return None - cdef np.ndarray pattern - cdef int n - if self.p.imgdata.idata.filters < 1000: - if self.p.imgdata.idata.filters == 0: - # black and white - n = 1 - elif self.p.imgdata.idata.filters == 1: - # Leaf Catchlight - n = 16 - elif self.p.imgdata.idata.filters == LIBRAW_XTRANS: - n = 6 - else: - raise NotImplementedError('filters: {}'.format(self.p.imgdata.idata.filters)) + self.ensure_unpack() + if self.p.imgdata.rawdata.raw_image == NULL: + return None + cdef np.ndarray pattern + cdef int n + if self.p.imgdata.idata.filters < 1000: + if self.p.imgdata.idata.filters == 0: + # black and white + n = 1 + elif self.p.imgdata.idata.filters == 1: + # Leaf Catchlight + n = 16 + elif self.p.imgdata.idata.filters == LIBRAW_XTRANS: + n = 6 else: - n = 4 - - pattern = np.empty((n, n), dtype=np.uint8) - cdef int y, x - for y in range(n): - for x in range(n): - pattern[y,x] = self.raw_color(y, x) - if n == 4: - if np.all(pattern[:2,:2] == pattern[:2,2:]) and \ - np.all(pattern[:2,:2] == pattern[2:,2:]) and \ - np.all(pattern[:2,:2] == pattern[2:,:2]): - pattern = pattern[:2,:2] - return pattern + raise NotImplementedError('filters: {}'.format(self.p.imgdata.idata.filters)) + else: + n = 4 + + pattern = np.empty((n, n), dtype=np.uint8) + cdef int y, x + for y in range(n): + for x in range(n): + pattern[y,x] = self.raw_color(y, x) + if n == 4: + if np.all(pattern[:2,:2] == pattern[:2,2:]) and \ + np.all(pattern[:2,:2] == pattern[2:,2:]) and \ + np.all(pattern[:2,:2] == pattern[2:,:2]): + pattern = pattern[:2,:2] + return pattern - property camera_whitebalance: + @property + def camera_whitebalance(self) -> List[float]: """ White balance coefficients (as shot). Either read from file or calculated. :rtype: list of length 4 """ - def __get__(self): - self.ensure_unpack() - return [self.p.imgdata.rawdata.color.cam_mul[0], - self.p.imgdata.rawdata.color.cam_mul[1], - self.p.imgdata.rawdata.color.cam_mul[2], - self.p.imgdata.rawdata.color.cam_mul[3]] + self.ensure_unpack() + return [self.p.imgdata.rawdata.color.cam_mul[0], + self.p.imgdata.rawdata.color.cam_mul[1], + self.p.imgdata.rawdata.color.cam_mul[2], + self.p.imgdata.rawdata.color.cam_mul[3]] - property daylight_whitebalance: + @property + def daylight_whitebalance(self) -> List[float]: """ White balance coefficients for daylight (daylight balance). Either read from file, or calculated on the basis of file data, @@ -741,14 +742,14 @@ cdef class RawPy: :rtype: list of length 4 """ - def __get__(self): - self.ensure_unpack() - return [self.p.imgdata.rawdata.color.pre_mul[0], - self.p.imgdata.rawdata.color.pre_mul[1], - self.p.imgdata.rawdata.color.pre_mul[2], - self.p.imgdata.rawdata.color.pre_mul[3]] + self.ensure_unpack() + return [self.p.imgdata.rawdata.color.pre_mul[0], + self.p.imgdata.rawdata.color.pre_mul[1], + self.p.imgdata.rawdata.color.pre_mul[2], + self.p.imgdata.rawdata.color.pre_mul[3]] - property auto_whitebalance: + @property + def auto_whitebalance(self) -> Optional[List[float]]: """ White balance coefficients used during postprocessing. @@ -766,69 +767,69 @@ cdef class RawPy: :rtype: list of length 4, or None if postprocessing hasn't been called yet """ - def __get__(self): - self.ensure_unpack() - if not self.dcraw_process_called: - return None - return [self.p.imgdata.color.pre_mul[0], - self.p.imgdata.color.pre_mul[1], - self.p.imgdata.color.pre_mul[2], - self.p.imgdata.color.pre_mul[3]] + self.ensure_unpack() + if not self.dcraw_process_called: + return None + return [self.p.imgdata.color.pre_mul[0], + self.p.imgdata.color.pre_mul[1], + self.p.imgdata.color.pre_mul[2], + self.p.imgdata.color.pre_mul[3]] - property black_level_per_channel: + @property + def black_level_per_channel(self) -> List[int]: """ Per-channel black level correction. :rtype: list of length 4 """ - def __get__(self): - self.ensure_unpack() - cdef libraw_colordata_black_level_t bl = adjust_bl_(self.p) - return [bl.cblack[0], - bl.cblack[1], - bl.cblack[2], - bl.cblack[3]] + self.ensure_unpack() + cdef libraw_colordata_black_level_t bl = adjust_bl_(self.p) + return [bl.cblack[0], + bl.cblack[1], + bl.cblack[2], + bl.cblack[3]] - property white_level: + @property + def white_level(self) -> int: """ Level at which the raw pixel value is considered to be saturated. """ - def __get__(self): - self.ensure_unpack() - return self.p.imgdata.rawdata.color.maximum + self.ensure_unpack() + return self.p.imgdata.rawdata.color.maximum - property camera_white_level_per_channel: + @property + def camera_white_level_per_channel(self) -> Optional[List[int]]: """ Per-channel saturation levels read from raw file metadata, if it exists. Otherwise None. :rtype: list of length 4, or None if metadata missing """ - def __get__(self): - self.ensure_unpack() - levels = [self.p.imgdata.rawdata.color.linear_max[0], - self.p.imgdata.rawdata.color.linear_max[1], - self.p.imgdata.rawdata.color.linear_max[2], - self.p.imgdata.rawdata.color.linear_max[3]] - if all(l > 0 for l in levels): - return levels - else: - return None + self.ensure_unpack() + levels = [self.p.imgdata.rawdata.color.linear_max[0], + self.p.imgdata.rawdata.color.linear_max[1], + self.p.imgdata.rawdata.color.linear_max[2], + self.p.imgdata.rawdata.color.linear_max[3]] + if all(l > 0 for l in levels): + return levels + else: + return None - property color_matrix: + @property + def color_matrix(self) -> NDArray[np.float32]: """ Color matrix, read from file for some cameras, calculated for others. :rtype: ndarray of shape (3,4) """ - def __get__(self): - self.ensure_unpack() - cdef np.ndarray matrix = np.empty((3, 4), dtype=np.float32) - for i in range(3): - for j in range(4): - matrix[i,j] = self.p.imgdata.rawdata.color.cmatrix[i][j] - return matrix + self.ensure_unpack() + cdef np.ndarray matrix = np.empty((3, 4), dtype=np.float32) + for i in range(3): + for j in range(4): + matrix[i,j] = self.p.imgdata.rawdata.color.cmatrix[i][j] + return matrix - property rgb_xyz_matrix: + @property + def rgb_xyz_matrix(self) -> NDArray[np.float32]: """ Camera RGB - XYZ conversion matrix. This matrix is constant (different for different models). @@ -836,28 +837,27 @@ cdef class RawPy: :rtype: ndarray of shape (4,3) """ - def __get__(self): - self.ensure_unpack() - cdef np.ndarray matrix = np.empty((4, 3), dtype=np.float32) - for i in range(4): - for j in range(3): - matrix[i,j] = self.p.imgdata.rawdata.color.cam_xyz[i][j] - return matrix + self.ensure_unpack() + cdef np.ndarray matrix = np.empty((4, 3), dtype=np.float32) + for i in range(4): + for j in range(3): + matrix[i,j] = self.p.imgdata.rawdata.color.cam_xyz[i][j] + return matrix - property tone_curve: + @property + def tone_curve(self) -> NDArray[np.uint16]: """ Camera tone curve, read from file for Nikon, Sony and some other cameras. :rtype: ndarray of length 65536 """ - def __get__(self): - self.ensure_unpack() - cdef np.npy_intp shape[1] - shape[0] = 65536 - return np.PyArray_SimpleNewFromData(1, shape, np.NPY_USHORT, - &self.p.imgdata.rawdata.color.curve) + self.ensure_unpack() + cdef np.npy_intp shape[1] + shape[0] = 65536 + return np.PyArray_SimpleNewFromData(1, shape, np.NPY_USHORT, + &self.p.imgdata.rawdata.color.curve) - def dcraw_process(self, params=None, **kw): + def dcraw_process(self, params: Optional[Params] = None, **kw) -> None: """ Postprocess the currently loaded RAW image. @@ -930,7 +930,7 @@ cdef class RawPy: else: raise NotImplementedError('thumb type: {}'.format(img.type)) - def extract_thumb(self): + def extract_thumb(self) -> Thumbnail: """ Extracts and returns the thumbnail/preview image (whichever is bigger) of the opened RAW image as :class:`rawpy.Thumbnail` object. @@ -963,7 +963,7 @@ cdef class RawPy: thumb = self.dcraw_make_mem_thumb() return thumb - def postprocess(self, params=None, **kw): + def postprocess(self, params: Optional[Params] = None, **kw) -> NDArray[np.uint8]: """ Postprocess the currently loaded RAW image and return the new resulting image as numpy array. @@ -1157,18 +1157,35 @@ class Params(object): """ A class that handles postprocessing parameters. """ - def __init__(self, demosaic_algorithm=None, half_size=False, four_color_rgb=False, - dcb_iterations=0, dcb_enhance=False, - fbdd_noise_reduction=FBDDNoiseReductionMode.Off, - noise_thr=None, median_filter_passes=0, - use_camera_wb=False, use_auto_wb=False, user_wb=None, - output_color=ColorSpace.sRGB, output_bps=8, - user_flip=None, user_black=None, user_cblack=None, user_sat=None, - no_auto_bright=False, auto_bright_thr=None, adjust_maximum_thr=0.75, - bright=1.0, highlight_mode=HighlightMode.Clip, - exp_shift=None, exp_preserve_highlights=0.0, no_auto_scale=False, - gamma=None, chromatic_aberration=None, - bad_pixels_path=None): + def __init__(self, + demosaic_algorithm: Optional[DemosaicAlgorithm] = None, + half_size: bool = False, + four_color_rgb: bool = False, + dcb_iterations: int = 0, + dcb_enhance: bool = False, + fbdd_noise_reduction: FBDDNoiseReductionMode = FBDDNoiseReductionMode.Off, + noise_thr: Optional[float] = None, + median_filter_passes: int = 0, + use_camera_wb: bool = False, + use_auto_wb: bool = False, + user_wb: Optional[List[float]] = None, + output_color: ColorSpace = ColorSpace.sRGB, + output_bps: int = 8, + user_flip: Optional[int] = None, + user_black: Optional[int] = None, + user_cblack: Optional[List[int]] = None, + user_sat: Optional[int] = None, + no_auto_bright: bool = False, + auto_bright_thr: Optional[float] = None, + adjust_maximum_thr: float = 0.75, + bright: float = 1.0, + highlight_mode: Union[HighlightMode, int] = HighlightMode.Clip, + exp_shift: Optional[float] = None, + exp_preserve_highlights: float = 0.0, + no_auto_scale: bool = False, + gamma: Optional[Tuple[float, float]] = None, + chromatic_aberration: Optional[Tuple[float, float]] = None, + bad_pixels_path: Optional[str] = None) -> None: """ If use_camera_wb and use_auto_wb are False and user_wb is None, then @@ -1303,9 +1320,4 @@ cdef class processed_image_wrapper: def __dealloc__(self): self.raw.p.dcraw_clear_mem(self.processed_image) - -def _chars(s): - if isinstance(s, unicode): - # convert unicode to chars - s = (s).encode('UTF-8') - return s + \ No newline at end of file diff --git a/rawpy/enhance.py b/rawpy/enhance.py index 7d6ef52b..27f23adb 100644 --- a/rawpy/enhance.py +++ b/rawpy/enhance.py @@ -9,22 +9,25 @@ import warnings from functools import partial import numpy as np +from numpy.typing import NDArray + +from typing import Optional, Callable, Any, cast try: - from skimage.filters.rank import median + from skimage.filters.rank import median as median_func except ImportError: try: - from skimage.filter.rank import median + from skimage.filter.rank import median as median_func # type: ignore except ImportError as e: warnings.warn('scikit-image not found, will use OpenCV (error: ' + str(e) + ')') - median = None + median_func = cast(Optional[Callable[..., Any]], None) # type: ignore try: import cv2 except ImportError as e: warnings.warn('OpenCV not found, install for faster processing (error: ' + str(e) + ')') cv2 = None -if median is None and cv2 is None: +if median_func is None and cv2 is None: raise ImportError('Either scikit-image or OpenCV must be installed to use rawpy.enhance') import rawpy @@ -79,17 +82,19 @@ def find_bad_pixels(paths, find_hot=True, find_dead=True, confirm_ratio=0.9): isCandidate = partial(_is_candidate, find_hot=find_hot, find_dead=find_dead, thresh=thresh) coords.extend(_find_bad_pixel_candidates(raw, isCandidate)) - coords = np.vstack(coords) + coords_array: NDArray[np.int_] = np.vstack(coords) if len(paths) == 1: - return coords + return coords_array # select candidates that appear on most input images # count how many times a coordinate appears + assert width is not None, "width must be set from at least one image" + # first we convert y,x to array offset such that we have an array of integers - offset = coords[:,0]*width - offset += coords[:,1] + offset: NDArray[np.int_] = coords_array[:,0]*width + offset += coords_array[:,1] # now we count how many times each offset occurs counts = _groupcount(offset) @@ -116,7 +121,7 @@ def _find_bad_pixel_candidates(raw, isCandidateFn): return coords def _find_bad_pixel_candidates_generic(raw, isCandidateFn): - if median is None: + if median_func is None: raise RuntimeError('scikit-image is required if the Bayer pattern is not 2x2') color_masks = _colormasks(raw) @@ -131,7 +136,7 @@ def _find_bad_pixel_candidates_generic(raw, isCandidateFn): # There exist O(log(r)) and O(1) algorithms, see https://nomis80.org/ctmf.pdf. # Also, we only need the median values for the masked pixels. # Currently, they are calculated for all pixels for each color. - med = median(rawimg, kernel, mask=mask) + med = median_func(rawimg, kernel, mask=mask) # detect possible bad pixels candidates = isCandidateFn(rawimg, med) @@ -161,8 +166,9 @@ def _find_bad_pixel_candidates_bayer2x2(raw, isCandidateFn): if cv2 is not None: median_ = partial(cv2.medianBlur, ksize=r) else: + assert median_func is not None kernel = np.ones((r,r)) - median_ = partial(median, footprint=kernel) + median_ = partial(median_func, footprint=kernel) coords = [] @@ -223,7 +229,7 @@ def repair_bad_pixels(raw, coords, method='median'): #raw.raw_image_visible[coords[:,0], coords[:,1]] = 0 def _repair_bad_pixels_generic(raw, coords, method='median'): - if median is None: + if median_func is None: raise RuntimeError('scikit-image is required for repair_bad_pixels if the Bayer pattern is not 2x2') color_masks = _colormasks(raw) @@ -247,7 +253,7 @@ def _repair_bad_pixels_generic(raw, coords, method='median'): # bad pixels won't influence the median in most cases and just using # the color mask prevents bad pixel clusters from producing # bad interpolated values (NaNs) - smooth = median(rawimg, kernel, mask=color_mask) + smooth = median_func(rawimg, kernel, mask=color_mask) else: raise ValueError @@ -264,8 +270,9 @@ def _repair_bad_pixels_bayer2x2(raw, coords, method='median'): if cv2 is not None: median_ = partial(cv2.medianBlur, ksize=r) else: + assert median_func is not None kernel = np.ones((r,r)) - median_ = partial(median, footprint=kernel) + median_ = partial(median_func, footprint=kernel) # we have 4 colors (two greens are always seen as two colors) for offset_y in [0,1]: diff --git a/rawpy/py.typed b/rawpy/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/scripts/build_dist.sh b/scripts/build_dist.sh new file mode 100755 index 00000000..4bc8ec0b --- /dev/null +++ b/scripts/build_dist.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Build sdist and wheel artifacts for local testing. +# +# This uses build isolation (like CI), so the result closely matches +# what gets published to PyPI. Output goes to dist/. +# +# Usage: +# bash scripts/build_dist.sh +# RAWPY_USE_SYSTEM_LIBRAW=1 bash scripts/build_dist.sh +# Builds the wheel against the system libraw instead of the bundled +# source. The sdist is unaffected. +# +# To build with a different Python version: +# bash scripts/setup_python.sh 3.12 +# bash scripts/build_dist.sh + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +# Require venv +if [ ! -d "$VENV_DIR" ]; then + echo "ERROR: .venv not found." + echo "Run 'bash scripts/setup_dev_env.sh' first." + exit 1 +fi + +source "$VENV_DIR/bin/activate" +cd "$PROJECT_ROOT" + +# Ensure 'build' package is available +if ! python -m build --help &>/dev/null; then + echo "Installing 'build' package..." + pip install build -q +fi + +# Clean previous artifacts +rm -rf dist/ + +# Ensure submodules are initialized (needed for sdist to include LibRaw source) +if [ ! -f "external/LibRaw/README.md" ]; then + echo "Initializing git submodules..." + git submodule update --init --recursive +fi + +echo "=== Building sdist + wheel ===" +echo "" +python -m build + +echo "" +echo "=== Build complete ===" +echo "Artifacts in dist/:" +ls -lh dist/ diff --git a/scripts/dev_check.sh b/scripts/dev_check.sh new file mode 100755 index 00000000..1e7f29a1 --- /dev/null +++ b/scripts/dev_check.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +# Auto-activate venv if not active +if [ -z "$VIRTUAL_ENV" ]; then + if [ -d "$VENV_DIR" ]; then + echo "Activating virtual environment..." + source "$VENV_DIR/bin/activate" + else + echo "ERROR: No virtual environment found at $VENV_DIR" + echo "Run 'bash scripts/setup_dev_env.sh' first." + exit 1 + fi +fi + +echo "=== Development Environment Check ===" +echo "" + +# Step 1: Verify rawpy can be imported +echo "1. Checking rawpy import..." +if ! python -c "import rawpy; print(f' rawpy {rawpy.__version__}')"; then + echo " FAILED: Cannot import rawpy" + echo "" + echo " The Cython extension may not be built." + echo " Run: bash scripts/rebuild.sh" + exit 1 +fi +echo " OK" +echo "" + +# Step 2: Type checking +echo "2. Checking types (mypy)..." +cd "$PROJECT_ROOT" +if python -m mypy rawpy --no-error-summary 2>/dev/null; then + echo " OK" +else + echo " WARNINGS (non-fatal, review output above)" +fi +echo "" + +# Step 3: Quick runtime test +echo "3. Running quick test..." +if pytest test/test_basic.py::testFileOpenAndPostProcess -v --tb=short 2>/dev/null; then + echo " OK" +else + echo " FAILED" + exit 1 +fi + +echo "" +echo "=== All Checks Passed ===" diff --git a/scripts/rebuild.sh b/scripts/rebuild.sh new file mode 100755 index 00000000..2ac97cae --- /dev/null +++ b/scripts/rebuild.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Rebuilds the rawpy Cython extension. +# Use this after changing .pyx files, C++ source, or headers. + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +# Require venv +if [ ! -d "$VENV_DIR" ]; then + echo "ERROR: .venv not found." + echo "Run 'bash scripts/setup_dev_env.sh' first." + exit 1 +fi + +source "$VENV_DIR/bin/activate" +cd "$PROJECT_ROOT" + +# Delete stale _rawpy.cpp so cythonize() regenerates it (see AGENTS.md) +rm -f rawpy/_rawpy.cpp + +echo "Rebuilding rawpy..." +# --no-build-isolation: reuses current env's numpy/cython (faster) +# -e: editable install (.py changes apply immediately) +pip install --no-build-isolation -e . -q + +# Verify the build succeeded +echo "" +if python -c "import rawpy; print(f'rawpy {rawpy.__version__} rebuilt successfully')"; then + exit 0 +else + echo "ERROR: Build completed but import failed." + exit 1 +fi diff --git a/scripts/setup_dev_env.sh b/scripts/setup_dev_env.sh new file mode 100755 index 00000000..0dfb106f --- /dev/null +++ b/scripts/setup_dev_env.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +# Optional: specify a Python version (e.g. 3.12) +PYTHON_VERSION="${1:-}" + +echo "=== Development Environment Setup ===" +echo "" + +# If a specific Python version was requested, install it first +if [ -n "$PYTHON_VERSION" ]; then + echo "Requested Python $PYTHON_VERSION" + bash "$SCRIPT_DIR/setup_python.sh" "$PYTHON_VERSION" + echo "" +fi + +# Check system dependencies first +echo "Checking system dependencies..." + +# Check for C++ compiler +if command -v g++ &> /dev/null; then + echo " C++ compiler: g++ $(g++ --version | head -1)" +elif command -v clang++ &> /dev/null; then + echo " C++ compiler: clang++ $(clang++ --version | head -1)" +else + echo "ERROR: No C++ compiler found (g++ or clang++)." + echo " Ubuntu/Debian: sudo apt install g++" + echo " macOS: xcode-select --install" + exit 1 +fi + +echo "" + +# Create or reuse venv +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment..." + python3 -m venv "$VENV_DIR" +else + echo "Using existing virtual environment." +fi + +source "$VENV_DIR/bin/activate" + +echo " Python: $(python --version)" +echo " pip: $(pip --version | cut -d' ' -f1-2)" +echo "" + +echo "Upgrading pip..." +pip install --upgrade pip -q + +echo "Initializing git submodules..." +cd "$PROJECT_ROOT" +git submodule update --init --recursive + +echo "Installing Python dependencies..." +# This includes setuptools since --no-build-isolation skips build-system.requires. +pip install -r "$PROJECT_ROOT/dev-requirements.txt" -q + +echo "Building and installing rawpy (this may take a minute)..." +# Delete stale _rawpy.cpp so cythonize() regenerates it (see AGENTS.md) +rm -f rawpy/_rawpy.cpp +pip install -e "$PROJECT_ROOT" --no-build-isolation -q + +# Verify the installation +echo "" +echo "Verifying installation..." +if python -c "import rawpy; print(f' rawpy {rawpy.__version__} installed successfully')" 2>/dev/null; then + echo "" + echo "=== Setup Complete ===" + echo "" + echo "To activate this environment:" + echo " source .venv/bin/activate" + echo "" + echo "Quick verification:" + echo " bash scripts/dev_check.sh" +else + echo "" + echo "ERROR: rawpy import failed after installation." + echo "Check the build output above for errors." + exit 1 +fi diff --git a/scripts/setup_numpy.sh b/scripts/setup_numpy.sh new file mode 100755 index 00000000..c1251303 --- /dev/null +++ b/scripts/setup_numpy.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Switch the active venv to a specific numpy version. +# +# Usage: +# bash scripts/setup_numpy.sh 2.0.2 +# bash scripts/setup_numpy.sh '2.1.*' +# +# After this, use pytest / mypy / any command as normal. +# The Cython extension does NOT need rebuilding — NumPy ABI 2.0+ is +# forward-compatible at runtime, and the stubs ship with numpy itself. + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +NUMPY_VERSION="${1:?Usage: $0 (e.g. 2.0.2 or '2.0.*')}" + +if [ ! -d "$VENV_DIR" ]; then + echo "ERROR: .venv not found. Run 'bash scripts/setup_dev_env.sh' first." + exit 1 +fi +source "$VENV_DIR/bin/activate" + +BEFORE=$(python -c "import numpy; print(numpy.__version__)" 2>/dev/null || echo "none") +echo "Current numpy: $BEFORE" +echo "Installing numpy==$NUMPY_VERSION ..." +pip install "numpy==$NUMPY_VERSION" -q +AFTER=$(python -c "import numpy; print(numpy.__version__)") +echo "Now using numpy: $AFTER" diff --git a/scripts/setup_python.sh b/scripts/setup_python.sh new file mode 100755 index 00000000..56b57bd1 --- /dev/null +++ b/scripts/setup_python.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Install a specific Python version (Ubuntu only, via deadsnakes PPA) +# and create a venv with it. +# +# Usage: +# bash scripts/setup_python.sh 3.12 +# bash scripts/setup_python.sh 3.9 +# +# After this, run the normal setup: +# bash scripts/setup_dev_env.sh +# +# The venv at .venv will use the requested Python version. + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VENV_DIR="$PROJECT_ROOT/.venv" + +PYTHON_VERSION="${1:?Usage: $0 (e.g. 3.12)}" + +# Validate format +if ! [[ "$PYTHON_VERSION" =~ ^3\.[0-9]+$ ]]; then + echo "ERROR: Version must be in X.Y format (e.g. 3.12), got: $PYTHON_VERSION" + exit 1 +fi + +PYTHON_BIN="python${PYTHON_VERSION}" + +# Check if already available +if command -v "$PYTHON_BIN" &> /dev/null; then + echo "Python $PYTHON_VERSION already installed: $($PYTHON_BIN --version)" +else + echo "Python $PYTHON_VERSION not found, installing via deadsnakes PPA..." + echo "(This requires sudo on Ubuntu/Debian)" + echo "" + + if ! command -v apt-get &> /dev/null; then + echo "ERROR: apt-get not found. This script only supports Ubuntu/Debian." + echo "Install Python $PYTHON_VERSION manually, then re-run." + exit 1 + fi + + sudo apt-get update -qq + sudo apt-get install -y -qq software-properties-common + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get update -qq + sudo apt-get install -y -qq "${PYTHON_BIN}" "${PYTHON_BIN}-venv" "${PYTHON_BIN}-dev" + + echo "Installed: $($PYTHON_BIN --version)" +fi + +# Remove existing venv if it uses a different Python +if [ -d "$VENV_DIR" ]; then + CURRENT=$("$VENV_DIR/bin/python" --version 2>/dev/null | awk '{print $2}' | cut -d. -f1,2) + if [ "$CURRENT" = "$PYTHON_VERSION" ]; then + echo "Existing .venv already uses Python $PYTHON_VERSION, keeping it." + exit 0 + fi + echo "Removing existing .venv (Python $CURRENT) to replace with $PYTHON_VERSION..." + rm -rf "$VENV_DIR" +fi + +echo "Creating .venv with Python $PYTHON_VERSION..." +"$PYTHON_BIN" -m venv "$VENV_DIR" + +echo "" +echo "Done. .venv now uses $("$VENV_DIR/bin/python" --version)." +echo "" +echo "Next step — run the full environment setup:" +echo " bash scripts/setup_dev_env.sh" diff --git a/scripts/test_dist.sh b/scripts/test_dist.sh new file mode 100755 index 00000000..d4964493 --- /dev/null +++ b/scripts/test_dist.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# Install a built artifact (wheel or sdist) into a clean temporary venv +# and run the test suite against it. This validates that the package works +# as an end-user would experience it — no editable install, no source tree +# on sys.path. +# +# Usage: +# bash scripts/test_dist.sh sdist # test the sdist +# bash scripts/test_dist.sh wheel # test the wheel +# bash scripts/test_dist.sh sdist 2.0.2 # test sdist with numpy 2.0.2 +# bash scripts/test_dist.sh wheel 2.0.2 # test wheel with numpy 2.0.2 +# +# RAWPY_USE_SYSTEM_LIBRAW=1 bash scripts/test_dist.sh sdist +# Install the sdist using system libraw and verify linkage. +# Prerequisites: sudo apt install libraw-dev pkg-config +# Note: Requires system LibRaw >= 0.21. Ubuntu 22.04 ships 0.20.2 +# which is too old. Use Ubuntu 24.04+ or install from source. +# +# The script creates a temporary venv (.venv-test) using the same Python +# as .venv, installs the artifact, runs pytest, and cleans up on exit. +# +# To test with a different Python version: +# bash scripts/setup_python.sh 3.12 # recreates .venv with Python 3.12 +# bash scripts/build_dist.sh # build artifacts +# bash scripts/test_dist.sh sdist # test it +# +# Prerequisites: +# bash scripts/build_dist.sh # build the artifact(s) first + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +TEST_VENV="$PROJECT_ROOT/.venv-test" +DIST_DIR="$PROJECT_ROOT/dist" + +ARTIFACT_TYPE="${1:?Usage: $0 [numpy-version]}" +NUMPY_VERSION="${2:-}" # optional, e.g. "2.0.2" +VENV_DIR="$PROJECT_ROOT/.venv" + +# Use the same Python as .venv (set up by setup_python.sh / setup_dev_env.sh) +if [ -x "$VENV_DIR/bin/python" ]; then + PYTHON_BIN="$VENV_DIR/bin/python" +else + echo "ERROR: .venv not found. Run 'bash scripts/setup_dev_env.sh' first." + exit 1 +fi + +# --- Validation --- + +case "$ARTIFACT_TYPE" in + sdist|wheel) ;; + *) + echo "Usage: $0 [numpy-version]" + echo "" + echo " sdist — install and test the .tar.gz source distribution" + echo " wheel — install and test the .whl wheel" + echo "" + echo " numpy-version — optional, e.g. 2.0.2 or '2.1.*'" + exit 1 + ;; +esac + +if [ ! -d "$DIST_DIR" ]; then + echo "ERROR: dist/ directory not found." + echo "Run 'bash scripts/build_dist.sh' first." + exit 1 +fi + +# --- Find the artifact --- + +find_artifact() { + local pattern="$1" + local matches + matches=$(find "$DIST_DIR" -maxdepth 1 -name "$pattern" | head -1) + echo "$matches" +} + +case "$ARTIFACT_TYPE" in + sdist) + ARTIFACT=$(find_artifact "rawpy-*.tar.gz") + if [ -z "$ARTIFACT" ]; then + echo "ERROR: No sdist found in dist/. Run 'bash scripts/build_dist.sh'." + exit 1 + fi + ;; + wheel) + ARTIFACT=$(find_artifact "rawpy-*.whl") + if [ -z "$ARTIFACT" ]; then + echo "ERROR: No wheel found in dist/. Run 'bash scripts/build_dist.sh'." + exit 1 + fi + ;; +esac + +echo "=== Testing Distribution Artifact ===" +echo " Artifact: $(basename "$ARTIFACT")" +echo " Python: $($PYTHON_BIN --version 2>&1) (from .venv)" +if [ -n "$NUMPY_VERSION" ]; then + echo " NumPy: $NUMPY_VERSION" +fi +if [ "$RAWPY_USE_SYSTEM_LIBRAW" = "1" ]; then + echo " LibRaw: system (RAWPY_USE_SYSTEM_LIBRAW=1)" +fi +echo "" + +# --- Cleanup handler --- + +cleanup() { + if [ -d "$TEST_VENV" ]; then + echo "" + echo "Cleaning up test venv..." + rm -rf "$TEST_VENV" + fi +} +trap cleanup EXIT + +# --- Create clean test venv --- + +echo "Creating clean test venv..." +rm -rf "$TEST_VENV" +"$PYTHON_BIN" -m venv "$TEST_VENV" +source "$TEST_VENV/bin/activate" + +pip install --upgrade pip -q + +# --- Install the artifact --- + +echo "Installing $(basename "$ARTIFACT")..." +if [[ "$ARTIFACT" == *.tar.gz ]]; then + # sdist: pip will build from source (with build isolation) + # This tests that the sdist contains everything needed to build. + pip install "$ARTIFACT[test]" --verbose +else + # wheel: direct install + pip install "$ARTIFACT[test]" -q +fi + +# --- Pin numpy version if requested --- + +if [ -n "$NUMPY_VERSION" ]; then + echo "Installing numpy==$NUMPY_VERSION..." + pip install "numpy==$NUMPY_VERSION" -q +fi + +# --- Verify import --- + +echo "" +echo "Verifying rawpy import..." +# Run from a temp directory so Python doesn't pick up the source tree's rawpy/ +VERIFY_DIR=$(mktemp -d) +cd "$VERIFY_DIR" + +RAWPY_VERSION=$(python -c "import rawpy; print(rawpy.__version__)") +echo " rawpy $RAWPY_VERSION imported successfully" + +RAWPY_LOCATION=$(python -c "import rawpy; print(rawpy.__file__)") +echo " Location: $RAWPY_LOCATION" + +# Sanity check: rawpy should NOT be loaded from the source tree +if [[ "$RAWPY_LOCATION" == "$PROJECT_ROOT/rawpy/"* ]]; then + echo " ERROR: rawpy is loaded from the source tree, not from the installed package." + echo " This means the test would not validate the artifact." + cd "$PROJECT_ROOT" + rm -rf "$VERIFY_DIR" + exit 1 +fi + +NUMPY_ACTUAL=$(python -c "import numpy; print(numpy.__version__)") +echo " numpy $NUMPY_ACTUAL" + +# --- Verify system libraw linkage (Linux only) --- + +if [ "$RAWPY_USE_SYSTEM_LIBRAW" = "1" ]; then + echo "" + echo "Verifying system libraw linkage..." + python -c " +import rawpy._rawpy as _rawpy +import os, subprocess, sys + +ext_path = _rawpy.__file__ +pkg_dir = os.path.dirname(ext_path) + +# Check 1: No bundled libraw_r.so in the package directory +bundled = [f for f in os.listdir(pkg_dir) if f.startswith('libraw_r.so')] +if bundled: + print(f' FAIL: Found bundled libraw files: {bundled}') + sys.exit(1) +print(' OK: No bundled libraw_r.so in package directory') + +# Check 2: ldd shows system libraw, not a local path +result = subprocess.run(['ldd', ext_path], capture_output=True, text=True) +libraw_lines = [l.strip() for l in result.stdout.splitlines() if 'libraw_r' in l] +if not libraw_lines: + print(' FAIL: libraw_r.so not found in ldd output') + sys.exit(1) +for line in libraw_lines: + print(f' ldd: {line}') + if pkg_dir in line: + print(' FAIL: libraw_r.so resolves to the package directory') + sys.exit(1) +print(' OK: libraw_r.so links to system library') +" +fi + +cd "$PROJECT_ROOT" +rm -rf "$VERIFY_DIR" + +# --- Run tests --- + +echo "" +echo "Running tests..." +# Run from a temp directory so Python doesn't pick up the source tree's rawpy/ +WORK_DIR=$(mktemp -d) +cd "$WORK_DIR" + +pytest --verbosity=3 -s "$PROJECT_ROOT/test" + +TEST_EXIT=$? + +cd "$PROJECT_ROOT" +rm -rf "$WORK_DIR" + +echo "" +if [ $TEST_EXIT -eq 0 ]; then + echo "=== All tests passed ===" +else + echo "=== Tests FAILED (exit code: $TEST_EXIT) ===" + exit $TEST_EXIT +fi diff --git a/setup.py b/setup.py index 26285c65..8e00dc34 100644 --- a/setup.py +++ b/setup.py @@ -1,45 +1,58 @@ from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext as _build_ext import subprocess import errno import os import shutil import sys -import zipfile import glob -from urllib.request import urlretrieve import numpy + from Cython.Build import cythonize +# --- Configuration --- + # As rawpy is distributed under the MIT license, it cannot use or distribute # GPL'd code. This is relevant only for the binary wheels which would have to # bundle the GPL'd code/algorithms (extra demosaic packs). -# Note: RAWPY_BUILD_GPL_CODE=1 only has an effect for macOS and Windows builds -# because libraw is built from source here, whereas for Linux we look -# for the library on the system. -# Note: Building GPL demosaic packs only works with libraw <= 0.18. -# See https://github.com/letmaik/rawpy/issues/72. -buildGPLCode = os.getenv('RAWPY_BUILD_GPL_CODE') == '1' -useSystemLibraw = os.getenv('RAWPY_USE_SYSTEM_LIBRAW') == '1' - -# don't treat mingw as Windows (https://stackoverflow.com/a/51200002) -isWindows = os.name == 'nt' and 'GCC' not in sys.version -isMac = sys.platform == 'darwin' +buildGPLCode = os.getenv("RAWPY_BUILD_GPL_CODE") == "1" +useSystemLibraw = os.getenv("RAWPY_USE_SYSTEM_LIBRAW") == "1" + +# Platform detection +isWindows = os.name == "nt" and "GCC" not in sys.version +isMac = sys.platform == "darwin" +isLinux = sys.platform.startswith("linux") is64Bit = sys.maxsize > 2**32 -# adapted from cffi's setup.py -# the following may be overridden if pkg-config exists -libraries = ['libraw_r'] -include_dirs = [] +# --- Compiler/Linker Flags --- + +libraries = ["libraw_r"] +include_dirs = [numpy.get_include()] # Always include numpy headers library_dirs = [] extra_compile_args = [] extra_link_args = [] +define_macros = [] + +if isWindows: + extra_compile_args += ["/DWIN32"] + +if isLinux: + # On Linux, we want the extension to find the bundled libraw_r.so in the same directory + extra_link_args += ["-Wl,-rpath,$ORIGIN"] + +if isMac: + # On macOS, @loader_path is the equivalent of $ORIGIN — it resolves to + # the directory containing the binary that references the dylib. + extra_link_args += ["-Wl,-rpath,@loader_path"] + +# --- Helper Functions --- + -def _ask_pkg_config(resultlist, option, result_prefix='', sysroot=False): - pkg_config = os.environ.get('PKG_CONFIG','pkg-config') +def _ask_pkg_config(resultlist, option, result_prefix="", sysroot=False): + pkg_config = os.environ.get("PKG_CONFIG", "pkg-config") try: - p = subprocess.Popen([pkg_config, option, 'libraw_r'], - stdout=subprocess.PIPE) + p = subprocess.Popen([pkg_config, option, "libraw_r"], stdout=subprocess.PIPE) except OSError as e: if e.errno != errno.ENOENT: raise @@ -50,269 +63,340 @@ def _ask_pkg_config(resultlist, option, result_prefix='', sysroot=False): # '-I/usr/...' -> '/usr/...' for x in res: assert x.startswith(result_prefix) - res = [x[len(result_prefix):] for x in res] + res = [x[len(result_prefix) :] for x in res] - sysroot = sysroot and os.environ.get('PKG_CONFIG_SYSROOT_DIR', '') + sysroot = sysroot and os.environ.get("PKG_CONFIG_SYSROOT_DIR", "") if sysroot: - # old versions of pkg-config don't support this env var, - # so here we emulate its effect if needed - res = [path if path.startswith(sysroot) - else sysroot + path - for path in res] + res = [ + path if path.startswith(sysroot) else sysroot + path for path in res + ] resultlist[:] = res -def use_pkg_config(): - _ask_pkg_config(include_dirs, '--cflags-only-I', '-I', sysroot=True) - _ask_pkg_config(extra_compile_args, '--cflags-only-other') - _ask_pkg_config(library_dirs, '--libs-only-L', '-L', sysroot=True) - _ask_pkg_config(extra_link_args, '--libs-only-other') - _ask_pkg_config(libraries, '--libs-only-l', '-l') - -# Some thoughts on bundling LibRaw in Linux installs: -# Compiling and bundling libraw.so like in the Windows wheels is likely not -# easily possible for Linux. This is due to the fact that the dynamic linker ld -# doesn't search for libraw.so in the directory where the Python extension is in. -# The -rpath with $ORIGIN method can not be used in this case as $ORIGIN is always -# relative to the executable and not the shared library, -# see https://stackoverflow.com/q/6323603. -# But note that this was never tested and may actually still work somehow. -# matplotlib works around such problems by including external libraries as pure -# Python extensions, partly rewriting their sources and removing any dependency -# on a configure script, or cmake or other build infrastructure. -# A possible work-around could be to statically link against libraw. - -if (isWindows or isMac) and not useSystemLibraw: - external_dir = os.path.abspath('external') - libraw_dir = os.path.join(external_dir, 'LibRaw') - cmake_build = os.path.join(external_dir, 'LibRaw-cmake', 'build') - install_dir = os.path.join(cmake_build, 'install') - - include_dirs += [os.path.join(install_dir, 'include', 'libraw')] - library_dirs += [os.path.join(install_dir, 'lib')] - libraries = ['raw_r'] - - # for Windows and Mac we use cmake, so libraw_config.h will always exist - libraw_config_found = True -else: - use_pkg_config() - - # check if libraw_config.h exists - # this header is only installed when using cmake - libraw_config_found = False - for include_dir in include_dirs: - if 'libraw_config.h' in os.listdir(include_dir): - libraw_config_found = True - break -define_macros = [('_HAS_LIBRAW_CONFIG_H', '1' if libraw_config_found else '0')] +def use_pkg_config(): + pkg_config = os.environ.get("PKG_CONFIG", "pkg-config") + if subprocess.call([pkg_config, "--atleast-version=0.21", "libraw_r"]) != 0: + raise SystemExit("ERROR: System LibRaw is too old or not found. rawpy requires LibRaw >= 0.21.") + _ask_pkg_config(include_dirs, "--cflags-only-I", "-I", sysroot=True) + _ask_pkg_config(extra_compile_args, "--cflags-only-other") + _ask_pkg_config(library_dirs, "--libs-only-L", "-L", sysroot=True) + _ask_pkg_config(extra_link_args, "--libs-only-other") + _ask_pkg_config(libraries, "--libs-only-l", "-l") -if isWindows: - extra_compile_args += ['/DWIN32'] - -# this must be after use_pkg_config()! -include_dirs += [numpy.get_include()] def clone_submodules(): - if not os.path.exists('external/LibRaw/README.md'): - print('LibRaw git submodule is not cloned yet, will invoke "git submodule update --init" now') - if os.system('git submodule update --init') != 0: - raise Exception('git failed') - + if not os.path.exists("external/LibRaw/libraw/libraw.h"): + print( + 'LibRaw git submodule is not cloned yet, will invoke "git submodule update --init" now' + ) + if os.system("git submodule update --init") != 0: + raise Exception("git failed") + + +def get_cmake_build_dir(): + external_dir = os.path.abspath("external") + return os.path.join(external_dir, "LibRaw-cmake", "build") + + +def get_install_dir(): + return os.path.join(get_cmake_build_dir(), "install") + + def windows_libraw_compile(): clone_submodules() - - # download cmake to compile libraw - # the cmake zip contains a cmake-3.12.4-win32-x86 folder when extracted - cmake_url = 'https://cmake.org/files/v3.12/cmake-3.12.4-win32-x86.zip' - cmake = os.path.abspath('external/cmake-3.12.4-win32-x86/bin/cmake.exe') - - files = [(cmake_url, 'external', cmake)] - - for url, extractdir, extractcheck in files: - if not os.path.exists(extractcheck): - path = 'external/' + os.path.basename(url) - if not os.path.exists(path): - print('Downloading', url) - try: - urlretrieve(url, path) - except: - # repeat once in case of network issues - urlretrieve(url, path) - - with zipfile.ZipFile(path) as z: - print('Extracting', path, 'into', extractdir) - z.extractall(extractdir) - - if not os.path.exists(path): - raise RuntimeError(path + ' not found!') - + + cmake = "cmake" + # openmp dll # VS 2017 and higher - vc_redist_dir = os.getenv('VCToolsRedistDir') - vs_target_arch = os.getenv('VSCMD_ARG_TGT_ARCH') + vc_redist_dir = os.getenv("VCToolsRedistDir") + vs_target_arch = os.getenv("VSCMD_ARG_TGT_ARCH") if not vc_redist_dir: # VS 2015 - vc_redist_dir = os.path.join(os.environ['VCINSTALLDIR'], 'redist') - vs_target_arch = 'x64' if is64Bit else 'x86' - - omp_glob = os.path.join(vc_redist_dir, vs_target_arch, 'Microsoft.VC*.OpenMP', 'vcomp*.dll') - omp_dlls = glob.glob(omp_glob) + if "VCINSTALLDIR" in os.environ: + vc_redist_dir = os.path.join(os.environ["VCINSTALLDIR"], "redist") + vs_target_arch = "x64" if is64Bit else "x86" + else: + vc_redist_dir = None + + if vc_redist_dir and vs_target_arch: + omp_glob = os.path.join( + vc_redist_dir, vs_target_arch, "Microsoft.VC*.OpenMP", "vcomp*.dll" + ) + omp_dlls = glob.glob(omp_glob) + else: + omp_dlls = [] if len(omp_dlls) == 1: has_openmp_dll = True omp = omp_dlls[0] elif len(omp_dlls) > 1: - print('WARNING: disabling OpenMP because multiple runtime DLLs were found:') + print("WARNING: disabling OpenMP because multiple runtime DLLs were found:") for omp_dll in omp_dlls: print(omp_dll) has_openmp_dll = False else: - print('WARNING: disabling OpenMP because no runtime DLLs were found') + print("WARNING: disabling OpenMP because no runtime DLLs were found") has_openmp_dll = False - + # configure and compile libraw cwd = os.getcwd() + + cmake_build = get_cmake_build_dir() + install_dir = get_install_dir() + libraw_dir = os.path.join(os.path.abspath("external"), "LibRaw") + shutil.rmtree(cmake_build, ignore_errors=True) os.makedirs(cmake_build, exist_ok=True) os.chdir(cmake_build) - + # Important: always use Release build type, otherwise the library will depend on a # debug version of OpenMP which is not what we bundle it with, and then it would fail - enable_openmp_flag = 'ON' if has_openmp_dll else 'OFF' - cmds = [cmake + ' .. -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ' +\ - '-DCMAKE_PREFIX_PATH=' + os.environ['CMAKE_PREFIX_PATH'] + ' ' +\ - '-DLIBRAW_PATH=' + libraw_dir.replace('\\', '/') + ' ' +\ - '-DENABLE_X3FTOOLS=ON -DENABLE_6BY9RPI=ON ' +\ - '-DENABLE_EXAMPLES=OFF -DENABLE_OPENMP=' + enable_openmp_flag + ' -DENABLE_RAWSPEED=OFF ' +\ - ('-DENABLE_DEMOSAIC_PACK_GPL2=ON -DDEMOSAIC_PACK_GPL2_RPATH=../../LibRaw-demosaic-pack-GPL2 ' +\ - '-DENABLE_DEMOSAIC_PACK_GPL3=ON -DDEMOSAIC_PACK_GPL3_RPATH=../../LibRaw-demosaic-pack-GPL3 ' - if buildGPLCode else '') +\ - '-DCMAKE_INSTALL_PREFIX=install', - cmake + ' --build . --target install', - ] + enable_openmp_flag = "ON" if has_openmp_dll else "OFF" + cmds = [ + cmake + + ' .. -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ' + + "-DCMAKE_PREFIX_PATH=" + + os.environ["CMAKE_PREFIX_PATH"] + + " " + + "-DLIBRAW_PATH=" + + libraw_dir.replace("\\", "/") + + " " + + "-DENABLE_X3FTOOLS=ON -DENABLE_6BY9RPI=ON " + + "-DENABLE_EXAMPLES=OFF -DENABLE_OPENMP=" + + enable_openmp_flag + + " -DENABLE_RAWSPEED=OFF " + + ( + "-DENABLE_DEMOSAIC_PACK_GPL2=ON -DDEMOSAIC_PACK_GPL2_RPATH=../../LibRaw-demosaic-pack-GPL2 " + + "-DENABLE_DEMOSAIC_PACK_GPL3=ON -DDEMOSAIC_PACK_GPL3_RPATH=../../LibRaw-demosaic-pack-GPL3 " + if buildGPLCode + else "" + ) + + "-DCMAKE_INSTALL_PREFIX=install", + cmake + " --build . --target install", + ] for cmd in cmds: print(cmd) code = os.system(cmd) if code != 0: sys.exit(code) os.chdir(cwd) - + # bundle runtime dlls - dll_runtime_libs = [('raw_r.dll', os.path.join(install_dir, 'bin'))] - + dll_runtime_libs = [("raw_r.dll", os.path.join(install_dir, "bin"))] + if has_openmp_dll: # Check if OpenMP was enabled in the CMake build, independent of the flag we supplied. # If not, we don't have to bundle the DLL. - libraw_configh = os.path.join(install_dir, 'include', 'libraw', 'libraw_config.h') - match = '#define LIBRAW_USE_OPENMP 1' + libraw_configh = os.path.join( + install_dir, "include", "libraw", "libraw_config.h" + ) + match = "#define LIBRAW_USE_OPENMP 1" has_openmp_support = match in open(libraw_configh).read() if has_openmp_support: dll_runtime_libs.append((os.path.basename(omp), os.path.dirname(omp))) else: - print('WARNING: "#define LIBRAW_USE_OPENMP 1" not found even though OpenMP was enabled') - print('Will not bundle OpenMP runtime DLL') - + print( + 'WARNING: "#define LIBRAW_USE_OPENMP 1" not found even though OpenMP was enabled' + ) + print("Will not bundle OpenMP runtime DLL") + for filename, folder in dll_runtime_libs: src = os.path.join(folder, filename) - dest = 'rawpy/' + filename - print('copying', src, '->', dest) + dest = "rawpy/" + filename + print("copying", src, "->", dest) shutil.copyfile(src, dest) - -def mac_libraw_compile(): + + +def unix_libraw_compile(): + """Compiles LibRaw using CMake on macOS and Linux.""" clone_submodules() - - # configure and compile libraw + + external_dir = os.path.abspath("external") + libraw_dir = os.path.join(external_dir, "LibRaw") + cmake_build = get_cmake_build_dir() + install_dir = get_install_dir() + cwd = os.getcwd() if not os.path.exists(cmake_build): - os.mkdir(cmake_build) + os.makedirs(cmake_build, exist_ok=True) os.chdir(cmake_build) - - install_name_dir = os.path.join(install_dir, 'lib') - cmds = ['cmake .. -DCMAKE_BUILD_TYPE=Release ' +\ - '-DLIBRAW_PATH=' + libraw_dir + ' ' +\ - '-DENABLE_X3FTOOLS=ON -DENABLE_6BY9RPI=ON ' +\ - '-DENABLE_OPENMP=OFF ' +\ - '-DENABLE_EXAMPLES=OFF -DENABLE_RAWSPEED=OFF ' +\ - ('-DENABLE_DEMOSAIC_PACK_GPL2=ON -DDEMOSAIC_PACK_GPL2_RPATH=../../LibRaw-demosaic-pack-GPL2 ' +\ - '-DENABLE_DEMOSAIC_PACK_GPL3=ON -DDEMOSAIC_PACK_GPL3_RPATH=../../LibRaw-demosaic-pack-GPL3 ' - if buildGPLCode else '') +\ - '-DCMAKE_INSTALL_PREFIX=install -DCMAKE_INSTALL_NAME_DIR=' + install_name_dir, - 'cmake --build . --target install', + + # Use @rpath so the dylib's install name becomes @rpath/libraw_r..dylib. + # Combined with -rpath @loader_path on the extension, dyld will find the + # bundled dylib next to the .so at runtime. delocate (used in CI wheel + # builds) rewrites these paths anyway, so this is compatible with both + # plain pip installs and CI wheel builds. + install_name_dir = "@rpath" if isMac else os.path.join(install_dir, "lib") + + # OpenMP: enable on Linux (GCC supports it), disable on macOS + # (Apple Clang lacks OpenMP support out of the box). + enable_openmp = "ON" if isLinux else "OFF" + + # CMake arguments + cmake_args = [ + "cmake", + "..", + "-DCMAKE_BUILD_TYPE=Release", + "-DLIBRAW_PATH=" + libraw_dir, + "-DENABLE_X3FTOOLS=ON", + "-DENABLE_6BY9RPI=ON", + "-DENABLE_OPENMP=" + enable_openmp, + "-DENABLE_EXAMPLES=OFF", + "-DENABLE_RAWSPEED=OFF", + "-DCMAKE_INSTALL_PREFIX=install", + "-DCMAKE_INSTALL_LIBDIR=lib", + "-DCMAKE_INSTALL_NAME_DIR=" + install_name_dir, + ] + + if buildGPLCode: + cmake_args.extend( + [ + "-DENABLE_DEMOSAIC_PACK_GPL2=ON", + "-DDEMOSAIC_PACK_GPL2_RPATH=../../LibRaw-demosaic-pack-GPL2", + "-DENABLE_DEMOSAIC_PACK_GPL3=ON", + "-DDEMOSAIC_PACK_GPL3_RPATH=../../LibRaw-demosaic-pack-GPL3", ] + ) + + cmds = [" ".join(cmake_args), "cmake --build . --target install"] + for cmd in cmds: - print(cmd) - code = os.system(cmd) - if code != 0: - sys.exit(code) + print(f"Running: {cmd}") + if os.system(cmd) != 0: + sys.exit(f"Error executing: {cmd}") + os.chdir(cwd) - -package_data = {} - -# evil hack, check cmd line for relevant commands -# custom cmdclasses didn't work out in this case -cmdline = ''.join(sys.argv[1:]) -needsCompile = any(s in cmdline for s in ['install', 'bdist', 'build_ext']) and not useSystemLibraw -if isWindows and needsCompile: - windows_libraw_compile() - package_data['rawpy'] = ['*.dll'] - -elif isMac and needsCompile: - mac_libraw_compile() - -if any(s in cmdline for s in ['clean', 'sdist']): - # When running sdist after a previous run of bdist or build_ext - # then even with the 'clean' command the .egg-info folder stays. - # This folder contains SOURCES.txt which in turn is used by sdist - # to include package data files, but we don't want .dll's and .xml - # files in our source distribution. Therefore, to prevent accidents, - # we help a little... - egg_info = 'rawpy.egg-info' - print('removing', egg_info) - shutil.rmtree(egg_info, ignore_errors=True) - -extensions = cythonize([Extension("rawpy._rawpy", - include_dirs=include_dirs, - sources=[os.path.join('rawpy', '_rawpy.pyx')], - libraries=libraries, - library_dirs=library_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - )]) - -# make __version__ available (https://stackoverflow.com/a/16084844) -exec(open('rawpy/_version.py').read()) + + # When compiling LibRaw from source (not using system libraw), we + # copy the shared libraries into the package directory so they get + # bundled with the installed package (via package_data globs). + # The extension uses rpath ($ORIGIN on Linux, @loader_path on macOS) + # to find them at runtime. + # + # In CI, auditwheel (Linux) and delocate (macOS) further repair the + # wheel, but for editable installs and plain `pip install .` we need + # the libraries in-tree. + lib_dir = os.path.join(install_dir, "lib") + if isLinux: + libs = glob.glob(os.path.join(lib_dir, "libraw_r.so*")) + else: # macOS + libs = glob.glob(os.path.join(lib_dir, "libraw_r*.dylib")) + for lib in libs: + dest = os.path.join("rawpy", os.path.basename(lib)) + if os.path.islink(lib): + if os.path.lexists(dest): + os.remove(dest) + linkto = os.readlink(lib) + os.symlink(linkto, dest) + else: + shutil.copyfile(lib, dest) + print(f"Bundling {lib} -> {dest}") + + +# --- Main Logic --- + +# Determine if we need to compile LibRaw from source +# If using system libraw (e.g. installed via apt), we check pkg-config +libraw_config_found = False + +if (isWindows or isMac or isLinux) and not useSystemLibraw: + # Build from source + install_dir = get_install_dir() + include_dirs += [os.path.join(install_dir, "include", "libraw")] + library_dirs += [os.path.join(install_dir, "lib")] + libraries = ["raw_r"] + # If building from source, we know we have the config header + libraw_config_found = True +else: + # Use system library + use_pkg_config() + for include_dir in include_dirs: + if "libraw_config.h" in os.listdir(include_dir): + libraw_config_found = True + break + +# Ensure numpy headers are always included (use_pkg_config replaces the list) +if numpy.get_include() not in include_dirs: + include_dirs.insert(0, numpy.get_include()) + +define_macros.append(("_HAS_LIBRAW_CONFIG_H", "1" if libraw_config_found else "0")) + +# Package Data +# Always include platform-specific library globs — they harmlessly match +# nothing when libraries are not bundled (e.g. system libraw or sdist). +package_data = {"rawpy": ["py.typed", "*.pyi"]} +if isWindows: + package_data["rawpy"].append("*.dll") +elif isLinux: + package_data["rawpy"].append("*.so*") +elif isMac: + package_data["rawpy"].append("*.dylib") + + +# --- Custom build_ext --- +# Compile LibRaw from source before building the Cython extension. +# By putting this in build_ext.run(), it only runs when setuptools actually +# needs to build extensions — never during metadata-only commands like +# egg_info, sdist, or --version. This replaces the old sys.argv sniffing hack. + +class build_ext(_build_ext): + def run(self): + if not useSystemLibraw: + if isWindows: + windows_libraw_compile() + elif isMac or isLinux: + unix_libraw_compile() + super().run() + # Copy bundled shared libraries into the build output directory. + # build_py (which collects package_data) runs *before* build_ext, + # so the libraries compiled above aren't in build_lib yet. + if not useSystemLibraw: + self._copy_bundled_libs() + + def _copy_bundled_libs(self): + dest_dir = os.path.join(self.build_lib, "rawpy") + os.makedirs(dest_dir, exist_ok=True) + if isWindows: + libs = glob.glob("rawpy/*.dll") + elif isLinux: + libs = glob.glob("rawpy/libraw_r.so*") + elif isMac: + libs = glob.glob("rawpy/libraw_r*.dylib") + else: + return + for lib in libs: + dest = os.path.join(dest_dir, os.path.basename(lib)) + if os.path.islink(lib): + if os.path.lexists(dest): + os.remove(dest) + os.symlink(os.readlink(lib), dest) + else: + shutil.copyfile(lib, dest) + +# Extensions +extensions = cythonize( + [ + Extension( + "rawpy._rawpy", + include_dirs=include_dirs, + sources=[os.path.join("rawpy", "_rawpy.pyx")], + libraries=libraries, + library_dirs=library_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + ) + ] +) + +# Version +exec(open("rawpy/_version.py").read()) setup( - name = 'rawpy', - version = __version__, - description = 'RAW image processing for Python, a wrapper for libraw', - long_description = open('README.md').read(), - long_description_content_type='text/markdown', - author = 'Maik Riechert', - url = 'https://github.com/letmaik/rawpy', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Cython', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'Operating System :: MacOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Topic :: Multimedia :: Graphics', - 'Topic :: Software Development :: Libraries', - ], - packages = find_packages(), - ext_modules = extensions, - package_data = package_data, - install_requires=['numpy >= 1.26.0'] + version=__version__, + packages=find_packages(), + ext_modules=extensions, + package_data=package_data, + cmdclass={"build_ext": build_ext}, ) diff --git a/test/stubtest_allowlist.txt b/test/stubtest_allowlist.txt new file mode 100644 index 00000000..3ded4bff --- /dev/null +++ b/test/stubtest_allowlist.txt @@ -0,0 +1,12 @@ +# Allowlist for mypy stubtest +# These are internal implementation details that users shouldn't rely on + +# Cython-generated pickling support methods (module-level) +rawpy\._rawpy\.__reduce_cython__ +rawpy\._rawpy\.__setstate_cython__ + +# Internal wrapper class for LibRaw processed images +rawpy\._rawpy\.processed_image_wrapper + +# Doctest dictionary (not part of public API) +rawpy\._rawpy\.__test__ diff --git a/test/test_basic.py b/test/test_basic.py index 97d47756..f4f759e3 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -16,6 +16,14 @@ thisDir = os.path.dirname(__file__) +_x3f_supported = rawpy.flags is not None and rawpy.flags.get("X3FTOOLS", False) + +def _open_x3f(): + """Open X3F test file, skip test if format not supported by this LibRaw build.""" + if not _x3f_supported: + pytest.skip("X3F format not supported by this LibRaw build") + return rawpy.imread(raw4TestPath) + # Nikon D3S rawTestPath = os.path.join(thisDir, 'iss030e122639.NEF') badPixelsTestPath = os.path.join(thisDir, 'bad_pixels.gz') @@ -73,7 +81,7 @@ def testFileOpenAndPostProcess(): iio.imwrite('test_16daylight_linear.tiff', rgb) def testFoveonFileOpenAndPostProcess(): - raw = rawpy.imread(raw4TestPath) + raw = _open_x3f() assert_array_equal(raw.raw_image.shape, [1531, 2304, 3]) iio.imwrite('test_foveon_raw.tiff', raw.raw_image) @@ -140,13 +148,15 @@ def testThumbExtractJPEG(): with rawpy.imread(rawTestPath) as raw: thumb = raw.extract_thumb() assert thumb.format == rawpy.ThumbFormat.JPEG + assert isinstance(thumb.data, bytes) img = iio.imread(thumb.data) assert_array_equal(img.shape, [2832, 4256, 3]) def testThumbExtractBitmap(): - with rawpy.imread(raw4TestPath) as raw: + with _open_x3f() as raw: thumb = raw.extract_thumb() assert thumb.format == rawpy.ThumbFormat.BITMAP + assert isinstance(thumb.data, np.ndarray) assert_array_equal(thumb.data.shape, [378, 567, 3]) def testProperties(): @@ -169,11 +179,13 @@ def testBayerPattern(): for path in [rawTestPath, raw2TestPath]: raw = rawpy.imread(path) assert_equal(raw.color_desc, expected_desc) - assert_array_equal(raw.raw_pattern, [[0,1],[3,2]]) + assert raw.raw_pattern is not None + assert_array_equal(raw.raw_pattern, np.array([[0,1],[3,2]], dtype=np.uint8)) raw = rawpy.imread(raw3TestPath) assert_equal(raw.color_desc, expected_desc) - assert_array_equal(raw.raw_pattern, [[3,2],[0,1]]) + assert raw.raw_pattern is not None + assert_array_equal(raw.raw_pattern, np.array([[3,2],[0,1]], dtype=np.uint8)) def testAutoWhiteBalance(): # Test that auto_whitebalance returns None before postprocessing @@ -214,7 +226,7 @@ def getColorNeighbors(raw, y, x): # 5x5 area around coordinate masked by color of coordinate raw_colors = raw.raw_colors_visible raw_color = raw_colors[y, x] - masked = ma.masked_array(raw.raw_image_visible, raw_colors!=raw_color) + masked: ma.MaskedArray = ma.masked_array(raw.raw_image_visible, raw_colors!=raw_color) return masked[y-2:y+3,x-2:x+3].copy() bad_pixels = np.loadtxt(badPixelsTestPath, int) @@ -279,7 +291,7 @@ def testCropSizeCanon(): assert_equal(s.crop_height, 3744) def testCropSizeSigma(): - with rawpy.imread(raw4TestPath) as raw: + with _open_x3f() as raw: s = raw.sizes assert_equal(s.crop_left_margin, 0) assert_equal(s.crop_top_margin, 0) diff --git a/test/test_examples.py b/test/test_examples.py new file mode 100644 index 00000000..ff41bc2b --- /dev/null +++ b/test/test_examples.py @@ -0,0 +1,56 @@ +""" +Test that example scripts run without errors. + +Each example is executed as a subprocess so it runs exactly as a user would. +""" + +import os +import subprocess +import sys + +import pytest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +EXAMPLES_DIR = os.path.join(REPO_ROOT, "examples") +TEST_IMAGE = os.path.join(REPO_ROOT, "test", "iss030e122639.NEF") + +needs_test_image = pytest.mark.skipif( + not os.path.exists(TEST_IMAGE), + reason="test image not available", +) + +skip_examples = pytest.mark.skipif( + os.environ.get("RAWPY_SKIP_EXAMPLES", "") == "1", + reason="RAWPY_SKIP_EXAMPLES is set (e.g., slow QEMU emulation)", +) + +pytestmark = skip_examples + + +def run_example(script_name: str) -> subprocess.CompletedProcess: + """Run an example script and return the result.""" + script_path = os.path.join(EXAMPLES_DIR, script_name) + return subprocess.run( + [sys.executable, script_path], + capture_output=True, + text=True, + timeout=120, + ) + + +@needs_test_image +def test_basic_process(): + result = run_example("basic_process.py") + assert result.returncode == 0, result.stderr + + +@needs_test_image +def test_thumbnail_extract(): + result = run_example("thumbnail_extract.py") + assert result.returncode == 0, result.stderr + + +@needs_test_image +def test_bad_pixel_repair(): + result = run_example("bad_pixel_repair.py") + assert result.returncode == 0, result.stderr diff --git a/test/test_feature_flags.py b/test/test_feature_flags.py new file mode 100644 index 00000000..7678ed60 --- /dev/null +++ b/test/test_feature_flags.py @@ -0,0 +1,120 @@ +""" +Test that rawpy feature flags are present and that CI wheels have all +expected features enabled. + +The CI build scripts install zlib, libjpeg-turbo (jpeg8), libjasper, +and lcms2 on all platforms, so all codec-related flags should be True +in CI-built wheels. + +When running locally (editable install), some flags may be False because +the system libraries are not installed. The test for CI-required flags +is skipped in that case. +""" + +import os + +import pytest +import rawpy + + +@pytest.mark.skipif( + rawpy.flags is None, + reason="libraw_config.h not available (non-CMake LibRaw build)", +) +def test_flags_present(): + """rawpy.flags should be a dict (not None) with all known keys.""" + assert rawpy.flags is not None, ( + "rawpy.flags is None — libraw_config.h was not found at build time" + ) + expected_keys = { + "DNGDEFLATECODEC", + "DNGLOSSYCODEC", + "OPENMP", + "LCMS", + "REDCINECODEC", + "RAWSPEED", + "DEMOSAIC_PACK_GPL2", + "DEMOSAIC_PACK_GPL3", + "X3FTOOLS", + "6BY9RPI", + } + assert set(rawpy.flags.keys()) == expected_keys, ( + f"Unexpected flag keys: {set(rawpy.flags.keys()) ^ expected_keys}" + ) + + +@pytest.mark.skipif( + rawpy.flags is None, + reason="libraw_config.h not available (non-CMake LibRaw build)", +) +def test_flags_are_bool(): + """All flag values should be booleans.""" + assert rawpy.flags is not None + for key, value in rawpy.flags.items(): + assert isinstance(value, bool), f"flags[{key!r}] = {value!r}, expected bool" + + + +@pytest.mark.skipif( + rawpy.flags is None, + reason="libraw_config.h not available (non-CMake LibRaw build)", +) +@pytest.mark.skipif( + "CI" not in os.environ, + reason="only enforced in CI where all dependencies are installed", +) +def test_wheel_feature_flags(): + """CI wheels must have all expected feature flags enabled.""" + assert rawpy.flags is not None + + # Flags that must be True in every CI wheel. + required_true = { + "DNGDEFLATECODEC", # zlib + "DNGLOSSYCODEC", # libjpeg-turbo (jpeg8) + "LCMS", # lcms2 + "X3FTOOLS", # always enabled in setup.py + "6BY9RPI", # always enabled in setup.py + } + + # REDCINECODEC requires libjasper which is not available on all CI + # platforms (e.g. Ubuntu 24.04 has no libjasper-dev package). + # Only require it when RAWPY_CI_NO_JASPER is not set. + if not os.environ.get("RAWPY_CI_NO_JASPER"): + required_true.add("REDCINECODEC") + + # Flags that must be False (not enabled / not bundled). + required_false = { + "RAWSPEED", # never enabled + "DEMOSAIC_PACK_GPL2", # GPL, not bundled in MIT wheels + "DEMOSAIC_PACK_GPL3", # GPL, not bundled in MIT wheels + } + + # OpenMP: enabled on Linux (GCC) and Windows (when VC runtime DLL found), + # disabled on macOS (Apple Clang lacks OpenMP support). + import sys + + if sys.platform.startswith("linux"): + required_true.add("OPENMP") + elif sys.platform == "darwin": + required_false.add("OPENMP") + # On Windows, OpenMP depends on the VC runtime DLL being found, so we + # don't assert it either way. + + errors = [] + for flag in required_true: + if not rawpy.flags.get(flag): + errors.append(f"{flag} should be True but is {rawpy.flags.get(flag)}") + for flag in required_false: + if rawpy.flags.get(flag): + errors.append(f"{flag} should be False but is {rawpy.flags.get(flag)}") + + assert not errors, "Feature flag mismatches:\n" + "\n".join(f" - {e}" for e in errors) + + +def test_libraw_version(): + """rawpy.libraw_version should be a tuple of three ints >= 0.21.""" + ver = rawpy.libraw_version + assert isinstance(ver, tuple), f"Expected tuple, got {type(ver)}" + assert len(ver) == 3, f"Expected 3 elements, got {len(ver)}" + assert all(isinstance(v, int) for v in ver), f"Expected ints, got {ver}" + assert ver >= (0, 21, 0), f"LibRaw version {ver} is older than 0.21.0" diff --git a/test/test_mypy.py b/test/test_mypy.py new file mode 100644 index 00000000..8b24e0cc --- /dev/null +++ b/test/test_mypy.py @@ -0,0 +1,84 @@ +""" +Test that mypy type checking passes for rawpy package and tests. + +This validates that: +- Type annotations in rawpy module are correct +- Test files use types correctly +- No mypy errors in the codebase +""" + +import subprocess +import sys +import os +import pytest +import rawpy + +# These tests type-check the source tree (rawpy/ and test/ at repo root). +# When rawpy is installed from an artifact (site-packages), skip — the source +# tree's rawpy/ would shadow or conflict with the installed package. +# Note: .venv-test is inside the repo root, so we compare against rawpy/ subdir. +_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_rawpy_dir = os.path.join(os.path.abspath(_repo_root), "rawpy") +_is_editable = os.path.abspath(rawpy.__file__).startswith(_rawpy_dir) + + +@pytest.mark.skipif(not _is_editable, reason="requires editable install") +def test_mypy_all(): + """ + Run mypy on both rawpy/ package and test/ directory to validate type annotations. + + This ensures that: + - All type annotations in the package are correct and internally consistent + - Test files properly use the type-annotated rawpy API + """ + # Check if mypy is installed + result = subprocess.run( + [sys.executable, "-m", "mypy", "--version"], + capture_output=True, + text=True + ) + + if result.returncode != 0: + raise RuntimeError("mypy is not installed. Install with: pip install mypy") + + # Get repo root from test file location + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Run mypy on both rawpy package and test directory at once + # Use --install-types to automatically install missing type stubs + # Use --non-interactive to avoid prompts in CI + # Config is in pyproject.toml [tool.mypy], auto-discovered via cwd + result = subprocess.run( + [sys.executable, "-m", "mypy", + "--install-types", "--non-interactive", + "rawpy/", "test/"], + capture_output=True, + text=True, + cwd=repo_root + ) + + if result.returncode != 0: + error_msg = f""" +mypy found type errors! + +STDOUT: +{result.stdout} + +STDERR: +{result.stderr} + +To fix this, address the type errors shown above. +To run mypy manually: python -m mypy --install-types --non-interactive rawpy/ test/ +""" + raise AssertionError(error_msg) + + # Success + assert result.returncode == 0, "mypy should pass with no errors" + + +if __name__ == "__main__": + # Allow running the test directly for debugging + print("Running mypy on rawpy/ and test/ ...") + test_mypy_all() + print("✓ mypy passed on all checked directories!") + diff --git a/test/test_stubtest.py b/test/test_stubtest.py new file mode 100644 index 00000000..59dc3268 --- /dev/null +++ b/test/test_stubtest.py @@ -0,0 +1,88 @@ +""" +Test that the .pyi stub file matches the runtime signatures of the _rawpy module. + +This test uses mypy's stubtest tool, which is the industry standard for validating +that stub files accurately reflect runtime module signatures. + +Note: This test requires the rawpy module to be built and installed/importable. +""" + +import subprocess +import sys +import os +import pytest +import rawpy + +# stubtest validates that the .pyi stub matches the runtime module. +# When rawpy is installed from an artifact (site-packages), skip — the +# allowlist and stub source are tied to the editable/source-tree workflow. +# Note: .venv-test is inside the repo root, so we compare against rawpy/ subdir. +_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_rawpy_dir = os.path.join(os.path.abspath(_repo_root), "rawpy") +_is_editable = os.path.abspath(rawpy.__file__).startswith(_rawpy_dir) + + +@pytest.mark.skipif(not _is_editable, reason="requires editable install") +def test_stub_matches_runtime(): + """ + Use mypy stubtest to verify that rawpy/_rawpy.pyi matches the runtime signatures. + + stubtest compares the stub file against the actual runtime using Python's inspect + module, checking for: + - Missing or extra functions/methods/properties + - Signature mismatches (parameters, return types) + - Missing or extra class members + + This is the recommended approach from the Python typing community and is used + by typeshed to validate all their stubs. + + Internal implementation details (Cython-generated methods, internal classes) + are excluded via the stubtest_allowlist.txt file. + """ + # Import the module - this will fail if not built + import rawpy._rawpy + + # Get path to allowlist file + test_dir = os.path.dirname(__file__) + allowlist_path = os.path.join(test_dir, 'stubtest_allowlist.txt') + + # Run stubtest on the _rawpy module with allowlist + # The stub file rawpy/_rawpy.pyi will be automatically found by mypy + # Use --ignore-disjoint-bases to suppress disjoint base warnings for Cython classes + # (these are internal implementation details not relevant for type checking) + result = subprocess.run( + [sys.executable, "-m", "mypy.stubtest", "rawpy._rawpy", + "--allowlist", allowlist_path, + "--ignore-disjoint-bases"], + capture_output=True, + text=True + ) + + # Check if stubtest command exists + if "No module named mypy.stubtest" in result.stderr or "No module named mypy" in result.stderr: + pytest.fail("mypy is not installed. Install with: pip install mypy") + + # If there are mismatches, stubtest will return non-zero and output details + if result.returncode != 0: + error_msg = f""" +Stub file (rawpy/_rawpy.pyi) does not match runtime signatures! + +STDOUT: +{result.stdout} + +STDERR: +{result.stderr} + +To fix this, update rawpy/_rawpy.pyi to match the runtime signatures. +To run stubtest manually: python -m mypy.stubtest rawpy._rawpy +""" + pytest.fail(error_msg) + + # Success - stubs match runtime + assert result.returncode == 0, "Stubtest should pass with no mismatches" + + +if __name__ == "__main__": + # Allow running the test directly for debugging + test_stub_matches_runtime() + print("✓ Stub file matches runtime signatures!") diff --git a/test/test_type_imports.py b/test/test_type_imports.py new file mode 100644 index 00000000..e99f73d3 --- /dev/null +++ b/test/test_type_imports.py @@ -0,0 +1,56 @@ +""" +Test that type checkers can see all the types exported by rawpy. + +This validates that the TYPE_CHECKING imports in __init__.py include +all the types that are available at runtime via globals().update(). +""" + +import rawpy + +def test_runtime_imports_available() -> None: + """Test that all expected types are available at runtime.""" + # These should all be available at runtime due to globals().update() + assert hasattr(rawpy, 'RawPy') + assert hasattr(rawpy, 'Params') + assert hasattr(rawpy, 'ImageSizes') + assert hasattr(rawpy, 'Thumbnail') + assert hasattr(rawpy, 'RawType') + assert hasattr(rawpy, 'ThumbFormat') + assert hasattr(rawpy, 'DemosaicAlgorithm') + assert hasattr(rawpy, 'ColorSpace') + assert hasattr(rawpy, 'HighlightMode') + assert hasattr(rawpy, 'FBDDNoiseReductionMode') + # Exceptions + assert hasattr(rawpy, 'LibRawError') + assert hasattr(rawpy, 'LibRawIOError') + +def test_type_checker_sees_types() -> None: + """ + Test that type checkers can see the types. + + If mypy can type check this function without errors, then the types + are properly imported in the TYPE_CHECKING block. + """ + # These type annotations should be recognized by mypy + # Wrapped in if False to avoid UnboundLocalError at runtime + # while still allowing mypy to validate the type annotations + if False: # Never executed - only for type checking + sizes: rawpy.ImageSizes + thumb: rawpy.Thumbnail + raw_type: rawpy.RawType + thumb_fmt: rawpy.ThumbFormat + algo: rawpy.DemosaicAlgorithm + color: rawpy.ColorSpace + highlight: rawpy.HighlightMode + noise: rawpy.FBDDNoiseReductionMode + params: rawpy.Params + error: rawpy.LibRawError + +if __name__ == "__main__": + print("Testing runtime imports...") + test_runtime_imports_available() + print("✓ All runtime imports available") + + print("\nTesting type checker visibility...") + test_type_checker_sees_types() + print("✓ Type checker test passed") diff --git a/test/test_user_cblack.py b/test/test_user_cblack.py index f552c45d..aa2a54c2 100644 --- a/test/test_user_cblack.py +++ b/test/test_user_cblack.py @@ -7,6 +7,24 @@ thisDir = os.path.dirname(__file__) rawTestPath = os.path.join(thisDir, 'iss030e122639.NEF') +raw3TestPath = os.path.join(thisDir, 'RAW_CANON_5DMARK2_PREPROD.CR2') + + +def test_default_postprocess_color_balance(): + """Default postprocess must produce expected per-channel means. + + Uses the Canon 5D Mark II image which has non-zero per-channel black + levels (cblack=[1027, 1026, 1026, 1027]). If user_cblack is + accidentally set to zeros instead of left unset, the black level + override shifts the output and this test fails. + """ + with rawpy.imread(raw3TestPath) as raw: + rgb = raw.postprocess() + + mean = rgb.mean(axis=(0, 1)) + np.testing.assert_allclose(mean, [18.551, 19.079, 47.292], atol=0.01, + err_msg="Default postprocess color balance changed" + ) def test_user_cblack_parameter_acceptance():