From fd52ed23787aa817d7496324e2fa53242b76c43a Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 16:02:19 +0300 Subject: [PATCH 01/15] here we go again, my dear CI --- .github/workflows/release.yml | 10 +++++++++- .github/workflows/test_artifacts_io.yml | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c227ec..3c866fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,15 @@ jobs: with: path: dist + - name: Flatten dist directory + run: | + shopt -s globstar + mkdir -p flat + cp dist/**/* flat/ || true + rm -rf dist + mv flat dist + - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/test_artifacts_io.yml b/.github/workflows/test_artifacts_io.yml index 8e38e67..3d19c60 100644 --- a/.github/workflows/test_artifacts_io.yml +++ b/.github/workflows/test_artifacts_io.yml @@ -29,12 +29,20 @@ jobs: with: path: dist + - name: Flatten dist directory + run: | + shopt -s globstar + mkdir -p flat + cp dist/**/* flat/ || true + rm -rf dist + mv flat dist + - name: Show downloaded artifacts run: find dist/ - + - name: Assert files restored run: | - test -f dist/sdist/mock_sdist.tar.gz - test -f dist/wheels-ubuntu-22.04/mock_wheels_ubuntu-22.04.whl - test -f dist/wheels-windows-2022/mock_wheels_windows-2022.whl - test -f dist/wheels-macOS-13/mock_wheels_macOS-13.whl \ No newline at end of file + test -f dist/mock_sdist.tar.gz + test -f dist/mock_wheels_ubuntu-22.04.whl + test -f dist/mock_wheels_windows-2022.whl + test -f dist/mock_wheels_macOS-13.whl \ No newline at end of file From 958291da9630e86e35da6ffa50270ddec5cdb1d6 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 20:51:19 +0300 Subject: [PATCH 02/15] test_release instead of test_artifacts_io and test_build, complete release test --- .github/workflows/build-sdist.yml | 22 ------------ .github/workflows/build-wheels.yml | 9 ----- .github/workflows/check-version.yml | 31 ++++++++++++++++ .github/workflows/release.yml | 22 +----------- .github/workflows/test_artifacts_io.yml | 48 ------------------------- .github/workflows/test_build.yml | 18 ---------- .github/workflows/test_release.yml | 48 +++++++++++++++++++++++++ 7 files changed, 80 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/check-version.yml delete mode 100644 .github/workflows/test_artifacts_io.yml delete mode 100644 .github/workflows/test_build.yml create mode 100644 .github/workflows/test_release.yml diff --git a/.github/workflows/build-sdist.yml b/.github/workflows/build-sdist.yml index 7671943..0e1f2e3 100644 --- a/.github/workflows/build-sdist.yml +++ b/.github/workflows/build-sdist.yml @@ -2,15 +2,6 @@ name: build-sdist on: workflow_call: - inputs: - mock: - required: false - type: boolean - default: false - upload_artifacts: - required: false - type: boolean - default: false env: MODULE_NAME: imops @@ -22,31 +13,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Mock sdist - if: ${{ inputs.mock }} - run: | - mkdir -p dist - echo "Create mock sdist" > "dist/mock_sdist.tar.gz" - - name: Set up Python 3.9 - if: ${{ inputs.mock == false }} uses: actions/setup-python@v5 with: python-version: 3.9 - name: Build - if: ${{ inputs.mock == false }} run: | pip install build wheel python -m build --sdist - name: Upload sdist - if: ${{ inputs.upload_artifacts }} uses: actions/upload-artifact@v4 with: name: sdist path: dist/*.tar.gz - - - name: Cleanup all local files - if: ${{ inputs.mock }} - run: rm -rf dist \ No newline at end of file diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index e90a5f2..408fa95 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -10,14 +10,6 @@ on: required: false type: string default: "" - mock: - required: false - type: boolean - default: false - upload_artifacts: - required: false - type: boolean - default: false env: MODULE_NAME: imops @@ -88,7 +80,6 @@ jobs: fi - name: Upload wheels - if: ${{ inputs.upload_artifacts }} uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }} diff --git a/.github/workflows/check-version.yml b/.github/workflows/check-version.yml new file mode 100644 index 0000000..d8a9acc --- /dev/null +++ b/.github/workflows/check-version.yml @@ -0,0 +1,31 @@ +name: check-version + +on: + workflow_call: + +env: + MODULE_NAME: imops + +jobs: + check_version: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - id: get_version + name: Get the release version + uses: Simply007/get-version-action@v2 + + - name: Check the version + run: | + RELEASE=${{ steps.get_version.outputs.version-without-v }} + VERSION=$(python -c "from pathlib import Path; import runpy; folder, = {d.parent for d in Path().resolve().glob('*/__init__.py') if d.parent.is_dir() and (d.parent / '__version__.py').exists()}; print(runpy.run_path(folder / '__version__.py')['__version__'])") + MATCH=$(pip index versions $MODULE_NAME | grep "Available versions:" | grep $VERSION) || echo + echo $MATCH + if [ "$GITHUB_BASE_REF" = "master" ] && [ "$MATCH" != "" ]; then echo "Version $VERSION already present" && exit 1; fi + if [ "$VERSION" != "$RELEASE" ]; then echo "$VERSION vs $RELEASE" && exit 1; fi \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c866fb..09e1b0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,27 +9,7 @@ env: jobs: check_version: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - - id: get_version - name: Get the release version - uses: Simply007/get-version-action@v2 - - - name: Check the version - run: | - RELEASE=${{ steps.get_version.outputs.version-without-v }} - VERSION=$(python -c "from pathlib import Path; import runpy; folder, = {d.parent for d in Path().resolve().glob('*/__init__.py') if d.parent.is_dir() and (d.parent / '__version__.py').exists()}; print(runpy.run_path(folder / '__version__.py')['__version__'])") - MATCH=$(pip index versions $MODULE_NAME | grep "Available versions:" | grep $VERSION) || echo - echo $MATCH - if [ "$GITHUB_BASE_REF" = "master" ] && [ "$MATCH" != "" ]; then echo "Version $VERSION already present" && exit 1; fi - if [ "$VERSION" != "$RELEASE" ]; then echo "$VERSION vs $RELEASE" && exit 1; fi + uses: ./.github/workflows/check-version.yml build_wheels: uses: ./.github/workflows/build-wheels.yml diff --git a/.github/workflows/test_artifacts_io.yml b/.github/workflows/test_artifacts_io.yml deleted file mode 100644 index 3d19c60..0000000 --- a/.github/workflows/test_artifacts_io.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Test artifacts IO - -on: [ pull_request ] - -jobs: - sdist_mock: - uses: ./.github/workflows/build-sdist.yml - with: - mock: true - upload_artifacts: true - - wheels_mock: - uses: ./.github/workflows/build-wheels.yml - with: - cibw_build: "" - mock: true - upload_artifacts: true - - test_artifacts: - needs: [ sdist_mock, wheels_mock ] - runs-on: ubuntu-latest - - steps: - - name: Create dist directory - run: mkdir -p dist - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: dist - - - name: Flatten dist directory - run: | - shopt -s globstar - mkdir -p flat - cp dist/**/* flat/ || true - rm -rf dist - mv flat dist - - - name: Show downloaded artifacts - run: find dist/ - - - name: Assert files restored - run: | - test -f dist/mock_sdist.tar.gz - test -f dist/mock_wheels_ubuntu-22.04.whl - test -f dist/mock_wheels_windows-2022.whl - test -f dist/mock_wheels_macOS-13.whl \ No newline at end of file diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml deleted file mode 100644 index e54d839..0000000 --- a/.github/workflows/test_build.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Test build - -on: [ pull_request ] - -jobs: - build_sdist: - uses: ./.github/workflows/build-sdist.yml - with: - mock: false - upload_artifacts: false - - build_wheels: - uses: ./.github/workflows/build-wheels.yml - with: - cibw_build: cp37-* cp39-* cp312-* - cibw_skip: "*manylinux_x86_64" - mock: false - upload_artifacts: false \ No newline at end of file diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml new file mode 100644 index 0000000..596bcb4 --- /dev/null +++ b/.github/workflows/test_release.yml @@ -0,0 +1,48 @@ +name: Test build + +on: [ pull_request ] + +env: + MODULE_NAME: imops + +jobs: + check_version: + uses: ./.github/workflows/check-version.yml + + build_wheels: + uses: ./.github/workflows/build-wheels.yml + needs: [ check_version ] + with: + cibw_build: cp37-* cp39-* cp312-* + cibw_skip: "*manylinux_x86_64 *musllinux_x86_64" + upload_artifacts: true + + build_sdist: + uses: ./.github/workflows/build-sdist.yml + needs: [ check_version ] + with: + upload_artifacts: true + + publish_testpypi: + needs: [ build_wheels, build_sdist ] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + + - name: Flatten dist directory + run: | + shopt -s globstar + mkdir -p flat + cp dist/**/* flat/ || true + rm -rf dist + mv flat dist + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + skip-existing: true \ No newline at end of file From 75dedd112ceac3634d111de35baf2a7e726107a8 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 20:53:17 +0300 Subject: [PATCH 03/15] fix build-wheels --- .github/workflows/build-wheels.yml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 408fa95..4a3fcf0 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -24,37 +24,28 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Mock wheels - if: ${{ inputs.mock }} - run: | - New-Item -ItemType Directory -Force -Path wheelhouse | Out-Null - "Create mock wheels" | Out-File -Encoding ascii "wheelhouse/mock_wheels_${{ matrix.os }}.whl" - shell: pwsh - - name: Set up Python 3.9 - if: ${{ inputs.mock == false }} uses: actions/setup-python@v5 with: python-version: '3.9' - name: Install cibuildwheel - if: ${{ inputs.mock == false }} run: python -m pip install cibuildwheel==2.17.0 - name: Install llvm for mac - if: ${{ matrix.os == 'macOS-13' && inputs.mock == false }} + if: ${{ matrix.os == 'macOS-13' }} run: | brew install llvm - name: Install g++-11 for ubuntu - if: ${{ matrix.os == 'ubuntu-22.04' && inputs.mock == false }} + if: ${{ matrix.os == 'ubuntu-22.04' }} id: install_cc uses: rlalik/setup-cpp-compiler@master with: compiler: g++-11 - name: Check compilers for ubuntu - if: ${{ matrix.os == 'ubuntu-22.04' && inputs.mock == false }} + if: ${{ matrix.os == 'ubuntu-22.04' }} run: | ls /usr/bin/gcc* ls /usr/bin/g++* @@ -64,7 +55,6 @@ jobs: gcc --version - name: Build wheels - if: ${{ inputs.mock == false }} run: | python -m pip install --upgrade pip python -m pip install build wheel @@ -83,9 +73,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }} - path: ./wheelhouse/*.whl - - - name: Cleanup all local files - if: ${{ inputs.mock }} - run: Remove-Item -Recurse -Force wheelhouse - shell: pwsh \ No newline at end of file + path: ./wheelhouse/*.whl \ No newline at end of file From 215d68af82ef9c9eaaa8890f83c6db8f1d69f295 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 20:54:48 +0300 Subject: [PATCH 04/15] remove args --- .github/workflows/release.yml | 3 --- .github/workflows/test_release.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09e1b0c..27dffcf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,13 +16,10 @@ jobs: needs: [ check_version ] with: cibw_build: cp37-* cp38-* cp39-* cp310-* cp311-* cp312-* - upload_artifacts: true build_sdist: uses: ./.github/workflows/build-sdist.yml needs: [ check_version ] - with: - upload_artifacts: true release: needs: [ build_wheels, build_sdist ] diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index 596bcb4..3321cc2 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -15,13 +15,10 @@ jobs: with: cibw_build: cp37-* cp39-* cp312-* cibw_skip: "*manylinux_x86_64 *musllinux_x86_64" - upload_artifacts: true build_sdist: uses: ./.github/workflows/build-sdist.yml needs: [ check_version ] - with: - upload_artifacts: true publish_testpypi: needs: [ build_wheels, build_sdist ] From 7e43eb01643943a57769e6cfcea9a738aa828183 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 20:58:17 +0300 Subject: [PATCH 05/15] don't check version on merge --- .github/workflows/check-version.yml | 31 ----------------------------- .github/workflows/release.yml | 22 +++++++++++++++++++- .github/workflows/test_release.yml | 5 ----- 3 files changed, 21 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/check-version.yml diff --git a/.github/workflows/check-version.yml b/.github/workflows/check-version.yml deleted file mode 100644 index d8a9acc..0000000 --- a/.github/workflows/check-version.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: check-version - -on: - workflow_call: - -env: - MODULE_NAME: imops - -jobs: - check_version: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - - id: get_version - name: Get the release version - uses: Simply007/get-version-action@v2 - - - name: Check the version - run: | - RELEASE=${{ steps.get_version.outputs.version-without-v }} - VERSION=$(python -c "from pathlib import Path; import runpy; folder, = {d.parent for d in Path().resolve().glob('*/__init__.py') if d.parent.is_dir() and (d.parent / '__version__.py').exists()}; print(runpy.run_path(folder / '__version__.py')['__version__'])") - MATCH=$(pip index versions $MODULE_NAME | grep "Available versions:" | grep $VERSION) || echo - echo $MATCH - if [ "$GITHUB_BASE_REF" = "master" ] && [ "$MATCH" != "" ]; then echo "Version $VERSION already present" && exit 1; fi - if [ "$VERSION" != "$RELEASE" ]; then echo "$VERSION vs $RELEASE" && exit 1; fi \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 27dffcf..36c542a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,27 @@ env: jobs: check_version: - uses: ./.github/workflows/check-version.yml + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - id: get_version + name: Get the release version + uses: Simply007/get-version-action@v2 + + - name: Check the version + run: | + RELEASE=${{ steps.get_version.outputs.version-without-v }} + VERSION=$(python -c "from pathlib import Path; import runpy; folder, = {d.parent for d in Path().resolve().glob('*/__init__.py') if d.parent.is_dir() and (d.parent / '__version__.py').exists()}; print(runpy.run_path(folder / '__version__.py')['__version__'])") + MATCH=$(pip index versions $MODULE_NAME | grep "Available versions:" | grep $VERSION) || echo + echo $MATCH + if [ "$GITHUB_BASE_REF" = "master" ] && [ "$MATCH" != "" ]; then echo "Version $VERSION already present" && exit 1; fi + if [ "$VERSION" != "$RELEASE" ]; then echo "$VERSION vs $RELEASE" && exit 1; fi build_wheels: uses: ./.github/workflows/build-wheels.yml diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index 3321cc2..e26b67d 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -6,19 +6,14 @@ env: MODULE_NAME: imops jobs: - check_version: - uses: ./.github/workflows/check-version.yml - build_wheels: uses: ./.github/workflows/build-wheels.yml - needs: [ check_version ] with: cibw_build: cp37-* cp39-* cp312-* cibw_skip: "*manylinux_x86_64 *musllinux_x86_64" build_sdist: uses: ./.github/workflows/build-sdist.yml - needs: [ check_version ] publish_testpypi: needs: [ build_wheels, build_sdist ] From 547527208130d53fe11ff37ce13d1df70f16f939 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 21:12:20 +0300 Subject: [PATCH 06/15] workflow name --- .github/workflows/test_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index e26b67d..b743874 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -1,4 +1,4 @@ -name: Test build +name: Test release on: [ pull_request ] From e8563650e6d25aa7fa357c45d595108572ef0c9c Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Thu, 4 Dec 2025 21:14:21 +0300 Subject: [PATCH 07/15] fewer tests --- tests/test_argmax.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_argmax.py b/tests/test_argmax.py index 694747c..7ed85b4 100644 --- a/tests/test_argmax.py +++ b/tests/test_argmax.py @@ -14,17 +14,17 @@ def dtype(request): return request.param -@pytest.fixture(params=[1, 7, 54, 72, 128, 256]) +@pytest.fixture(params=[1, 2, 7, 54, 128]) def pre_dim(request): return request.param -@pytest.fixture(params=[1, 7, 54, 72, 128, 256]) +@pytest.fixture(params=[1, 2, 7, 54, 128]) def post_dim(request): return request.param -@pytest.fixture(params=[1, 2, 7, 11, 18, 33, 57, 129]) +@pytest.fixture(params=[1, 2, 7, 11, 57, 129]) def argmax_dim(request): return request.param From 6aadd019ccd40aa708dcdb48946fd4e9a68da7c2 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Fri, 5 Dec 2025 14:26:44 +0300 Subject: [PATCH 08/15] fewer stress --- tests/test_convex_hull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_convex_hull.py b/tests/test_convex_hull.py index 8d83273..31cee86 100644 --- a/tests/test_convex_hull.py +++ b/tests/test_convex_hull.py @@ -10,7 +10,7 @@ np.random.seed(1337) -N_STRESS = 1000 +N_STRESS = 200 @pytest.fixture(params=[False, True]) From 2854cd8bb9597e153dcea096c6e7f1a419bf9120 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Fri, 5 Dec 2025 18:35:54 +0300 Subject: [PATCH 09/15] try pytest-xdist with num_workers=4 --- .github/workflows/tests.yml | 2 +- tests/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7fd0c5a..091ff27 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,7 @@ jobs: - name: Test with pytest run: | - pytest tests -m "not nonumba" --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch + pytest tests -n 4 -m "not nonumba" --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch pip uninstall numba -y pytest tests/test_backend.py -m nonumba - name: Generate coverage report diff --git a/tests/requirements.txt b/tests/requirements.txt index f358f3f..84edb3d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ pytest pytest-subtests pytest-cov +pytest-xdist scikit-image numba deli From 8b3af60c1c84d46b0f8419182a153fdcc0b6992f Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Sat, 6 Dec 2025 00:24:09 +0300 Subject: [PATCH 10/15] less threads in test_numeric, pytest num_workers: 2 -> 4, test release naming --- .github/workflows/{test_release.yml => test-release.yml} | 0 .github/workflows/tests.yml | 2 +- tests/test_numeric.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{test_release.yml => test-release.yml} (100%) diff --git a/.github/workflows/test_release.yml b/.github/workflows/test-release.yml similarity index 100% rename from .github/workflows/test_release.yml rename to .github/workflows/test-release.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 091ff27..eeb58fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,7 @@ jobs: - name: Test with pytest run: | - pytest tests -n 4 -m "not nonumba" --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch + pytest tests -n 2 -m "not nonumba" --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch pip uninstall numba -y pytest tests/test_backend.py -m nonumba - name: Generate coverage report diff --git a/tests/test_numeric.py b/tests/test_numeric.py index 34b77a7..a7aeedf 100644 --- a/tests/test_numeric.py +++ b/tests/test_numeric.py @@ -27,7 +27,7 @@ def backend(request): return request.param -@pytest.fixture(params=range(1, 9)) +@pytest.fixture(params=range(1, 5)) def num_threads(request): return request.param From 39aab8028f1bfc64d897d0bfc91376a75b16f07e Mon Sep 17 00:00:00 2001 From: Nikita Ushakov Date: Sat, 13 Dec 2025 05:25:33 -0800 Subject: [PATCH 11/15] Goodbye Numba... --- .github/workflows/tests.yml | 6 +- README.md | 54 ++-- asv.conf.json | 4 +- benchmarks/benchmark_interp1d.py | 3 +- benchmarks/benchmark_zoom.py | 3 +- docs/index.md | 1 - imops/__init__.py | 2 +- imops/_configs.py | 6 +- imops/backend.py | 18 +- imops/interp1d.py | 23 +- imops/src/_numba_zoom.py | 503 ------------------------------- imops/utils.py | 9 - imops/zoom.py | 43 +-- pyproject.toml | 6 - setup.py | 1 - tests/requirements.txt | 1 - tests/test_backend.py | 17 +- tests/test_interp1d.py | 8 - tests/test_zoom.py | 7 - 19 files changed, 45 insertions(+), 670 deletions(-) delete mode 100644 imops/src/_numba_zoom.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eeb58fd..4a635b8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,12 +45,10 @@ jobs: - name: Test with pytest run: | - pytest tests -n 2 -m "not nonumba" --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch - pip uninstall numba -y - pytest tests/test_backend.py -m nonumba + pytest tests -n 2 --junitxml=reports/junit-${{ matrix.python-version }}.xml --cov="$MODULE_PARENT/$MODULE_NAME" --cov-report=xml --cov-branch - name: Generate coverage report run: | - coverage xml -o reports/coverage-${{ matrix.python-version }}.xml --omit=imops/src/_numba_zoom.py + coverage xml -o reports/coverage-${{ matrix.python-version }}.xml sed -i -e "s|$MODULE_PARENT/||g" reports/coverage-${{ matrix.python-version }}.xml sed -i -e "s|$(echo $MODULE_PARENT/ | tr "/" .)||g" reports/coverage-${{ matrix.python-version }}.xml diff --git a/README.md b/README.md index 2aba638..c0d5ea0 100644 --- a/README.md +++ b/README.md @@ -13,24 +13,23 @@ Efficient parallelizable algorithms for multidimensional arrays to speed up your ```shell pip install imops # default install with Cython backend -pip install imops[numba] # additionally install Numba backend ``` # How fast is it? Time comparisons (ms) for Intel(R) Xeon(R) Silver 4114 CPU @ 2.20GHz using 8 threads. All inputs are C-contiguous NumPy arrays. For morphology functions `bool` dtype is used and `float64` for all others. -| function / backend | Scipy() | Cython(fast=False) | Cython(fast=True) | Numba() | -|:----------------------:|:-----------:|:----------------------:|:---------------------:|:-----------:| -| `zoom(..., order=0)` | 2072 | 1114 | **867** | 3590 | -| `zoom(..., order=1)` | 6527 | 596 | **575** | 3757 | -| `interp1d` | 780 | 149 | **146** | 420 | -| `radon` | 59711 | 5982 | **4837** | - | -| `inverse_radon` | 52928 | 8254 | **6535** | - | -| `binary_dilation` | 2207 | 310 | **298** | - | -| `binary_erosion` | 2296 | 326 | **304** | - | -| `binary_closing` | 4158 | 544 | **469** | - | -| `binary_opening` | 4410 | 567 | **522** | - | -| `center_of_mass` | 2237 | **64** | **64** | - | +| function / backend | Scipy() | Cython(fast=False) | Cython(fast=True) | +|:----------------------:|:-----------:|:----------------------:|:---------------------:| +| `zoom(..., order=0)` | 2072 | 1114 | **867** | +| `zoom(..., order=1)` | 6527 | 596 | **575** | +| `interp1d` | 780 | 149 | **146** | +| `radon` | 59711 | 5982 | **4837** | +| `inverse_radon` | 52928 | 8254 | **6535** | +| `binary_dilation` | 2207 | 310 | **298** | +| `binary_erosion` | 2296 | 326 | **304** | +| `binary_closing` | 4158 | 544 | **469** | +| `binary_opening` | 4410 | 567 | **522** | +| `center_of_mass` | 2237 | **64** | **64** | We use [`airspeed velocity`](https://asv.readthedocs.io/en/stable/) to benchmark our code. For detailed results visit [benchmark page](https://neuro-ml.github.io/imops/benchmarks/). @@ -121,36 +120,33 @@ labeled, num_components = label(x, background=1, return_num=True) # Backends For all heavy image routines except `label` you can specify which backend to use. Backend can be specified by a string or by an instance of `Backend` class. The latter allows you to customize some backend options: ```python -from imops import Cython, Numba, Scipy, zoom +from imops import Cython, Scipy, zoom y = zoom(x, 2, backend='Cython') y = zoom(x, 2, backend=Cython(fast=False)) # same as previous y = zoom(x, 2, backend=Cython(fast=True)) # -ffast-math compiled cython backend y = zoom(x, 2, backend=Scipy()) # use scipy original implementation -y = zoom(x, 2, backend='Numba') -y = zoom(x, 2, backend=Numba(parallel=True, nogil=True, cache=True)) # same as previous ``` Also backend can be specified globally or locally: ```python from imops import imops_backend, set_backend, zoom -set_backend('Numba') # sets Numba as default backend +set_backend('Scipy') # sets Scipy as default backend with imops_backend('Cython'): # sets Cython backend via context manager zoom(x, 2) ``` -Note that for `Numba` backend setting `num_threads` argument has no effect for now and you should use `NUMBA_NUM_THREADS` environment variable. Available backends: -| function / backend | Scipy | Cython | Numba | -|:-------------------:|:---------:|:---------:|:---------:| -| `zoom` | ✓ | ✓ | ✓ | -| `interp1d` | ✓ | ✓ | ✓ | -| `radon` | ✗ | ✓ | ✗ | -| `inverse_radon` | ✗ | ✓ | ✗ | -| `binary_dilation` | ✓ | ✓ | ✗ | -| `binary_erosion` | ✓ | ✓ | ✗ | -| `binary_closing` | ✓ | ✓ | ✗ | -| `binary_opening` | ✓ | ✓ | ✗ | -| `center_of_mass` | ✓ | ✓ | ✗ | +| function / backend | Scipy | Cython | +|:-------------------:|:---------:|:---------:| +| `zoom` | ✓ | ✓ | +| `interp1d` | ✓ | ✓ | +| `radon` | ✗ | ✓ | +| `inverse_radon` | ✗ | ✓ | +| `binary_dilation` | ✓ | ✓ | +| `binary_erosion` | ✓ | ✓ | +| `binary_closing` | ✓ | ✓ | +| `binary_opening` | ✓ | ✓ | +| `center_of_mass` | ✓ | ✓ | # Acknowledgements diff --git a/asv.conf.json b/asv.conf.json index 8dcee20..6d34980 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -15,14 +15,12 @@ "Cython": ["3.0.10"], "scipy": [], "scikit-image": [], - "numba": [], "pip+connected-components-3d": [], "pip+fastremap": [], "pybind11": [] }, "env_nobuild": { - "OMP_NUM_THREADS": "8", - "NUMBA_NUM_THREADS": "8" + "OMP_NUM_THREADS": "8" } }, "env_dir": ".asv/env", diff --git a/benchmarks/benchmark_interp1d.py b/benchmarks/benchmark_interp1d.py index 64b0f3e..759aeee 100644 --- a/benchmarks/benchmark_interp1d.py +++ b/benchmarks/benchmark_interp1d.py @@ -7,12 +7,11 @@ try: from imops._configs import interp1d_configs except ModuleNotFoundError: - from imops.backend import Cython, Numba, Scipy + from imops.backend import Cython, Scipy interp1d_configs = [ Scipy(), *[Cython(fast) for fast in [False, True]], - *[Numba(*flags) for flags in product([False, True], repeat=3)], ] from imops.interp1d import interp1d diff --git a/benchmarks/benchmark_zoom.py b/benchmarks/benchmark_zoom.py index e21ba80..4813d1b 100644 --- a/benchmarks/benchmark_zoom.py +++ b/benchmarks/benchmark_zoom.py @@ -6,12 +6,11 @@ try: from imops._configs import zoom_configs except ModuleNotFoundError: - from imops.backend import Cython, Numba, Scipy + from imops.backend import Cython, Scipy zoom_configs = [ Scipy(), *[Cython(fast) for fast in [False, True]], - *[Numba(*flags) for flags in product([False, True], repeat=3)], ] from imops.zoom import zoom diff --git a/docs/index.md b/docs/index.md index 64879ee..8ee269f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,6 @@ Efficient parallelizable algorithms for multidimensional arrays to speed up your ```shell pip install imops # default install with Cython backend -pip install imops[numba] # additionally install Numba backend ``` ## Functions diff --git a/imops/__init__.py b/imops/__init__.py index 233dae6..79c1773 100644 --- a/imops/__init__.py +++ b/imops/__init__.py @@ -1,6 +1,6 @@ from .__version__ import __version__ from .argmax import argmax -from .backend import Cython, Numba, Scipy, imops_backend, set_backend +from .backend import Cython, Scipy, imops_backend, set_backend from .crop import crop_to_box, crop_to_shape from .interp1d import interp1d from .measure import label diff --git a/imops/_configs.py b/imops/_configs.py index 678f3dd..d3a542f 100644 --- a/imops/_configs.py +++ b/imops/_configs.py @@ -1,6 +1,4 @@ -from itertools import product - -from .backend import Cython, Numba, Scipy +from .backend import Cython, Scipy scipy_configs = [Scipy()] @@ -20,10 +18,8 @@ zoom_configs = [ Scipy(), *[Cython(fast) for fast in [False, True]], - *[Numba(*flags) for flags in product([False, True], repeat=3)], ] interp1d_configs = [ Scipy(), *[Cython(fast) for fast in [False, True]], - *[Numba(*flags) for flags in product([False, True], repeat=3)], ] diff --git a/imops/backend.py b/imops/backend.py index 16f7a30..29d775b 100644 --- a/imops/backend.py +++ b/imops/backend.py @@ -18,7 +18,6 @@ def name(self): return type(self).__name__ Cython: 'Cython' - Numba: 'Numba' Scipy: 'Scipy' @@ -64,21 +63,6 @@ def imops_backend(backend: BackendLike): set_backend(previous) -# implementations -# TODO: Investigate whether it is safe to use -ffast-math in numba -@dataclass(frozen=True) -class Numba(Backend): - parallel: bool = True - nogil: bool = True - cache: bool = True - - def __post_init__(self): - try: - import numba # noqa: F401 - except ModuleNotFoundError: # pragma: no cover - raise ModuleNotFoundError('Install `numba` package (pip install numba) to use "numba" backend.') - - @dataclass(frozen=True) class Cython(Backend): fast: bool = False @@ -91,5 +75,5 @@ class Scipy(Backend): DEFAULT_BACKEND = Cython() -BACKEND_NAME2ENV_NUM_THREADS_VAR_NAME = {Cython.__name__: 'OMP_NUM_THREADS', Numba.__name__: 'NUMBA_NUM_THREADS'} +BACKEND_NAME2ENV_NUM_THREADS_VAR_NAME = {Cython.__name__: 'OMP_NUM_THREADS'} SINGLE_THREADED_BACKENDS = (Scipy.__name__,) diff --git a/imops/interp1d.py b/imops/interp1d.py index 861a0b6..23417c4 100644 --- a/imops/interp1d.py +++ b/imops/interp1d.py @@ -45,7 +45,7 @@ class interp1d: the number of threads to use for computation. Default = the cpu count. If negative value passed cpu count + num_threads + 1 threads will be used backend: BackendLike - which backend to use. `numba`, `cython` and `scipy` are available, `cython` is used by default + which backend to use. `cython` and `scipy` are available, `cython` is used by default Methods ------- @@ -78,7 +78,7 @@ def __init__( backend: BackendLike = None, ) -> None: backend = resolve_backend(backend, warn_stacklevel=3) - if backend.name not in ('Scipy', 'Numba', 'Cython'): + if backend.name not in ('Scipy', 'Cython'): raise ValueError(f'Unsupported backend "{backend.name}".') self.backend = backend @@ -131,14 +131,6 @@ def __init__( if backend.name == 'Cython': self.src_interp1d = cython_fast_interp1d if backend.fast else cython_interp1d - if backend.name == 'Numba': - from numba import njit - - from .src._numba_zoom import _interp1d as numba_interp1d - - njit_kwargs = {kwarg: getattr(backend, kwarg) for kwarg in backend.__dataclass_fields__.keys()} - self.src_interp1d = njit(**njit_kwargs)(numba_interp1d) - def __call__(self, x_new: np.ndarray) -> np.ndarray: """ Evaluate the interpolant @@ -160,13 +152,7 @@ def __call__(self, x_new: np.ndarray) -> np.ndarray: return self.scipy_interp1d(x_new) extrapolate = self.fill_value == 'extrapolate' - args = () if self.backend.name in ('Numba',) else (num_threads,) - if self.backend.name == 'Numba': - from numba import get_num_threads, set_num_threads - - old_num_threads = get_num_threads() - set_num_threads(num_threads) # TODO: Figure out how to properly handle multiple type signatures in Cython and remove `.astype`-s out = self.src_interp1d( self.y, @@ -176,12 +162,9 @@ def __call__(self, x_new: np.ndarray) -> np.ndarray: 0.0 if extrapolate else self.fill_value, extrapolate, self.assume_sorted, - *args, + num_threads ) - if self.backend.name == 'Numba': - set_num_threads(old_num_threads) - out = out.astype(max(self.y.dtype, self.x.dtype, x_new.dtype, key=lambda x: x.type(0).itemsize), copy=False) if self.n_dummy: diff --git a/imops/src/_numba_zoom.py b/imops/src/_numba_zoom.py deleted file mode 100644 index e8de135..0000000 --- a/imops/src/_numba_zoom.py +++ /dev/null @@ -1,503 +0,0 @@ -from typing import Union - -import numpy as np -from numba import njit, prange - - -float_or_int = Union[float, int] - - -def _interp1d( - input: np.ndarray, - old_locations: np.ndarray, - new_locations: np.ndarray, - bounds_error: bool, - fill_value: float, - extrapolate: bool, - assume_sorted: bool, -) -> np.ndarray: - rows, cols, dims = input.shape[0], input.shape[1], len(new_locations) - contiguous_input = np.ascontiguousarray(input) - - dtype = input.dtype - interpolated = np.zeros((rows, cols, dims), dtype=dtype) - dd = np.zeros(dims) - - old_dims = len(old_locations) - sort_permutation = np.arange(old_dims) if assume_sorted else np.argsort(old_locations) - max_idxs = np.searchsorted(old_locations[sort_permutation], new_locations) - - extr = np.zeros(dims, dtype=np.int8) - - for k in prange(dims): - if max_idxs[k] == 0: - if new_locations[k] < old_locations[sort_permutation[max_idxs[k]]]: - extr[k] = -1 - else: - max_idxs[k] = 1 - - if max_idxs[k] >= old_dims: - extr[k] = 1 - - if extr[k] == 0: - dd[k] = (new_locations[k] - old_locations[sort_permutation[max_idxs[k] - 1]]) / ( - old_locations[sort_permutation[max_idxs[k]]] - old_locations[sort_permutation[max_idxs[k] - 1]] - ) - - if bounds_error and np.any(extr): - raise ValueError('A value in x_new is out of the interpolation range.') - - if np.any(extr) and extrapolate: - slope_left = np.zeros((rows, cols)) - slope_right = np.zeros((rows, cols)) - bias_left = np.zeros((rows, cols)) - bias_right = np.zeros((rows, cols)) - - slope_left = get_slope( - old_locations[sort_permutation[0]], - contiguous_input[..., sort_permutation[0]], - old_locations[sort_permutation[1]], - contiguous_input[..., sort_permutation[1]], - ) - slope_right = get_slope( - old_locations[sort_permutation[old_dims - 1]], - contiguous_input[..., sort_permutation[old_dims - 1]], - old_locations[sort_permutation[old_dims - 2]], - contiguous_input[..., sort_permutation[old_dims - 2]], - ) - - bias_left = contiguous_input[..., sort_permutation[0]] - slope_left * old_locations[sort_permutation[0]] - bias_right = ( - contiguous_input[..., sort_permutation[old_dims - 1]] - - slope_right * old_locations[sort_permutation[old_dims - 1]] - ) - - for i in prange(rows): - for j in prange(cols): - for k in prange(dims): - if extr[k] == 0: - interpolated[i, j, k] = ( - contiguous_input[i, j, sort_permutation[max_idxs[k] - 1]] * (1 - dd[k]) - + contiguous_input[i, j, sort_permutation[max_idxs[k]]] * dd[k] - ) - elif extrapolate: - if extr[k] == 1: - interpolated[i, j, k] = slope_right[i, j] * new_locations[k] + bias_right[i, j] - else: - interpolated[i, j, k] = slope_left[i, j] * new_locations[k] + bias_left[i, j] - else: - interpolated[i, j, k] = fill_value - - return interpolated - - -@njit(nogil=True) -def get_slope(x1: np.ndarray, y1: np.ndarray, x2: np.ndarray, y2: np.ndarray) -> np.ndarray: - return (y2 - y1) / (x2 - x1) - - -@njit(nogil=True) -def get_pixel3d( - input: np.ndarray, rows: int, cols: int, dims: int, r: int, c: int, d: int, cval: float_or_int -) -> float_or_int: - if 0 <= r < rows and 0 <= c < cols and 0 <= d < dims: - return input[r, c, d] - - return cval - - -@njit(nogil=True) -def get_pixel4d( - input: np.ndarray, - dim1: int, - dim2: int, - dim3: int, - dim4: int, - c1: int, - c2: int, - c3: int, - c4: int, - cval: float_or_int, -) -> float_or_int: - if 0 <= c1 < dim1 and 0 <= c2 < dim2 and 0 <= c3 < dim3 and 0 <= c4 < dim4: - return input[c1, c2, c3, c4] - - return cval - - -@njit(nogil=True) -def adjusted_coef(old_n: int, new_n: int) -> float: - if new_n == 1: - return old_n - return (np.float64(old_n) - 1) / (np.float64(new_n) - 1) - - -@njit(nogil=True) -def distance3d(x1: float, y1: float, z1: float, x2: float, y2: float, z2: float) -> float: - return ((x1 - x2) ** 2 + (y1 - y2) ** 2 + (z1 - z2) ** 2) ** 0.5 - - -@njit(nogil=True) -def distance4d(x1: float, y1: float, z1: float, d1: float, x2: float, y2: float, z2: float, d2: float) -> float: - return ((x1 - x2) ** 2 + (y1 - y2) ** 2 + (z1 - z2) ** 2 + (d1 - d2) ** 2) ** 0.5 - - -@njit(nogil=True) -def interpolate3d_linear( - input: np.ndarray, rows: int, cols: int, dims: int, r: float, c: float, d: float, cval: float -) -> float: - minr = int(r) - minc = int(c) - mind = int(d) - maxr = minr + 1 - maxc = minc + 1 - maxd = mind + 1 - - dr = r - minr - dc = c - minc - dd = d - mind - - c000 = get_pixel3d(input, rows, cols, dims, minr, minc, mind, cval) - c001 = get_pixel3d(input, rows, cols, dims, minr, minc, maxd, cval) - c010 = get_pixel3d(input, rows, cols, dims, minr, maxc, mind, cval) - c011 = get_pixel3d(input, rows, cols, dims, minr, maxc, maxd, cval) - c100 = get_pixel3d(input, rows, cols, dims, maxr, minc, mind, cval) - c101 = get_pixel3d(input, rows, cols, dims, maxr, minc, maxd, cval) - c110 = get_pixel3d(input, rows, cols, dims, maxr, maxc, mind, cval) - c111 = get_pixel3d(input, rows, cols, dims, maxr, maxc, maxd, cval) - - c00 = c000 * (1 - dr) + c100 * dr - c01 = c001 * (1 - dr) + c101 * dr - c10 = c010 * (1 - dr) + c110 * dr - c11 = c011 * (1 - dr) + c111 * dr - - c0 = c00 * (1 - dc) + c10 * dc - c1 = c01 * (1 - dc) + c11 * dc - - return c0 * (1 - dd) + c1 * dd - - -@njit(nogil=True) -def interpolate3d_nearest( - input: np.ndarray, rows: int, cols: int, dims: int, r: float, c: float, d: float, cval: float_or_int -) -> float_or_int: - min_distance = 3.0 - i_nearest, j_nearest, k_nearest = -1, -1, -1 - - minr = int(r) - minc = int(c) - mind = int(d) - maxr = minr + 1 - maxc = minc + 1 - maxd = mind + 1 - - for i in range(2): - curr = maxr if i else minr - if curr >= rows: - continue - for j in range(2): - curc = maxc if j else minc - if curc >= cols: - continue - for k in range(2): - curd = maxd if k else mind - if curd >= dims: - continue - - distance = distance3d(r, c, d, curr, curc, curd) - - if distance <= min_distance: - i_nearest = i - j_nearest = j - k_nearest = k - min_distance = distance - - if i_nearest == -1 or j_nearest == -1 or k_nearest == -1: - return cval - - return get_pixel3d( - input, - rows, - cols, - dims, - maxr if i_nearest else minr, - maxc if j_nearest else minc, - maxd if k_nearest else mind, - cval, - ) - - -@njit(nogil=True) -def interpolate4d_linear( - input: np.ndarray, - dim1: int, - dim2: int, - dim3: int, - dim4: int, - c1: int, - c2: int, - c3: int, - c4: int, - cval: float, -) -> float: - minc1 = int(c1) - minc2 = int(c2) - minc3 = int(c3) - minc4 = int(c4) - maxc1 = minc1 + 1 - maxc2 = minc2 + 1 - maxc3 = minc3 + 1 - maxc4 = minc4 + 1 - - dc1 = c1 - minc1 - dc2 = c2 - minc2 - dc3 = c3 - minc3 - dc4 = c4 - minc4 - - c0000 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, minc2, minc3, minc4, cval) - c0001 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, minc2, minc3, maxc4, cval) - c0010 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, minc2, maxc3, minc4, cval) - c0011 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, minc2, maxc3, maxc4, cval) - c0100 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, maxc2, minc3, minc4, cval) - c0101 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, maxc2, minc3, maxc4, cval) - c0110 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, maxc2, maxc3, minc4, cval) - c0111 = get_pixel4d(input, dim1, dim2, dim3, dim4, minc1, maxc2, maxc3, maxc4, cval) - c1000 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, minc2, minc3, minc4, cval) - c1001 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, minc2, minc3, maxc4, cval) - c1010 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, minc2, maxc3, minc4, cval) - c1011 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, minc2, maxc3, maxc4, cval) - c1100 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, maxc2, minc3, minc4, cval) - c1101 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, maxc2, minc3, maxc4, cval) - c1110 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, maxc2, maxc3, minc4, cval) - c1111 = get_pixel4d(input, dim1, dim2, dim3, dim4, maxc1, maxc2, maxc3, maxc4, cval) - - c000 = c0000 * (1 - dc1) + c1000 * dc1 - c001 = c0001 * (1 - dc1) + c1001 * dc1 - c010 = c0010 * (1 - dc1) + c1010 * dc1 - c011 = c0011 * (1 - dc1) + c1011 * dc1 - c100 = c0100 * (1 - dc1) + c1100 * dc1 - c101 = c0101 * (1 - dc1) + c1101 * dc1 - c110 = c0110 * (1 - dc1) + c1110 * dc1 - c111 = c0111 * (1 - dc1) + c1111 * dc1 - - c00 = c000 * (1 - dc2) + c100 * dc2 - c01 = c001 * (1 - dc2) + c101 * dc2 - c10 = c010 * (1 - dc2) + c110 * dc2 - c11 = c011 * (1 - dc2) + c111 * dc2 - - c0_ = c00 * (1 - dc3) + c10 * dc3 - c1_ = c01 * (1 - dc3) + c11 * dc3 - - return c0_ * (1 - dc4) + c1_ * dc4 - - -@njit(nogil=True) -def interpolate4d_nearest( - input: np.ndarray, - dim1: int, - dim2: int, - dim3: int, - dim4: int, - c1: float, - c2: float, - c3: float, - c4: float, - cval: float_or_int, -) -> float_or_int: - min_distance = 3.0 - i1_nearest, i2_nearest, i3_nearest, i4_nearest = -1, -1, -1, -1 - minc1 = int(c1) - minc2 = int(c2) - minc3 = int(c3) - minc4 = int(c4) - maxc1 = minc1 + 1 - maxc2 = minc2 + 1 - maxc3 = minc3 + 1 - maxc4 = minc4 + 1 - - for i1 in range(2): - curc1 = maxc1 if i1 else minc1 - if curc1 >= dim1: - continue - for i2 in range(2): - curc2 = maxc2 if i2 else minc2 - if curc2 >= dim2: - continue - for i3 in range(2): - curc3 = maxc3 if i3 else minc3 - if curc3 >= dim3: - continue - for i4 in range(2): - curc4 = maxc4 if i4 else minc4 - if curc4 >= dim4: - continue - - distance = distance4d(c1, c2, c3, c4, curc1, curc2, curc3, curc4) - if distance <= min_distance: - i1_nearest = i1 - i2_nearest = i2 - i3_nearest = i3 - i4_nearest = i4 - min_distance = distance - - if i1_nearest == -1 or i2_nearest == -1 or i3_nearest == -1 or i4_nearest == -1: - return cval - - return get_pixel4d( - input, - dim1, - dim2, - dim3, - dim4, - maxc1 if i1_nearest else minc1, - maxc2 if i2_nearest else minc2, - maxc3 if i3_nearest else minc3, - maxc4 if i4_nearest else minc4, - cval, - ) - - -def _zoom3d_linear(input: np.ndarray, zoom: np.ndarray, cval: float) -> np.ndarray: - contiguous_input = np.ascontiguousarray(input) - - old_rows, old_cols, old_dims = input.shape - row_coef, col_coef, dim_coef = zoom - - new_shape = (round(old_rows * row_coef), round(old_cols * col_coef), round(old_dims * dim_coef)) - new_rows, new_cols, new_dims = new_shape - - zoomed = np.zeros(new_shape, dtype=input.dtype) - - adjusted_row_coef = adjusted_coef(old_rows, new_rows) - adjusted_col_coef = adjusted_coef(old_cols, new_cols) - adjusted_dim_coef = adjusted_coef(old_dims, new_dims) - - for i in prange(new_rows): - for j in prange(new_cols): - for k in prange(new_dims): - zoomed[i, j, k] = interpolate3d_linear( - contiguous_input, - old_rows, - old_cols, - old_dims, - i * adjusted_row_coef, - j * adjusted_col_coef, - k * adjusted_dim_coef, - cval, - ) - - return zoomed - - -def _zoom3d_nearest(input: np.ndarray, zoom: np.ndarray, cval: float_or_int) -> np.ndarray: - contiguous_input = np.ascontiguousarray(input) - - old_rows, old_cols, old_dims = input.shape - row_coef, col_coef, dim_coef = zoom - - new_shape = (round(old_rows * row_coef), round(old_cols * col_coef), round(old_dims * dim_coef)) - new_rows, new_cols, new_dims = new_shape - - zoomed = np.zeros(new_shape, dtype=input.dtype) - - adjusted_row_coef = adjusted_coef(old_rows, new_rows) - adjusted_col_coef = adjusted_coef(old_cols, new_cols) - adjusted_dim_coef = adjusted_coef(old_dims, new_dims) - - for i in prange(new_rows): - for j in prange(new_cols): - for k in prange(new_dims): - zoomed[i, j, k] = interpolate3d_nearest( - contiguous_input, - old_rows, - old_cols, - old_dims, - i * adjusted_row_coef, - j * adjusted_col_coef, - k * adjusted_dim_coef, - cval, - ) - - return zoomed - - -def _zoom4d_linear(input: np.ndarray, zoom: np.ndarray, cval: float) -> np.ndarray: - contiguous_input = np.ascontiguousarray(input) - - old_dim1, old_dim2, old_dim3, old_dim4 = input.shape - dim1_coef, dim2_coef, dim3_coef, dim4_coef = zoom - - new_shape = ( - round(old_dim1 * dim1_coef), - round(old_dim2 * dim2_coef), - round(old_dim3 * dim3_coef), - round(old_dim4 * dim4_coef), - ) - new_dim1, new_dim2, new_dim3, new_dim4 = new_shape - - zoomed = np.zeros(new_shape, dtype=input.dtype) - - adjusted_dim1_coef = adjusted_coef(old_dim1, new_dim1) - adjusted_dim2_coef = adjusted_coef(old_dim2, new_dim2) - adjusted_dim3_coef = adjusted_coef(old_dim3, new_dim3) - adjusted_dim4_coef = adjusted_coef(old_dim4, new_dim4) - - for i1 in prange(new_dim1): - for i2 in prange(new_dim2): - for i3 in prange(new_dim3): - for i4 in prange(new_dim4): - zoomed[i1, i2, i3, i4] = interpolate4d_linear( - contiguous_input, - old_dim1, - old_dim2, - old_dim3, - old_dim4, - i1 * adjusted_dim1_coef, - i2 * adjusted_dim2_coef, - i3 * adjusted_dim3_coef, - i4 * adjusted_dim4_coef, - cval, - ) - - return zoomed - - -def _zoom4d_nearest(input: np.ndarray, zoom: np.ndarray, cval: float_or_int) -> np.ndarray: - contiguous_input = np.ascontiguousarray(input) - - old_dim1, old_dim2, old_dim3, old_dim4 = input.shape - dim1_coef, dim2_coef, dim3_coef, dim4_coef = zoom - - new_shape = ( - round(old_dim1 * dim1_coef), - round(old_dim2 * dim2_coef), - round(old_dim3 * dim3_coef), - round(old_dim4 * dim4_coef), - ) - new_dim1, new_dim2, new_dim3, new_dim4 = new_shape - - zoomed = np.zeros(new_shape, dtype=input.dtype) - - adjusted_dim1_coef = adjusted_coef(old_dim1, new_dim1) - adjusted_dim2_coef = adjusted_coef(old_dim2, new_dim2) - adjusted_dim3_coef = adjusted_coef(old_dim3, new_dim3) - adjusted_dim4_coef = adjusted_coef(old_dim4, new_dim4) - - for i1 in prange(new_dim1): - for i2 in prange(new_dim2): - for i3 in prange(new_dim3): - for i4 in prange(new_dim4): - zoomed[i1, i2, i3, i4] = interpolate4d_nearest( - contiguous_input, - old_dim1, - old_dim2, - old_dim3, - old_dim4, - i1 * adjusted_dim1_coef, - i2 * adjusted_dim2_coef, - i3 * adjusted_dim3_coef, - i4 * adjusted_dim4_coef, - cval, - ) - - return zoomed diff --git a/imops/utils.py b/imops/utils.py index 4b1bd97..7219598 100644 --- a/imops/utils.py +++ b/imops/utils.py @@ -59,15 +59,6 @@ def normalize_num_threads(num_threads: int, backend: Backend, warn_stacklevel: i max_num_threads = min(filter(bool, [IMOPS_NUM_THREADS, env_num_threads, num_available_cpus])) if num_threads >= 0: - # FIXME - if backend.name == 'Numba': - warn( - 'Setting `num_threads` has no effect with "Numba" backend. ' - 'Use `NUMBA_NUM_THREADS` environment variable.', - stacklevel=warn_stacklevel, - ) - return num_threads - if num_threads > max_num_threads: if max_num_threads == IMOPS_NUM_THREADS: warn( diff --git a/imops/zoom.py b/imops/zoom.py index b5a0bde..932d87e 100644 --- a/imops/zoom.py +++ b/imops/zoom.py @@ -52,23 +52,6 @@ def _choose_cython_zoom(ndim: int, order: int, fast: bool) -> Callable: return cython_fast_zoom4d_linear if fast else cython_zoom4d_linear -def _choose_numba_zoom(ndim: int, order: int) -> Callable: - assert ndim <= 4, ndim - assert order in (0, 1), order - - if ndim <= 3: - if order == 0: - from .src._numba_zoom import _zoom3d_nearest as numba_zoom - else: - from .src._numba_zoom import _zoom3d_linear as numba_zoom - elif order == 0: - from .src._numba_zoom import _zoom4d_nearest as numba_zoom - else: - from .src._numba_zoom import _zoom4d_linear as numba_zoom - - return numba_zoom - - def zoom( x: np.ndarray, scale_factor: AxesParams, @@ -100,7 +83,7 @@ def zoom( the number of threads to use for computation. Default = the cpu count. If negative value passed cpu count + num_threads + 1 threads will be used backend: BackendLike - which backend to use. `numba`, `cython` and `scipy` are available, `cython` is used by default + which backend to use. `cython` and `scipy` are available, `cython` is used by default Returns ------- @@ -157,7 +140,7 @@ def zoom_to_shape( the number of threads to use for computation. Default = the cpu count. If negative value passed cpu count + num_threads + 1 threads will be used backend: BackendLike - which backend to use. `numba`, `cython` and `scipy` are available, `cython` is used by default + which backend to use. `cython` and `scipy` are available, `cython` is used by default Returns ------- @@ -207,13 +190,13 @@ def _zoom( Works faster only for ndim <= 4. Shares interface with `scipy.ndimage.zoom` except for - `num_threads` argument defining how many threads to use (all available threads are used by default). - - `backend` argument defining which backend to use. `numba`, `cython` and `scipy` are available, + - `backend` argument defining which backend to use. `cython` and `scipy` are available, `cython` is used by default. See `https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.zoom.html` """ backend = resolve_backend(backend, warn_stacklevel=4) - if backend.name not in ('Scipy', 'Numba', 'Cython'): + if backend.name not in ('Scipy', 'Cython'): raise ValueError(f'Unsupported backend "{backend.name}".') ndim = image.ndim @@ -252,15 +235,6 @@ def _zoom( if backend.name == 'Cython': src_zoom = _choose_cython_zoom(ndim, order, backend.fast) - if backend.name == 'Numba': - from numba import get_num_threads, njit, set_num_threads - - old_num_threads = get_num_threads() - set_num_threads(num_threads) - - njit_kwargs = {kwarg: getattr(backend, kwarg) for kwarg in backend.__dataclass_fields__.keys()} - src_zoom = njit(**njit_kwargs)(_choose_numba_zoom(ndim, order)) - n_dummy = 3 - ndim if ndim <= 3 else 0 if n_dummy: @@ -270,7 +244,6 @@ def _zoom( zoom = np.array(zoom, dtype=np.float64) is_contiguous = image.data.c_contiguous c_contiguous_permutaion = None - args = () if backend.name in ('Numba',) else (num_threads,) if not is_contiguous: c_contiguous_permutaion = get_c_contiguous_permutaion(image) @@ -279,19 +252,17 @@ def _zoom( np.transpose(image, c_contiguous_permutaion), zoom[c_contiguous_permutaion], cval, - *args, + num_threads ) else: warn("Input array can't be represented as C-contiguous, performance can drop a lot.", stacklevel=3) - out = src_zoom(image, zoom, cval, *args) + out = src_zoom(image, zoom, cval, num_threads) else: - out = src_zoom(image, zoom, cval, *args) + out = src_zoom(image, zoom, cval, num_threads) if c_contiguous_permutaion is not None: out = np.transpose(out, inverse_permutation(c_contiguous_permutaion)) if n_dummy: out = out[(0,) * n_dummy] - if backend.name == 'Numba': - set_num_threads(old_num_threads) return out diff --git a/pyproject.toml b/pyproject.toml index a1da3ac..2682bb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,6 @@ classifiers = [ [options] include_package_data = true -[project.optional-dependencies] -numba = ['numba'] - [project.urls] 'Homepage' = 'https://github.com/neuro-ml/imops' 'Issues' = 'https://github.com/neuro-ml/imops/issues' @@ -44,9 +41,6 @@ numba = ['numba'] line-length = 120 skip-string-normalization = true -[tool.pytest.ini_options] -markers = ['nonumba'] - [tool.isort] line_length = 120 lines_after_imports = 2 diff --git a/setup.py b/setup.py index 3adc876..d919370 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,6 @@ ], classifiers=classifiers, install_requires=requirements, - extras_require={'numba': ['numba'], 'all': ['numba']}, setup_requires=[ 'setuptools<69.0.0', 'numpy<3.0.0', diff --git a/tests/requirements.txt b/tests/requirements.txt index 84edb3d..d751415 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,5 +3,4 @@ pytest-subtests pytest-cov pytest-xdist scikit-image -numba deli diff --git a/tests/test_backend.py b/tests/test_backend.py index b2a6f30..c655425 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -2,13 +2,13 @@ import pytest -from imops.backend import Backend, Cython, Numba, Scipy, imops_backend, resolve_backend, set_backend +from imops.backend import Backend, Cython, Scipy, imops_backend, resolve_backend, set_backend def test_resolve(): assert resolve_backend(None) == Cython() - for cls in [Numba, Cython, Scipy]: + for cls in [Cython, Scipy]: assert resolve_backend(cls) == cls() assert resolve_backend(cls.__name__) == cls() @@ -18,8 +18,6 @@ def test_resolve(): def test_backend_change(): assert resolve_backend(None) == Cython() - set_backend('Numba') - assert resolve_backend(None) == Numba() set_backend(Cython(fast=True)) assert resolve_backend(None) == Cython(fast=True) set_backend('Cython') @@ -32,11 +30,6 @@ def test_backend_change(): def test_existing_backend(): - with pytest.raises(ValueError): - - class Numba(Backend): - pass - with pytest.raises(ValueError): class Cython(Backend): @@ -48,12 +41,6 @@ class Scipy(Backend): pass -@pytest.mark.nonumba -def test_error_without_numba(): - with pytest.raises(ModuleNotFoundError): - Numba() - - # TODO: come up with more comprehensive tests to check that imops doesn't affect global FPU state def test_not_affecting_subnormals(): import imops # noqa: F401 diff --git a/tests/test_interp1d.py b/tests/test_interp1d.py index 30438a1..7289d95 100644 --- a/tests/test_interp1d.py +++ b/tests/test_interp1d.py @@ -57,14 +57,6 @@ def test_extrapolate_error(backend): interp1d(x, y, fill_value='extrapolate', bounds_error=True, backend=backend) -def test_numba_num_threads(): - x = np.array([1.0, 2.0, 3.0]) - y = np.array([1.0, 2.0, 3.0]) - - with pytest.warns(UserWarning): - interp1d(x, y, axis=0, fill_value=0, num_threads=2, backend='Numba')(x) - - def test_extrapolation_exception(backend): x = np.array([1.0, 2.0, 3.0]) x_new = np.array([0.0, 1.0, 2.0]) diff --git a/tests/test_zoom.py b/tests/test_zoom.py index 83cb155..3d7a936 100644 --- a/tests/test_zoom.py +++ b/tests/test_zoom.py @@ -59,13 +59,6 @@ def test_single_threaded_warning(order): zoom(inp, 2, order=order, num_threads=2, backend='Scipy') -def test_numba_num_threads(order): - inp = np.random.randn(32, 32, 32) - - with pytest.warns(UserWarning): - zoom(inp, 2, order=order, num_threads=2, backend='Numba') - - def test_callable_fill_value(backend, order): inp = np.random.randn(64, 64, 64) scale = np.random.uniform(0.5, 1.5, size=inp.ndim) From d3a97603bef13c4c808fecaaed47404024b3a89d Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Sat, 13 Dec 2025 05:28:47 -0800 Subject: [PATCH 12/15] lint --- benchmarks/benchmark_interp1d.py | 2 -- benchmarks/benchmark_zoom.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/benchmarks/benchmark_interp1d.py b/benchmarks/benchmark_interp1d.py index 759aeee..b360378 100644 --- a/benchmarks/benchmark_interp1d.py +++ b/benchmarks/benchmark_interp1d.py @@ -1,5 +1,3 @@ -from itertools import product - import numpy as np diff --git a/benchmarks/benchmark_zoom.py b/benchmarks/benchmark_zoom.py index 4813d1b..907bcd6 100644 --- a/benchmarks/benchmark_zoom.py +++ b/benchmarks/benchmark_zoom.py @@ -1,5 +1,3 @@ -from itertools import product - import numpy as np From 18c2ee6b35b6182e60a0f99f2e003ca2c9ea1ddf Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Sun, 14 Dec 2025 07:49:30 -0800 Subject: [PATCH 13/15] linters --- imops/interp1d.py | 2 +- imops/zoom.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/imops/interp1d.py b/imops/interp1d.py index 23417c4..b8183ff 100644 --- a/imops/interp1d.py +++ b/imops/interp1d.py @@ -162,7 +162,7 @@ def __call__(self, x_new: np.ndarray) -> np.ndarray: 0.0 if extrapolate else self.fill_value, extrapolate, self.assume_sorted, - num_threads + num_threads, ) out = out.astype(max(self.y.dtype, self.x.dtype, x_new.dtype, key=lambda x: x.type(0).itemsize), copy=False) diff --git a/imops/zoom.py b/imops/zoom.py index 932d87e..19197ff 100644 --- a/imops/zoom.py +++ b/imops/zoom.py @@ -249,10 +249,7 @@ def _zoom( c_contiguous_permutaion = get_c_contiguous_permutaion(image) if c_contiguous_permutaion is not None: out = src_zoom( - np.transpose(image, c_contiguous_permutaion), - zoom[c_contiguous_permutaion], - cval, - num_threads + np.transpose(image, c_contiguous_permutaion), zoom[c_contiguous_permutaion], cval, num_threads ) else: warn("Input array can't be represented as C-contiguous, performance can drop a lot.", stacklevel=3) From 853f83e26fc8ff0d338bdf0bb585daee2e473919 Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Sun, 14 Dec 2025 07:50:45 -0800 Subject: [PATCH 14/15] macOS-13 -> macOS-15 --- .github/workflows/build-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 4a3fcf0..43256fb 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -18,7 +18,7 @@ jobs: build_wheels: strategy: matrix: - os: [ ubuntu-22.04, windows-2022, macOS-13 ] + os: [ ubuntu-22.04, windows-2022, macOS-15 ] name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: @@ -33,7 +33,7 @@ jobs: run: python -m pip install cibuildwheel==2.17.0 - name: Install llvm for mac - if: ${{ matrix.os == 'macOS-13' }} + if: ${{ matrix.os == 'macOS-15' }} run: | brew install llvm From 70ed46c93ad15752ee35bd821d17e82ce0dff7ca Mon Sep 17 00:00:00 2001 From: AnihilatorGun Date: Sun, 14 Dec 2025 18:09:43 +0100 Subject: [PATCH 15/15] version --- imops/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imops/__version__.py b/imops/__version__.py index 4033697..9d1bb72 100644 --- a/imops/__version__.py +++ b/imops/__version__.py @@ -1 +1 @@ -__version__ = '0.9.4' +__version__ = '0.10.0'