From 13fb2badf30259c0dfaa443ab6824e4faacd9615 Mon Sep 17 00:00:00 2001 From: MugundanMCW Date: Fri, 12 Dec 2025 14:46:31 +0530 Subject: [PATCH 1/4] Add support for building PyArrow library on Windows ARM64 --- .../python_wheel_windows_arm64_build.bat | 142 ++++++++++++++++++ ci/vcpkg/vcpkg.json | 8 +- .../python-wheels/github.windows.arm64.yml | 72 +++++++++ dev/tasks/tasks.yml | 10 ++ 4 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 ci/scripts/python_wheel_windows_arm64_build.bat create mode 100644 dev/tasks/python-wheels/github.windows.arm64.yml diff --git a/ci/scripts/python_wheel_windows_arm64_build.bat b/ci/scripts/python_wheel_windows_arm64_build.bat new file mode 100644 index 00000000000..875b52fccf5 --- /dev/null +++ b/ci/scripts/python_wheel_windows_arm64_build.bat @@ -0,0 +1,142 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +echo "Building windows wheel..." + +@REM List installed Pythons +py -0p + +%PYTHON_CMD% -m sysconfig || exit /B 1 + +call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat" +@echo on + +echo "=== (%PYTHON%) Building Arrow C++ libraries ===" +set ARROW_SRC=%GITHUB_WORKSPACE%\arrow +set ARROW_DIST=%GITHUB_WORKSPACE%\arrow-dist +set ARROW_ACERO=ON +set ARROW_DATASET=ON +set ARROW_FLIGHT=ON +set ARROW_GANDIVA=OFF +set ARROW_GCS=ON +set ARROW_HDFS=ON +set ARROW_ORC=ON +set ARROW_PARQUET=ON +set PARQUET_REQUIRE_ENCRYPTION=ON +set ARROW_MIMALLOC=ON +set ARROW_SUBSTRAIT=OFF +set ARROW_S3=ON +set ARROW_TENSORFLOW=OFF +set CMAKE_GENERATOR=Visual Studio 17 2022 +set CMAKE_PLATFORM=ARM64 +set VCPKG_ROOT=C:\vcpkg +set VCPKG_FEATURE_FLAGS=-manifests +set VCPKG_TARGET_TRIPLET=arm64-windows + +mkdir C:\arrow-build +pushd C:\arrow-build +cmake ^ + -G "Visual Studio 17 2022" -A ARM64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_DIST% ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_DEPENDENCY_USE_SHARED=ON ^ + -DARROW_PROTOBUF_USE_SHARED=ON ^ + -Dutf8proc_SOURCE=BUNDLED ^ + -DARROW_SIMD_LEVEL=NONE ^ + -DARROW_RUNTIME_SIMD_LEVEL=NONE ^ + -DARROW_USE_XSIMD=OFF ^ + -DARROW_WITH_UTF8PROC=OFF ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=OFF ^ + -DARROW_ACERO=%ARROW_ACERO% ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=%ARROW_DATASET% ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_FLIGHT=%ARROW_FLIGHT% ^ + -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ + -DARROW_HDFS=%ARROW_HDFS% ^ + -DARROW_JSON=ON ^ + -DVCPKG_MANIFEST_MODE=OFF ^ + -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ + -DARROW_ORC=%ARROW_ORC% ^ + -DARROW_PACKAGE_KIND="python-wheel-windows" ^ + -DARROW_PARQUET=%ARROW_PARQUET% ^ + -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ + -DARROW_S3=%ARROW_S3% ^ + -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ + -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + "%ARROW_SRC%\cpp" || exit /B 1 +cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 +popd + +echo "=== (%PYTHON%) Building wheel ===" +set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% +set CMAKE_GENERATOR=Visual Studio 17 2022 +set CMAKE_PLATFORM=ARM64 +set PYARROW_BUILD_VERBOSE=1 +set PYARROW_BUNDLE_ARROW_CPP=ON +set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% +set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%" +set PYARROW_WITH_ACERO=%ARROW_ACERO% +set PYARROW_WITH_DATASET=%ARROW_DATASET% +set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% +set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% +set PYARROW_WITH_GCS=%ARROW_GCS% +set PYARROW_WITH_HDFS=%ARROW_HDFS% +set PYARROW_WITH_ORC=%ARROW_ORC% +set PYARROW_WITH_PARQUET=%ARROW_PARQUET% +set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% +set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% +set PYARROW_WITH_S3=%ARROW_S3% +set Arrow_DIR=%ARROW_DIST%\lib\cmake\arrow +set CMAKE_PREFIX_PATH=%ARROW_DIST% +set ARROW_HOME=%ARROW_DIST% + +pushd %ARROW_SRC%\python + +@REM Install Python dependencies +echo "=== Installing Python dependencies ===" +%PYTHON_CMD% -m pip install --upgrade pip || exit /B 1 +%PYTHON_CMD% -m pip install cython numpy setuptools_scm setuptools || exit /B 1 + +@REM Build wheel +%PYTHON_CMD% setup.py bdist_wheel || exit /B 1 + +@REM Repair the wheel with delvewheel +%PYTHON_CMD% -m pip install delvewheel || exit /B 1 + +for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 +echo "Wheel name: %WHEEL_NAME%" + +%PYTHON_CMD% -m delvewheel repair -vv --add-path "%ARROW_DIST%\bin" --add-path "C:\vcpkg\installed\arm64-windows\bin" --ignore-existing --with-mangle -w repaired_wheels %WHEEL_NAME% || exit /B 1 + +popd + diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index 5dfe61a0c60..ed6a59f1d8d 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -3,7 +3,7 @@ "version-string": "", "description": "Cross-language development platform for in-memory analytics", "homepage": "https://arrow.apache.org", - "supports": "x64 | (arm64 & !windows)", + "supports": "x64 | arm64", "dependencies": [ { "name": "boost-multiprecision", @@ -19,7 +19,10 @@ "re2", "snappy", "utf8proc", - "xsimd", + { + "name": "xsimd", + "platform": "!(arm64 & windows)" + }, "zlib", "zstd", { @@ -143,3 +146,4 @@ } } } + diff --git a/dev/tasks/python-wheels/github.windows.arm64.yml b/dev/tasks/python-wheels/github.windows.arm64.yml new file mode 100644 index 00000000000..065d6bba513 --- /dev/null +++ b/dev/tasks/python-wheels/github.windows.arm64.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Windows ARM64" + runs-on: "{{ runs_on }}" + timeout-minutes: 240 + env: + PYTHON_CMD: "py -{{ python_version }}" + CMAKE_BUILD_TYPE: release + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "{{ python_version }}" + architecture: "ARM64" + + - name: Cache vcpkg dependencies + uses: actions/cache@v4 + with: + path: C:\vcpkg\installed + key: "{% raw %}vcpkg-installed-windows-arm64-${{ hashFiles('arrow/ci/vcpkg/vcpkg.json', 'arrow/ci/vcpkg/arm64-windows-*.cmake') }}{% endraw %}" + restore-keys: vcpkg-installed-windows-arm64- + + - name: Install vcpkg dependencies + shell: cmd + run: | + cd arrow\ci\vcpkg + C:\vcpkg\vcpkg.exe install ^ + --binarysource="clear;x-gha,readwrite" ^ + --x-install-root=C:\vcpkg\installed ^ + --x-manifest-root=. ^ + --x-feature=flight ^ + --x-feature=json ^ + --x-feature=gcs ^ + --x-feature=orc ^ + --x-feature=parquet ^ + --x-feature=s3 + - name: Build Arrow C++ and PyArrow wheel + shell: cmd + env: + SETUPTOOLS_SCM_PRETEND_VERSION: "{{ arrow.no_rc_version }}" + run: | + call "arrow\ci\scripts\python_wheel_windows_arm64_build.bat" + + - uses: actions/upload-artifact@v4 + with: + name: wheel + path: arrow/python/repaired_wheels/*.whl + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} + diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e2bd5c6d8e5..c4dc9a5845d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -249,6 +249,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl + wheel-windows-{{ python_tag }}-{{ abi_tag }}-arm64: + ci: github + template: python-wheels/github.windows.arm64.yml + params: + python_version: "{{ python_version }}" + python_abi_tag: "{{ abi_tag }}" + runs_on: "windows-11-arm" + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_arm64.whl + {% endfor %} {############################ Python sdist ####################################} From bc3020dd9f17c792a84b9e949c04e9e7e21497ca Mon Sep 17 00:00:00 2001 From: MUGUNDAN Date: Tue, 16 Dec 2025 13:53:33 +0530 Subject: [PATCH 2/4] Fix indentation for Windows ARM64 Job --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c4dc9a5845d..4abb99e11d2 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -249,7 +249,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl - wheel-windows-{{ python_tag }}-{{ abi_tag }}-arm64: + wheel-windows-{{ python_tag }}-{{ abi_tag }}-arm64: ci: github template: python-wheels/github.windows.arm64.yml params: From 495e26af76eb797e5ed68370fb69dcd8836812af Mon Sep 17 00:00:00 2001 From: MUGUNDAN Date: Tue, 16 Dec 2025 14:05:22 +0530 Subject: [PATCH 3/4] Fix architecture for Python setup actions --- dev/tasks/python-wheels/github.windows.arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/github.windows.arm64.yml b/dev/tasks/python-wheels/github.windows.arm64.yml index 065d6bba513..4ac41b1e467 100644 --- a/dev/tasks/python-wheels/github.windows.arm64.yml +++ b/dev/tasks/python-wheels/github.windows.arm64.yml @@ -33,7 +33,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "{{ python_version }}" - architecture: "ARM64" + architecture: "arm64" - name: Cache vcpkg dependencies uses: actions/cache@v4 From 28731a74692e6067445af211db34375a328b0b12 Mon Sep 17 00:00:00 2001 From: MugundanMCW Date: Tue, 23 Dec 2025 09:53:59 +0530 Subject: [PATCH 4/4] Add support for building PyArrow library on Windows ARM64 --- .../python_wheel_windows_arm64_build.bat | 142 ----------- ci/scripts/python_wheel_windows_build.bat | 226 ++++++++++++------ .../python-wheels/github.windows.arm64.yml | 5 +- 3 files changed, 160 insertions(+), 213 deletions(-) delete mode 100644 ci/scripts/python_wheel_windows_arm64_build.bat diff --git a/ci/scripts/python_wheel_windows_arm64_build.bat b/ci/scripts/python_wheel_windows_arm64_build.bat deleted file mode 100644 index 875b52fccf5..00000000000 --- a/ci/scripts/python_wheel_windows_arm64_build.bat +++ /dev/null @@ -1,142 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -echo "Building windows wheel..." - -@REM List installed Pythons -py -0p - -%PYTHON_CMD% -m sysconfig || exit /B 1 - -call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat" -@echo on - -echo "=== (%PYTHON%) Building Arrow C++ libraries ===" -set ARROW_SRC=%GITHUB_WORKSPACE%\arrow -set ARROW_DIST=%GITHUB_WORKSPACE%\arrow-dist -set ARROW_ACERO=ON -set ARROW_DATASET=ON -set ARROW_FLIGHT=ON -set ARROW_GANDIVA=OFF -set ARROW_GCS=ON -set ARROW_HDFS=ON -set ARROW_ORC=ON -set ARROW_PARQUET=ON -set PARQUET_REQUIRE_ENCRYPTION=ON -set ARROW_MIMALLOC=ON -set ARROW_SUBSTRAIT=OFF -set ARROW_S3=ON -set ARROW_TENSORFLOW=OFF -set CMAKE_GENERATOR=Visual Studio 17 2022 -set CMAKE_PLATFORM=ARM64 -set VCPKG_ROOT=C:\vcpkg -set VCPKG_FEATURE_FLAGS=-manifests -set VCPKG_TARGET_TRIPLET=arm64-windows - -mkdir C:\arrow-build -pushd C:\arrow-build -cmake ^ - -G "Visual Studio 17 2022" -A ARM64 ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake ^ - -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_DIST% ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_DEPENDENCY_USE_SHARED=ON ^ - -DARROW_PROTOBUF_USE_SHARED=ON ^ - -Dutf8proc_SOURCE=BUNDLED ^ - -DARROW_SIMD_LEVEL=NONE ^ - -DARROW_RUNTIME_SIMD_LEVEL=NONE ^ - -DARROW_USE_XSIMD=OFF ^ - -DARROW_WITH_UTF8PROC=OFF ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=OFF ^ - -DARROW_ACERO=%ARROW_ACERO% ^ - -DARROW_COMPUTE=ON ^ - -DARROW_CSV=ON ^ - -DARROW_DATASET=%ARROW_DATASET% ^ - -DARROW_FILESYSTEM=ON ^ - -DARROW_FLIGHT=%ARROW_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_GANDIVA% ^ - -DARROW_GCS=%ARROW_GCS% ^ - -DARROW_HDFS=%ARROW_HDFS% ^ - -DARROW_JSON=ON ^ - -DVCPKG_MANIFEST_MODE=OFF ^ - -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ - -DARROW_ORC=%ARROW_ORC% ^ - -DARROW_PACKAGE_KIND="python-wheel-windows" ^ - -DARROW_PARQUET=%ARROW_PARQUET% ^ - -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ - -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ - -DARROW_WITH_BROTLI=ON ^ - -DARROW_WITH_BZ2=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - "%ARROW_SRC%\cpp" || exit /B 1 -cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 -popd - -echo "=== (%PYTHON%) Building wheel ===" -set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% -set CMAKE_GENERATOR=Visual Studio 17 2022 -set CMAKE_PLATFORM=ARM64 -set PYARROW_BUILD_VERBOSE=1 -set PYARROW_BUNDLE_ARROW_CPP=ON -set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% -set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%" -set PYARROW_WITH_ACERO=%ARROW_ACERO% -set PYARROW_WITH_DATASET=%ARROW_DATASET% -set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% -set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% -set PYARROW_WITH_GCS=%ARROW_GCS% -set PYARROW_WITH_HDFS=%ARROW_HDFS% -set PYARROW_WITH_ORC=%ARROW_ORC% -set PYARROW_WITH_PARQUET=%ARROW_PARQUET% -set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% -set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% -set PYARROW_WITH_S3=%ARROW_S3% -set Arrow_DIR=%ARROW_DIST%\lib\cmake\arrow -set CMAKE_PREFIX_PATH=%ARROW_DIST% -set ARROW_HOME=%ARROW_DIST% - -pushd %ARROW_SRC%\python - -@REM Install Python dependencies -echo "=== Installing Python dependencies ===" -%PYTHON_CMD% -m pip install --upgrade pip || exit /B 1 -%PYTHON_CMD% -m pip install cython numpy setuptools_scm setuptools || exit /B 1 - -@REM Build wheel -%PYTHON_CMD% setup.py bdist_wheel || exit /B 1 - -@REM Repair the wheel with delvewheel -%PYTHON_CMD% -m pip install delvewheel || exit /B 1 - -for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 -echo "Wheel name: %WHEEL_NAME%" - -%PYTHON_CMD% -m delvewheel repair -vv --add-path "%ARROW_DIST%\bin" --add-path "C:\vcpkg\installed\arm64-windows\bin" --ignore-existing --with-mangle -w repaired_wheels %WHEEL_NAME% || exit /B 1 - -popd - diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index b4b7fed99fd..cf976e346b8 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -24,16 +24,35 @@ py -0p %PYTHON_CMD% -m sysconfig || exit /B 1 -call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" +@REM Detect architecture if not set +if "%arch%"=="" set arch=x64 + +@REM Set architecture-specific options +if "%arch%"=="ARM64" ( + set CMAKE_PLATFORM=ARM64 + set VCVARS_BAT=C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat + set VCPKG_TARGET_TRIPLET=arm64-windows + set ARROW_SRC=%GITHUB_WORKSPACE%\arrow + set ARROW_DIST=%GITHUB_WORKSPACE%\arrow-dist +) else ( + set VCVARS_BAT=C:\Program Files ^(x86^)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat + set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% + set ARROW_SRC=C:\arrow + set ARROW_DIST=C:\arrow-dist +) + +call "%VCVARS_BAT%" @echo on -echo "=== (%PYTHON%) Clear output directories and leftovers ===" -del /s /q C:\arrow-build -del /s /q C:\arrow-dist -del /s /q C:\arrow\python\dist -del /s /q C:\arrow\python\build -del /s /q C:\arrow\python\pyarrow\*.so -del /s /q C:\arrow\python\pyarrow\*.so.* +if "%arch%"=="x64" ( + echo "=== (%PYTHON%) Clear output directories and leftovers ===" + del /s /q C:\arrow-build + del /s /q C:\arrow-dist + del /s /q C:\arrow\python\dist + del /s /q C:\arrow\python\build + del /s /q C:\arrow\python\pyarrow\*.so + del /s /q C:\arrow\python\pyarrow\*.so.* +) echo "=== (%PYTHON%) Building Arrow C++ libraries ===" set ARROW_ACERO=ON @@ -46,67 +65,119 @@ set ARROW_ORC=ON set ARROW_PARQUET=ON set PARQUET_REQUIRE_ENCRYPTION=ON set ARROW_MIMALLOC=ON -set ARROW_SUBSTRAIT=ON set ARROW_S3=ON -set ARROW_TENSORFLOW=ON -set ARROW_WITH_BROTLI=ON -set ARROW_WITH_BZ2=ON -set ARROW_WITH_LZ4=ON -set ARROW_WITH_SNAPPY=ON -set ARROW_WITH_ZLIB=ON -set ARROW_WITH_ZSTD=ON -set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 -set CMAKE_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests -set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% + +if "%arch%"=="ARM64" ( + set ARROW_SUBSTRAIT=OFF + set ARROW_TENSORFLOW=OFF +) else ( + set ARROW_SUBSTRAIT=ON + set ARROW_TENSORFLOW=ON + set ARROW_WITH_BROTLI=ON + set ARROW_WITH_BZ2=ON + set ARROW_WITH_LZ4=ON + set ARROW_WITH_SNAPPY=ON + set ARROW_WITH_ZLIB=ON + set ARROW_WITH_ZSTD=ON + set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON + set CMAKE_UNITY_BUILD=ON +) mkdir C:\arrow-build pushd C:\arrow-build -cmake ^ - -DARROW_ACERO=%ARROW_ACERO% ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=OFF ^ - -DARROW_COMPUTE=ON ^ - -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=%ARROW_DATASET% ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_DEPENDENCY_USE_SHARED=OFF ^ - -DARROW_FILESYSTEM=ON ^ - -DARROW_FLIGHT=%ARROW_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_GANDIVA% ^ - -DARROW_GCS=%ARROW_GCS% ^ - -DARROW_HDFS=%ARROW_HDFS% ^ - -DARROW_JSON=ON ^ - -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ - -DARROW_ORC=%ARROW_ORC% ^ - -DARROW_PACKAGE_KIND="python-wheel-windows" ^ - -DARROW_PARQUET=%ARROW_PARQUET% ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ - -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ - -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ - -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ - -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ - -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ - -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ - -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ - -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION% ^ - -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ - -DMSVC_LINK_VERBOSE=ON ^ - -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ - -DVCPKG_MANIFEST_MODE=OFF ^ - -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ - -Dxsimd_SOURCE=BUNDLED ^ - -G "%CMAKE_GENERATOR%" ^ - -A "%CMAKE_PLATFORM%" ^ - C:\arrow\cpp || exit /B 1 + +if "%arch%"=="ARM64" ( + cmake ^ + -G "Visual Studio 17 2022" -A ARM64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_DIST% ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_DEPENDENCY_USE_SHARED=ON ^ + -DARROW_PROTOBUF_USE_SHARED=ON ^ + -Dutf8proc_SOURCE=BUNDLED ^ + -DARROW_SIMD_LEVEL=NONE ^ + -DARROW_RUNTIME_SIMD_LEVEL=NONE ^ + -DARROW_USE_XSIMD=OFF ^ + -DARROW_WITH_UTF8PROC=OFF ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=OFF ^ + -DARROW_ACERO=%ARROW_ACERO% ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=%ARROW_DATASET% ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_FLIGHT=%ARROW_FLIGHT% ^ + -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ + -DARROW_HDFS=%ARROW_HDFS% ^ + -DARROW_JSON=ON ^ + -DVCPKG_MANIFEST_MODE=OFF ^ + -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ + -DARROW_ORC=%ARROW_ORC% ^ + -DARROW_PACKAGE_KIND="python-wheel-windows" ^ + -DARROW_PARQUET=%ARROW_PARQUET% ^ + -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ + -DARROW_S3=%ARROW_S3% ^ + -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ + -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + "%ARROW_SRC%\cpp" || exit /B 1 +) else ( + cmake ^ + -DARROW_ACERO=%ARROW_ACERO% ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=OFF ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_CXXFLAGS="/MP" ^ + -DARROW_DATASET=%ARROW_DATASET% ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_DEPENDENCY_USE_SHARED=OFF ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_FLIGHT=%ARROW_FLIGHT% ^ + -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ + -DARROW_HDFS=%ARROW_HDFS% ^ + -DARROW_JSON=ON ^ + -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ + -DARROW_ORC=%ARROW_ORC% ^ + -DARROW_PACKAGE_KIND="python-wheel-windows" ^ + -DARROW_PARQUET=%ARROW_PARQUET% ^ + -DARROW_S3=%ARROW_S3% ^ + -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ + -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ + -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ + -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ + -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ + -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ + -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ + -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ + -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ + -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION% ^ + -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ + -DMSVC_LINK_VERBOSE=ON ^ + -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ + -DVCPKG_MANIFEST_MODE=OFF ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ + -Dxsimd_SOURCE=BUNDLED ^ + -G "%CMAKE_GENERATOR%" ^ + -A "%CMAKE_PLATFORM%" ^ + C:\arrow\cpp || exit /B 1 +) + cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 popd @@ -127,10 +198,21 @@ set PYARROW_WITH_PARQUET=%ARROW_PARQUET% set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% set PYARROW_WITH_S3=%ARROW_S3% -set ARROW_HOME=C:\arrow-dist -set CMAKE_PREFIX_PATH=C:\arrow-dist +set ARROW_HOME=%ARROW_DIST% +set CMAKE_PREFIX_PATH=%ARROW_DIST% + +if "%arch%"=="ARM64" ( + set Arrow_DIR=%ARROW_DIST%\lib\cmake\arrow +) -pushd C:\arrow\python +pushd %ARROW_SRC%\python + +if "%arch%"=="ARM64" ( + @REM Install Python dependencies for Win-ARM64 + echo "=== Installing Python dependencies ===" + %PYTHON_CMD% -m pip install --upgrade pip || exit /B 1 + %PYTHON_CMD% -m pip install cython numpy setuptools_scm setuptools wheel || exit /B 1 +) @REM Build wheel %PYTHON_CMD% setup.py bdist_wheel || exit /B 1 @@ -145,8 +227,14 @@ pushd C:\arrow\python for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 echo "Wheel name: %WHEEL_NAME%" -%PYTHON_CMD% -m delvewheel repair -vv ^ - --ignore-existing --with-mangle ^ - -w repaired_wheels %WHEEL_NAME% || exit /B 1 +if "%arch%"=="ARM64" ( + %PYTHON_CMD% -m delvewheel repair -vv --add-path "%ARROW_DIST%\bin" ^ + --add-path "C:\vcpkg\installed\arm64-windows\bin" --ignore-existing ^ + --with-mangle -w repaired_wheels %WHEEL_NAME% || exit /B 1 +) else ( + %PYTHON_CMD% -m delvewheel repair -vv ^ + --ignore-existing --with-mangle ^ + -w repaired_wheels %WHEEL_NAME% || exit /B 1 +) -popd +popd \ No newline at end of file diff --git a/dev/tasks/python-wheels/github.windows.arm64.yml b/dev/tasks/python-wheels/github.windows.arm64.yml index 4ac41b1e467..7e2023b9738 100644 --- a/dev/tasks/python-wheels/github.windows.arm64.yml +++ b/dev/tasks/python-wheels/github.windows.arm64.yml @@ -25,7 +25,7 @@ jobs: runs-on: "{{ runs_on }}" timeout-minutes: 240 env: - PYTHON_CMD: "py -{{ python_version }}" + PYTHON_CMD: "py -{{ python_version }}-arm64" CMAKE_BUILD_TYPE: release steps: {{ macros.github_checkout_arrow()|indent }} @@ -59,9 +59,10 @@ jobs: - name: Build Arrow C++ and PyArrow wheel shell: cmd env: + arch: "ARM64" SETUPTOOLS_SCM_PRETEND_VERSION: "{{ arrow.no_rc_version }}" run: | - call "arrow\ci\scripts\python_wheel_windows_arm64_build.bat" + call "arrow\ci\scripts\python_wheel_windows_build.bat" - uses: actions/upload-artifact@v4 with: