diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index b4b7fed99fd..cf976e346b8 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -24,16 +24,35 @@ py -0p %PYTHON_CMD% -m sysconfig || exit /B 1 -call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" +@REM Detect architecture if not set +if "%arch%"=="" set arch=x64 + +@REM Set architecture-specific options +if "%arch%"=="ARM64" ( + set CMAKE_PLATFORM=ARM64 + set VCVARS_BAT=C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat + set VCPKG_TARGET_TRIPLET=arm64-windows + set ARROW_SRC=%GITHUB_WORKSPACE%\arrow + set ARROW_DIST=%GITHUB_WORKSPACE%\arrow-dist +) else ( + set VCVARS_BAT=C:\Program Files ^(x86^)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat + set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% + set ARROW_SRC=C:\arrow + set ARROW_DIST=C:\arrow-dist +) + +call "%VCVARS_BAT%" @echo on -echo "=== (%PYTHON%) Clear output directories and leftovers ===" -del /s /q C:\arrow-build -del /s /q C:\arrow-dist -del /s /q C:\arrow\python\dist -del /s /q C:\arrow\python\build -del /s /q C:\arrow\python\pyarrow\*.so -del /s /q C:\arrow\python\pyarrow\*.so.* +if "%arch%"=="x64" ( + echo "=== (%PYTHON%) Clear output directories and leftovers ===" + del /s /q C:\arrow-build + del /s /q C:\arrow-dist + del /s /q C:\arrow\python\dist + del /s /q C:\arrow\python\build + del /s /q C:\arrow\python\pyarrow\*.so + del /s /q C:\arrow\python\pyarrow\*.so.* +) echo "=== (%PYTHON%) Building Arrow C++ libraries ===" set ARROW_ACERO=ON @@ -46,67 +65,119 @@ set ARROW_ORC=ON set ARROW_PARQUET=ON set PARQUET_REQUIRE_ENCRYPTION=ON set ARROW_MIMALLOC=ON -set ARROW_SUBSTRAIT=ON set ARROW_S3=ON -set ARROW_TENSORFLOW=ON -set ARROW_WITH_BROTLI=ON -set ARROW_WITH_BZ2=ON -set ARROW_WITH_LZ4=ON -set ARROW_WITH_SNAPPY=ON -set ARROW_WITH_ZLIB=ON -set ARROW_WITH_ZSTD=ON -set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 -set CMAKE_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests -set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% + +if "%arch%"=="ARM64" ( + set ARROW_SUBSTRAIT=OFF + set ARROW_TENSORFLOW=OFF +) else ( + set ARROW_SUBSTRAIT=ON + set ARROW_TENSORFLOW=ON + set ARROW_WITH_BROTLI=ON + set ARROW_WITH_BZ2=ON + set ARROW_WITH_LZ4=ON + set ARROW_WITH_SNAPPY=ON + set ARROW_WITH_ZLIB=ON + set ARROW_WITH_ZSTD=ON + set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON + set CMAKE_UNITY_BUILD=ON +) mkdir C:\arrow-build pushd C:\arrow-build -cmake ^ - -DARROW_ACERO=%ARROW_ACERO% ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=OFF ^ - -DARROW_COMPUTE=ON ^ - -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=%ARROW_DATASET% ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_DEPENDENCY_USE_SHARED=OFF ^ - -DARROW_FILESYSTEM=ON ^ - -DARROW_FLIGHT=%ARROW_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_GANDIVA% ^ - -DARROW_GCS=%ARROW_GCS% ^ - -DARROW_HDFS=%ARROW_HDFS% ^ - -DARROW_JSON=ON ^ - -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ - -DARROW_ORC=%ARROW_ORC% ^ - -DARROW_PACKAGE_KIND="python-wheel-windows" ^ - -DARROW_PARQUET=%ARROW_PARQUET% ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ - -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ - -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ - -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ - -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ - -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ - -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ - -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ - -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION% ^ - -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ - -DMSVC_LINK_VERBOSE=ON ^ - -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ - -DVCPKG_MANIFEST_MODE=OFF ^ - -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ - -Dxsimd_SOURCE=BUNDLED ^ - -G "%CMAKE_GENERATOR%" ^ - -A "%CMAKE_PLATFORM%" ^ - C:\arrow\cpp || exit /B 1 + +if "%arch%"=="ARM64" ( + cmake ^ + -G "Visual Studio 17 2022" -A ARM64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_DIST% ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_DEPENDENCY_USE_SHARED=ON ^ + -DARROW_PROTOBUF_USE_SHARED=ON ^ + -Dutf8proc_SOURCE=BUNDLED ^ + -DARROW_SIMD_LEVEL=NONE ^ + -DARROW_RUNTIME_SIMD_LEVEL=NONE ^ + -DARROW_USE_XSIMD=OFF ^ + -DARROW_WITH_UTF8PROC=OFF ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=OFF ^ + -DARROW_ACERO=%ARROW_ACERO% ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=%ARROW_DATASET% ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_FLIGHT=%ARROW_FLIGHT% ^ + -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ + -DARROW_HDFS=%ARROW_HDFS% ^ + -DARROW_JSON=ON ^ + -DVCPKG_MANIFEST_MODE=OFF ^ + -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ + -DARROW_ORC=%ARROW_ORC% ^ + -DARROW_PACKAGE_KIND="python-wheel-windows" ^ + -DARROW_PARQUET=%ARROW_PARQUET% ^ + -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ + -DARROW_S3=%ARROW_S3% ^ + -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ + -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + "%ARROW_SRC%\cpp" || exit /B 1 +) else ( + cmake ^ + -DARROW_ACERO=%ARROW_ACERO% ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=OFF ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_CXXFLAGS="/MP" ^ + -DARROW_DATASET=%ARROW_DATASET% ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_DEPENDENCY_USE_SHARED=OFF ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_FLIGHT=%ARROW_FLIGHT% ^ + -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ + -DARROW_HDFS=%ARROW_HDFS% ^ + -DARROW_JSON=ON ^ + -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ + -DARROW_ORC=%ARROW_ORC% ^ + -DARROW_PACKAGE_KIND="python-wheel-windows" ^ + -DARROW_PARQUET=%ARROW_PARQUET% ^ + -DARROW_S3=%ARROW_S3% ^ + -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^ + -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ + -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ + -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ + -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ + -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ + -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ + -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ + -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ + -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION% ^ + -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ + -DMSVC_LINK_VERBOSE=ON ^ + -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ + -DVCPKG_MANIFEST_MODE=OFF ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ + -Dxsimd_SOURCE=BUNDLED ^ + -G "%CMAKE_GENERATOR%" ^ + -A "%CMAKE_PLATFORM%" ^ + C:\arrow\cpp || exit /B 1 +) + cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 popd @@ -127,10 +198,21 @@ set PYARROW_WITH_PARQUET=%ARROW_PARQUET% set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% set PYARROW_WITH_S3=%ARROW_S3% -set ARROW_HOME=C:\arrow-dist -set CMAKE_PREFIX_PATH=C:\arrow-dist +set ARROW_HOME=%ARROW_DIST% +set CMAKE_PREFIX_PATH=%ARROW_DIST% + +if "%arch%"=="ARM64" ( + set Arrow_DIR=%ARROW_DIST%\lib\cmake\arrow +) -pushd C:\arrow\python +pushd %ARROW_SRC%\python + +if "%arch%"=="ARM64" ( + @REM Install Python dependencies for Win-ARM64 + echo "=== Installing Python dependencies ===" + %PYTHON_CMD% -m pip install --upgrade pip || exit /B 1 + %PYTHON_CMD% -m pip install cython numpy setuptools_scm setuptools wheel || exit /B 1 +) @REM Build wheel %PYTHON_CMD% setup.py bdist_wheel || exit /B 1 @@ -145,8 +227,14 @@ pushd C:\arrow\python for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 echo "Wheel name: %WHEEL_NAME%" -%PYTHON_CMD% -m delvewheel repair -vv ^ - --ignore-existing --with-mangle ^ - -w repaired_wheels %WHEEL_NAME% || exit /B 1 +if "%arch%"=="ARM64" ( + %PYTHON_CMD% -m delvewheel repair -vv --add-path "%ARROW_DIST%\bin" ^ + --add-path "C:\vcpkg\installed\arm64-windows\bin" --ignore-existing ^ + --with-mangle -w repaired_wheels %WHEEL_NAME% || exit /B 1 +) else ( + %PYTHON_CMD% -m delvewheel repair -vv ^ + --ignore-existing --with-mangle ^ + -w repaired_wheels %WHEEL_NAME% || exit /B 1 +) -popd +popd \ No newline at end of file diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index 5dfe61a0c60..ed6a59f1d8d 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -3,7 +3,7 @@ "version-string": "", "description": "Cross-language development platform for in-memory analytics", "homepage": "https://arrow.apache.org", - "supports": "x64 | (arm64 & !windows)", + "supports": "x64 | arm64", "dependencies": [ { "name": "boost-multiprecision", @@ -19,7 +19,10 @@ "re2", "snappy", "utf8proc", - "xsimd", + { + "name": "xsimd", + "platform": "!(arm64 & windows)" + }, "zlib", "zstd", { @@ -143,3 +146,4 @@ } } } + diff --git a/dev/tasks/python-wheels/github.windows.arm64.yml b/dev/tasks/python-wheels/github.windows.arm64.yml new file mode 100644 index 00000000000..7e2023b9738 --- /dev/null +++ b/dev/tasks/python-wheels/github.windows.arm64.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Windows ARM64" + runs-on: "{{ runs_on }}" + timeout-minutes: 240 + env: + PYTHON_CMD: "py -{{ python_version }}-arm64" + CMAKE_BUILD_TYPE: release + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "{{ python_version }}" + architecture: "arm64" + + - name: Cache vcpkg dependencies + uses: actions/cache@v4 + with: + path: C:\vcpkg\installed + key: "{% raw %}vcpkg-installed-windows-arm64-${{ hashFiles('arrow/ci/vcpkg/vcpkg.json', 'arrow/ci/vcpkg/arm64-windows-*.cmake') }}{% endraw %}" + restore-keys: vcpkg-installed-windows-arm64- + + - name: Install vcpkg dependencies + shell: cmd + run: | + cd arrow\ci\vcpkg + C:\vcpkg\vcpkg.exe install ^ + --binarysource="clear;x-gha,readwrite" ^ + --x-install-root=C:\vcpkg\installed ^ + --x-manifest-root=. ^ + --x-feature=flight ^ + --x-feature=json ^ + --x-feature=gcs ^ + --x-feature=orc ^ + --x-feature=parquet ^ + --x-feature=s3 + - name: Build Arrow C++ and PyArrow wheel + shell: cmd + env: + arch: "ARM64" + SETUPTOOLS_SCM_PRETEND_VERSION: "{{ arrow.no_rc_version }}" + run: | + call "arrow\ci\scripts\python_wheel_windows_build.bat" + + - uses: actions/upload-artifact@v4 + with: + name: wheel + path: arrow/python/repaired_wheels/*.whl + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} + diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e2bd5c6d8e5..4abb99e11d2 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -249,6 +249,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl + wheel-windows-{{ python_tag }}-{{ abi_tag }}-arm64: + ci: github + template: python-wheels/github.windows.arm64.yml + params: + python_version: "{{ python_version }}" + python_abi_tag: "{{ abi_tag }}" + runs_on: "windows-11-arm" + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_arm64.whl + {% endfor %} {############################ Python sdist ####################################}