diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 76f380c94bb8..3a928f2a3229 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3121,10 +3121,11 @@ class BeamModulePlugin implements Plugin { // TODO: https://github.com/apache/beam/issues/29022 // pip 23.3 is failing due to Hash mismatch between expected SHA of the packaged and actual SHA. // until it is resolved on pip's side, don't use pip's cache. - // pip 25.1 casues :sdks:python:installGcpTest stuck. Pin to 25.0.1 for now. + // Use pip 26.0.1 for improved resolution performance and bug fixes. See #34798. args '-c', ". ${project.ext.envdir}/bin/activate && " + - "pip install --pre --retries 10 --upgrade pip==25.0.1 --no-cache-dir && " + - "pip install --pre --retries 10 --upgrade tox --no-cache-dir" + "pip install --pre --retries 10 --upgrade pip==26.0.1 --no-cache-dir && " + + "pip install --pre --retries 10 --upgrade tox --no-cache-dir && " + + "pip install uv" } } // Gradle will delete outputs whenever it thinks they are stale. Putting a @@ -3169,9 +3170,19 @@ class BeamModulePlugin implements Plugin { packages += ",${extra}" } + def pythonSdkDir = project.project(":sdks:python").projectDir + def constraintsPath = "${pythonSdkDir}/constraints.txt" + def constraintFile = project.file(constraintsPath) + def constraintFlag = constraintFile.exists() ? "--constraint ${constraintsPath}" : "" + + // Use uv instead of pip - pip was hitting resolution-too-deep on tensorflow->keras->namex/optree. + // Include namex/optree as explicit deps to constrain resolution. + // --prerelease allow: envoy-data-plane depends on betterproto==2.0.0b6 (beta). + def anchorPkgs = "namex==0.0.9 optree==0.16.0" + def installCmd = ". ${project.ext.envdir}/bin/activate && uv pip install --prerelease allow ${constraintFlag} ${anchorPkgs} ${distTarBall}[${packages}]".replaceAll(/ +/, ' ').trim() project.exec { executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install --pre --retries 10 ${distTarBall}[${packages}]" + args '-c', installCmd } } } diff --git a/sdks/python/constraints.txt b/sdks/python/constraints.txt new file mode 100644 index 000000000000..c591d73e48d6 --- /dev/null +++ b/sdks/python/constraints.txt @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pins for installGcpTest. numpy handled via setup.py python_version markers. +# Use google-api-core 2.29.0 for py313 compat (2.16.2 lacks it). +googleapis-common-protos==1.72.0 +grpc-google-iam-v1>=0.12.4,<1.0.0 +google-api-core==2.29.0 +optree==0.16.0 +namex==0.0.9 diff --git a/sdks/python/container/ml/py310/base_image_requirements.txt b/sdks/python/container/ml/py310/base_image_requirements.txt index 9817ce616a6c..a89189c903d0 100644 --- a/sdks/python/container/ml/py310/base_image_requirements.txt +++ b/sdks/python/container/ml/py310/base_image_requirements.txt @@ -145,7 +145,7 @@ multidict==6.7.1 namex==0.1.0 networkx==3.4.2 nltk==3.9.2 -numpy==2.2.6 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/container/ml/py310/gpu_image_requirements.txt b/sdks/python/container/ml/py310/gpu_image_requirements.txt index f43dd2745aba..f02261d99584 100644 --- a/sdks/python/container/ml/py310/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py310/gpu_image_requirements.txt @@ -172,7 +172,7 @@ networkx==3.4.2 ninja==1.13.0 nltk==3.9.2 numba==0.61.2 -numpy==2.2.6 +numpy==1.26.4 nvidia-cublas-cu12==12.6.4.1 nvidia-cuda-cupti-cu12==12.6.80 nvidia-cuda-nvrtc-cu12==12.6.77 diff --git a/sdks/python/container/ml/py311/base_image_requirements.txt b/sdks/python/container/ml/py311/base_image_requirements.txt index 571cdb28c767..78297cd5925c 100644 --- a/sdks/python/container/ml/py311/base_image_requirements.txt +++ b/sdks/python/container/ml/py311/base_image_requirements.txt @@ -143,7 +143,7 @@ multidict==6.7.1 namex==0.1.0 networkx==3.6.1 nltk==3.9.2 -numpy==2.4.2 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/container/ml/py311/gpu_image_requirements.txt b/sdks/python/container/ml/py311/gpu_image_requirements.txt index f44a8ff9be00..bb9469364878 100644 --- a/sdks/python/container/ml/py311/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py311/gpu_image_requirements.txt @@ -170,7 +170,7 @@ networkx==3.6.1 ninja==1.13.0 nltk==3.9.2 numba==0.61.2 -numpy==2.2.6 +numpy==1.26.4 nvidia-cublas-cu12==12.6.4.1 nvidia-cuda-cupti-cu12==12.6.80 nvidia-cuda-nvrtc-cu12==12.6.77 diff --git a/sdks/python/container/ml/py312/base_image_requirements.txt b/sdks/python/container/ml/py312/base_image_requirements.txt index fdc80fd0ccb4..6f505c817a3b 100644 --- a/sdks/python/container/ml/py312/base_image_requirements.txt +++ b/sdks/python/container/ml/py312/base_image_requirements.txt @@ -142,7 +142,7 @@ multidict==6.7.1 namex==0.1.0 networkx==3.6.1 nltk==3.9.2 -numpy==2.4.2 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/container/ml/py312/gpu_image_requirements.txt b/sdks/python/container/ml/py312/gpu_image_requirements.txt index a9b13f834d1a..5c711e7fcf18 100644 --- a/sdks/python/container/ml/py312/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py312/gpu_image_requirements.txt @@ -169,7 +169,7 @@ networkx==3.6.1 ninja==1.13.0 nltk==3.9.2 numba==0.61.2 -numpy==2.2.6 +numpy==1.26.4 nvidia-cublas-cu12==12.6.4.1 nvidia-cuda-cupti-cu12==12.6.80 nvidia-cuda-nvrtc-cu12==12.6.77 diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index 1f78a4935f73..8f290074d385 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -126,7 +126,7 @@ mock==5.2.0 more-itertools==10.8.0 multidict==6.7.1 nltk==3.9.2 -numpy==2.2.6 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index e41f72af87e3..ec796606536e 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -124,7 +124,7 @@ mock==5.2.0 more-itertools==10.8.0 multidict==6.7.1 nltk==3.9.2 -numpy==2.4.2 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index 2f7820d01d68..03958cf374e2 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -123,7 +123,7 @@ mock==5.2.0 more-itertools==10.8.0 multidict==6.7.1 nltk==3.9.2 -numpy==2.4.2 +numpy==1.26.4 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.39.1 diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index c705dcb438cf..d17bffb8c14e 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -26,8 +26,9 @@ requires = [ "mypy-protobuf==3.5.0", # Avoid https://github.com/pypa/virtualenv/issues/2006 "distlib==0.3.9", - # Numpy headers - "numpy>=1.14.3,<2.5.0", # Update setup.py as well. + # Numpy headers. py313 requires 2.1+; py<3.13 use 1.x for pandas ABI compat. + "numpy>=1.26.0,<2.0.0; python_version < '3.13'", + "numpy>=2.1.0; python_version >= '3.13'", # having cython here will create wheels that are platform dependent. "cython>=3.0,<4", ## deps for generating external transform wrappers: diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 19aab312f274..828333444064 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -384,9 +384,10 @@ def get_portability_package_data(): 'grpcio>=1.67.0; python_version >= "3.13"', 'httplib2>=0.8,<0.32.0', 'jsonpickle>=3.0.0,<4.0.0', - # numpy can have breaking changes in minor versions. - # Use a strict upper bound. - 'numpy>=1.14.3,<2.5.0', # Update pyproject.toml as well. + # numpy: py310-312 use 1.x; py313 needs 2.x (1.x unsupported; avoids # pylint: disable=line-too-long + # pandas ABI mismatch). + 'numpy>=1.26.0,<2.0.0; python_version < "3.13"', + 'numpy>=2.1.0; python_version >= "3.13"', 'objsize>=0.6.1,<0.8.0', 'packaging>=22.0', 'pillow', @@ -541,6 +542,9 @@ def get_portability_package_data(): # tensorflow-transform requires dill, but doesn't set dill as a # hard requirement in setup.py. 'dill', + # namex/optree: pin to avoid resolver issues (lack version bounds) # pylint: disable=line-too-long + 'namex==0.0.9', + 'optree==0.16.0', 'tensorflow-transform', # Comment out xgboost as it is breaking presubmit python ml # tests due to tag check introduced since pip 24.2 @@ -549,8 +553,13 @@ def get_portability_package_data(): ] + ml_base, 'p312_ml_test': [ 'datatable', + 'namex==0.0.9', + 'optree==0.16.0', + ] + ml_base, + 'p313_ml_test': [ + 'namex==0.0.9', + 'optree==0.16.0', ] + ml_base, - 'p313_ml_test': ml_base, 'aws': ['boto3>=1.9,<2'], 'azure': [ 'azure-storage-blob>=12.3.2,<13',