From 0ecdc3cf14c3b0f313f134fdbe03643a0e994c8c Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Thu, 8 May 2025 13:41:58 -0400 Subject: [PATCH 1/4] Allow psycopg2 2.9.10 --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 2b21d0463c98..6d3596069a41 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -417,7 +417,7 @@ def get_portability_package_data(): 'scikit-learn>=0.20.0', 'setuptools', 'sqlalchemy>=1.3,<3.0', - 'psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10', + 'psycopg2-binary>=2.8.5,<3.0.0', 'testcontainers[mysql]>=3.0.3,<4.0.0', 'cryptography>=41.0.2', 'hypothesis>5.0.0,<7.0.0', From a1efa8eb67f9c11eed4349a23bf6d0f0f67d6889 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Thu, 8 May 2025 14:01:54 -0400 Subject: [PATCH 2/4] try dependency specifieres --- sdks/python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 6d3596069a41..734072dcabe3 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -417,7 +417,8 @@ def get_portability_package_data(): 'scikit-learn>=0.20.0', 'setuptools', 'sqlalchemy>=1.3,<3.0', - 'psycopg2-binary>=2.8.5,<3.0.0', + 'psycopg2-binary>=2.8.5,<3.0.0;', + 'psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10;python_version=="3.9" and os_name=="macos" and platform_machine=="arm64"', 'testcontainers[mysql]>=3.0.3,<4.0.0', 'cryptography>=41.0.2', 'hypothesis>5.0.0,<7.0.0', From 8c0da77f9365a0dcc57a0e3d993bfa78ee2174fd Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Thu, 8 May 2025 14:22:44 -0400 Subject: [PATCH 3/4] split out logic to own function + formatting --- sdks/python/setup.py | 52 +++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 734072dcabe3..9bfc7c8ba953 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -159,6 +159,14 @@ def cythonize(*args, **kwargs): 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3', ] +# psycopg2-binary did not release a MacOS Python 3.9 arm64 wheel for 2.9.10 for +# some reason, so we'll handle that distinction here. +if sys.platform == 'darwin' and (sys.version_info.major, + sys.version_info.minor) == (3, 9): + psycopg2_dependency = ['psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10'] +else: + psycopg2_dependency = ['psycopg2-binary>=2.8.5,<3.0.0'] + def find_by_ext(root_dir, ext): for root, _, files in os.walk(root_dir): @@ -222,18 +230,18 @@ def generate_external_transform_wrappers(): script_exists = os.path.exists( os.path.join(sdk_dir, 'gen_xlang_wrappers.py')) config_exists = os.path.exists( - os.path.join(os.path.dirname(sdk_dir), - 'standard_external_transforms.yaml')) + os.path.join( + os.path.dirname(sdk_dir), 'standard_external_transforms.yaml')) # we need both the script and the standard transforms config file. # at build time, we don't have access to apache_beam to discover and # retrieve external transforms, so the config file has to already exist if not script_exists or not config_exists: generated_transforms_dir = os.path.join( - sdk_dir, 'apache_beam', 'transforms', 'xlang') + sdk_dir, 'apache_beam', 'transforms', 'xlang') # if exists, this directory will have at least its __init__.py file if (not os.path.exists(generated_transforms_dir) or - len(os.listdir(generated_transforms_dir)) <= 1): + len(os.listdir(generated_transforms_dir)) <= 1): message = 'External transform wrappers have not been generated ' if not script_exists: message += 'and the generation script `gen_xlang_wrappers.py`' @@ -251,13 +259,16 @@ def generate_external_transform_wrappers(): os.path.join(sdk_dir, 'gen_xlang_wrappers.py'), '--cleanup', '--transforms-config-source', - os.path.join(os.path.dirname(sdk_dir), - 'standard_external_transforms.yaml') - ], capture_output=True, check=True) + os.path.join( + os.path.dirname(sdk_dir), 'standard_external_transforms.yaml') + ], + capture_output=True, + check=True) except subprocess.CalledProcessError as err: raise RuntimeError( 'Could not generate external transform wrappers due to ' - 'error: %s', err.stderr) + 'error: %s', + err.stderr) def get_portability_package_data(): @@ -417,13 +428,11 @@ def get_portability_package_data(): 'scikit-learn>=0.20.0', 'setuptools', 'sqlalchemy>=1.3,<3.0', - 'psycopg2-binary>=2.8.5,<3.0.0;', - 'psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10;python_version=="3.9" and os_name=="macos" and platform_machine=="arm64"', 'testcontainers[mysql]>=3.0.3,<4.0.0', 'cryptography>=41.0.2', 'hypothesis>5.0.0,<7.0.0', 'virtualenv-clone>=0.5,<1.0', - ], + ] + psycopg2_dependency, 'gcp': [ 'cachetools>=3.1.0,<6', 'google-api-core>=2.0.0,<3', @@ -547,20 +556,14 @@ def get_portability_package_data(): # in https://github.com/apache/beam/blob/master/sdks/python/tox.ini # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing - 'torch': [ - 'torch<=1.13.0,<=2.0.0' - ], - 'tensorflow': [ - 'tensorflow>=2.12rc1,<2.13' - ], + 'torch': ['torch<=1.13.0,<=2.0.0'], + 'tensorflow': ['tensorflow>=2.12rc1,<2.13'], 'transformers': [ 'transformers>=4.28.0,<4.49.0', 'tensorflow==2.12.0', 'torch>=1.9.0,<2.1.0' ], - 'tft': [ - 'tensorflow_transform>=1.14.0,<1.15.0' - ], + 'tft': ['tensorflow_transform>=1.14.0,<1.15.0'], 'onnx': [ 'onnxruntime==1.13.1', 'torch==1.13.1', @@ -569,13 +572,8 @@ def get_portability_package_data(): 'skl2onnx==1.13', 'transformers==4.25.1' ], - 'xgboost': [ - 'xgboost>=1.6.0,<2.1.3', - 'datatable==1.0.0' - ], - 'tensorflow-hub': [ - 'tensorflow-hub>=0.14.0,<0.16.0' - ] + 'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'], + 'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'] }, zip_safe=False, # PyPI package information. From d7b1bd23718864aadbb30b92aca32f0574fd5937 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 13 May 2025 15:22:44 -0400 Subject: [PATCH 4/4] swap to more general psycopg2 bounds --- sdks/python/setup.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 9bfc7c8ba953..80ec0323b86c 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -159,14 +159,6 @@ def cythonize(*args, **kwargs): 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3', ] -# psycopg2-binary did not release a MacOS Python 3.9 arm64 wheel for 2.9.10 for -# some reason, so we'll handle that distinction here. -if sys.platform == 'darwin' and (sys.version_info.major, - sys.version_info.minor) == (3, 9): - psycopg2_dependency = ['psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10'] -else: - psycopg2_dependency = ['psycopg2-binary>=2.8.5,<3.0.0'] - def find_by_ext(root_dir, ext): for root, _, files in os.walk(root_dir): @@ -428,11 +420,13 @@ def get_portability_package_data(): 'scikit-learn>=0.20.0', 'setuptools', 'sqlalchemy>=1.3,<3.0', + 'psycopg2-binary>=2.8.5,<2.9.10; python_version <= "3.9"', + 'psycopg2-binary>=2.8.5,<3.0; python_version >= "3.10"', 'testcontainers[mysql]>=3.0.3,<4.0.0', 'cryptography>=41.0.2', 'hypothesis>5.0.0,<7.0.0', 'virtualenv-clone>=0.5,<1.0', - ] + psycopg2_dependency, + ], 'gcp': [ 'cachetools>=3.1.0,<6', 'google-api-core>=2.0.0,<3',