From 4a2e1a447e4516f3ee2f957ebbb87dfe23d08165 Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Thu, 26 Sep 2024 13:38:58 +0200 Subject: [PATCH 1/3] CI: add Python 3.12 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5247e7..6b1cef5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python: ['3.8', '3.9', '3.10', '3.11'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] runs-on: ${{ matrix.os }} steps: From 7b360f2f8ef2ebaa1d694700226e7658608cbf15 Mon Sep 17 00:00:00 2001 From: mbaak Date: Fri, 29 Nov 2024 14:20:14 +0100 Subject: [PATCH 2/3] FIX: remove python 3.11 requirement for spark --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5d3721..c1367a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,7 @@ spark = [ # https://issues.apache.org/jira/browse/SPARK-41718 # 3.4 is needed for python 3.11 # https://github.com/apache/spark/pull/38987 - "pyspark>=3.1; python_version < '3.11'", - "numpy<1.24.0", + "pyspark>=3.1", ] dev = [ "pre-commit", From adc1f1d28628bb8f58883dbc572ce38a8cd1bf9f Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Fri, 29 Nov 2024 16:57:56 +0100 Subject: [PATCH 3/3] PKG: add `setuptools` for `spark` 3.5+ compatibility --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c1367a2..15965ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,10 @@ spark = [ # 3.4 is needed for python 3.11 # https://github.com/apache/spark/pull/38987 "pyspark>=3.1", + # setuptools is only necessary because Python 3.12 removed the distutils + # package. pyspark 3.5.X still imports this package, so we need it on + # Python 3.12 to run the tests. + "setuptools", ] dev = [ "pre-commit",