From d35cbfa1227a5abd92ce90e1f8e712500d71f5d3 Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Wed, 7 Jul 2021 04:45:13 -0700 Subject: [PATCH 1/4] update python deps --- python/setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 54026875..90061944 100644 --- a/python/setup.py +++ b/python/setup.py @@ -90,14 +90,14 @@ def run(self): copy2(SCRIPT_PATH, SCRIPT_TARGET) install_requires = [ - "numpy", - "typing", - "pandas == 1.1.4", - "psutil", - "pyarrow >= 0.10", - "ray == 1.4.0", - "pyspark >= 3.0.0, < 3.1.0", - "netifaces" + "numpy < 2.0.0", + "typing < 4.0.0", + "pandas >= 1.1.4, < 2.0.0", + "psutil < 6.0.0", + "pyarrow >= 0.10, < 5.0.0", + "ray >= 1.4.0, < 2.0.0", + "pyspark >= 3.0.0, < 4.0.0", + "netifaces < 1.0.0" ] _packages = find_packages() From a34469e184c353c7e3b147bdb3c08df3e6fec1f2 Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Wed, 7 Jul 2021 04:51:09 -0700 Subject: [PATCH 2/4] add main to RayDP CI branches --- .github/workflows/raydp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/raydp.yml b/.github/workflows/raydp.yml index 25f81858..341b5a81 100644 --- a/.github/workflows/raydp.yml +++ b/.github/workflows/raydp.yml @@ -19,9 +19,9 @@ name: RayDP CI on: push: - branches: [ master ] + branches: [ main, master ] pull_request: - branches: [ master ] + branches: [ main, master ] jobs: build-and-test: From b3e713501cfd956e78420a8b952887f4905bea9f Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Fri, 9 Jul 2021 02:27:18 -0700 Subject: [PATCH 3/4] test multiple pyspark versions --- .github/workflows/raydp.yml | 3 ++- core/pom.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/raydp.yml b/.github/workflows/raydp.yml index 341b5a81..8782ccd7 100644 --- a/.github/workflows/raydp.yml +++ b/.github/workflows/raydp.yml @@ -29,6 +29,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest ] + pyspark-version: [3.0.0, 3.1.2] python-version: [3.6, 3.7, 3.8] runs-on: ${{ matrix.os }} @@ -51,7 +52,6 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get install -y mpich - - name: Cache pip - Ubuntu if: matrix.os == 'ubuntu-latest' uses: actions/cache@v2 @@ -87,6 +87,7 @@ jobs: env: GITHUB_CI: 1 run: | + pip install pyspark==${{ matrix.pyspark-version }} ./build.sh pip install dist/raydp-*.whl - name: Lint diff --git a/core/pom.xml b/core/pom.xml index eda2c780..2ff89068 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -152,7 +152,7 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 3.3.3 scala-compile-first From 46d0f23697b3171b0f8b51ebcbf8c074d7770f47 Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Fri, 9 Jul 2021 08:11:25 -0700 Subject: [PATCH 4/4] less changes --- python/setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 90061944..1484a2cc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -90,14 +90,14 @@ def run(self): copy2(SCRIPT_PATH, SCRIPT_TARGET) install_requires = [ - "numpy < 2.0.0", - "typing < 4.0.0", - "pandas >= 1.1.4, < 2.0.0", - "psutil < 6.0.0", - "pyarrow >= 0.10, < 5.0.0", - "ray >= 1.4.0, < 2.0.0", - "pyspark >= 3.0.0, < 4.0.0", - "netifaces < 1.0.0" + "numpy", + "typing", + "pandas >= 1.1.4", + "psutil", + "pyarrow >= 0.10", + "ray >= 1.4.0, < 1.5.0", + "pyspark >= 3.0.0", + "netifaces" ] _packages = find_packages()