diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d07ccab0..06ba671d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,14 +36,43 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Restore Maven packages cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}-${{ hashFiles('pom.xml') }} + restore-keys: + ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}-${{ hashFiles('pom.xml') }} + ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}- - name: Setup JDK uses: actions/setup-java@v4 with: java-version: 17 distribution: 'zulu' + - name: Set Spark and Scala versions run: | ./scripts/set-version.sh "${{ matrix.spark_version }}" "${{ matrix.scala_version }}" git diff - - name: Build - run: mvn install + - name: Fetch mvn dependencies + run: mvn --batch-mode dependency:go-offline + - name: Build package + run: mvn --batch-mode install + + - name: Checkout Spark sources + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + uses: actions/checkout@v4 + with: + repository: apache/spark + ref: "v${{ matrix.spark_version }}" + path: spark + - name: Build Spark + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + run: | + cd spark + ./build/mvn package -Pkubernetes -DskipTests -Dtest=none + - name: Build image + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + env: + SPARK_HOME: ${{ github.workspace }}/spark + run: ./scripts/createImage.sh