From a5c858dbfc3a0e0fcf7ce326fbcbd8eb7a36cd77 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Mon, 17 Mar 2025 07:29:15 +0100 Subject: [PATCH 1/2] Cache depdendencies, fetch before compile Signed-off-by: Enrico Minack --- .github/workflows/ci.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d07ccab0..4f2750a2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,6 +36,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Restore Maven packages cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}-${{ hashFiles('pom.xml') }} + restore-keys: + ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}-${{ hashFiles('pom.xml') }} + ${{ runner.os }}-mvn-build-${{ matrix.spark-version }}-${{ matrix.scala-version }}- - name: Setup JDK uses: actions/setup-java@v4 with: @@ -45,5 +53,7 @@ jobs: run: | ./scripts/set-version.sh "${{ matrix.spark_version }}" "${{ matrix.scala_version }}" git diff + - name: Fetch mvn dependencies + run: mvn --batch-mode dependency:go-offline - name: Build - run: mvn install + run: mvn --batch-mode install \ No newline at end of file From 033292118b6040de95df9580300c15bdc3e2aa9d Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Mon, 17 Mar 2025 21:44:02 +0100 Subject: [PATCH 2/2] Build image Signed-off-by: Enrico Minack --- .github/workflows/ci.yaml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f2750a2..06ba671d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,11 +49,30 @@ jobs: with: java-version: 17 distribution: 'zulu' + - name: Set Spark and Scala versions run: | ./scripts/set-version.sh "${{ matrix.spark_version }}" "${{ matrix.scala_version }}" git diff - name: Fetch mvn dependencies run: mvn --batch-mode dependency:go-offline - - name: Build - run: mvn --batch-mode install \ No newline at end of file + - name: Build package + run: mvn --batch-mode install + + - name: Checkout Spark sources + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + uses: actions/checkout@v4 + with: + repository: apache/spark + ref: "v${{ matrix.spark_version }}" + path: spark + - name: Build Spark + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + run: | + cd spark + ./build/mvn package -Pkubernetes -DskipTests -Dtest=none + - name: Build image + if: startsWith(matrix.scala_version, '2.13') && ! contains(matrix.spark_version, '-SNAPSHOT') + env: + SPARK_HOME: ${{ github.workspace }}/spark + run: ./scripts/createImage.sh