From dd3e8cc3603876ddca806f735bb38b9783899d0d Mon Sep 17 00:00:00 2001 From: Chang chen Date: Thu, 26 Feb 2026 10:20:15 +0000 Subject: [PATCH 1/2] [GLUTEN-11658][CORE] Restore scala.recompile.mode default to 'all' and introduce fast-build profile - Restore scala.recompile.mode default from 'incremental' back to 'all' to fix CI/clean build failures caused by Zinc's incremental compiler not tracking transitive Spark dependencies correctly - Rename the 'bloop' Maven profile (added in #11645) to 'fast-build' and add scala.recompile.mode=incremental inside it; used by bloop-setup.sh and run-scala-test.sh where Zinc analysis is warm and reliable - Simplify run-scala-test.sh: replace 5 redundant -D skip flags with -Pfast-build profile activation - Add pathing JAR support in run-scala-test.sh to avoid OS command-line length limits when classpath is very long Closes #11658 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/velox_backend_x86.yml | 10 +++ dev/bloop-setup.sh | 2 +- dev/run-scala-test.sh | 106 +++++++++++++++++------- docs/developers/bloop-integration.md | 2 +- pom.xml | 30 +++++-- 5 files changed, 111 insertions(+), 39 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index a91617eff9c2..8d5bc71aec6f 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1338,6 +1338,16 @@ jobs: ccache -s " + build-fast-build-test: + runs-on: ubuntu-22.04 + container: apache/gluten:centos-8-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Build with fast-build profile (Spark 4.0, Java 17) + run: | + cd $GITHUB_WORKSPACE/ + $MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13 -Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build + spark-test-spark40: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 diff --git a/dev/bloop-setup.sh b/dev/bloop-setup.sh index e633e2453b25..8a2eb0b78c0b 100755 --- a/dev/bloop-setup.sh +++ b/dev/bloop-setup.sh @@ -190,7 +190,7 @@ fi # Skip style checks for faster generation log_step "Running generate-sources + bloopInstall..." ./build/mvn generate-sources bloop:bloopInstall \ - -P"${PROFILES}",bloop \ + -P"${PROFILES}",fast-build \ -DskipTests if [[ ! -d ".bloop" ]]; then diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index 6f3f8688d3c7..cfe195f9399f 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -527,14 +527,9 @@ fi ${MVN_CMD} ${MVN_GOALS} \ -T 1C -q \ -pl "${MODULE}" -am \ - -P"${PROFILES}" \ + -P"${PROFILES},fast-build" \ -DincludeScope=test \ -Dmdep.outputFile="${CLASSPATH_FILE}" \ - -Dspotless.check.skip=true \ - -Dscalastyle.skip=true \ - -Dcheckstyle.skip=true \ - -Dmaven.gitcommitid.skip=true \ - -Dremoteresources.skip=true \ ${EXTRA_MVN_ARGS} if [[ ! -f "${CLASSPATH_FILE}" ]]; then @@ -617,14 +612,82 @@ TIMING_STEP3=$(timer_elapsed $TIMER_STEP3_START $TIMER_STEP3_END) log_timing "Step 3 - Resolve classpath" "$TIMING_STEP3" # ============================================================================= -# Step 3.5: Export-only mode (if requested) +# Step 3.5: Create pathing JAR +# ============================================================================= +# A "pathing JAR" is a thin JAR whose MANIFEST.MF Class-Path header lists all +# the real classpath entries as file: URIs. This avoids exceeding OS command-line +# length limits. Works on Java 8+ (unlike @argfile which requires Java 9+). +# Also includes META-INF/classpath.txt for human-readable review. +# ============================================================================= + +PATHING_JAR="/tmp/gluten-test-pathing-$$.jar" +rm -f "${PATHING_JAR}" + +PATHING_MANIFEST_DIR="/tmp/gluten-test-manifest-$$" +mkdir -p "${PATHING_MANIFEST_DIR}/META-INF" + +# Convert colon-separated classpath to space-separated file: URIs for manifest. +# Also write a human-readable classpath listing (one entry per line, same order). +CP_URIS="" +IFS=':' read -ra _CP_ITEMS <<< "${RESOLVED_CLASSPATH}" +: > "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + +for _item in "${_CP_ITEMS[@]}"; do + [[ -z "$_item" ]] && continue + echo "$_item" >> "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + # Ensure directories end with / (required by Class-Path spec for directories) + [[ -d "$_item" && "$_item" != */ ]] && _item="${_item}/" + CP_URIS="${CP_URIS} file://${_item}" +done +CP_URIS="${CP_URIS# }" + +# Write manifest with proper 72-byte line wrapping. +# MANIFEST.MF spec: max 72 bytes per line; continuation lines start with +# a single space character. +{ + echo "Manifest-Version: 1.0" + _mf_header="Class-Path: ${CP_URIS}" + echo "${_mf_header:0:72}" + _mf_rest="${_mf_header:72}" + while [[ -n "$_mf_rest" ]]; do + echo " ${_mf_rest:0:71}" + _mf_rest="${_mf_rest:71}" + done + echo "" +} > "${PATHING_MANIFEST_DIR}/META-INF/MANIFEST.MF" + +# Build the pathing JAR (manifest + human-readable classpath listing) +(cd "${PATHING_MANIFEST_DIR}" && jar cfm "${PATHING_JAR}" META-INF/MANIFEST.MF META-INF/classpath.txt) +rm -rf "${PATHING_MANIFEST_DIR}" +log_info "Created pathing JAR: ${PATHING_JAR} ($(du -h "${PATHING_JAR}" | cut -f1))" +log_info " Review classpath: unzip -p ${PATHING_JAR} META-INF/classpath.txt" + +# ============================================================================= +# Build java command (shared by export-only and run modes) +# ============================================================================= + +JAVA_CMD="${JAVA_HOME}/bin/java" +[[ ! -x "$JAVA_CMD" ]] && JAVA_CMD="java" + +JAVA_ARGS=( + ${JVM_ARGS} + "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties" + -cp "${PATHING_JAR}" + org.scalatest.tools.Runner + -oDF + -s "${SUITE}" +) +[[ -n "$TEST_METHOD" ]] && JAVA_ARGS+=(-t "${TEST_METHOD}") + +# ============================================================================= +# Step 3.6: Export-only mode (if requested) # ============================================================================= if [[ "$EXPORT_ONLY" == "true" ]]; then - EXPORT_FILE="/tmp/gluten-classpath-exported.txt" - echo "$RESOLVED_CLASSPATH" > "$EXPORT_FILE" - log_info "✓ Classpath exported to: ${EXPORT_FILE}" - log_info " Use with: java -cp \"\$(cat ${EXPORT_FILE})\" org.scalatest.tools.Runner -s " + echo "" + echo -e "${YELLOW}# Run the test with:${NC}" + echo "${JAVA_CMD} ${JAVA_ARGS[*]}" + echo "" print_timing_summary false exit 0 fi @@ -635,12 +698,6 @@ fi log_step "Step 4: Running ScalaTest..." -# Find Java -JAVA_CMD="${JAVA_HOME}/bin/java" -if [[ ! -x "$JAVA_CMD" ]]; then - JAVA_CMD="java" -fi - log_info "Suite: ${SUITE}" [[ -n "$TEST_METHOD" ]] && log_info "Test method: ${TEST_METHOD}" @@ -650,21 +707,12 @@ echo "Running ScalaTest" echo "==========================================" echo "" -# Build test method args conditionally -TEST_METHOD_ARGS=() -[[ -n "$TEST_METHOD" ]] && TEST_METHOD_ARGS=(-t "${TEST_METHOD}") - TIMER_STEP4_START=$(timer_now) TEST_EXIT_CODE=0 -${JAVA_CMD} \ - ${JVM_ARGS} \ - -Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties \ - -cp "${RESOLVED_CLASSPATH}" \ - org.scalatest.tools.Runner \ - -s "${SUITE}" \ - "${TEST_METHOD_ARGS[@]}" \ - -oDF || TEST_EXIT_CODE=$? +${JAVA_CMD} "${JAVA_ARGS[@]}" || TEST_EXIT_CODE=$? + +rm -f "${PATHING_JAR}" TIMER_STEP4_END=$(timer_now) TIMING_STEP4=$(timer_elapsed $TIMER_STEP4_START $TIMER_STEP4_END) diff --git a/docs/developers/bloop-integration.md b/docs/developers/bloop-integration.md index 2364b8f596c7..c5efdbbd528f 100644 --- a/docs/developers/bloop-integration.md +++ b/docs/developers/bloop-integration.md @@ -65,7 +65,7 @@ The `-Pbloop` profile automatically skips style checks during configuration gene ./dev/bloop-setup.sh -Pspark-3.5,scala-2.12,backends-velox # Manual invocation with profile -./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,bloop -DskipTests +./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,fast-build -DskipTests ``` The bloop profile sets these properties automatically: diff --git a/pom.xml b/pom.xml index 9af9927f9eee..891598172f00 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ --> unknown - incremental + all 2.15.0 4.13.1 @@ -571,11 +571,6 @@ UTF-8 1024m - - true - true @@ -2147,18 +2142,37 @@ - bloop + fast-build false - + true true true true true + + incremental + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + true + true + + + + + From 9adfadf2a74b2a76d0a4d62661393e4dcd755a5d Mon Sep 17 00:00:00 2001 From: Chang chen Date: Fri, 27 Feb 2026 03:52:16 +0000 Subject: [PATCH 2/2] Fix Java 8 build failure caused by scala-maven-plugin 4.9.2 Problem: scala-maven-plugin 4.9.2 changed behavior for Scala >= 2.12 builds. When maven.compiler.target is set (e.g. "1.8"), it now passes the value to computeBytecodeVersionOptions() [1] which injects '-release 1.8' into scalac args. The scalac '-release' flag is only supported on Java 9+, so building under JDK 8 with recompileMode=all (restored by the parent commit) fails with: scalac error: -release is only supported on Java 9 and higher Root cause: PR #11645 (which introduced scala-maven-plugin 4.9.2) also set recompileMode=incremental, which routes through Zinc's SbtIncrementalCompiler and bypasses computeBytecodeVersionOptions(). Restoring recompileMode=all exposes the 4.9.2 regression for Java 8 builds. Fix: in the java-8 profile, pin scala.compiler.version to 4.8.0, which does not have this regression. Java 9+ builds continue to use 4.9.2 unaffected. [1] https://github.com/davidB/scala-maven-plugin/blob/4.9.2/src/main/java/scala_maven/ScalaMojoSupport.java#L620-L648 Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 891598172f00..d34633ecf69d 100644 --- a/pom.xml +++ b/pom.xml @@ -1100,6 +1100,9 @@ 1.8 + + 4.8.0