[GLUTEN-11658][CORE] Restore scala.recompile.mode default to 'all' and introduce fast-build profile

.github/workflows/velox_backend_x86.yml

@@ -1338,6 +1338,16 @@ jobs:
           ccache -s
           "
 
+  build-fast-build-test:
+    runs-on: ubuntu-22.04
+    container: apache/gluten:centos-8-jdk17
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build with fast-build profile (Spark 4.0, Java 17)
+        run: |
+          cd $GITHUB_WORKSPACE/
+          $MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13 -Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build
+
   spark-test-spark40:
     needs: build-native-lib-centos-7
     runs-on: ubuntu-22.04

dev/bloop-setup.sh

@@ -190,7 +190,7 @@ fi
 # Skip style checks for faster generation
 log_step "Running generate-sources + bloopInstall..."
 ./build/mvn generate-sources bloop:bloopInstall \
-  -P"${PROFILES}",bloop \
+  -P"${PROFILES}",fast-build \
   -DskipTests
 
 if [[ ! -d ".bloop" ]]; then

dev/run-scala-test.sh

@@ -527,14 +527,9 @@ fi
 ${MVN_CMD} ${MVN_GOALS} \
   -T 1C -q \
   -pl "${MODULE}" -am \
-  -P"${PROFILES}" \
+  -P"${PROFILES},fast-build" \
   -DincludeScope=test \
   -Dmdep.outputFile="${CLASSPATH_FILE}" \
-  -Dspotless.check.skip=true \
-  -Dscalastyle.skip=true \
-  -Dcheckstyle.skip=true \
-  -Dmaven.gitcommitid.skip=true \
-  -Dremoteresources.skip=true \
   ${EXTRA_MVN_ARGS}
 
 if [[ ! -f "${CLASSPATH_FILE}" ]]; then
@@ -617,14 +612,82 @@ TIMING_STEP3=$(timer_elapsed $TIMER_STEP3_START $TIMER_STEP3_END)
 log_timing "Step 3 - Resolve classpath" "$TIMING_STEP3"
 
 # =============================================================================
-# Step 3.5: Export-only mode (if requested)
+# Step 3.5: Create pathing JAR
+# =============================================================================
+# A "pathing JAR" is a thin JAR whose MANIFEST.MF Class-Path header lists all
+# the real classpath entries as file: URIs. This avoids exceeding OS command-line
+# length limits. Works on Java 8+ (unlike @argfile which requires Java 9+).
+# Also includes META-INF/classpath.txt for human-readable review.
+# =============================================================================
+
+PATHING_JAR="/tmp/gluten-test-pathing-$$.jar"
+rm -f "${PATHING_JAR}"
+
+PATHING_MANIFEST_DIR="/tmp/gluten-test-manifest-$$"
+mkdir -p "${PATHING_MANIFEST_DIR}/META-INF"
+
+# Convert colon-separated classpath to space-separated file: URIs for manifest.
+# Also write a human-readable classpath listing (one entry per line, same order).
+CP_URIS=""
+IFS=':' read -ra _CP_ITEMS <<< "${RESOLVED_CLASSPATH}"
+: > "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt"
+
+for _item in "${_CP_ITEMS[@]}"; do
+  [[ -z "$_item" ]] && continue
+  echo "$_item" >> "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt"
+  # Ensure directories end with / (required by Class-Path spec for directories)
+  [[ -d "$_item" && "$_item" != */ ]] && _item="${_item}/"
+  CP_URIS="${CP_URIS} file://${_item}"
+done
+CP_URIS="${CP_URIS# }"
+
+# Write manifest with proper 72-byte line wrapping.
+# MANIFEST.MF spec: max 72 bytes per line; continuation lines start with
+# a single space character.
+{
+  echo "Manifest-Version: 1.0"
+  _mf_header="Class-Path: ${CP_URIS}"
+  echo "${_mf_header:0:72}"
+  _mf_rest="${_mf_header:72}"
+  while [[ -n "$_mf_rest" ]]; do
+    echo " ${_mf_rest:0:71}"
+    _mf_rest="${_mf_rest:71}"
+  done
+  echo ""
+} > "${PATHING_MANIFEST_DIR}/META-INF/MANIFEST.MF"
+
+# Build the pathing JAR (manifest + human-readable classpath listing)
+(cd "${PATHING_MANIFEST_DIR}" && jar cfm "${PATHING_JAR}" META-INF/MANIFEST.MF META-INF/classpath.txt)
+rm -rf "${PATHING_MANIFEST_DIR}"
+log_info "Created pathing JAR: ${PATHING_JAR} ($(du -h "${PATHING_JAR}" | cut -f1))"
+log_info "  Review classpath: unzip -p ${PATHING_JAR} META-INF/classpath.txt"
+
+# =============================================================================
+# Build java command (shared by export-only and run modes)
+# =============================================================================
+
+JAVA_CMD="${JAVA_HOME}/bin/java"
+[[ ! -x "$JAVA_CMD" ]] && JAVA_CMD="java"
+
+JAVA_ARGS=(
+  ${JVM_ARGS}
+  "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties"
+  -cp "${PATHING_JAR}"
+  org.scalatest.tools.Runner
+  -oDF
+  -s "${SUITE}"
+)
+[[ -n "$TEST_METHOD" ]] && JAVA_ARGS+=(-t "${TEST_METHOD}")
+
+# =============================================================================
+# Step 3.6: Export-only mode (if requested)
 # =============================================================================
 
 if [[ "$EXPORT_ONLY" == "true" ]]; then
-  EXPORT_FILE="/tmp/gluten-classpath-exported.txt"
-  echo "$RESOLVED_CLASSPATH" > "$EXPORT_FILE"
-  log_info "✓ Classpath exported to: ${EXPORT_FILE}"
-  log_info "  Use with: java <jvm-args> -cp \"\$(cat ${EXPORT_FILE})\" org.scalatest.tools.Runner -s <suite>"
+  echo ""
+  echo -e "${YELLOW}# Run the test with:${NC}"
+  echo "${JAVA_CMD} ${JAVA_ARGS[*]}"
+  echo ""
   print_timing_summary false
   exit 0
 fi
@@ -635,12 +698,6 @@ fi
 
 log_step "Step 4: Running ScalaTest..."
 
-# Find Java
-JAVA_CMD="${JAVA_HOME}/bin/java"
-if [[ ! -x "$JAVA_CMD" ]]; then
-  JAVA_CMD="java"
-fi
-
 log_info "Suite: ${SUITE}"
 [[ -n "$TEST_METHOD" ]] && log_info "Test method: ${TEST_METHOD}"
 
@@ -650,21 +707,12 @@ echo "Running ScalaTest"
 echo "=========================================="
 echo ""
 
-# Build test method args conditionally
-TEST_METHOD_ARGS=()
-[[ -n "$TEST_METHOD" ]] && TEST_METHOD_ARGS=(-t "${TEST_METHOD}")
-
 TIMER_STEP4_START=$(timer_now)
 
 TEST_EXIT_CODE=0
-${JAVA_CMD} \
-  ${JVM_ARGS} \
-  -Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties \
-  -cp "${RESOLVED_CLASSPATH}" \
-  org.scalatest.tools.Runner \
-  -s "${SUITE}" \
-  "${TEST_METHOD_ARGS[@]}" \
-  -oDF || TEST_EXIT_CODE=$?
+${JAVA_CMD} "${JAVA_ARGS[@]}" || TEST_EXIT_CODE=$?
+
+rm -f "${PATHING_JAR}"
 
 TIMER_STEP4_END=$(timer_now)
 TIMING_STEP4=$(timer_elapsed $TIMER_STEP4_START $TIMER_STEP4_END)

docs/developers/bloop-integration.md

@@ -65,7 +65,7 @@ The `-Pbloop` profile automatically skips style checks during configuration gene
 ./dev/bloop-setup.sh -Pspark-3.5,scala-2.12,backends-velox
 
 # Manual invocation with profile
-./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,bloop -DskipTests
+./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,fast-build -DskipTests
 ```
 
 The bloop profile sets these properties automatically:

pom.xml

@@ -97,7 +97,7 @@
     -->
     <os.full.name>unknown</os.full.name>
     <!-- To build built-in backend c++ codes -->
-    <scala.recompile.mode>incremental</scala.recompile.mode>
+    <scala.recompile.mode>all</scala.recompile.mode>
     <!-- For unit tests -->
     <fasterxml.version>2.15.0</fasterxml.version>
     <junit.version>4.13.1</junit.version>
@@ -571,11 +571,6 @@
           <configuration>
             <encoding>UTF-8</encoding>
             <maxmem>1024m</maxmem>
-            <!-- Skip javac entirely. In incremental mode (recompileMode=incremental),
-                 Zinc compiles both Scala and Java sources together, making
-                 maven-compiler-plugin redundant. Same approach as Apache Spark. -->
-            <skipMain>true</skipMain>
-            <skip>true</skip>
           </configuration>
         </plugin>
         <plugin>
@@ -1105,6 +1100,9 @@
       </activation>
       <properties>
         <java.version>1.8</java.version>
+        <!-- scala-maven-plugin 4.9.2 injects '-release 1.8' into scalac args,
+             which fails under JDK 8. Pin to 4.8.0 where this bug does not exist. -->
+        <scala.compiler.version>4.8.0</scala.compiler.version>
       </properties>
     </profile>
     <profile>
@@ -2147,18 +2145,37 @@
       </build>
     </profile>
     <profile>
-      <id>bloop</id>
+      <id>fast-build</id>
       <activation>
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
-        <!-- Skip style checks during bloop configuration generation -->
+        <!-- Skip style checks for faster local builds -->
         <spotless.check.skip>true</spotless.check.skip>
         <scalastyle.skip>true</scalastyle.skip>
         <checkstyle.skip>true</checkstyle.skip>
         <maven.gitcommitid.skip>true</maven.gitcommitid.skip>
         <remoteresources.skip>true</remoteresources.skip>
+        <!-- Use incremental Scala compilation (Zinc state is warm in local/dev contexts) -->
+        <scala.recompile.mode>incremental</scala.recompile.mode>
       </properties>
+      <build>
+        <pluginManagement>
+          <plugins>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-compiler-plugin</artifactId>
+              <configuration>
+                <!-- Skip javac entirely. In incremental mode (recompileMode=incremental),
+                     Zinc compiles both Scala and Java sources together, making
+                     maven-compiler-plugin redundant. Same approach as Apache Spark. -->
+                <skipMain>true</skipMain>
+                <skip>true</skip>
+              </configuration>
+            </plugin>
+          </plugins>
+        </pluginManagement>
+      </build>
     </profile>
 
     <!-- Profiles for different platforms -->