diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index a91617eff9c2..8d5bc71aec6f 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1338,6 +1338,16 @@ jobs: ccache -s " + build-fast-build-test: + runs-on: ubuntu-22.04 + container: apache/gluten:centos-8-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Build with fast-build profile (Spark 4.0, Java 17) + run: | + cd $GITHUB_WORKSPACE/ + $MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13 -Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build + spark-test-spark40: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 diff --git a/dev/bloop-setup.sh b/dev/bloop-setup.sh index e633e2453b25..8a2eb0b78c0b 100755 --- a/dev/bloop-setup.sh +++ b/dev/bloop-setup.sh @@ -190,7 +190,7 @@ fi # Skip style checks for faster generation log_step "Running generate-sources + bloopInstall..." ./build/mvn generate-sources bloop:bloopInstall \ - -P"${PROFILES}",bloop \ + -P"${PROFILES}",fast-build \ -DskipTests if [[ ! -d ".bloop" ]]; then diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index 6f3f8688d3c7..cfe195f9399f 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -527,14 +527,9 @@ fi ${MVN_CMD} ${MVN_GOALS} \ -T 1C -q \ -pl "${MODULE}" -am \ - -P"${PROFILES}" \ + -P"${PROFILES},fast-build" \ -DincludeScope=test \ -Dmdep.outputFile="${CLASSPATH_FILE}" \ - -Dspotless.check.skip=true \ - -Dscalastyle.skip=true \ - -Dcheckstyle.skip=true \ - -Dmaven.gitcommitid.skip=true \ - -Dremoteresources.skip=true \ ${EXTRA_MVN_ARGS} if [[ ! -f "${CLASSPATH_FILE}" ]]; then @@ -617,14 +612,82 @@ TIMING_STEP3=$(timer_elapsed $TIMER_STEP3_START $TIMER_STEP3_END) log_timing "Step 3 - Resolve classpath" "$TIMING_STEP3" # ============================================================================= -# Step 3.5: Export-only mode (if requested) +# Step 3.5: Create pathing JAR +# ============================================================================= +# A "pathing JAR" is a thin JAR whose MANIFEST.MF Class-Path header lists all +# the real classpath entries as file: URIs. This avoids exceeding OS command-line +# length limits. Works on Java 8+ (unlike @argfile which requires Java 9+). +# Also includes META-INF/classpath.txt for human-readable review. +# ============================================================================= + +PATHING_JAR="/tmp/gluten-test-pathing-$$.jar" +rm -f "${PATHING_JAR}" + +PATHING_MANIFEST_DIR="/tmp/gluten-test-manifest-$$" +mkdir -p "${PATHING_MANIFEST_DIR}/META-INF" + +# Convert colon-separated classpath to space-separated file: URIs for manifest. +# Also write a human-readable classpath listing (one entry per line, same order). +CP_URIS="" +IFS=':' read -ra _CP_ITEMS <<< "${RESOLVED_CLASSPATH}" +: > "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + +for _item in "${_CP_ITEMS[@]}"; do + [[ -z "$_item" ]] && continue + echo "$_item" >> "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + # Ensure directories end with / (required by Class-Path spec for directories) + [[ -d "$_item" && "$_item" != */ ]] && _item="${_item}/" + CP_URIS="${CP_URIS} file://${_item}" +done +CP_URIS="${CP_URIS# }" + +# Write manifest with proper 72-byte line wrapping. +# MANIFEST.MF spec: max 72 bytes per line; continuation lines start with +# a single space character. +{ + echo "Manifest-Version: 1.0" + _mf_header="Class-Path: ${CP_URIS}" + echo "${_mf_header:0:72}" + _mf_rest="${_mf_header:72}" + while [[ -n "$_mf_rest" ]]; do + echo " ${_mf_rest:0:71}" + _mf_rest="${_mf_rest:71}" + done + echo "" +} > "${PATHING_MANIFEST_DIR}/META-INF/MANIFEST.MF" + +# Build the pathing JAR (manifest + human-readable classpath listing) +(cd "${PATHING_MANIFEST_DIR}" && jar cfm "${PATHING_JAR}" META-INF/MANIFEST.MF META-INF/classpath.txt) +rm -rf "${PATHING_MANIFEST_DIR}" +log_info "Created pathing JAR: ${PATHING_JAR} ($(du -h "${PATHING_JAR}" | cut -f1))" +log_info " Review classpath: unzip -p ${PATHING_JAR} META-INF/classpath.txt" + +# ============================================================================= +# Build java command (shared by export-only and run modes) +# ============================================================================= + +JAVA_CMD="${JAVA_HOME}/bin/java" +[[ ! -x "$JAVA_CMD" ]] && JAVA_CMD="java" + +JAVA_ARGS=( + ${JVM_ARGS} + "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties" + -cp "${PATHING_JAR}" + org.scalatest.tools.Runner + -oDF + -s "${SUITE}" +) +[[ -n "$TEST_METHOD" ]] && JAVA_ARGS+=(-t "${TEST_METHOD}") + +# ============================================================================= +# Step 3.6: Export-only mode (if requested) # ============================================================================= if [[ "$EXPORT_ONLY" == "true" ]]; then - EXPORT_FILE="/tmp/gluten-classpath-exported.txt" - echo "$RESOLVED_CLASSPATH" > "$EXPORT_FILE" - log_info "✓ Classpath exported to: ${EXPORT_FILE}" - log_info " Use with: java -cp \"\$(cat ${EXPORT_FILE})\" org.scalatest.tools.Runner -s " + echo "" + echo -e "${YELLOW}# Run the test with:${NC}" + echo "${JAVA_CMD} ${JAVA_ARGS[*]}" + echo "" print_timing_summary false exit 0 fi @@ -635,12 +698,6 @@ fi log_step "Step 4: Running ScalaTest..." -# Find Java -JAVA_CMD="${JAVA_HOME}/bin/java" -if [[ ! -x "$JAVA_CMD" ]]; then - JAVA_CMD="java" -fi - log_info "Suite: ${SUITE}" [[ -n "$TEST_METHOD" ]] && log_info "Test method: ${TEST_METHOD}" @@ -650,21 +707,12 @@ echo "Running ScalaTest" echo "==========================================" echo "" -# Build test method args conditionally -TEST_METHOD_ARGS=() -[[ -n "$TEST_METHOD" ]] && TEST_METHOD_ARGS=(-t "${TEST_METHOD}") - TIMER_STEP4_START=$(timer_now) TEST_EXIT_CODE=0 -${JAVA_CMD} \ - ${JVM_ARGS} \ - -Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties \ - -cp "${RESOLVED_CLASSPATH}" \ - org.scalatest.tools.Runner \ - -s "${SUITE}" \ - "${TEST_METHOD_ARGS[@]}" \ - -oDF || TEST_EXIT_CODE=$? +${JAVA_CMD} "${JAVA_ARGS[@]}" || TEST_EXIT_CODE=$? + +rm -f "${PATHING_JAR}" TIMER_STEP4_END=$(timer_now) TIMING_STEP4=$(timer_elapsed $TIMER_STEP4_START $TIMER_STEP4_END) diff --git a/docs/developers/bloop-integration.md b/docs/developers/bloop-integration.md index 2364b8f596c7..c5efdbbd528f 100644 --- a/docs/developers/bloop-integration.md +++ b/docs/developers/bloop-integration.md @@ -65,7 +65,7 @@ The `-Pbloop` profile automatically skips style checks during configuration gene ./dev/bloop-setup.sh -Pspark-3.5,scala-2.12,backends-velox # Manual invocation with profile -./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,bloop -DskipTests +./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,fast-build -DskipTests ``` The bloop profile sets these properties automatically: diff --git a/pom.xml b/pom.xml index 9af9927f9eee..d34633ecf69d 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ --> unknown - incremental + all 2.15.0 4.13.1 @@ -571,11 +571,6 @@ UTF-8 1024m - - true - true @@ -1105,6 +1100,9 @@ 1.8 + + 4.8.0 @@ -2147,18 +2145,37 @@ - bloop + fast-build false - + true true true true true + + incremental + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + true + true + + + + +