[GLUTEN-10574][1.5] Backport #10793 and #10807 to update documents and automate release process

.github/workflows/build_release.yml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build Release
+
+# Only triggered when new tag like v1.5.0 is created.
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build-velox-backend-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build
+        run: |
+          docker pull apache/gluten:vcpkg-centos-7
+          docker run -v $GITHUB_WORKSPACE:/workspace -w /work apache/gluten:vcpkg-centos-7 bash -c "
+            set -e
+            cd /workspace
+            bash dev/release/build-release.sh
+          "
+      - name: Upload packages
+        if: ${{ success() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-packages
+          path: ${{ github.workspace }}/package/target/gluten-velox-bundle-*.jar

dev/release/build-release.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+source /opt/rh/devtoolset-11/enable
+source /opt/rh/rh-git227/enable
+
+CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+GLUTEN_HOME=${CURRENT_DIR}/../../
+cd ${GLUTEN_HOME}
+
+# Enable static build with support for S3, GCS, HDFS, and ABFS.
+./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
+                           --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
+
+JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
+
+if [[ $JAVA_VERSION == 1.8* ]]; then
+  echo "Java 8 is being used."
+else
+  echo "Error: Java 8 is required. Current version is $JAVA_VERSION."
+  exit 1
+fi
+
+# Build Gluten for Spark 3.2 and 3.3 with Java 8. All feature modules are enabled.
+for spark_version in 3.2 3.3
+do
+  mvn clean install -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \
+                    -Piceberg,delta,hudi,paimon -DskipTests
+done
+
+sudo curl -Lo /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo
+sudo yum install -y java-17-amazon-corretto-devel
+export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto
+export PATH=$JAVA_HOME/bin:$PATH
+
+JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
+if [[ $JAVA_VERSION == 17* ]]; then
+  echo "Java 17 is being used."
+else
+  echo "Error: Java 17 is required. Current version is $JAVA_VERSION."
+  exit 1
+fi
+
+# Build Gluten for Spark 3.4 and 3.5 with Java 17. The version of Iceberg being used requires Java 11 or higher.
+# All feature modules are enabled.
+for spark_version in 3.4 3.5
+do
+  mvn clean install -Pjava-17 -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \
+                    -Piceberg,delta,hudi,paimon -DskipTests
+done

dev/release/package-release.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: Manually create $GLUTEN_HOME/release/ and place the release JARs inside.
+#       Also set TAG manually (e.g., v1.5.0).
+
+set -eu
+
+TAG=""
+
+if [[ -z "$TAG" ]]; then
+  echo "TAG is not set. Please set TAG=vX.Y.Z before running."
+  exit 1
+fi
+
+VERSION=${TAG#v}
+
+CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+GLUTEN_HOME=${CURRENT_DIR}/../../
+if [ ! -d "$GLUTEN_HOME/release/" ]; then
+  echo "Release directory does not exist."
+fi
+pushd $GLUTEN_HOME/release/
+
+SPARK_VERSIONS="3.2 3.3 3.4 3.5"
+
+for v in $SPARK_VERSIONS; do
+  JAR="gluten-velox-bundle-spark${v}_2.12-linux_amd64-${VERSION}.jar"
+  if [[ ! -f "$JAR" ]]; then
+    echo "Missing Gluten release JAR under $GLUTEN_HOME/release/ for Spark $v: $JAR"
+    exit 1
+  fi
+  echo "Packaging for Spark $v..."
+  tar -czf apache-gluten-$VERSION-incubating-bin-spark-${v}.tar.gz \
+      ${GLUTEN_HOME}/DISCLAIMER \
+      $JAR
+done
+
+SRC_ZIP="${TAG}.zip"
+SRC_DIR="incubator-gluten-${VERSION}"
+
+echo "Packaging source code..."
+wget https://github.com/apache/incubator-gluten/archive/refs/tags/${SRC_ZIP}
+unzip -q ${SRC_ZIP}
+tar -czf apache-gluten-$VERSION-incubating-src.tar.gz ${SRC_DIR}
+rm -r ${SRC_ZIP} ${SRC_DIR}
+
+popd
+echo "Finished packaging release binaries and source code."