diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml new file mode 100644 index 000000000000..cec50c92b1f4 --- /dev/null +++ b/.github/workflows/build_release.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build Release + +# Only triggered when new tag like v1.5.0 is created. +on: + push: + tags: + - 'v*' + +jobs: + build-velox-backend-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build + run: | + docker pull apache/gluten:vcpkg-centos-7 + docker run -v $GITHUB_WORKSPACE:/workspace -w /work apache/gluten:vcpkg-centos-7 bash -c " + set -e + cd /workspace + bash dev/release/build-release.sh + " + - name: Upload packages + if: ${{ success() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-packages + path: ${{ github.workspace }}/package/target/gluten-velox-bundle-*.jar diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh new file mode 100755 index 000000000000..ec4a509c729d --- /dev/null +++ b/dev/release/build-release.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +source /opt/rh/devtoolset-11/enable +source /opt/rh/rh-git227/enable + +CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) +GLUTEN_HOME=${CURRENT_DIR}/../../ +cd ${GLUTEN_HOME} + +# Enable static build with support for S3, GCS, HDFS, and ABFS. +./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON + +JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}') + +if [[ $JAVA_VERSION == 1.8* ]]; then + echo "Java 8 is being used." +else + echo "Error: Java 8 is required. Current version is $JAVA_VERSION." + exit 1 +fi + +# Build Gluten for Spark 3.2 and 3.3 with Java 8. All feature modules are enabled. +for spark_version in 3.2 3.3 +do + mvn clean install -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \ + -Piceberg,delta,hudi,paimon -DskipTests +done + +sudo curl -Lo /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo +sudo yum install -y java-17-amazon-corretto-devel +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto +export PATH=$JAVA_HOME/bin:$PATH + +JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}') +if [[ $JAVA_VERSION == 17* ]]; then + echo "Java 17 is being used." +else + echo "Error: Java 17 is required. Current version is $JAVA_VERSION." + exit 1 +fi + +# Build Gluten for Spark 3.4 and 3.5 with Java 17. The version of Iceberg being used requires Java 11 or higher. +# All feature modules are enabled. +for spark_version in 3.4 3.5 +do + mvn clean install -Pjava-17 -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \ + -Piceberg,delta,hudi,paimon -DskipTests +done diff --git a/dev/release/package-release.sh b/dev/release/package-release.sh new file mode 100755 index 000000000000..bb0de30259c7 --- /dev/null +++ b/dev/release/package-release.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note: Manually create $GLUTEN_HOME/release/ and place the release JARs inside. +# Also set TAG manually (e.g., v1.5.0). + +set -eu + +TAG="" + +if [[ -z "$TAG" ]]; then + echo "TAG is not set. Please set TAG=vX.Y.Z before running." + exit 1 +fi + +VERSION=${TAG#v} + +CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) +GLUTEN_HOME=${CURRENT_DIR}/../../ +if [ ! -d "$GLUTEN_HOME/release/" ]; then + echo "Release directory does not exist." +fi +pushd $GLUTEN_HOME/release/ + +SPARK_VERSIONS="3.2 3.3 3.4 3.5" + +for v in $SPARK_VERSIONS; do + JAR="gluten-velox-bundle-spark${v}_2.12-linux_amd64-${VERSION}.jar" + if [[ ! -f "$JAR" ]]; then + echo "Missing Gluten release JAR under $GLUTEN_HOME/release/ for Spark $v: $JAR" + exit 1 + fi + echo "Packaging for Spark $v..." + tar -czf apache-gluten-$VERSION-incubating-bin-spark-${v}.tar.gz \ + ${GLUTEN_HOME}/DISCLAIMER \ + $JAR +done + +SRC_ZIP="${TAG}.zip" +SRC_DIR="incubator-gluten-${VERSION}" + +echo "Packaging source code..." +wget https://github.com/apache/incubator-gluten/archive/refs/tags/${SRC_ZIP} +unzip -q ${SRC_ZIP} +tar -czf apache-gluten-$VERSION-incubating-src.tar.gz ${SRC_DIR} +rm -r ${SRC_ZIP} ${SRC_DIR} + +popd +echo "Finished packaging release binaries and source code."