Skip to content

Support for latest Spark version in oozie workflow xml #110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sushrutikhar wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Support for latest Spark version in oozie workflow xml #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6f5db41
latest spark changes
Jul 23, 2018
0ae0de6
pom snapshot version change revert
Jul 23, 2018
e88aa4e
merge conflicts resolved
Jul 23, 2018
6f45b0e
update pom version
Jul 25, 2018
2637d26
spark duplicate uri patch
Jul 26, 2018
be531e2
commons lang3
Oct 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
public class SparkActionExecutor extends JavaActionExecutor {
public static final String SPARK_MAIN_CLASS_NAME_1 = "org.apache.oozie.action.hadoop.SparkMain";
public static final String SPARK_MAIN_CLASS_NAME_2 = "org.apache.oozie.action.hadoop.SparkMain2";
public static final String SPARK_MAIN_CLASS_NAME_LATEST = "org.apache.oozie.action.hadoop.SparkMainLatest";

public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; // hadoop-2
public static final String TASK_USER_CLASSPATH_PRECEDENCE = "mapreduce.user.classpath.first"; // hadoop-1
Expand Down Expand Up @@ -143,9 +144,11 @@ Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appP
public List<Class> getLauncherClasses() {
List<Class> classes = new ArrayList<Class>();
try {
if("2".equals(sparkVersion)){
if("0".equals(sparkVersion)){
classes.add(Class.forName(SPARK_MAIN_CLASS_NAME_LATEST));
} else if("2".equals(sparkVersion)){
classes.add(Class.forName(SPARK_MAIN_CLASS_NAME_2));
}else {
} else {
classes.add(Class.forName(SPARK_MAIN_CLASS_NAME_1));
}
} catch (ClassNotFoundException e) {
Expand All @@ -163,19 +166,24 @@ public List<Class> getLauncherClasses() {
*/
@Override
protected String getDefaultShareLibName(Element actionXml) {
if("2".equals(sparkVersion)) {
if("0".equals(sparkVersion)) {
return "sparkLatest";
} else if("2".equals(sparkVersion)) {
return "spark2";
}else{
} else{
return "spark";
}
}

@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
if("2".equals(sparkVersion)){
if("0".equals(sparkVersion)){
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME_LATEST);

} else if("2".equals(sparkVersion)){
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME_2);

}else {
} else {
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME_1);
}
}
Expand Down
8 changes: 6 additions & 2 deletions distro/src/deb/control/postinst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ if [ "$DEPLOYMENT_MODE" == "ONLY_SHARELIB" ]; then
fi

DISTRO=$(ls /opt/inmobi/${OOZIE_VERSION}|grep distro|grep tar.gz|grep $OOZIE_VERSION)
sudo mkdir $OOZIE_BASE/$OOZIE_SHARELIB
sudo mkdir -p $OOZIE_BASE/$OOZIE_SHARELIB
sudo tar -xvf /opt/inmobi/${OOZIE_VERSION}/$DISTRO -C $OOZIE_BASE/$OOZIE_SHARELIB

cd $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION
Expand All @@ -118,6 +118,7 @@ if [ "$DEPLOYMENT_MODE" == "ONLY_SHARELIB" ]; then
sudo mkdir -p $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/hive2/
sudo mkdir -p $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/spark/
sudo mkdir -p $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/spark2/
sudo mkdir -p $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/sparkLatest/
cd $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION

sudo cp $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/share/lib/oozie/* $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/oozie/ || true
Expand All @@ -126,6 +127,7 @@ if [ "$DEPLOYMENT_MODE" == "ONLY_SHARELIB" ]; then
sudo cp $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/share/lib/hive2/* $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/hive2/ || true
sudo cp $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/share/lib/spark/* $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/spark/ || true
sudo cp $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/share/lib/spark2/* $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/spark2/ || true
sudo cp $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/share/lib/sparkLatest/* $OOZIE_BASE/$OOZIE_SHARELIB/$OOZIE_VERSION/temp_sharelib/lib/sparkLatest/ || true

sudo chown -R oozie:oozie $OOZIE_CURRENT/ || true
sudo chown -R oozie $OOZIE_CURRENT/ || true
Expand All @@ -152,7 +154,7 @@ else
sudo mv $OOZIE_BASE/$OOZIE_VERSION $OOZIE_CURRENT

cd $OOZIE_CURRENT
sudo mkdir $OOZIE_CURRENT/libext
sudo mkdir -p $OOZIE_CURRENT/libext
cd $OOZIE_CURRENT/libext
sudo cp $HADOOP_CLIENT/*.jar . || true
sudo cp $HADOOP_CLIENT/lib/*.jar . || true
Expand Down Expand Up @@ -189,6 +191,7 @@ else
sudo mkdir -p $OOZIE_CURRENT/temp_sharelib/lib/hive2/
sudo mkdir -p $OOZIE_CURRENT/temp_sharelib/lib/spark/
sudo mkdir -p $OOZIE_CURRENT/temp_sharelib/lib/spark2/
sudo mkdir -p $OOZIE_CURRENT/temp_sharelib/lib/sparkLatest/
sudo mkdir -p $OOZIE_CURRENT/temp_sharelib/lib/distcp/

cd $OOZIE_CURRENT
Expand All @@ -199,6 +202,7 @@ else
sudo cp $OOZIE_CURRENT/share/lib/hive2/* $OOZIE_CURRENT/temp_sharelib/lib/hive2/ || true
sudo cp $OOZIE_CURRENT/share/lib/spark/* $OOZIE_CURRENT/temp_sharelib/lib/spark/ || true
sudo cp $OOZIE_CURRENT/share/lib/spark2/* $OOZIE_CURRENT/temp_sharelib/lib/spark2/ || true
sudo cp $OOZIE_CURRENT/share/lib/sparkLatest/* $OOZIE_CURRENT/temp_sharelib/lib/sparkLatest/ || true
sudo cp $OOZIE_CURRENT/libext/s4*jar $OOZIE_CURRENT/temp_sharelib/lib/oozie/ || true
sudo cp $OOZIE_CURRENT/libext/s4*jar $OOZIE_CURRENT/libext/ || true
sudo cp $OOZIE_CURRENT/libext/falcon*jar $OOZIE_CURRENT/libext/ || true
Expand Down
12 changes: 12 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@
<spark.bagel.version.2>1.6.3</spark.bagel.version.2>
<spark.scala.binary.version.2>2.11</spark.scala.binary.version.2>

<!-- spark latest version -->
<spark.version.latest>2.3.1</spark.version.latest>
<spark.streaming.kafka.version.latest>1.6.3</spark.streaming.kafka.version.latest>
<spark.bagel.version.latest>1.6.3</spark.bagel.version.latest>
<spark.scala.binary.version.latest>2.11</spark.scala.binary.version.latest>
<spark.commons.lang3.version.latest>3.5</spark.commons.lang3.version.latest>


<spark.guava.version>14.0.1</spark.guava.version>

Expand Down Expand Up @@ -294,6 +301,11 @@
<artifactId>oozie-sharelib-spark2</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-sharelib-sparkLatest</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-docs</artifactId>
Expand Down
1 change: 1 addition & 0 deletions sharelib/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
<module>distcp</module>
<module>spark</module>
<module>spark2</module>
<module>sparkLatest</module>
</modules>

<build>
Expand Down
Loading