Set tmpjars for hadoop

hraven-core/src/main/java/com/twitter/hraven/Constants.java

@@ -426,4 +426,8 @@ public class Constants {
 
   /** name of the properties file used for cluster to cluster identifier mapping */
   public static final String HRAVEN_CLUSTER_PROPERTIES_FILENAME = "hRavenClusters.properties";
+
+  public static final String HRAVEN_HDFS_LIB_PATH_CONF = "hraven.conf.libpath";
+
+  public static final String HADOOP_TMP_JARS_CONF = "tmpjars";
 }

hraven-etl/src/main/java/com/twitter/hraven/etl/HadoopUtil.java

@@ -0,0 +1,27 @@
+package com.twitter.hraven.etl;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.twitter.hraven.Constants;
+
+public class HadoopUtil {
+  public static void setTmpJars(String libPathConf, Configuration conf) throws IOException {
+    StringBuilder tmpjars = new StringBuilder();
+    if (conf.get(libPathConf) != null) {
+      FileSystem fs = FileSystem.get(conf);
+      FileStatus[] files = fs.listStatus(new Path(conf.get(libPathConf)));
+      if (files != null) {
+        for (FileStatus file : files) {
+          if (!tmpjars.toString().isEmpty()) tmpjars = tmpjars.append(",");
+          tmpjars = tmpjars.append(file.getPath());
+        }
+        conf.set(Constants.HADOOP_TMP_JARS_CONF, tmpjars.toString());
+      }
+    }
+  }
+}

hraven-etl/src/main/java/com/twitter/hraven/etl/JobFileProcessor.java

@@ -603,6 +603,9 @@ private Job getProcessingJob(Configuration conf, Scan scan, int totalJobCount)
     // Note: must be BEFORE the job construction with the new mapreduce API.
     confClone.setBoolean("mapred.map.tasks.speculative.execution", false);
 
+    //Set tmpjars for hadoop to be able to find hraven-core and other required libs
+    HadoopUtil.setTmpJars(Constants.HRAVEN_HDFS_LIB_PATH_CONF, confClone);
+
     // Set up job
     Job job = new Job(confClone, getJobName(totalJobCount));
 

hraven-etl/src/main/java/com/twitter/hraven/etl/JobFileRawLoader.java

@@ -282,6 +282,9 @@ private boolean runRawLoaderJob(Configuration myHBaseConf, String input,
     // Note: must be BEFORE the job construction with the new mapreduce API.
     myHBaseConf.setBoolean("mapred.map.tasks.speculative.execution", false);
 
+    // Set tmpjars for hadoop to be able to find hraven-core and other required libs
+    HadoopUtil.setTmpJars(Constants.HRAVEN_HDFS_LIB_PATH_CONF, myHBaseConf);
+
     // Set up job
     Job job = new Job(myHBaseConf, getJobName(totalJobCount));
     job.setJarByClass(JobFileRawLoader.class);