diff --git a/src/main/java/com/scienaptic/Application.java b/src/main/java/com/scienaptic/Application.java index 6908556..fb930b8 100644 --- a/src/main/java/com/scienaptic/Application.java +++ b/src/main/java/com/scienaptic/Application.java @@ -11,15 +11,14 @@ public class Application extends MistFn { @Override public Handle handle() { - return withArgs(stringArg("inputPath")). + return withArgs(stringArg("inputPath")). //Give complete hdfs path hdfs:///filepath for parameter inputPath while running the job withMistExtras(). - onSparkSession((path, extras, SparkSession) -> { - String filePath = "file://" + path; + onSparkSession((filePath, extras, SparkSession) -> { Dataset df = SparkSession.read() .format("csv") .option("header",true) .load(filePath); - String fileName = filePath.substring(0,path.lastIndexOf('.')); + String fileName = filePath.substring(0,filePath.lastIndexOf('.')); String outputPath = fileName + ".parquet"; df.write().mode("overwrite").parquet(outputPath); return outputPath;