Implementing Run Spark Job all in a single task

.gitignore

@@ -85,4 +85,6 @@ package-lock.json
 npm-debug.log
 .override-file.json
 tasks/ExecuteNotebook/ExecuteNotebookV1/job-configuration.json
-.taskkey
+.taskkey
+tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/job-configuration.json
+tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/last-run.txt

azure-pipelines-cd.yml

@@ -1,4 +1,4 @@
-name: 0.5$(Rev:.r)
+name: 0.6$(Rev:.r)
 trigger:
   branches:
     include:

azure-pipelines.yml

@@ -1,13 +1,14 @@
-name: 0.5$(Rev:.r)
+name: 0.6$(Rev:.r)
 trigger:
   branches:
-    include:
-    - '*'
     exclude:
     - master
   paths:
     exclude:
     - README.md
+    - CONTRIBUTING.md
+    - LICENSE
+    - 'docs/*'
 
 pool:
   vmImage: "ubuntu-latest"

package.json

@@ -24,8 +24,8 @@
     "build.scala.runSbtTests": "tsc -p ./tasks/Scala/RunSbtTests/RunSbtTestsV1",
     "build.scala.startCluster": "tsc -p ./tasks/Scala/StartCluster/StartClusterV1",
     "build.scala.uninstallCodeFromCluster": "tsc -p ./tasks/Scala/UninstallCodeFromCluster/UninstallCodeFromClusterV1",
-    "build.scala.waitForClusterReboot": "tsc -p ./tasks/Scala/WaitForClusterReboot/WaitForClusterRebootV1",
-    "build": "npm run recursive-install && npm run build.deployNotebook && npm run build.configureDatabricks && npm run build.executeNotebook && npm run build.waitExecution && npm run build.scala.installScalaTools && npm run build.scala.installSpark && npm run build.scala.runSbtTests && npm run build.scala.startCluster && npm run build.scala.uninstallCodeFromCluster && npm run build.scala.waitForClusterReboot",
+    "build.executeDatabricksJob": "tsc -p ./tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1",
+    "build": "npm run recursive-install && npm run build.deployNotebook && npm run build.configureDatabricks && npm run build.executeNotebook && npm run build.waitExecution && npm run build.scala.installScalaTools && npm run build.scala.installSpark && npm run build.scala.runSbtTests && npm run build.scala.startCluster && npm run build.scala.uninstallCodeFromCluster && npm run build.executeDatabricksJob",
     "test.deployNotebook": "npm run build.deployNotebook && mocha ./DeployNotebooksTask/DeployNotebooksTaskV1/tests/_suite.js",
     "test": "npm run test.deployNotebook",
     "package.dev": "npm run test && tfx extension create --manifest-globs vss-extension.json --overrides-file --output-path out",

tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/executedatabricksjob.js

@@ -0,0 +1,57 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = require("path");
+const tl = require("azure-pipelines-task-lib/task");
+const shell = require("shelljs");
+const clusterid = tl.getInput('clusterid', true);
+const failOnStderr = tl.getBoolInput('failOnStderr', false);
+function runJarJob() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const packageName = tl.getInput('packageName', true);
+        const mainClassName = tl.getInput('mainClassName', true);
+        const jarParameters = tl.getInput('jarParameters', false);
+        let jarParametersJson = JSON.stringify(jarParameters);
+        let fileName = 'executedatabricksjob.sh';
+        let filePath = path.join(__dirname, fileName);
+        let runJobExec = shell.exec(`bash ${filePath} ${clusterid} ${packageName} ${mainClassName} ${jarParametersJson}`.trim());
+        if (runJobExec.code != 0) {
+            tl.setResult(tl.TaskResult.Failed, `Error while executing command: ${runJobExec.stderr}`);
+        }
+        if (failOnStderr && runJobExec.stderr != "") {
+            tl.setResult(tl.TaskResult.Failed, `Command wrote to stderr: ${runJobExec.stderr}`);
+        }
+    });
+}
+function runNotebookJob() {
+    return __awaiter(this, void 0, void 0, function* () {
+    });
+}
+function run() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            tl.setResourcePath(path.join(__dirname, 'task.json'));
+            const targetType = tl.getInput('targetType');
+            if (targetType.toUpperCase() == "JARJOB") {
+                yield runJarJob();
+            }
+            else if (targetType.toUpperCase() == "NOTEBOOKJOB") {
+                yield runNotebookJob();
+            }
+            else {
+                tl.setResult(tl.TaskResult.Failed, "Could not retrieve Job Type.");
+            }
+        }
+        catch (err) {
+            tl.setResult(tl.TaskResult.Failed, err.message);
+        }
+    });
+}
+run();

tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/executedatabricksjob.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+# ===================================================================================
+#
+# FILE:  1-run-jar.sh
+#
+# USAGE:  bash 1-run-jar.sh ...
+#
+#   DESCRIPTION: Uses Databricks API to launch Spark Job.
+#                Relies on an existing JAR file being present.
+#                Uses API api/2.0/jobs/create
+#                         api/2.0/jobs/run-now
+#                Results in a Run Id that is needed later to validate SUCCESS
+#
+# NOTES:  ---
+# AUTHOR:  Bruno Terkaly
+# VERSION:  1.0
+# CREATED:  June 10, 2019
+#===================================================================================
+
+#---------Create job
+
+clusterid=$1
+packagename=$2
+mainclassname=$3
+additionalparams=$4
+jobrunid=-1
+
+createAndRunJob() {
+  echo "Run a job"
+  cat > job-configuration.json << EOF
+{
+  "name": "MySparkJob",
+  "existing_cluster_id": "$clusterid",
+  "libraries": [
+    {
+      "jar": "dbfs:/jar/$packagename.jar"
+    }
+  ],
+  "spark_jar_task": {
+    "main_class_name": "$mainclassname"
+  }
+}
+EOF
+  cat job-configuration.json
+
+  result=$(databricks jobs create --json-file job-configuration.json --profile AZDO)
+  echo "result = $result"
+  echo "Finished creating Databricks Job"
+
+  jobid=$(echo $result | jq -r ".job_id")
+  echo "=================================="
+  echo "Job id "$jobid
+  echo "=================================="
+
+  #---------Run the job
+
+  echo "Additional params: $additionalparams"
+
+  if [ "$additionalparams" == "" ]; then
+      echo "No additional params passed."
+      result=$(databricks jobs run-now --job-id $jobid --profile AZDO)
+  else
+      result=$(databricks jobs run-now --job-id $jobid --jar-params "$additionalparams" --profile AZDO)
+  fi
+  echo "result = $result"
+  runid=`echo $result | jq -r ".run_id"`
+  number_in_job=`echo $result | jq ".number_in_job"`
+  echo "number_in_job = "$number_in_job
+
+  echo "=================================="
+  echo "Run id = "$runid
+  echo "Number in Job = "$number_in_job
+  echo "=================================="
+  jobrunid=$runid
+}
+
+waitJobExecution() {
+  echo "run_id = "$jobrunid
+
+  result=$(databricks runs get --run-id $jobrunid --profile AZDO | jq -r '.state.result_state')
+
+  if [ "$result" == "null" ]
+  then
+    while [ "$result" == "null" ]
+    do
+      echo "Job still running..."
+      result=$(databricks runs get --run-id $jobrunid --profile AZDO | jq -r '.state.result_state')
+      sleep 10
+    done
+  fi
+
+  echo "result = $result"
+  if [ "$result" == "SUCCESS" ]
+  then
+    echo "-------------------------------"
+    echo "Success for last run of "$jobrunid
+    echo "-------------------------------"
+  else
+    echo "-------------------------------"
+    echo "Failure for last run of "$jobrunid
+    echo "-------------------------------"
+    exit 1
+  fi
+}
+
+createAndRunJob
+waitJobExecution

tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/executedatabricksjob.ts

@@ -0,0 +1,53 @@
+import path = require('path');
+import tl = require('azure-pipelines-task-lib/task');
+import shell = require('shelljs');
+import { async } from 'q';
+
+const clusterid: string = tl.getInput('clusterid', true);
+const failOnStderr: boolean = tl.getBoolInput('failOnStderr', false);
+
+async function runJarJob(){
+    const packageName: string = tl.getInput('packageName', true);
+    const mainClassName: string = tl.getInput('mainClassName', true);
+    const jarParameters: string = tl.getInput('jarParameters', false);
+
+    let jarParametersJson = JSON.stringify(jarParameters);
+
+    let fileName = 'executedatabricksjob.sh';
+    let filePath = path.join(__dirname, fileName);
+
+    let runJobExec = shell.exec(`bash ${filePath} ${clusterid} ${packageName} ${mainClassName} ${jarParametersJson}`.trim());
+
+    if(runJobExec.code != 0) {
+        tl.setResult(tl.TaskResult.Failed, `Error while executing command: ${runJobExec.stderr}`);
+    }
+
+    if(failOnStderr && runJobExec.stderr != "") {
+        tl.setResult(tl.TaskResult.Failed, `Command wrote to stderr: ${runJobExec.stderr}`);
+    }
+}
+
+async function runNotebookJob() {
+
+}
+
+async function run() {
+    try {
+        tl.setResourcePath(path.join(__dirname, 'task.json'));
+
+        const targetType: string = tl.getInput('targetType');
+
+        if(targetType.toUpperCase() == "JARJOB"){
+            await runJarJob();
+        } else if(targetType.toUpperCase() == "NOTEBOOKJOB"){
+            await runNotebookJob();
+        } else {
+            tl.setResult(tl.TaskResult.Failed, "Could not retrieve Job Type.");
+        }
+    }
+    catch (err) {
+        tl.setResult(tl.TaskResult.Failed, err.message);
+    }
+}
+
+run();

tasks/Scala/WaitForClusterReboot/WaitForClusterRebootV1/package.json → tasks/ExecuteDatabricksJob/ExecuteDatabricksJobV1/package.json

@@ -1,8 +1,8 @@
 {
-  "name": "waitforclusterreboot",
+  "name": "executedatabricksjob",
   "version": "1.0.0",
-  "description": "Waits for a given cluster to be running",
-  "main": "waitforclusterreboot.js",
+  "description": "Executes a Databricks Job",
+  "main": "executedatabricksjob.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -13,8 +13,13 @@
   "keywords": [
     "azure",
     "databricks",
+    "spark",
     "devops",
-    "cluster"
+    "notebook",
+    "python",
+    "scala",
+    "jar",
+    "job"
   ],
   "author": "Microsoft DevLabs",
   "license": "MIT",
@@ -23,10 +28,11 @@
   },
   "homepage": "https://github.com/microsoft/azdo-databricks#readme",
   "dependencies": {
+    "@types/shelljs": "^0.8.5",
     "azure-pipelines-task-lib": "^2.8.0"
   },
   "devDependencies": {
-    "@types/node": "^12.6.2",
+    "@types/node": "^12.6.8",
     "@types/q": "^1.5.2"
   }
 }