Adding new pipeline variable, updating dependencies.

azure-pipelines-cd.yml

@@ -13,7 +13,7 @@ pool:
 steps:
   - task: NodeTool@0
     inputs:
-      versionSpec: "10.x"
+      versionSpec: "18.x"
     displayName: "Install Node.js"
 
   - script: |

azure-pipelines.yml

@@ -15,7 +15,7 @@ pool:
 steps:
   - task: NodeTool@0
     inputs:
-      versionSpec: "10.x"
+      versionSpec: "18.x"
     displayName: "Install Node.js"
 
   - script: |

package.json

@@ -3,14 +3,14 @@
   "version": "0.1.0",
   "description": "A set of Pipeline tasks to help add DevOps practices to a Databricks development cycle.",
   "dependencies": {
-    "azure-pipelines-task-lib": "^2.8.0",
+    "azure-pipelines-task-lib": "^4.1.0",
     "recursive-install": "^1.4.0"
   },
   "devDependencies": {
     "@types/mocha": "^5.2.7",
     "@types/node": "^12.0.8",
     "@types/q": "^1.5.2",
-    "tfx-cli": "^0.7.6"
+    "tfx-cli": "^0.12.0"
   },
   "scripts": {
     "recursive-install": "npm-recursive-install",

tasks/ConfigureDatabricks/ConfigureDatabricksV1/index.ts

@@ -9,16 +9,16 @@ import { stringify } from 'querystring';
 
 async function run() {
     try {
-        const url: string = tl.getInput('url', true);
-        const token: string = tl.getInput('token', true);
+        const url: string = tl.getInput('url', true) ?? '';
+        const token: string = tl.getInput('token', true) ?? '';
 
         installDatabricksCli();
 
         configurePat(url, token);
 
         let homeDir = os.homedir();
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }
@@ -47,7 +47,7 @@ async function configurePat(url:string, token: string){
 
         fs.writeFileSync(databricksCfgPath, content, { flag: "w"});
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }

tasks/DeployNotebooksTask/DeployNotebooksTaskV1/index.ts

@@ -1,11 +1,10 @@
 import tl = require('azure-pipelines-task-lib/task');
 import fs = require('fs');
-import { fstat } from 'fs';
 
 async function run() {
     try {
-        const notebooksFolderPath: string = tl.getInput('notebooksFolderPath', true);
-        const workspaceFolder: string = tl.getInput('workspaceFolder', true);
+        const notebooksFolderPath: string = tl.getInput('notebooksFolderPath', true) ?? '';
+        const workspaceFolder: string = tl.getInput('workspaceFolder', true) ?? '';
 
         if (!isDirSync(notebooksFolderPath)){
             tl.setResult(tl.TaskResult.Failed, 'The specified path for Notebooks folder is a file.')
@@ -24,15 +23,15 @@ async function run() {
             console.log(importResult.stdout);
         }
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }
 
 function isDirSync(aPath: string) {
     try {
         return fs.statSync(aPath).isDirectory();
-    } catch (e) {
+    } catch (e: any) {
         if (e.code === 'ENOENT') {
             return false;
         } else {

tasks/ExecuteNotebook/ExecuteNotebookV1/executeNotebook.ts

@@ -1,15 +1,14 @@
 import tl = require('azure-pipelines-task-lib');
 import path = require('path');
 import fs = require('fs');
-import { utils } from 'mocha';
 
 async function run() {
     tl.setResourcePath(path.join(__dirname, 'task.json'));
 
     try {
-        const notebookPath: string = tl.getInput('notebookPath', true);
-        const executionParams: string = tl.getInput('executionParams', false);
-        const existingClusterId: string = tl.getInput('existingClusterId', false);
+        const notebookPath: string = tl.getInput('notebookPath', true) ?? '';
+        const executionParams: string = tl.getInput('executionParams', false) ?? '';
+        const existingClusterId: string = tl.getInput('existingClusterId', false) ?? '';
 
         let notebook = new Notebook(notebookPath, executionParams);
 
@@ -23,7 +22,7 @@ async function run() {
             notebook.execute(existingClusterId);
         }
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }
@@ -55,14 +54,16 @@ class Notebook{
     }
 
     isValid(){
-        if (!this.folder.startsWith("/")) {
+        if (this.folder === '' || !this.folder.startsWith("/")) {
             tl.setResult(tl.TaskResult.Failed, 'The Notebook path must start with a forward slash (/).');
             return false;
         }
 
         try {
-            let paramsJson = JSON.parse(this.params);
-        } catch (error) {
+            if (this.params != '') {
+                let paramsJson = JSON.parse(this.params);
+            }
+        } catch (error: any) {
             tl.setResult(tl.TaskResult.Failed, error);
 
             return false;
@@ -101,7 +102,7 @@ class Notebook{
             });
 
             return true;
-        } catch (err) {
+        } catch (err: any) {
             tl.setResult(tl.TaskResult.Failed, err);
         }
     }
@@ -116,6 +117,8 @@ class Notebook{
             return;
         }
 
+        tl.setVariable("AZDO_DATABRICKS_JOBID", job.job_id);
+
         let run = this.createRun(job);
 
         if(run == null){
@@ -128,7 +131,7 @@ class Notebook{
 
     private getJobConfigurationFile(existingClusterId: string) {
         let jobTemplatePath: string;
-        if (existingClusterId === null) {
+        if (existingClusterId === '') {
             jobTemplatePath = path.join(__dirname, "job-configuration/new-cluster.json");
         }
         else {

tasks/Scala/CompileInstallJar/CompileInstallJarV1/compileinstalljar.ts

@@ -4,14 +4,14 @@ import shell = require('shelljs');
 
 async function compileInstallJar(){
     const failOnStderr: boolean = tl.getBoolInput('failOnStderr', false);
-    const clusterid: string = tl.getInput('clusterid', true);
-    const workingDirectory: string = tl.getInput('workingDirectory', false);
-    const packageName: string = tl.getInput('packageName', true);
-    const packageVersion: string = tl.getInput('packageVersion', true);
-    const scalaVersion: string = tl.getInput('scalaVersion', true);
-    const sampleDataSetFilePath: string = tl.getInput('sampleDataSetFilePath', true);
-
-    if(workingDirectory){
+    const clusterid: string = tl.getInput('clusterid', true) ?? '';
+    const workingDirectory: string = tl.getInput('workingDirectory', false) ?? '';
+    const packageName: string = tl.getInput('packageName', true) ?? '';
+    const packageVersion: string = tl.getInput('packageVersion', true) ?? '';
+    const scalaVersion: string = tl.getInput('scalaVersion', true) ?? '';
+    const sampleDataSetFilePath: string = tl.getInput('sampleDataSetFilePath', true) ?? '';
+
+    if(workingDirectory !== ''){
         shell.cd(workingDirectory);
     }
 
@@ -41,7 +41,7 @@ async function run() {
         } else {
             await compileInstallJar();
         }
-    } catch(err){
+    } catch(err: any){
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }

tasks/Scala/InstallScalaTools/InstallScalaToolsV1/installScalaTools.js

@@ -1,9 +1,10 @@
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
         function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
         function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };

tasks/Scala/InstallScalaTools/InstallScalaToolsV1/installScalaTools.ts

@@ -57,7 +57,7 @@ async function run() {
 
         // Write the script to disk.
         tl.assertAgent('2.115.0');
-        let tempDirectory = tl.getVariable('agent.tempDirectory');
+        let tempDirectory = tl.getVariable('agent.tempDirectory')!;
         tl.checkPath(tempDirectory, `${tempDirectory} (agent.tempDirectory)`);
 
         let fileName = uuidV4() + '.sh';
@@ -123,7 +123,7 @@ async function run() {
 
         tl.setResult(result, "", true);
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message || 'run() failed', true);
     }
 }

tasks/Scala/InstallSpark/InstallSparkV1/installSpark.js

@@ -1,9 +1,10 @@
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
         function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
         function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };

tasks/Scala/InstallSpark/InstallSparkV1/installSpark.ts

@@ -55,7 +55,7 @@ async function run() {
 
         tl.setResult(result, "", true);
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }

tasks/Scala/RunSbtTests/RunSbtTestsV1/runsbttests.js

@@ -1,9 +1,10 @@
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
         function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
         function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
@@ -12,13 +13,14 @@ const path = require("path");
 const tl = require("azure-pipelines-task-lib");
 const shell = require("shelljs");
 function run() {
+    var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         try {
             tl.setResourcePath(path.join(__dirname, 'task.json'));
             const failOnStderr = tl.getBoolInput('failOnStderr', false);
-            const workingDirectory = tl.getInput('workingDirectory', false);
-            const additionalParameters = tl.getInput('additionalParameters', false);
-            if (workingDirectory != '') {
+            const workingDirectory = (_a = tl.getInput('workingDirectory', false)) !== null && _a !== void 0 ? _a : '';
+            const additionalParameters = (_b = tl.getInput('additionalParameters', false)) !== null && _b !== void 0 ? _b : '';
+            if (workingDirectory !== '') {
                 shell.cd(workingDirectory);
             }
             let fileName = 'runsbttests.sh';

tasks/Scala/RunSbtTests/RunSbtTestsV1/runsbttests.ts

@@ -7,10 +7,10 @@ async function run() {
         tl.setResourcePath(path.join(__dirname, 'task.json'));
 
         const failOnStderr: boolean = tl.getBoolInput('failOnStderr', false);
-        const workingDirectory: string = tl.getInput('workingDirectory', false);
-        const additionalParameters: string = tl.getInput('additionalParameters', false)
+        const workingDirectory: string = tl.getInput('workingDirectory', false) ?? '';
+        const additionalParameters: string = tl.getInput('additionalParameters', false) ?? '';
 
-        if(workingDirectory != ''){
+        if(workingDirectory !== ''){
             shell.cd(workingDirectory);
         }
 
@@ -27,7 +27,7 @@ async function run() {
             tl.setResult(tl.TaskResult.Failed, `Command wrote to stderr: ${runSbtExec.stderr}`);
         }
     }
-    catch (err) {
+    catch (err: any) {
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }

tasks/Scala/StartCluster/StartClusterV1/startCluster.js

@@ -1,9 +1,10 @@
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
         function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
         function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
@@ -25,10 +26,11 @@ function startCluster(clusterid, failOnStderr) {
     });
 }
 function run() {
+    var _a;
     return __awaiter(this, void 0, void 0, function* () {
         try {
             tl.setResourcePath(path.join(__dirname, 'task.json'));
-            const clusterid = tl.getInput('clusterid', true);
+            const clusterid = (_a = tl.getInput('clusterid', true)) !== null && _a !== void 0 ? _a : '';
             const failOnStderr = tl.getBoolInput('failOnStderr', false);
             if (!shell.which('databricks')) {
                 tl.setResult(tl.TaskResult.Failed, "databricks-cli was not found. Use the task 'Configure Databricks CLI' to install and configure it.");

tasks/Scala/StartCluster/StartClusterV1/startCluster.ts

@@ -1,7 +1,6 @@
 import shell = require('shelljs');
 import path = require('path');
 import tl = require('azure-pipelines-task-lib');
-import { async } from 'q';
 
 async function startCluster(clusterid: string, failOnStderr: boolean) {
     let fileName = 'startCluster.sh';
@@ -22,15 +21,15 @@ async function run() {
     try{
         tl.setResourcePath(path.join(__dirname, 'task.json'));
 
-        const clusterid: string = tl.getInput('clusterid', true);
+        const clusterid: string = tl.getInput('clusterid', true) ?? '';
         const failOnStderr: boolean = tl.getBoolInput('failOnStderr', false);
 
         if(!shell.which('databricks')){
             tl.setResult(tl.TaskResult.Failed, "databricks-cli was not found. Use the task 'Configure Databricks CLI' to install and configure it.");
         } else {
             await startCluster(clusterid, failOnStderr);
         }
-    } catch(err){
+    } catch(err: any){
         tl.setResult(tl.TaskResult.Failed, err.message);
     }
 }

tasks/Scala/UninstallCodeFromCluster/UninstallCodeFromClusterV1/uninstallcodefromcluster.js

@@ -1,9 +1,10 @@
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
         function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
         function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
@@ -12,10 +13,11 @@ const path = require("path");
 const tl = require("azure-pipelines-task-lib");
 const shell = require("shelljs");
 function uninstallLibsFromCluster() {
+    var _a, _b, _c;
     try {
-        const failOnStderr = tl.getBoolInput('failOnStderr', false);
-        const libraryfilename = tl.getInput('libraryfilename', true);
-        const clusterid = tl.getInput('clusterid', true);
+        const failOnStderr = (_a = tl.getBoolInput('failOnStderr', false)) !== null && _a !== void 0 ? _a : '';
+        const libraryfilename = (_b = tl.getInput('libraryfilename', true)) !== null && _b !== void 0 ? _b : '';
+        const clusterid = (_c = tl.getInput('clusterid', true)) !== null && _c !== void 0 ? _c : '';
         let fileName = 'uninstallcodefromcluster.sh';
         let filePath = path.join(__dirname, fileName);
         let uninstallExec = shell.exec(`bash ${filePath} ${clusterid} ${libraryfilename}`);