From 2cfc28db3bc3b32d574757763721818a1448ce9c Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Tue, 19 Feb 2019 14:05:27 -0600 Subject: [PATCH 1/7] Checking if Java, pip3, and Request module are installed --- mkcli.py | 342 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 235 insertions(+), 107 deletions(-) diff --git a/mkcli.py b/mkcli.py index f75a18b..f240eaf 100644 --- a/mkcli.py +++ b/mkcli.py @@ -3,121 +3,249 @@ import os import subprocess from urllib import request -import requests import json import urllib +import sys +import re -def run(args): - #Gets the value from the flags - print("Starting process") - field = args.field - value = args.value - noexec = args.noexec - route = 'src/test/groovy' - # muuktestRoute = 'http://ec2-3-17-71-29.us-east-2.compute.amazonaws.com:8081/' - # supportRoute = 'http://ec2-18-219-8-121.us-east-2.compute.amazonaws.com:8082/' - - muuktestRoute = 'http://localhost:8081/' - supportRoute = 'http://localhost:8082/' - - - dirname = os.path.dirname(__file__) - if dirname == "": - dirname = "." - - userId = '' - if field == "hashtag": - value = "#"+value - - valueArr = [] - valueArr.append(value) - valueArr.append("") - - # Getting the bearer token - path = dirname + '/key.pub' - token='' - try: - key_file = open(path,'r') - key = key_file.read() - r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) - responseObject = json.loads(r.content) - token = responseObject["token"] - userId = responseObject["userId"] - organizationId = responseObject["organizationId"] - except: - print("Key file was not found on the repository (Download it from the Muuktest portal)") - exit() - - auth = {'Authorization': 'Bearer ' + token} - - allowed_fields = ['tag','name', 'hashtag'] - if field in allowed_fields: - print("Downloading test") - # #Delete the old files - if os.path.exists("test.rar"): - os.remove('test.rar') - shutil.rmtree(route, ignore_errors=True) - if not os.path.exists(route): - os.makedirs(route) - - values = {'property': field, 'value': valueArr, 'userId': userId} - # This route downloads the scripts by the property. - url = muuktestRoute+'download_byproperty/' - data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') - - # now using urlopen get the file and store it locally - auth_request = request.Request(url,headers=auth, data=data) - auth_request.add_header('Authorization', 'Bearer '+token) - response = request.urlopen(auth_request) - - # response = request.urlopen(url,data) - file = response.read() - flag = False +def checkRequirements(): + + java_installed = False + pip_installed = False + requests_installed = False + isUbuntu = False; + + os_version = os.uname().version + if "Ubuntu" in os_version: + isUbuntu = True; + linkJava = 'https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html'; + + # Verifying if java is installed try: - decode_text = file.decode("utf-8") - json_decode = json.loads(file.decode("utf-8")) - print(json_decode["message"]) + java_version = subprocess.check_output('javac -version | awk -F \' \' \'{print $2}\'', shell=True).decode('utf-8') + + match = re.findall('(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)', java_version.splitlines()[0]) + if not match: + if isUbuntu == True: + while 1: + answer = input("Java 1.8 or newer. Do you want to update it? [Y/N]: "); + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + os.system('sudo add-apt-repository ppa:webupd8team/java') + os.system('sudo apt update; sudo apt install oracle-java8-set-default'); + out = os.system('java -version'); + javac_version = subprocess.check_output(['javac', '-version'], stderr=subprocess.STDOUT) + + if "javac 1.8" in javac_version: + java_installed = True + else: + print("We couldn't to update Java") + break + else: + print("Is empty") + else: + java_installed = True + + except Exception as e: + if isUbuntu == True: + while 1: + answer = input("Java is not installed. Do you want to install it? [Y/N]: "); + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + os.system('sudo add-apt-repository ppa:webupd8team/java') + os.system('sudo apt update; sudo apt install oracle-java8-installer'); + out = os.system('java -version'); + if out == 0: + java_installed = True + else: + print("We couldn't to install Java") + break + else: + print("Java is not installed. Java can be downloaded in the next link:") + print(linkJava) + + linkPip3 = "https://pip.pypa.io/en/stable/installing/" + # Verifying if pip3 is installed + try: + pip_version = subprocess.check_output(['pip3', '--version'], stderr=subprocess.STDOUT) + pip_installed = True except: - flag = True - - if (flag == True): - print("The test has been downloaded successfully") - fileobj = open('test.zip',"wb") - fileobj.write(file) - fileobj.close() - - # Unzip the file // the library needs the file to end in .rar for some reason - shutil.unpack_archive('test.zip', extract_dir=route, format='zip') - - os.system('chmod 544 ' + dirname + '/gradlew') - - # save the dowonloaded test entry to the database - payload = { - "action": 2, - "userId": userId, - "organizationId": organizationId, - "options": { - "executor": True - } - } - requests.post(supportRoute+"tracking_data", json=payload) - - if noexec == False : - #Execute the test - print("Executing test...") - os.system(dirname + '/gradlew clean test') - # save the execute test entry to the database - requests.post(supportRoute+"tracking_data", data={ - 'action': 3, - 'userId': userId, - 'organizationId': organizationId - }) - - else: - print(field+': is not an allowed property') + if isUbuntu == True: + while 1: + answer = input("Pip3 is not installed. Do you want to install it? [Y/N]: ") + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + os.system('sudo apt install python3-pip') + out = os.system('pip3 --version') + if out == 0: + pip_installed = True + else: + print("We couldn't to install Pip3") + break + else: + print("Pip3 is not installed. Pip3 can be downloaded in the next link:") + print(linkPip3) + + + linkRequests = "https://docs.python.org/3/installing/index.html" + #Verify if the request module is installed + if pip_installed: + import pip + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + installed_packages = [r.decode().split('==')[0] for r in reqs.split()] + if 'requests' in installed_packages: + requests_installed = True + else: + if isUbuntu == True: + while 1: + answer = input("Pip3 is not installed. Do you want to install it? [Y/N]: ") + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + os.system('sudo pip install requests') + out = os.system('pip3 --version') + if out == 0: + requests_installed = True + else: + print("We couldn't to install Pip3") + break + else: + print("Request Package is not installed. To know how to download it, go to the next link:") + print(linkRequests) + + + return java_installed and pip_installed and requests_installed + + +def run(args): + #Gets the value from the flags + print("Starting process") + # sqlv = subprocess.check_output(['mysql', '--version'], stderr=subprocess.STDOUT) + # print(sqlv); + + # out = subprocess.check_output(['sudo', 'apt-get' , 'update'], stderr=subprocess.STDOUT) + requirements = checkRequirements() + + if requirements == True: + import requests + + field = args.field + value = args.value + noexec = args.noexec + route = 'src/test/groovy' + + # muuktestRoute = 'http://ec2-3-17-71-29.us-east-2.compute.amazonaws.com:8081/' + # supportRoute = 'http://ec2-18-219-8-121.us-east-2.compute.amazonaws.com:8082/' + + muuktestRoute = 'http://localhost:8081/' + supportRoute = 'http://localhost:8082/' + + dirname = os.path.dirname(__file__) + + if dirname == "": + dirname = "." + + userId = '' + if field == "hashtag": + value = "#"+value + + valueArr = [] + valueArr.append(value) + valueArr.append("") + + # Getting the bearer token + path = dirname + '/key.pub' + token='' + + try: + key_file = open(path,'r') + key = key_file.read() + r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) + responseObject = json.loads(r.content) + token = responseObject["token"] + userId = responseObject["userId"] + organizationId = responseObject["organizationId"] + except: + print("Key file was not found on the repository (Download it from the Muuktest portal)") + exit() + + auth = {'Authorization': 'Bearer ' + token} + allowed_fields = ['tag','name', 'hashtag'] + + if field in allowed_fields: + print("Downloading test") + + #Delete the old files + if os.path.exists("test.rar"): + os.remove('test.rar') + + shutil.rmtree(route, ignore_errors=True) + + if not os.path.exists(route): + os.makedirs(route) + + values = {'property': field, 'value': valueArr, 'userId': userId} + + # This route downloads the scripts by the property. + url = muuktestRoute+'download_byproperty/' + data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') + + # now using urlopen get the file and store it locally + auth_request = request.Request(url,headers=auth, data=data) + auth_request.add_header('Authorization', 'Bearer '+token) + response = request.urlopen(auth_request) + + # response = request.urlopen(url,data) + file = response.read() + flag = False + + try: + decode_text = file.decode("utf-8") + json_decode = json.loads(file.decode("utf-8")) + print(json_decode["message"]) + except: + flag = True + + if (flag == True): + print("The test has been downloaded successfully") + fileobj = open('test.zip',"wb") + fileobj.write(file) + fileobj.close() + + # Unzip the file // the library needs the file to end in .rar for some reason + shutil.unpack_archive('test.zip', extract_dir=route, format='zip') + + os.system('chmod 544 ' + dirname + '/gradlew') + + # save the downloaded test entry to the database + payload = { + "action": 2, + "userId": userId, + "organizationId": organizationId, + "options": { + "executor": True + } + } + + requests.post(supportRoute+"tracking_data", json=payload) + + if noexec == False : + #Execute the test + print("Executing test...") + os.system(dirname + '/gradlew clean test') + # save the execute test entry to the database + requests.post(supportRoute+"tracking_data", data={ + 'action': 3, + 'userId': userId, + 'organizationId': organizationId + }) + else: + print(field+': is not an allowed property') + else: + print("There are not all the requirements to run the executor") + exit() def main(): From 1ef3a25f76b98523fc42896ba1a3b4f8616eff28 Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Tue, 19 Feb 2019 15:56:16 -0600 Subject: [PATCH 2/7] Minor change for cmd Java, for Windows --- mkcli.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/mkcli.py b/mkcli.py index f240eaf..8030cdb 100644 --- a/mkcli.py +++ b/mkcli.py @@ -15,17 +15,19 @@ def checkRequirements(): requests_installed = False isUbuntu = False; + platform = sys.platform os_version = os.uname().version - if "Ubuntu" in os_version: - isUbuntu = True; + + if platform == "linux": + if "Ubuntu" in os_version: + isUbuntu = True; linkJava = 'https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html'; # Verifying if java is installed try: - java_version = subprocess.check_output('javac -version | awk -F \' \' \'{print $2}\'', shell=True).decode('utf-8') - - match = re.findall('(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)', java_version.splitlines()[0]) + java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT).decode('utf-8') + match = re.findall('(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)', java_version.split('"')[1]) if not match: if isUbuntu == True: while 1: @@ -35,9 +37,9 @@ def checkRequirements(): os.system('sudo add-apt-repository ppa:webupd8team/java') os.system('sudo apt update; sudo apt install oracle-java8-set-default'); out = os.system('java -version'); - javac_version = subprocess.check_output(['javac', '-version'], stderr=subprocess.STDOUT) + javac_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) - if "javac 1.8" in javac_version: + if "1.8" in javac_version: java_installed = True else: print("We couldn't to update Java") @@ -120,10 +122,6 @@ def checkRequirements(): def run(args): #Gets the value from the flags print("Starting process") - # sqlv = subprocess.check_output(['mysql', '--version'], stderr=subprocess.STDOUT) - # print(sqlv); - - # out = subprocess.check_output(['sudo', 'apt-get' , 'update'], stderr=subprocess.STDOUT) requirements = checkRequirements() if requirements == True: From 4312c95aaf9b3197a3d1c2bc669c145fbe1f6cca Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Tue, 19 Feb 2019 18:05:38 -0600 Subject: [PATCH 3/7] Changing cmd inside of if statement, just for linux to use that cmd --- mkcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkcli.py b/mkcli.py index 8030cdb..28a1875 100644 --- a/mkcli.py +++ b/mkcli.py @@ -16,9 +16,9 @@ def checkRequirements(): isUbuntu = False; platform = sys.platform - os_version = os.uname().version if platform == "linux": + os_version = os.uname().version if "Ubuntu" in os_version: isUbuntu = True; From a685396c0f66684f56c2e0b6d411f5b639170abc Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Thu, 21 Feb 2019 21:05:00 -0600 Subject: [PATCH 4/7] Optimazing the code so we can reuse it --- mkcli.py | 250 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 139 insertions(+), 111 deletions(-) diff --git a/mkcli.py b/mkcli.py index 28a1875..b0438b9 100644 --- a/mkcli.py +++ b/mkcli.py @@ -8,117 +8,6 @@ import sys import re -def checkRequirements(): - - java_installed = False - pip_installed = False - requests_installed = False - isUbuntu = False; - - platform = sys.platform - - if platform == "linux": - os_version = os.uname().version - if "Ubuntu" in os_version: - isUbuntu = True; - - linkJava = 'https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html'; - - # Verifying if java is installed - try: - java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT).decode('utf-8') - match = re.findall('(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)', java_version.split('"')[1]) - if not match: - if isUbuntu == True: - while 1: - answer = input("Java 1.8 or newer. Do you want to update it? [Y/N]: "); - if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): - if (answer == 'Y' or answer == 'y') : - os.system('sudo add-apt-repository ppa:webupd8team/java') - os.system('sudo apt update; sudo apt install oracle-java8-set-default'); - out = os.system('java -version'); - javac_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) - - if "1.8" in javac_version: - java_installed = True - else: - print("We couldn't to update Java") - break - else: - print("Is empty") - else: - java_installed = True - - except Exception as e: - if isUbuntu == True: - while 1: - answer = input("Java is not installed. Do you want to install it? [Y/N]: "); - if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): - if (answer == 'Y' or answer == 'y') : - os.system('sudo add-apt-repository ppa:webupd8team/java') - os.system('sudo apt update; sudo apt install oracle-java8-installer'); - out = os.system('java -version'); - if out == 0: - java_installed = True - else: - print("We couldn't to install Java") - break - else: - print("Java is not installed. Java can be downloaded in the next link:") - print(linkJava) - - linkPip3 = "https://pip.pypa.io/en/stable/installing/" - # Verifying if pip3 is installed - try: - pip_version = subprocess.check_output(['pip3', '--version'], stderr=subprocess.STDOUT) - pip_installed = True - except: - if isUbuntu == True: - while 1: - answer = input("Pip3 is not installed. Do you want to install it? [Y/N]: ") - if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): - if (answer == 'Y' or answer == 'y') : - os.system('sudo apt install python3-pip') - out = os.system('pip3 --version') - if out == 0: - pip_installed = True - else: - print("We couldn't to install Pip3") - break - else: - print("Pip3 is not installed. Pip3 can be downloaded in the next link:") - print(linkPip3) - - - linkRequests = "https://docs.python.org/3/installing/index.html" - #Verify if the request module is installed - if pip_installed: - import pip - reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) - installed_packages = [r.decode().split('==')[0] for r in reqs.split()] - if 'requests' in installed_packages: - requests_installed = True - else: - if isUbuntu == True: - while 1: - answer = input("Pip3 is not installed. Do you want to install it? [Y/N]: ") - if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): - if (answer == 'Y' or answer == 'y') : - os.system('sudo pip install requests') - out = os.system('pip3 --version') - if out == 0: - requests_installed = True - else: - print("We couldn't to install Pip3") - break - else: - print("Request Package is not installed. To know how to download it, go to the next link:") - print(linkRequests) - - - return java_installed and pip_installed and requests_installed - - def run(args): #Gets the value from the flags print("Starting process") @@ -246,6 +135,145 @@ def run(args): exit() +# This function install or update the programs or pymodules +# The parameter element is the json with the info of the program or pymodules +# The parameter action is to determinate if we need to install or update +def installAndUpdate(element, action): + success = False + #We need to know if the os is Ubuntu + if sys.platform == "linux": + os_version = os.uname().version + if "Ubuntu" in os_version: + isUbuntu = True; + + name = element["name"] + + #We need to check if is program or pymodule + if element["type"] == "program": + # We need to check the action to determinate which cmd to use + if action == "update": + cmd = element["update_cmd"] + question = "You have a older " + name + "version. Do you want to update it? [Y/N]: " + else: + cmd = element["install_cmd"] + question = name + " is not installed. Do you want to install it? [Y/N]: " + else: + cmd = "sudo pip install " + name + question = name + " is not installed. Do you want to install it? [Y/N]: " + + + if isUbuntu: + while 1: + answer = input(question) + # Asking to the user if he wants to install or update it + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + # Installing or updating + os.system(cmd) + if action == "update": + split_cmd = element["version_cmd"].split(" ") + v = subprocess.check_output(split_cmd, stderr=subprocess.STDOUT).decode('utf-8') + + if element["minimum_version"] in v: + success = True + + else: + out = os.system(element[version_cmd]) + if out == 0: + success = True + + break + + else: + # If the OS is not Ubuntu, we send the link with the info to download it + print(name + " is not installed. It can be downloaded in the next link:") + if x["type"] == "pymodule": + print("https://docs.python.org/3/installing/index.html") + else: + print(element["link"]) + + return success + +# Check if the programs or modules need to install or update +# This function recieve a list of programs or pymodules to install +def checkAndInstall(list): + pip3_installed = False + result = True + + for x in list: + if x["type"] == "pymodule" and pip3_installed: + # Checking if the module is installed + import pip + # Retrieving if the mpython modules installed list + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + installed_packages = [r.decode().split('==')[0] for r in reqs.split()] + + # Check if the module is in the list + if x["name"] in installed_packages: + installed = True + else: + installed = False + + else: + try: + splitcmd = x["version_cmd"].split(" ") + # Getting the output of the command + version = subprocess.check_output(splitcmd, stderr=subprocess.STDOUT).decode('utf-8') + installed = True + except Exception as e: + # If it catch an exception the program doesn't exist + installed = False + + if installed: + if "minimum_version" in x: + # If the programs is installed, we need to check his version + match = re.findall(x["regex"], version.split('"')[1]) + if not match: + # If it doesn't find a match, we need to update it + installed = installAndUpdate(x, "update") + else: + if "Pip3" == x["name"]: + pip3_installed = True + else: + # If it isn't installed, we need to install + installed = installAndUpdate(x, "install") + + result &= installed + + return result + + +def checkRequirements(): + # Array of jsons, this contains the info of the programs or pymodules to install + programs = [ + { + "name": "Java", + "version_cmd": "java -version", + "install_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-installer", + "update_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-set-default", + "link": "https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html", + "minimum_version": "1.8", + "regex": "(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)", + "type": "program" + }, + { + "name": "Pip3", + "version_cmd": "pip3 --version", + "install_cmd": "sudo apt install python3-pip", + "link": "https://pip.pypa.io/en/stable/installing/", + "type":"program" + }, + { + "name": "requests", + "type": "pymodule" + } + ] + + # Check if the programs are install + check_reqs = checkAndInstall(programs) + + return check_reqs + def main(): parser=argparse.ArgumentParser(description="MuukTest cli to download tests from the cloud") parser.add_argument("-p",help="property to search the test for" ,dest="field", type=str, required=True) From 0641bda6c7e4a77843b4a49c4e1cb8d71631c0a6 Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Mon, 25 Feb 2019 14:35:43 -0600 Subject: [PATCH 5/7] Removing extra tabs --- mkcli.py | 505 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 258 insertions(+), 247 deletions(-) diff --git a/mkcli.py b/mkcli.py index b0438b9..92c0e2c 100644 --- a/mkcli.py +++ b/mkcli.py @@ -9,279 +9,290 @@ import re def run(args): - #Gets the value from the flags - print("Starting process") - requirements = checkRequirements() - - if requirements == True: - import requests + #Gets the value from the flags + print("Starting process") + requirements = checkRequirements() + + if requirements == True: + import requests + + field = args.field + value = args.value + noexec = args.noexec + route = 'src/test/groovy' + + # muuktestRoute = 'http://ec2-3-17-71-29.us-east-2.compute.amazonaws.com:8081/' + # supportRoute = 'http://ec2-18-219-8-121.us-east-2.compute.amazonaws.com:8082/' + + muuktestRoute = 'http://localhost:8081/' + supportRoute = 'http://localhost:8082/' + + dirname = os.path.dirname(__file__) + + if dirname == "": + dirname = "." + + userId = '' + if field == "hashtag": + value = "#"+value + + valueArr = [] + valueArr.append(value) + valueArr.append("") + + # Getting the bearer token + path = dirname + '/key.pub' + token='' + + try: + key_file = open(path,'r') + key = key_file.read() + r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) + responseObject = json.loads(r.content) + token = responseObject["token"] + userId = responseObject["userId"] + organizationId = responseObject["organizationId"] + except: + print("Key file was not found on the repository (Download it from the Muuktest portal)") + exit() + + auth = {'Authorization': 'Bearer ' + token} + allowed_fields = ['tag','name', 'hashtag'] + + if field in allowed_fields: + print("Downloading test") + + #Delete the old files + if os.path.exists("test.rar"): + os.remove('test.rar') + + shutil.rmtree(route, ignore_errors=True) + + if not os.path.exists(route): + os.makedirs(route) + + values = {'property': field, 'value': valueArr, 'userId': userId} + + # This route downloads the scripts by the property. + url = muuktestRoute+'download_byproperty/' + data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') + + # now using urlopen get the file and store it locally + auth_request = request.Request(url,headers=auth, data=data) + auth_request.add_header('Authorization', 'Bearer '+token) + response = request.urlopen(auth_request) + + # response = request.urlopen(url,data) + file = response.read() + flag = False + + try: + decode_text = file.decode("utf-8") + json_decode = json.loads(file.decode("utf-8")) + print(json_decode["message"]) + except: + flag = True + + if (flag == True): + print("The test has been downloaded successfully") + fileobj = open('test.zip',"wb") + fileobj.write(file) + fileobj.close() + + # Unzip the file // the library needs the file to end in .rar for some reason + shutil.unpack_archive('test.zip', extract_dir=route, format='zip') + + os.system('chmod 544 ' + dirname + '/gradlew') + + # save the downloaded test entry to the database + payload = { + "action": 2, + "userId": userId, + "organizationId": organizationId, + "options": { + "executor": True + } + } + + requests.post(supportRoute+"tracking_data", json=payload) - field = args.field - value = args.value - noexec = args.noexec - route = 'src/test/groovy' - - # muuktestRoute = 'http://ec2-3-17-71-29.us-east-2.compute.amazonaws.com:8081/' - # supportRoute = 'http://ec2-18-219-8-121.us-east-2.compute.amazonaws.com:8082/' - - muuktestRoute = 'http://localhost:8081/' - supportRoute = 'http://localhost:8082/' - - dirname = os.path.dirname(__file__) - - if dirname == "": - dirname = "." - - userId = '' - if field == "hashtag": - value = "#"+value - - valueArr = [] - valueArr.append(value) - valueArr.append("") - - # Getting the bearer token - path = dirname + '/key.pub' - token='' - - try: - key_file = open(path,'r') - key = key_file.read() - r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) - responseObject = json.loads(r.content) - token = responseObject["token"] - userId = responseObject["userId"] - organizationId = responseObject["organizationId"] - except: - print("Key file was not found on the repository (Download it from the Muuktest portal)") - exit() - - auth = {'Authorization': 'Bearer ' + token} - allowed_fields = ['tag','name', 'hashtag'] - - if field in allowed_fields: - print("Downloading test") - - #Delete the old files - if os.path.exists("test.rar"): - os.remove('test.rar') - - shutil.rmtree(route, ignore_errors=True) - - if not os.path.exists(route): - os.makedirs(route) - - values = {'property': field, 'value': valueArr, 'userId': userId} - - # This route downloads the scripts by the property. - url = muuktestRoute+'download_byproperty/' - data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') - - # now using urlopen get the file and store it locally - auth_request = request.Request(url,headers=auth, data=data) - auth_request.add_header('Authorization', 'Bearer '+token) - response = request.urlopen(auth_request) - - # response = request.urlopen(url,data) - file = response.read() - flag = False - - try: - decode_text = file.decode("utf-8") - json_decode = json.loads(file.decode("utf-8")) - print(json_decode["message"]) - except: - flag = True - - if (flag == True): - print("The test has been downloaded successfully") - fileobj = open('test.zip',"wb") - fileobj.write(file) - fileobj.close() - - # Unzip the file // the library needs the file to end in .rar for some reason - shutil.unpack_archive('test.zip', extract_dir=route, format='zip') - - os.system('chmod 544 ' + dirname + '/gradlew') - - # save the downloaded test entry to the database - payload = { - "action": 2, - "userId": userId, - "organizationId": organizationId, - "options": { - "executor": True - } - } - - requests.post(supportRoute+"tracking_data", json=payload) - - if noexec == False : - #Execute the test - print("Executing test...") - os.system(dirname + '/gradlew clean test') - # save the execute test entry to the database - requests.post(supportRoute+"tracking_data", data={ - 'action': 3, - 'userId': userId, - 'organizationId': organizationId - }) - - else: - print(field+': is not an allowed property') + if noexec == False : + #Execute the test + print("Executing test...") + os.system(dirname + '/gradlew clean test') + # save the execute test entry to the database + requests.post(supportRoute+"tracking_data", data={ + 'action': 3, + 'userId': userId, + 'organizationId': organizationId + }) else: - print("There are not all the requirements to run the executor") - exit() + print(field+': is not an allowed property') + + else: + print("There are not all the requirements to run the executor") + exit() + + +def checkPyModule(module): + # Checking if the module is installed + import pip + # Retrieving if the mpython modules installed list + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + installed_packages = [r.decode().split('==')[0] for r in reqs.split()] + # Check if the module is in the list + if module["name"] in installed_packages: + m_installed = True + else: + m_installed = False + + return m_installed # This function install or update the programs or pymodules # The parameter element is the json with the info of the program or pymodules # The parameter action is to determinate if we need to install or update def installAndUpdate(element, action): - success = False - #We need to know if the os is Ubuntu - if sys.platform == "linux": - os_version = os.uname().version - if "Ubuntu" in os_version: - isUbuntu = True; - - name = element["name"] - - #We need to check if is program or pymodule - if element["type"] == "program": - # We need to check the action to determinate which cmd to use - if action == "update": - cmd = element["update_cmd"] - question = "You have a older " + name + "version. Do you want to update it? [Y/N]: " - else: - cmd = element["install_cmd"] - question = name + " is not installed. Do you want to install it? [Y/N]: " + success = False + #We need to know if the os is Ubuntu + if sys.platform == "linux": + os_version = os.uname().version + if "Ubuntu" in os_version: + isUbuntu = True; + + name = element["name"] + + #We need to check if is program or pymodule + if element["type"] == "program": + # We need to check the action to determinate which cmd to use + if action == "update": + cmd = element["update_cmd"] + question = "You have a older " + name + "version. Do you want to update it? [Y/N]: " else: - cmd = "sudo pip install " + name - question = name + " is not installed. Do you want to install it? [Y/N]: " - - - if isUbuntu: - while 1: - answer = input(question) - # Asking to the user if he wants to install or update it - if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): - if (answer == 'Y' or answer == 'y') : - # Installing or updating - os.system(cmd) - if action == "update": - split_cmd = element["version_cmd"].split(" ") - v = subprocess.check_output(split_cmd, stderr=subprocess.STDOUT).decode('utf-8') - - if element["minimum_version"] in v: - success = True + cmd = element["install_cmd"] + question = name + " is not installed. Do you want to install it? [Y/N]: " + else: + cmd = "sudo pip install " + name + question = name + " is not installed. Do you want to install it? [Y/N]: " + + + if isUbuntu: + while 1: + answer = input(question) + # Asking to the user if he wants to install or update it + if ( answer == 'Y' or answer == 'N' or answer == 'y' or answer == 'n'): + if (answer == 'Y' or answer == 'y') : + # Installing or updating + os.system(cmd) + if action == "update": + split_cmd = element["version_cmd"].split(" ") + v = subprocess.check_output(split_cmd, stderr=subprocess.STDOUT).decode('utf-8') + + if element["minimum_version"] in v: + success = True + + else: + out = -1 + + if element["type"] == "pymodule": + if checkPyModule(element): + out = 0 + else: + out = os.system(element["version_cmd"]) - else: - out = os.system(element[version_cmd]) - if out == 0: - success = True + if out == 0: + success = True - break + break + else: + # If the OS is not Ubuntu, we send the link with the info to download it + print(name + " is not installed. It can be downloaded in the next link:") + if x["type"] == "pymodule": + print("https://docs.python.org/3/installing/index.html") else: - # If the OS is not Ubuntu, we send the link with the info to download it - print(name + " is not installed. It can be downloaded in the next link:") - if x["type"] == "pymodule": - print("https://docs.python.org/3/installing/index.html") - else: - print(element["link"]) + print(element["link"]) - return success + return success # Check if the programs or modules need to install or update # This function recieve a list of programs or pymodules to install def checkAndInstall(list): - pip3_installed = False - result = True - - for x in list: - if x["type"] == "pymodule" and pip3_installed: - # Checking if the module is installed - import pip - # Retrieving if the mpython modules installed list - reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) - installed_packages = [r.decode().split('==')[0] for r in reqs.split()] - - # Check if the module is in the list - if x["name"] in installed_packages: - installed = True - else: - installed = False - - else: - try: - splitcmd = x["version_cmd"].split(" ") - # Getting the output of the command - version = subprocess.check_output(splitcmd, stderr=subprocess.STDOUT).decode('utf-8') - installed = True - except Exception as e: - # If it catch an exception the program doesn't exist - installed = False - - if installed: - if "minimum_version" in x: - # If the programs is installed, we need to check his version - match = re.findall(x["regex"], version.split('"')[1]) - if not match: - # If it doesn't find a match, we need to update it - installed = installAndUpdate(x, "update") - else: - if "Pip3" == x["name"]: - pip3_installed = True - else: - # If it isn't installed, we need to install - installed = installAndUpdate(x, "install") + pip3_installed = False + result = True - result &= installed - - return result + for x in list: + if x["type"] == "pymodule" and pip3_installed: + installed = checkPyModule(x) + else: + try: + splitcmd = x["version_cmd"].split(" ") + # Getting the output of the command + version = subprocess.check_output(splitcmd, stderr=subprocess.STDOUT).decode('utf-8') + installed = True + except Exception as e: + # If it catch an exception the program doesn't exist + installed = False + + if installed: + if "minimum_version" in x: + # If the programs is installed, we need to check his version + match = re.findall(x["regex"], version.split('"')[1]) + if not match: + # If it doesn't find a match, we need to update it + installed = installAndUpdate(x, "update") + else: + if "Pip3" == x["name"]: + pip3_installed = True + else: + # If it isn't installed, we need to install + installed = installAndUpdate(x, "install") + result &= installed -def checkRequirements(): - # Array of jsons, this contains the info of the programs or pymodules to install - programs = [ - { - "name": "Java", - "version_cmd": "java -version", - "install_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-installer", - "update_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-set-default", - "link": "https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html", - "minimum_version": "1.8", - "regex": "(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)", - "type": "program" - }, - { - "name": "Pip3", - "version_cmd": "pip3 --version", - "install_cmd": "sudo apt install python3-pip", - "link": "https://pip.pypa.io/en/stable/installing/", - "type":"program" - }, - { - "name": "requests", - "type": "pymodule" - } - ] + return result - # Check if the programs are install - check_reqs = checkAndInstall(programs) - return check_reqs +def checkRequirements(): + # Array of jsons, this contains the info of the programs or pymodules to install + programs = [ + { + "name": "Java", + "version_cmd": "java -version", + "install_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-installer", + "update_cmd": "sudo add-apt-repository ppa:webupd8team/java; sudo apt update; sudo apt install oracle-java8-set-default", + "link": "https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html", + "minimum_version": "1.8", + "regex": "(((10|11)|1\\.8|1\\.9)([\\.+\\d+\\.*])*)", + "type": "program" + }, + { + "name": "Pip3", + "version_cmd": "pip3 --version", + "install_cmd": "sudo apt install python3-pip", + "link": "https://pip.pypa.io/en/stable/installing/", + "type":"program" + }, + { + "name": "requests", + "type": "pymodule" + } + ] + + # Check if the programs are install + check_reqs = checkAndInstall(programs) + + return check_reqs def main(): - parser=argparse.ArgumentParser(description="MuukTest cli to download tests from the cloud") - parser.add_argument("-p",help="property to search the test for" ,dest="field", type=str, required=True) - parser.add_argument("-t",help="value of the test or hashtag field" ,dest="value", type=str, required=True) - parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") - parser.set_defaults(func=run) - args=parser.parse_args() - args.func(args) + parser=argparse.ArgumentParser(description="MuukTest cli to download tests from the cloud") + parser.add_argument("-p",help="property to search the test for" ,dest="field", type=str, required=True) + parser.add_argument("-t",help="value of the test or hashtag field" ,dest="value", type=str, required=True) + parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") + parser.set_defaults(func=run) + args=parser.parse_args() + args.func(args) if __name__=="__main__": main() From 717bc8b2da7096e14e16384170522cd9db673889 Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Mon, 25 Feb 2019 15:18:12 -0600 Subject: [PATCH 6/7] Move verification of pip3 --- mkcli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mkcli.py b/mkcli.py index 92c0e2c..9df6680 100644 --- a/mkcli.py +++ b/mkcli.py @@ -242,13 +242,13 @@ def checkAndInstall(list): if not match: # If it doesn't find a match, we need to update it installed = installAndUpdate(x, "update") - else: - if "Pip3" == x["name"]: - pip3_installed = True else: # If it isn't installed, we need to install installed = installAndUpdate(x, "install") + if "Pip3" == x["name"] and installed: + pip3_installed = True + result &= installed return result From 4ef0d4780906ceae463758fe9e544feb94d617ee Mon Sep 17 00:00:00 2001 From: Armando Martinez Mandujano Date: Mon, 25 Feb 2019 16:37:11 -0600 Subject: [PATCH 7/7] Add message if they want to install pymodules without pip3 --- mkcli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mkcli.py b/mkcli.py index 9df6680..9bfa1e6 100644 --- a/mkcli.py +++ b/mkcli.py @@ -221,7 +221,6 @@ def installAndUpdate(element, action): def checkAndInstall(list): pip3_installed = False result = True - for x in list: if x["type"] == "pymodule" and pip3_installed: installed = checkPyModule(x) @@ -243,8 +242,11 @@ def checkAndInstall(list): # If it doesn't find a match, we need to update it installed = installAndUpdate(x, "update") else: - # If it isn't installed, we need to install - installed = installAndUpdate(x, "install") + if x["type"] == "program" or pip3_installed: + # If it isn't installed, we need to install + installed = installAndUpdate(x, "install") + else: + print('The python3 module "' + x["name"] + '" needs to have Pip3 installed.') if "Pip3" == x["name"] and installed: pip3_installed = True