From 5304fe13b1aaed812ef335ad4790a95c702c2fdc Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Mon, 12 Oct 2020 16:22:08 -0500 Subject: [PATCH 01/74] adding changes to support video and add ssl cert ignore flags --- mkcli.py | 12 ++ mkvideo.py | 211 ++++++++++++++++++++++++++++ src/test/resources/GebConfig.groovy | 19 ++- 3 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 mkvideo.py diff --git a/mkcli.py b/mkcli.py index 5dcf2b0..721efd1 100644 --- a/mkcli.py +++ b/mkcli.py @@ -7,6 +7,7 @@ import json import urllib import xml.etree.ElementTree +import mkvideo from time import strftime from mkcloud import gatherScreenshots, resizeImages #import ssl @@ -197,13 +198,19 @@ def run(args): if noexec == False : #Execute the test + videoNameFile = str(organizationId) + "_" + str(executionNumber) + ".mp4" + print("File name for video: " + videoNameFile) print("Executing test...") try: + v = Video() + v.checkAndStartRecording(videoNameFile) exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) + v.checkAndStopRecording() + del v testsExecuted = gatherFeedbackData(browserName) url = muuktestRoute+'feedback/' values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber)} @@ -215,8 +222,13 @@ def run(args): filesData = gatherScreenshots(browserName) try: if filesData != {}: + print("Upload the screenshots: ") requests.post(muuktestRoute + 'upload_cloud_steps_images/', headers=hed, files = filesData) #requests.post(muuktestRoute + 'upload_cloud_steps_images/', data={'cloudKey': cloudKey}, headers=hed, files = filesData, verify=False) + print("Upload the Video: ") + videoFile = open(videoNameFile, 'rb') + files = {'file': videoFile} + requests.post(muuktestRoute + 'upload_video/', headers=hed, files=files) else: print ("filesData empty.. cannot send screenshots") except Exception as e: diff --git a/mkvideo.py b/mkvideo.py new file mode 100644 index 0000000..6115aa3 --- /dev/null +++ b/mkvideo.py @@ -0,0 +1,211 @@ +import os +import subprocess +import time + +# This class comprise the methos to handle the recording of a video. +# It requires ffmpeg and tmux libraries to be running in the Linux box. +# Currently it only supports linux (ubuntu?) +class Video: + # This is the constructor, input might change when we support windows (gdigrab/desktop) + # or mac(avfoundation?) . Check https://www.ffmpeg.org/ffmpeg.html + # + # Keep in mind that the default port we have used is :99 , that is specified when + # starting the Xvfb process (ie. Xvfb :99 1366x768x16) + # FYI TMUX reference: https://gist.github.com/henrik/1967800 + # + def __init__(self, port= ":99.0", dimension="1366x768", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + self.port = port + self.dimension = dimension + self.nameFile = nameFile + self.session = session + self.input = input + + # this function should get the OS + # def getOs(): + # pass + def setNameFile(self, nameFile): + self.nameFile = nameFile + + # This function checks the libraries to be able to record the video. + # returns: + # Boolean whether it satisfies or not. + def checkLibraries (self): + status = True + + cmds = ["tmux -V | grep tmux", "ffmpeg -version | grep version"] + notFoundStr = "command not found" + for x in cmds : + #print("Cmd: " , x) + res = self.executeCmd(x) + if res == "" or notFoundStr in str(res): + print("checkLibraries - The library is not installed : " , x) + status = False + break + + return status + + # this function checks whether there is an active tmux session + # This is important because we cannot start more than one session with + # the same name. + # returns: + # Boolean whether there is an active session or not + def checkActiveSession(self): + status = False + cmd = "tmux ls | grep " + self.session + + result = self.executeCmd(cmd) + # Verify if there is an existing session + if result != "" and self.session in str(result): + print("checkActiveSession - There is an active sessions") + status = True + + return status + + # this function starts the tmux session. It does not check whether there is + # already an active session though + # returns: + # Boolean whether there was able to start the session or not + def startSession(self): + status = True + cmd = "tmux new-session -d -s "+self.session + + res = self.executeCmd(cmd) + + return status + + #This function kills a tmux active session. It assumes you have already + # confirmed that there is an active session. + # returns: + # Boolean whether there was able to kill the session or not + def killSession(self): + status = True + cmd = "tmux kill-session -t "+self.session + + res = self.executeCmd(cmd) + + return status + + # This function starts the recording of the video in a new TMUX sessions. + # which means that there should not be an active session already. This + # must be verified prior calling this function + # Returns: + # Boolean whether there was able to start the recording + def startRecording(self): + status = True + # The following command starts the recording creating a new TMUX session. + # such session should not exist. All parameters are required and the -y + # means that it will overwrite the file in case that exists. + cmd = "tmux new-session -d -s "+ self.session +" 'ffmpeg -video_size "+ self.dimension + " -framerate 25 -f "+self.input + " -i "+self.port +" -y " + self.nameFile+"'" + print("startRecoring - start the recording : " , cmd) + + resStr = self.executeCmd(cmd) + # This means there was an error... + if resStr != "": + status = True + else: + status = False + print("startRecoring - There was an error starting the recording") + + return status + + # This function stops the recording of the video. Basically it sends + # the "q" key to the tmux terminal. This does not mean that kills the tmux + # session though + # Returns: + # Boolean whether there was able to stop the recording + def stopRecording(self): + status = True + cmd = "tmux send-keys -t "+self.session+ " q " + print("stopRecording - start the recording : " , cmd) + + resStr = self.executeCmd(cmd) + # This means there was an error... + if resStr != "": + status = True + else: + status = False + + time.sleep(1) + return status + + # This function issues the command via subprocess check_output which returns + # the stdout. + # params: + # command to issue (string) + # returns: + # result which contains the stout (string) + def executeCmd(self, cmd=""): + result = "" + if cmd != "" : + try: + result= subprocess.check_output(cmd, shell = True) + except subprocess.CalledProcessError as notZero: + print ("executeCmd - returned a non zero response , this means there was an error") + result ="" + except Exception as e : + print ("executeCmd - Execution returned ") + result = "" + print(e) + else : + print ("executeCmd - No command provided... Nothing to do") + + print("executeCmd - result: " , result) + return result + + + # this function starts the recording of the video in a safe way. + # params: + # Name of the file (string) + # returns + # Boolean + def checkAndStartRecording(self, nameFile=""): + status = True + # only perform this function if we have the libraries.... + + status = self.checkLibraries() + if status == True: + + if nameFile != "": + self.setNameFile(nameFile) + + status = self.checkActiveSession() + # this means there is an existing active session, something must have happened + # but we need to kill it before we continue + if status == True: + status = self.killSession() + + # ok, lets the fun begin... + status = self.startRecording() + if status == True: + print("checkAndStartRercording - Recording started successfully") + else: + print("checkAndStartRercording - There was a failure during the recording") + else: + print("checkAndStartRercording - Libraries not installed... nothing to do") + + return status + + # this function stops the recording of the video in a safe way + # returns + # Boolean + def checkAndStopRecording(self): + status = True + + status = self.checkActiveSession() + # this means there is an existing active session, something must have happened + # but we need to kill it before we continue + if status == True: + status = self.stopRecording() + if status == True: + print("checkAndStopRercording - Recording stopped successfully") + else: + print("checkAndStopRercording - There was a failure during the recording") + else: + print("checkAndStopRercording - No active sessions... nothing to stop") + + return status + +def startRecording(): + status = True + + return status diff --git a/src/test/resources/GebConfig.groovy b/src/test/resources/GebConfig.groovy index c4bb162..b9d1150 100644 --- a/src/test/resources/GebConfig.groovy +++ b/src/test/resources/GebConfig.groovy @@ -5,6 +5,8 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.firefox.FirefoxDriver +import org.openqa.selenium.remote.DesiredCapabilities +import org.openqa.selenium.remote.CapabilityType waiting { timeout = 20 @@ -14,10 +16,17 @@ environments { // run via “./gradlew chromeTest” // See: http://code.google.com/p/selenium/wiki/ChromeDriver chrome { - ChromeOptions o = new ChromeOptions() - o.addArguments('--no-sandbox'); - o.addArguments('--disable-dev-shm-usage'); - driver = { new ChromeDriver(o) } + driver{ + ChromeOptions o = new ChromeOptions() + o.addArguments('--no-sandbox'); + o.addArguments('--disable-dev-shm-usage'); + o.addArguments("--ignore-certificate-errors"); + DesiredCapabilities cap=DesiredCapabilities.chrome(); + cap.setCapability(ChromeOptions.CAPABILITY, o); + cap.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); + cap.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS, true); + new ChromeDriver(cap); + } } // run via “./gradlew chromeHeadlessTest” @@ -29,7 +38,7 @@ environments { new ChromeDriver(o) } } - + // run via “./gradlew firefoxTest” // See: http://code.google.com/p/selenium/wiki/FirefoxDriver firefox { From 68b8ea48ad0d9dc871e44a34992f2ab83538482a Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Mon, 12 Oct 2020 17:09:27 -0500 Subject: [PATCH 02/74] fixing header --- mkcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkcli.py b/mkcli.py index 721efd1..35de4c9 100644 --- a/mkcli.py +++ b/mkcli.py @@ -7,9 +7,9 @@ import json import urllib import xml.etree.ElementTree -import mkvideo from time import strftime from mkcloud import gatherScreenshots, resizeImages +from mkvideo import Video #import ssl def gatherFeedbackData(browserName): From fa153e276e6068efd6913d4fed0c6c7982f7e4b8 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 10:41:59 -0500 Subject: [PATCH 03/74] changes to verify the recording --- mkcli.py | 1 + src/test/resources/GebConfig.groovy | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mkcli.py b/mkcli.py index 35de4c9..1fb37a2 100644 --- a/mkcli.py +++ b/mkcli.py @@ -204,6 +204,7 @@ def run(args): try: v = Video() v.checkAndStartRecording(videoNameFile) + v.checkActiveSession() exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") diff --git a/src/test/resources/GebConfig.groovy b/src/test/resources/GebConfig.groovy index b9d1150..002d452 100644 --- a/src/test/resources/GebConfig.groovy +++ b/src/test/resources/GebConfig.groovy @@ -21,11 +21,12 @@ environments { o.addArguments('--no-sandbox'); o.addArguments('--disable-dev-shm-usage'); o.addArguments("--ignore-certificate-errors"); - DesiredCapabilities cap=DesiredCapabilities.chrome(); - cap.setCapability(ChromeOptions.CAPABILITY, o); - cap.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); - cap.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS, true); - new ChromeDriver(cap); + //DesiredCapabilities cap=DesiredCapabilities.chrome(); + //cap.setCapability(ChromeOptions.CAPABILITY, o); + //cap.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); + //cap.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS, true); + //new ChromeDriver(cap); + new ChromeDriver(o); } } From 8d2299ef8148fc2532ef08d6e57694b93aba83bb Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 10:46:07 -0500 Subject: [PATCH 04/74] fixing gebconfig file --- src/test/resources/GebConfig.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/GebConfig.groovy b/src/test/resources/GebConfig.groovy index 002d452..133365f 100644 --- a/src/test/resources/GebConfig.groovy +++ b/src/test/resources/GebConfig.groovy @@ -16,11 +16,11 @@ environments { // run via “./gradlew chromeTest” // See: http://code.google.com/p/selenium/wiki/ChromeDriver chrome { - driver{ + driver = { ChromeOptions o = new ChromeOptions() o.addArguments('--no-sandbox'); o.addArguments('--disable-dev-shm-usage'); - o.addArguments("--ignore-certificate-errors"); + //o.addArguments("--ignore-certificate-errors"); //DesiredCapabilities cap=DesiredCapabilities.chrome(); //cap.setCapability(ChromeOptions.CAPABILITY, o); //cap.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); From 9b5786811905f2ae609538506487a7f87cd4300b Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 11:06:57 -0500 Subject: [PATCH 05/74] adding changes for video --- mkvideo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mkvideo.py b/mkvideo.py index 6115aa3..cabcb40 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -138,6 +138,7 @@ def executeCmd(self, cmd=""): result = "" if cmd != "" : try: + print("executeCmd - command received: ", cmd) result= subprocess.check_output(cmd, shell = True) except subprocess.CalledProcessError as notZero: print ("executeCmd - returned a non zero response , this means there was an error") From 5c464f01124d6bbfd5e56b88327ba9a4a3d5b3ab Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 18:18:51 -0500 Subject: [PATCH 06/74] adding logs for debugging --- mkcli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkcli.py b/mkcli.py index 1fb37a2..670f0d5 100644 --- a/mkcli.py +++ b/mkcli.py @@ -205,11 +205,14 @@ def run(args): v = Video() v.checkAndStartRecording(videoNameFile) v.checkActiveSession() + v.executeCmd("ps -ef | grep ffmpeg") + v.executeCmd("ps -ef | grep tmux") exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) + v.executeCmd("ps -ef | grep ffmpeg") v.checkAndStopRecording() del v testsExecuted = gatherFeedbackData(browserName) From 9f9b000f32832959f3b8e9d50fb5152477c9afb5 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 18:31:48 -0500 Subject: [PATCH 07/74] testing --- mkcli.py | 3 ++- mkvideo.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mkcli.py b/mkcli.py index 670f0d5..f77748a 100644 --- a/mkcli.py +++ b/mkcli.py @@ -206,13 +206,14 @@ def run(args): v.checkAndStartRecording(videoNameFile) v.checkActiveSession() v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ps -ef | grep tmux") + v.executeCmd("ls -ltr ") exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) v.executeCmd("ps -ef | grep ffmpeg") + v.executeCmd("ls -ltr ") v.checkAndStopRecording() del v testsExecuted = gatherFeedbackData(browserName) diff --git a/mkvideo.py b/mkvideo.py index cabcb40..3693d26 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -13,7 +13,7 @@ class Video: # starting the Xvfb process (ie. Xvfb :99 1366x768x16) # FYI TMUX reference: https://gist.github.com/henrik/1967800 # - def __init__(self, port= ":99.0", dimension="1366x768", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + def __init__(self, port= ":99.0", dimension="800x600", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): self.port = port self.dimension = dimension self.nameFile = nameFile From 4a4d8ef08d1e3244f8fbc4bdcf2138c548475f5e Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 18:37:03 -0500 Subject: [PATCH 08/74] testing --- mkcli.py | 4 ++-- mkvideo.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mkcli.py b/mkcli.py index f77748a..6bb7dce 100644 --- a/mkcli.py +++ b/mkcli.py @@ -206,14 +206,14 @@ def run(args): v.checkAndStartRecording(videoNameFile) v.checkActiveSession() v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ls -ltr ") + v.executeCmd("ls -ltr | grep *.mp4") exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ls -ltr ") + v.executeCmd("ls -ltr | grep *.mp4") v.checkAndStopRecording() del v testsExecuted = gatherFeedbackData(browserName) diff --git a/mkvideo.py b/mkvideo.py index 3693d26..64c457e 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -13,7 +13,7 @@ class Video: # starting the Xvfb process (ie. Xvfb :99 1366x768x16) # FYI TMUX reference: https://gist.github.com/henrik/1967800 # - def __init__(self, port= ":99.0", dimension="800x600", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + def __init__(self, port= ":99.0", dimension="1280x1024", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): self.port = port self.dimension = dimension self.nameFile = nameFile From 1f7801a5c62e6931ea75d6a714f6d46ea3a7d899 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 18:46:56 -0500 Subject: [PATCH 09/74] testing --- mkcli.py | 2 +- mkvideo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mkcli.py b/mkcli.py index 6bb7dce..c24a245 100644 --- a/mkcli.py +++ b/mkcli.py @@ -213,8 +213,8 @@ def run(args): print(e) v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ls -ltr | grep *.mp4") v.checkAndStopRecording() + v.executeCmd("ls -ltr | grep *.mp4") del v testsExecuted = gatherFeedbackData(browserName) url = muuktestRoute+'feedback/' diff --git a/mkvideo.py b/mkvideo.py index 64c457e..cabcb40 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -13,7 +13,7 @@ class Video: # starting the Xvfb process (ie. Xvfb :99 1366x768x16) # FYI TMUX reference: https://gist.github.com/henrik/1967800 # - def __init__(self, port= ":99.0", dimension="1280x1024", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + def __init__(self, port= ":99.0", dimension="1366x768", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): self.port = port self.dimension = dimension self.nameFile = nameFile From 5e3fc28f7946ac904f7699043c0fdac40ceccbe3 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Tue, 13 Oct 2020 19:08:52 -0500 Subject: [PATCH 10/74] testing --- mkvideo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkvideo.py b/mkvideo.py index cabcb40..bfc9492 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -13,7 +13,7 @@ class Video: # starting the Xvfb process (ie. Xvfb :99 1366x768x16) # FYI TMUX reference: https://gist.github.com/henrik/1967800 # - def __init__(self, port= ":99.0", dimension="1366x768", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + def __init__(self, port= ":99.0", dimension="1365x767", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): self.port = port self.dimension = dimension self.nameFile = nameFile From 1c212dadf456f511f2890e375c6e89b0f40edc96 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Wed, 14 Oct 2020 11:39:36 -0500 Subject: [PATCH 11/74] default settings --- mkvideo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkvideo.py b/mkvideo.py index bfc9492..cabcb40 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -13,7 +13,7 @@ class Video: # starting the Xvfb process (ie. Xvfb :99 1366x768x16) # FYI TMUX reference: https://gist.github.com/henrik/1967800 # - def __init__(self, port= ":99.0", dimension="1365x767", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): + def __init__(self, port= ":99.0", dimension="1366x768", nameFile = "default.mp4", session = "Muukrecording", input ="x11grab"): self.port = port self.dimension = dimension self.nameFile = nameFile From da1da43973d9760a54f1c0b8c0d6c3af6d061051 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Wed, 14 Oct 2020 18:01:17 -0500 Subject: [PATCH 12/74] changes --- src/test/resources/GebConfig.groovy | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/test/resources/GebConfig.groovy b/src/test/resources/GebConfig.groovy index c4bb162..58aadda 100644 --- a/src/test/resources/GebConfig.groovy +++ b/src/test/resources/GebConfig.groovy @@ -14,10 +14,17 @@ environments { // run via “./gradlew chromeTest” // See: http://code.google.com/p/selenium/wiki/ChromeDriver chrome { - ChromeOptions o = new ChromeOptions() - o.addArguments('--no-sandbox'); - o.addArguments('--disable-dev-shm-usage'); - driver = { new ChromeDriver(o) } + driver = { + ChromeOptions o = new ChromeOptions() + o.addArguments('--no-sandbox'); + o.addArguments('--disable-dev-shm-usage'); + o.addArguments("--ignore-certificate-errors"); + DesiredCapabilities cap=DesiredCapabilities.chrome(); + cap.setCapability(ChromeOptions.CAPABILITY, o); + cap.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); + cap.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS, true); + new ChromeDriver(cap); + } } // run via “./gradlew chromeHeadlessTest” @@ -29,7 +36,7 @@ environments { new ChromeDriver(o) } } - + // run via “./gradlew firefoxTest” // See: http://code.google.com/p/selenium/wiki/FirefoxDriver firefox { From 0fd2ffbbb078d9894e33308c5e3db8a8187401cb Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Wed, 14 Oct 2020 18:05:38 -0500 Subject: [PATCH 13/74] headers --- src/test/resources/GebConfig.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/resources/GebConfig.groovy b/src/test/resources/GebConfig.groovy index 58aadda..5fe535f 100644 --- a/src/test/resources/GebConfig.groovy +++ b/src/test/resources/GebConfig.groovy @@ -5,6 +5,8 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.firefox.FirefoxDriver +import org.openqa.selenium.remote.DesiredCapabilities +import org.openqa.selenium.remote.CapabilityType waiting { timeout = 20 From 764105f2be30c1edd60fa9d3d3b516a4ca7f3247 Mon Sep 17 00:00:00 2001 From: Renan Ugalde Date: Wed, 14 Oct 2020 18:25:54 -0500 Subject: [PATCH 14/74] adding threads --- mkvideo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkvideo.py b/mkvideo.py index cabcb40..a9f3316 100644 --- a/mkvideo.py +++ b/mkvideo.py @@ -95,7 +95,7 @@ def startRecording(self): # The following command starts the recording creating a new TMUX session. # such session should not exist. All parameters are required and the -y # means that it will overwrite the file in case that exists. - cmd = "tmux new-session -d -s "+ self.session +" 'ffmpeg -video_size "+ self.dimension + " -framerate 25 -f "+self.input + " -i "+self.port +" -y " + self.nameFile+"'" + cmd = "tmux new-session -d -s "+ self.session +" 'ffmpeg -video_size "+ self.dimension + " -framerate 25 -f "+self.input + " -threads 4 -i "+self.port +" -y " + self.nameFile+"'" print("startRecoring - start the recording : " , cmd) resStr = self.executeCmd(cmd) From 470544c3efeac58dd262c93517d68d3d84c2766c Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Tue, 1 Jun 2021 09:06:10 -0700 Subject: [PATCH 15/74] Adding MuukReport to mkcli for a better error reporting. --- mkcli.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/mkcli.py b/mkcli.py index 5dcf2b0..977891a 100644 --- a/mkcli.py +++ b/mkcli.py @@ -9,7 +9,9 @@ import xml.etree.ElementTree from time import strftime from mkcloud import gatherScreenshots, resizeImages -#import ssl +import ssl +from bs4 import BeautifulSoup +import pprint def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) @@ -20,6 +22,7 @@ def gatherFeedbackData(browserName): for filename in os.listdir(path): testSuccess = True error = '' + failureMessage = '' if filename.endswith('.xml'): e = xml.etree.ElementTree.parse('build/test-results/'+browserName+'/' + filename).getroot() @@ -30,7 +33,8 @@ def gatherFeedbackData(browserName): if e.find('testcase') is not None : if e.find('testcase').find('failure') is not None : error = e.find('testcase').find('failure').attrib['message'] - + failureMessage = e.find('testcase').find('failure').text + testResult = { "className": e.attrib['name'] if e.attrib['name'] is not None else "", "success": testSuccess, @@ -38,12 +42,16 @@ def gatherFeedbackData(browserName): "hostname": e.attrib['hostname'] if e.attrib['hostname'] is not None else "", "executionTime": e.attrib['time'] if e.attrib['time'] is not None else "", "error": error, - "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "" + "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "", + "systemerror": e.find('system-err').text if e.find('system-err') is not None else "", + "failureMessage": failureMessage, } + testResult["muukReport"] = createMuukReport(testResult.get("className")) feedbackData.append(testResult) else: print("gatherFeedbackData - path does not exists ") testResult = { + "muukReport": {}, "success" : False, #"executionAt": "", "error" : "Test failed during execution. This could be compilation error", @@ -53,6 +61,50 @@ def gatherFeedbackData(browserName): return(feedbackData) +def obtainFeedbackFromDOM(classname, stepId, ntagselector): + path = 'build/reports/geb/firefoxTest/' + filename = path + classname + "_" + str(stepId) + ".html" + domData = {} + parents = [] + if os.path.exists(filename): + try: + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + selectors = soup.select(ntagselector) + domData["numberOfElementsWithSameSelector"] = len(selectors) + #domData["elementsWithSameSelector"] = selectors + for selector in selectors: + parents.append(selector.parent) + domData["elementsWithSameSelectorAndParents"] = parents + except Exception as ex: + print("Failed to open file " + filename) + print (ex) + + return domData + + +def createMuukReport(classname): + path = 'build/reports/' + filename = path + classname + ".json" + muukReport = {} + steps = [] + if(os.path.exists(filename)): + try: + jsonFile = open(filename, 'r') + elements = json.load(jsonFile) + for element in elements['stepsFeedback']: + domInfo = obtainFeedbackFromDOM(classname, element.get("id"), element.get("selector")) + element["numberOfElementsWithSameSelector"] = domInfo.get("numberOfElementsWithSameSelector") + element["elementsWithSameSelectorAndParents"] = str(domInfo.get("elementsWithSameSelectorAndParents")) + steps.append(element) + except ValueError as err: + print("Invalid json file found") + + # Closing file + jsonFile.close() + + muukReport["steps"] = steps + return muukReport def run(args): #Gets the value from the flags From 88a6ea70c135eae701020bdce89cbfd73bc66f4c Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 9 Jun 2021 10:07:35 -0700 Subject: [PATCH 16/74] Handle DOM parsing on mkcli side. --- domparser.py | 215 +++++++++++++++++++++++++++++++++++++++++++++++++++ mkcli.py | 48 +----------- 2 files changed, 216 insertions(+), 47 deletions(-) create mode 100644 domparser.py diff --git a/domparser.py b/domparser.py new file mode 100644 index 0000000..b204044 --- /dev/null +++ b/domparser.py @@ -0,0 +1,215 @@ + +import os +import json +import pprint +from bs4 import BeautifulSoup + +def parseImageSelector(selectors, expectedValue, expectedIndex): + jsonObject = {} + elements = [] + index = 0 + for selector in selectors: + if(selector['src'] == expectedValue and expectedIndex == index): + element = {} + element["index"] = index + element["value"] = selector['src'] + element["expectedSelector"] = "true" + elements.append(element) + index+=1 + + # If the correct selector was not found, we need to return the selector that we think was selected + # using the index from the object. + if(len(elements) == 0 and expectedIndex <= len(selectors)): + selector = selectors[expectedIndex] + if(selector): + element = {} + element["index"] = expectedIndex + element["value"] = selector['src'] + element["expectedSelector"] = "false" + elements.append(element) + + jsonObject["selectors"] = elements + return jsonObject + +def parseHypertextSelector(selectors, expectedValue, expectedIndex): + + jsonObject = {} + elements = [] + index = 0 + for selector in selectors: + if(selector and ("href" in selector)): + if(selector['href'] == expectedValue and expectedIndex == index): + element = {} + element["index"] = index + element["value"] = selector['href'] + element["expectedSelector"] = "true" + elements.append(element) + index+=1 + + # If the correct selector was not found, we need to return the selector that we think was selected + # using the index from the object. + # If the correct selector was not found, we need to return the selector that we think was selected + # using the index from the object. + if(len(elements) == 0 and expectedIndex <= len(selectors)): + selector = selectors[expectedIndex] + if(selector): + element = {} + element["index"] = expectedIndex + if("href" in selector): + element["value"] = selector['href'] + element["expectedSelector"] = "false" + else: + element["value"] = expectedValue + element["expectedSelector"] = "true" + elements.append(element) + + jsonObject["selectors"] = elements + return jsonObject + +def parseTextSelector(selectors, expectedValue, expectedIndex): + jsonObject = {} + elements = [] + index = 0 + for selector in selectors: + if((selector.text).strip() == expectedValue.strip() and expectedIndex == index ): + element = {} + element["index"] = index + element["value"] = selector.text + element["expectedSelector"] = "true" + elements.append(element) + index+=1 + + # If the correct selector was not found, we need to return the selector that we think was selected + # using the index from the object. + if(len(elements) == 0 and expectedIndex <= len(selectors)): + selector = selectors[expectedIndex] + if(selector): + element = {} + element["index"] = expectedIndex + element["value"] = selector.text + element["expectedSelector"] = "false" + elements.append(element) + + jsonObject["selectors"] = elements + return jsonObject + +def parseValueSelector(selectors, expectedValue, expectedIndex, type): + jsonObject = {} + elements = [] + index = 0 + + for selector in selectors: + if(selector['value'] == expectedValue and expectedIndex <= len(selectors)): + element = {} + element["index"] = index + element["value"] = expectedValue + element["expectedSelector"] = "true" + elements.append(element) + index+=1 + + # If the correct selector was not found, we need to return the selector that we think was selected + # using the index from the object. + if(len(elements) == 0 and expectedIndex <= len(selectors)): + selector = selectors[expectedIndex] + if(selector): + element = {} + element["index"] = expectedIndex + element["value"] = expectedValue + element["expectedSelector"] = "false" + elements.append(element) + + jsonObject["selectors"] = elements + return jsonObject + +def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType): + jsonObject = {} + elements = [] + path = 'build/reports/geb/firefoxTest/' + filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): + try: + print("\n============= Step " + str(stepId) + "=============") + print("Tag " + tag) + print("Search by " + searchType) + print("index " + str(index)) + print("action " + str(action)) + + if(action == "assignment" or action == "mouseover"): + jsonObject["selectors"] = [] + numberSelectorsFound = 0 + else: + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + selectorsFound = soup.select(ntagselector) + numberSelectorsFound = len(selectorsFound) + print("Selectors found: " + str(numberSelectorsFound)) + if(numberSelectorsFound == 0 ): + jsonObject["selectors"] = [] + elif(numberSelectorsFound > 1 ): + if(searchType == "value"): + jsonObject = parseValueSelector(selectorsFound, value, index, type) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(selectorsFound, value, index) + elif(searchType == "text"): + jsonObject = parseTextSelector(selectorsFound, value, index) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(selectorsFound, value, index) + else: + # Backend sent an undef searchType, we will return no info + jsonObject["selectors"] = [] + numberSelectorsFound = 0 + elif(numberSelectorsFound == 1 ): + element = {} + element["index"] = index + element["value"] = value + element["expectedSelector"] = "true" + elements.append(element) + jsonObject["selectors"] = elements + + jsonObject["numberOfElementsWithSameSelector"] = len(jsonObject["selectors"]) + pprint.pprint(jsonObject) + print("==============================================") + except Exception as ex: + print("Failed to open file " + ex) + print (ex) + + return jsonObject + + +def createMuukReport(classname): + print("createMuukReport") + path = 'build/reports/' + filename = path + classname + ".json" + muukReport = {} + steps = [] + if(os.path.exists(filename)): + try: + jsonFile = open(filename, 'r') + elements = json.load(jsonFile) + index = 0 + for element in elements['stepsFeedback']: + valueData = json.loads(element.get("value")) + domInfo = obtainFeedbackFromDOM(classname, element.get("id"), + element.get("selector"), valueData["value"], + element.get("index"), element.get("tag"), + element.get("objectType"), + element.get("action"), + valueData["searchType"]) + if(domInfo): + element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] + element["selectors"] = domInfo["selectors"] + steps.append(element) + + except Exception as ex: + print("Exception found during DOM parsing. Exception = " + str(ex)) + + # Closing file + jsonFile.close() + + muukReport["steps"] = steps + print("createMuukReport - exit ") + pprint.pprint(steps) + + return muukReport + +#createMuukReport("muuktestElorusCom593c0d63") diff --git a/mkcli.py b/mkcli.py index 977891a..30a1a63 100644 --- a/mkcli.py +++ b/mkcli.py @@ -10,8 +10,7 @@ from time import strftime from mkcloud import gatherScreenshots, resizeImages import ssl -from bs4 import BeautifulSoup -import pprint +from domparser import createMuukReport def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) @@ -61,51 +60,6 @@ def gatherFeedbackData(browserName): return(feedbackData) -def obtainFeedbackFromDOM(classname, stepId, ntagselector): - path = 'build/reports/geb/firefoxTest/' - filename = path + classname + "_" + str(stepId) + ".html" - domData = {} - parents = [] - if os.path.exists(filename): - try: - text = open(filename, 'r').read() - soup = BeautifulSoup(text, 'html.parser') - selectors = soup.select(ntagselector) - domData["numberOfElementsWithSameSelector"] = len(selectors) - #domData["elementsWithSameSelector"] = selectors - for selector in selectors: - parents.append(selector.parent) - domData["elementsWithSameSelectorAndParents"] = parents - except Exception as ex: - print("Failed to open file " + filename) - print (ex) - - return domData - - -def createMuukReport(classname): - path = 'build/reports/' - filename = path + classname + ".json" - muukReport = {} - steps = [] - if(os.path.exists(filename)): - try: - jsonFile = open(filename, 'r') - elements = json.load(jsonFile) - for element in elements['stepsFeedback']: - domInfo = obtainFeedbackFromDOM(classname, element.get("id"), element.get("selector")) - element["numberOfElementsWithSameSelector"] = domInfo.get("numberOfElementsWithSameSelector") - element["elementsWithSameSelectorAndParents"] = str(domInfo.get("elementsWithSameSelectorAndParents")) - steps.append(element) - except ValueError as err: - print("Invalid json file found") - - # Closing file - jsonFile.close() - - muukReport["steps"] = steps - return muukReport - def run(args): #Gets the value from the flags print("Starting process") From d153fa2eb2a924d27f43898f629a7f399ea18d5b Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 9 Jun 2021 20:34:05 -0700 Subject: [PATCH 17/74] Add more attributes to the steps. --- domparser.py | 304 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 197 insertions(+), 107 deletions(-) diff --git a/domparser.py b/domparser.py index b204044..691dce3 100644 --- a/domparser.py +++ b/domparser.py @@ -4,128 +4,205 @@ import pprint from bs4 import BeautifulSoup -def parseImageSelector(selectors, expectedValue, expectedIndex): +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_INCORRECT_INDEX = 12 + + +def processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, attribute): jsonObject = {} elements = [] - index = 0 - for selector in selectors: - if(selector['src'] == expectedValue and expectedIndex == index): + + # After we filter the selectors we have 3 options: + # 1) No selectors were found having the value we are expecting. On this case, + # information returned will be the element with the index that was expected. + # + # 2) We found only one selector, we have two options here: + # a) Found the correct selector: Return the original element. + # b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. + # + # 3) We found two or more selectors with the same src. We have two options here: + # a) The correct selector was found. Return the original element. + # b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. + # + if(selectorsFound == 0): + print("No selectors were found with the expected src!!!") + element = {} + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "found" + elements.append(element) + returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE + elif(selectorsFound == 1): + if(expectedIndex in selectorIndexes): + print("The expected selector was found and it is the only selector.") element = {} - element["index"] = index - element["value"] = selector['src'] - element["expectedSelector"] = "true" - elements.append(element) - index+=1 - - # If the correct selector was not found, we need to return the selector that we think was selected - # using the index from the object. - if(len(elements) == 0 and expectedIndex <= len(selectors)): - selector = selectors[expectedIndex] - if(selector): + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + else: + # The expected selector was not found, we need to return the original selector (using expected index) + # and the found selector. + print("The incorrect selector was found and this is the only selector with the expected src") element = {} element["index"] = expectedIndex - element["value"] = selector['src'] - element["expectedSelector"] = "false" - elements.append(element) - - jsonObject["selectors"] = elements - return jsonObject + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" + elements.append(element) -def parseHypertextSelector(selectors, expectedValue, expectedIndex): - - jsonObject = {} - elements = [] - index = 0 - for selector in selectors: - if(selector and ("href" in selector)): - if(selector['href'] == expectedValue and expectedIndex == index): - element = {} - element["index"] = index - element["value"] = selector['href'] - element["expectedSelector"] = "true" - elements.append(element) - index+=1 - - # If the correct selector was not found, we need to return the selector that we think was selected - # using the index from the object. - # If the correct selector was not found, we need to return the selector that we think was selected - # using the index from the object. - if(len(elements) == 0 and expectedIndex <= len(selectors)): - selector = selectors[expectedIndex] - if(selector): + element = {} + element["index"] = selectorIndexes[selectorsFound -1] + if(attribute == "text"): + element["value"] = selectors[selectorIndexes[selectorsFound -1]].text + else: + element["value"] = selectors[selectorIndexes[selectorsFound -1]][attribute] + element["selector"] = "found" + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + elif(selectorsFound > 1): + print("Several selectors were found with same src " + str(selectorIndexes)) + if(expectedIndex in selectorIndexes): + print("The expected element " + str(expectedIndex) + " was found on the selectors") element = {} element["index"] = expectedIndex - if("href" in selector): - element["value"] = selector['href'] - element["expectedSelector"] = "false" - else: - element["value"] = expectedValue - element["expectedSelector"] = "true" + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" elements.append(element) - + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_CORRECT_INDEX + else: + print("The expected element " + str(expectedIndex) + " was NOT found on the selectors") + element = {} + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["index"] = expectedIndex + element["selector"] = "original" + elements.append(element) + + element = {} + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["index"] = str(selectorIndexes) + element["selector"] = "found" + elements.append(element) + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_INCORRECT_INDEX + + jsonObject["numberOfElementsWithSameSelectorAndValue"] = selectorsFound jsonObject["selectors"] = elements + jsonObject["rc"] = returnCode + return jsonObject def parseTextSelector(selectors, expectedValue, expectedIndex): jsonObject = {} - elements = [] - index = 0 + selectorIndexes = [] + counter = 0 + selectorsFound = 0 + + if(expectedValue == ""): + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + for selector in selectors: - if((selector.text).strip() == expectedValue.strip() and expectedIndex == index ): - element = {} - element["index"] = index - element["value"] = selector.text - element["expectedSelector"] = "true" - elements.append(element) - index+=1 + if((selector.text).strip() == expectedValue.strip()): + selectorsFound += 1 + selectorIndexes.append(counter) + counter+=1 - # If the correct selector was not found, we need to return the selector that we think was selected - # using the index from the object. - if(len(elements) == 0 and expectedIndex <= len(selectors)): - selector = selectors[expectedIndex] - if(selector): - element = {} - element["index"] = expectedIndex - element["value"] = selector.text - element["expectedSelector"] = "false" - elements.append(element) + return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "text") + + +def parseImageSelector(selectors, expectedValue, expectedIndex): + selectorIndexes = [] + counter = 0 + selectorsFound = 0 + for selector in selectors: + if(selector['src'] == expectedValue ): + selectorsFound += 1 + selectorIndexes.append(counter) + counter+=1 + + return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "src") + +# +# This method will be called when two or more selectors are found with . +# the same ntagselector value. This method will use the expected value (href) +# to filter the selctors and try to find the one that was used by the test. +# +def parseHypertextSelector(selectors, expectedValue, expectedIndex): + jsonObject = {} + selectorIndexes = [] + counter = 0 + selectorsFound = 0 + + if(expectedValue == ""): + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for selector in selectors: + if(selector and selector.has_attr('href')): + if(selector['href'] == expectedValue): + selectorsFound += 1 + selectorIndexes.append(counter) + counter+=1 + + return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "href") - jsonObject["selectors"] = elements - return jsonObject def parseValueSelector(selectors, expectedValue, expectedIndex, type): jsonObject = {} - elements = [] - index = 0 + selectorIndexes = [] + counter = 0 + selectorsFound = 0 for selector in selectors: - if(selector['value'] == expectedValue and expectedIndex <= len(selectors)): - element = {} - element["index"] = index - element["value"] = expectedValue - element["expectedSelector"] = "true" - elements.append(element) - index+=1 - - # If the correct selector was not found, we need to return the selector that we think was selected - # using the index from the object. - if(len(elements) == 0 and expectedIndex <= len(selectors)): - selector = selectors[expectedIndex] - if(selector): - element = {} - element["index"] = expectedIndex - element["value"] = expectedValue - element["expectedSelector"] = "false" - elements.append(element) + if(selector['value'] == expectedValue ): + selectorsFound += 1 + selectorIndexes.append(counter) + counter+=1 + + jsonObject = processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "value") - jsonObject["selectors"] = elements return jsonObject + + def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType): jsonObject = {} elements = [] path = 'build/reports/geb/firefoxTest/' filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): try: print("\n============= Step " + str(stepId) + "=============") @@ -133,16 +210,26 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty print("Search by " + searchType) print("index " + str(index)) print("action " + str(action)) + + #if(stepId != 7): + #return - if(action == "assignment" or action == "mouseover"): + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + selectorsFound = soup.select(ntagselector) + #print("Selectors: " + str(selectorsFound)) + numberSelectorsFound = len(selectorsFound) + print("Selectors found: " + str(numberSelectorsFound)) + + if(index > numberSelectorsFound): + jsonObject["selectors"] = [] + jsonObject["rc"] = STEP_INDEX_GREATER_THAN_SELECTORS_FOUND + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elif(action == "assignment" or action == "mouseover"): jsonObject["selectors"] = [] - numberSelectorsFound = 0 + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 else: - text = open(filename, 'r').read() - soup = BeautifulSoup(text, 'html.parser') - selectorsFound = soup.select(ntagselector) - numberSelectorsFound = len(selectorsFound) - print("Selectors found: " + str(numberSelectorsFound)) if(numberSelectorsFound == 0 ): jsonObject["selectors"] = [] elif(numberSelectorsFound > 1 ): @@ -157,16 +244,20 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty else: # Backend sent an undef searchType, we will return no info jsonObject["selectors"] = [] - numberSelectorsFound = 0 + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elif(numberSelectorsFound == 1 ): element = {} element["index"] = index element["value"] = value - element["expectedSelector"] = "true" + element["selector"] = "original" elements.append(element) jsonObject["selectors"] = elements + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["rc"] = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR - jsonObject["numberOfElementsWithSameSelector"] = len(jsonObject["selectors"]) + jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound pprint.pprint(jsonObject) print("==============================================") except Exception as ex: @@ -177,7 +268,6 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty def createMuukReport(classname): - print("createMuukReport") path = 'build/reports/' filename = path + classname + ".json" muukReport = {} @@ -186,7 +276,6 @@ def createMuukReport(classname): try: jsonFile = open(filename, 'r') elements = json.load(jsonFile) - index = 0 for element in elements['stepsFeedback']: valueData = json.loads(element.get("value")) domInfo = obtainFeedbackFromDOM(classname, element.get("id"), @@ -197,6 +286,8 @@ def createMuukReport(classname): valueData["searchType"]) if(domInfo): element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] + element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] + element["rc"] = domInfo["rc"] element["selectors"] = domInfo["selectors"] steps.append(element) @@ -204,12 +295,11 @@ def createMuukReport(classname): print("Exception found during DOM parsing. Exception = " + str(ex)) # Closing file - jsonFile.close() + jsonFile.close() muukReport["steps"] = steps - print("createMuukReport - exit ") pprint.pprint(steps) return muukReport -#createMuukReport("muuktestElorusCom593c0d63") +#createMuukReport("devFushosoftCom86e660e3") From f0b2590e4ab8eb7a86e77d89d2f94ad2621b91b7 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Thu, 10 Jun 2021 09:40:33 -0700 Subject: [PATCH 18/74] Update error codes. --- domparser.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/domparser.py b/domparser.py index 691dce3..835a829 100644 --- a/domparser.py +++ b/domparser.py @@ -9,16 +9,16 @@ NO_VALUE_PROVIDED_BY_BE = 3 NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 ACTION_NOT_VALID_FOR_ANALYSIS = 5 -STEP_INDEX_GREATER_THAN_SELECTORS_FOUND = 6 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 -MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_CORRECT_INDEX = 11 -MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_INCORRECT_INDEX = 12 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 -def processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, attribute): +def processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, attribute): jsonObject = {} elements = [] @@ -91,7 +91,7 @@ def processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, at element["value"] = selectors[expectedIndex][attribute] element["selector"] = "original" elements.append(element) - returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_CORRECT_INDEX + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX else: print("The expected element " + str(expectedIndex) + " was NOT found on the selectors") element = {} @@ -105,13 +105,13 @@ def processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, at element = {} if(attribute == "text"): - element["value"] = selectors[expectedIndex].text + element["value"] = expectedValue else: - element["value"] = selectors[expectedIndex][attribute] + element["value"] = expectedValue element["index"] = str(selectorIndexes) element["selector"] = "found" elements.append(element) - returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPCTED_VALUE_INCORRECT_INDEX + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX jsonObject["numberOfElementsWithSameSelectorAndValue"] = selectorsFound jsonObject["selectors"] = elements @@ -137,7 +137,7 @@ def parseTextSelector(selectors, expectedValue, expectedIndex): selectorIndexes.append(counter) counter+=1 - return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "text") + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "text") def parseImageSelector(selectors, expectedValue, expectedIndex): @@ -150,7 +150,7 @@ def parseImageSelector(selectors, expectedValue, expectedIndex): selectorIndexes.append(counter) counter+=1 - return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "src") + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "src") # # This method will be called when two or more selectors are found with . @@ -176,7 +176,7 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): selectorIndexes.append(counter) counter+=1 - return processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "href") + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") def parseValueSelector(selectors, expectedValue, expectedIndex, type): @@ -191,7 +191,7 @@ def parseValueSelector(selectors, expectedValue, expectedIndex, type): selectorIndexes.append(counter) counter+=1 - jsonObject = processResults(selectors, expectedIndex, selectorsFound, selectorIndexes, "value") + jsonObject = processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") return jsonObject @@ -206,14 +206,15 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty if os.path.exists(filename): try: print("\n============= Step " + str(stepId) + "=============") + + if(stepId == 6): + index = 2 + print("Tag " + tag) print("Search by " + searchType) print("index " + str(index)) print("action " + str(action)) - #if(stepId != 7): - #return - text = open(filename, 'r').read() soup = BeautifulSoup(text, 'html.parser') selectorsFound = soup.select(ntagselector) @@ -223,7 +224,7 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty if(index > numberSelectorsFound): jsonObject["selectors"] = [] - jsonObject["rc"] = STEP_INDEX_GREATER_THAN_SELECTORS_FOUND + jsonObject["rc"] = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 elif(action == "assignment" or action == "mouseover"): jsonObject["selectors"] = [] @@ -254,7 +255,7 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty element["selector"] = "original" elements.append(element) jsonObject["selectors"] = elements - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound jsonObject["rc"] = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound @@ -302,4 +303,4 @@ def createMuukReport(classname): return muukReport -#createMuukReport("devFushosoftCom86e660e3") +createMuukReport("devFushosoftCom86e660e3") From 31e710ff16ebb2aad2726ba33370f5ad821aafac Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Thu, 10 Jun 2021 09:43:41 -0700 Subject: [PATCH 19/74] Remove code used to run locally. --- domparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domparser.py b/domparser.py index 835a829..6c82fc0 100644 --- a/domparser.py +++ b/domparser.py @@ -303,4 +303,4 @@ def createMuukReport(classname): return muukReport -createMuukReport("devFushosoftCom86e660e3") +#createMuukReport("devFushosoftCom86e660e3") From bccfb3a71f13b89d58def051db1f9dd3c9b10429 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Fri, 11 Jun 2021 12:49:53 -0700 Subject: [PATCH 20/74] add json simple to gradle --- build.gradle | 3 +++ domparser.py | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index afdc692..55d6a67 100644 --- a/build.gradle +++ b/build.gradle @@ -39,6 +39,9 @@ dependencies { testCompile "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion" compile group: 'org.seleniumhq.selenium', name: 'selenium-support', version: '3.14.0' + + // JSON simple for MuukReport + implementation group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1' } diff --git a/domparser.py b/domparser.py index 6c82fc0..dd6880c 100644 --- a/domparser.py +++ b/domparser.py @@ -296,11 +296,9 @@ def createMuukReport(classname): print("Exception found during DOM parsing. Exception = " + str(ex)) # Closing file - jsonFile.close() + jsonFile.close() muukReport["steps"] = steps pprint.pprint(steps) return muukReport - -#createMuukReport("devFushosoftCom86e660e3") From d5e8432e0fe10c23893429133426ff8a1e26db76 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Fri, 11 Jun 2021 15:43:28 -0700 Subject: [PATCH 21/74] Pass browser name to domparser to have the correct path during execution. --- domparser.py | 15 ++++++++------- mkcli.py | 39 +++++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/domparser.py b/domparser.py index dd6880c..e9c043c 100644 --- a/domparser.py +++ b/domparser.py @@ -197,19 +197,16 @@ def parseValueSelector(selectors, expectedValue, expectedIndex, type): -def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType): +def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType, browserName): jsonObject = {} elements = [] - path = 'build/reports/geb/firefoxTest/' + path = 'build/reports/geb/' + browserName + '/' filename = path + classname + "_" + str(stepId) + ".html" if os.path.exists(filename): try: print("\n============= Step " + str(stepId) + "=============") - if(stepId == 6): - index = 2 - print("Tag " + tag) print("Search by " + searchType) print("index " + str(index)) @@ -268,7 +265,8 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty return jsonObject -def createMuukReport(classname): +def createMuukReport(classname, browserName): + print("starting Muuk Report on " + str(browserName)) path = 'build/reports/' filename = path + classname + ".json" muukReport = {} @@ -284,7 +282,8 @@ def createMuukReport(classname): element.get("index"), element.get("tag"), element.get("objectType"), element.get("action"), - valueData["searchType"]) + valueData["searchType"], + browserName) if(domInfo): element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] @@ -297,6 +296,8 @@ def createMuukReport(classname): # Closing file jsonFile.close() + else: + print("Muuk Report does not exists!") muukReport["steps"] = steps pprint.pprint(steps) diff --git a/mkcli.py b/mkcli.py index 30a1a63..e7011d4 100644 --- a/mkcli.py +++ b/mkcli.py @@ -45,7 +45,7 @@ def gatherFeedbackData(browserName): "systemerror": e.find('system-err').text if e.find('system-err') is not None else "", "failureMessage": failureMessage, } - testResult["muukReport"] = createMuukReport(testResult.get("className")) + testResult["muukReport"] = createMuukReport(testResult.get("className"), browserName) feedbackData.append(testResult) else: print("gatherFeedbackData - path does not exists ") @@ -79,12 +79,12 @@ def run(args): exitCode = 1 #Check if we received a browser and get the string for the gradlew task command browserName = getBrowserName(browser) - muuktestRoute = 'https://portal.muuktest.com:8081/' - supportRoute = 'https://testing.muuktest.com:8082/' + #muuktestRoute = 'https://portal.muuktest.com:8081/' + #supportRoute = 'https://testing.muuktest.com:8082/' - #muuktestRoute = 'https://localhost:8081/' - #supportRoute = 'https://localhost:8082/' + muuktestRoute = 'https://localhost:8081/' + supportRoute = 'https://localhost:8082/' @@ -105,8 +105,8 @@ def run(args): try: key_file = open(path,'r') key = key_file.read() - r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) - #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) + #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) + r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) responseObject = json.loads(r.content) token = responseObject["token"] userId = responseObject["userId"] @@ -143,14 +143,14 @@ def run(args): # This route downloads the scripts by the property. url = muuktestRoute+'download_byproperty/' - #context = ssl._create_unverified_context() + context = ssl._create_unverified_context() data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') #now using urlopen get the file and store it locally auth_request = request.Request(url,headers=auth, data=data) auth_request.add_header('Authorization', 'Bearer '+token) - response = request.urlopen(auth_request) - #response = request.urlopen(auth_request, context=context) + #response = request.urlopen(auth_request) + response = request.urlopen(auth_request, context=context) #response = request.urlopen(url,data) file = response.read() @@ -195,8 +195,8 @@ def run(args): } try: - requests.post(supportRoute+"tracking_data", json=payload) - # equests.post(supportRoute+"tracking_data", json=payload, verify=False) + #requests.post(supportRoute+"tracking_data", json=payload) + requests.post(supportRoute+"tracking_data", json=payload, verify=False) except Exception as e: print("No connection to support Data Base") @@ -205,6 +205,7 @@ def run(args): #Execute the test print("Executing test...") try: + exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") @@ -217,12 +218,12 @@ def run(args): #CLOUD SCREENSHOTS STARTS # resizeImages(browserName) - #cloudKey = getCloudKey() + cloudKey = getCloudKey() filesData = gatherScreenshots(browserName) try: if filesData != {}: - requests.post(muuktestRoute + 'upload_cloud_steps_images/', headers=hed, files = filesData) - #requests.post(muuktestRoute + 'upload_cloud_steps_images/', data={'cloudKey': cloudKey}, headers=hed, files = filesData, verify=False) + #requests.post(muuktestRoute + 'upload_cloud_steps_images/', headers=hed, files = filesData) + requests.post(muuktestRoute + 'upload_cloud_steps_images/', data={'cloudKey': cloudKey}, headers=hed, files = filesData, verify=False) else: print ("filesData empty.. cannot send screenshots") except Exception as e: @@ -231,8 +232,8 @@ def run(args): #CLOUD SCREENSHOTS ENDS try: #Executions feedback - requests.post(url, json=values, headers=hed) - #requests.post(url, json=values, headers=hed, verify=False) + #requests.post(url, json=values, headers=hed) + requests.post(url, json=values, headers=hed, verify=False) #save the executed test entry to the database requests.post(supportRoute+"tracking_data", data={ @@ -262,7 +263,9 @@ def getBrowserName(browser): #select a browser from the list or return firefox as default return switcher.get(browser,"firefoxTest") - +def getCloudKey(): + return('lkussv97qginpidrtoey-1ec3r3678pe7n0wulh1v9k-zthz8er1ukl59ux1k0epop-mioxpeadsu5iegbjhd4j4') + def main(): parser=argparse.ArgumentParser(description="MuukTest cli to download tests from the cloud") parser.add_argument("-p",help="property to search the test for" ,dest="field", type=str, required=True) From 61d3d0b2833ebc1c0c6105784e02bc6765dbf818 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 16 Jun 2021 15:11:47 -0700 Subject: [PATCH 22/74] Fix exception. --- domparser.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/domparser.py b/domparser.py index e9c043c..890a2b4 100644 --- a/domparser.py +++ b/domparser.py @@ -16,6 +16,7 @@ SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 def processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, attribute): @@ -153,7 +154,7 @@ def parseImageSelector(selectors, expectedValue, expectedIndex): return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "src") # -# This method will be called when two or more selectors are found with . +# This method will be called when two or more selectors are found with # the same ntagselector value. This method will use the expected value (href) # to filter the selctors and try to find the one that was used by the test. # @@ -178,7 +179,11 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") - +# +# This method will be called when two or more selectors are found with +# the same ntagselector value. This method will use the expected value +# to filter the selctors and try to find the one that was used by the test. +# def parseValueSelector(selectors, expectedValue, expectedIndex, type): jsonObject = {} selectorIndexes = [] @@ -230,6 +235,8 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty else: if(numberSelectorsFound == 0 ): jsonObject["selectors"] = [] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR elif(numberSelectorsFound > 1 ): if(searchType == "value"): jsonObject = parseValueSelector(selectorsFound, value, index, type) @@ -272,7 +279,7 @@ def createMuukReport(classname, browserName): muukReport = {} steps = [] if(os.path.exists(filename)): - try: + #try: jsonFile = open(filename, 'r') elements = json.load(jsonFile) for element in elements['stepsFeedback']: @@ -291,11 +298,11 @@ def createMuukReport(classname, browserName): element["selectors"] = domInfo["selectors"] steps.append(element) - except Exception as ex: - print("Exception found during DOM parsing. Exception = " + str(ex)) + #except Exception as ex: + #print("Exception found during DOM parsing. Exception = " + str(ex)) # Closing file - jsonFile.close() + jsonFile.close() else: print("Muuk Report does not exists!") From 35272dbda7978d366aa13d1d6d6f97b712788892 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Mon, 21 Jun 2021 17:40:56 -0700 Subject: [PATCH 23/74] Update code comments. --- domparser.py | 181 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 117 insertions(+), 64 deletions(-) diff --git a/domparser.py b/domparser.py index 890a2b4..813ebec 100644 --- a/domparser.py +++ b/domparser.py @@ -19,24 +19,31 @@ NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +# Description: +# This method will be called to handle the result of filter operation done +# on the selectors found. +# +# There are 3 options for the result: +# 1) No selectors were found having the value we are expecting. On this case, +# information returned will be the element with the index that was expected. +# +# 2) We found only one selector, we have two options here: +# a) Found the correct selector: Return the original element. +# b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. +# +# 3) We found two or more selectors with the same src. We have two options here: +# a) The correct selector was found. Return the original element. +# b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +# def processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, attribute): jsonObject = {} elements = [] - # After we filter the selectors we have 3 options: - # 1) No selectors were found having the value we are expecting. On this case, - # information returned will be the element with the index that was expected. - # - # 2) We found only one selector, we have two options here: - # a) Found the correct selector: Return the original element. - # b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. - # - # 3) We found two or more selectors with the same src. We have two options here: - # a) The correct selector was found. Return the original element. - # b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. - # if(selectorsFound == 0): - print("No selectors were found with the expected src!!!") + # No selectors were found with the expected value element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -48,7 +55,7 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE elif(selectorsFound == 1): if(expectedIndex in selectorIndexes): - print("The expected selector was found and it is the only selector.") + # The expected selector was found and it is the only selector. element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -59,9 +66,7 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele elements.append(element) returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX else: - # The expected selector was not found, we need to return the original selector (using expected index) - # and the found selector. - print("The incorrect selector was found and this is the only selector with the expected src") + # The incorrect selector was found and this is the only selector with the expected value element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -81,9 +86,9 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele elements.append(element) returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX elif(selectorsFound > 1): - print("Several selectors were found with same src " + str(selectorIndexes)) + # Several selectors were found with same value if(expectedIndex in selectorIndexes): - print("The expected element " + str(expectedIndex) + " was found on the selectors") + # The expected element was found on the selectors element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -94,7 +99,7 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele elements.append(element) returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX else: - print("The expected element " + str(expectedIndex) + " was NOT found on the selectors") + # The expected element was NOT found on the selectors element = {} if(attribute == "text"): element["value"] = selectors[expectedIndex].text @@ -120,10 +125,22 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele return jsonObject +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the text value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# expectedValue: The value that is expected to be found (value captured by the extension). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. def parseTextSelector(selectors, expectedValue, expectedIndex): jsonObject = {} selectorIndexes = [] - counter = 0 + selectorIndex = 0 selectorsFound = 0 if(expectedValue == ""): @@ -135,33 +152,51 @@ def parseTextSelector(selectors, expectedValue, expectedIndex): for selector in selectors: if((selector.text).strip() == expectedValue.strip()): selectorsFound += 1 - selectorIndexes.append(counter) - counter+=1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "text") - +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the src value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# expectedValue: The value that is expected to be found (value captured by the extension). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. def parseImageSelector(selectors, expectedValue, expectedIndex): selectorIndexes = [] - counter = 0 + selectorIndex = 0 selectorsFound = 0 for selector in selectors: if(selector['src'] == expectedValue ): selectorsFound += 1 - selectorIndexes.append(counter) - counter+=1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "src") +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the href value attribute +# to filter the selctors and try to find the one that was used by the test. # -# This method will be called when two or more selectors are found with -# the same ntagselector value. This method will use the expected value (href) -# to filter the selctors and try to find the one that was used by the test. +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# expectedValue: The value that is expected to be found (value captured by the extension). +# expectedIndex: The index that is expected to contain the expected value. # +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. def parseHypertextSelector(selectors, expectedValue, expectedIndex): jsonObject = {} selectorIndexes = [] - counter = 0 + selectorIndex = 0 selectorsFound = 0 if(expectedValue == ""): @@ -174,34 +209,45 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): if(selector and selector.has_attr('href')): if(selector['href'] == expectedValue): selectorsFound += 1 - selectorIndexes.append(counter) - counter+=1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the value attribute +# to filter the selctors and try to find the one that was used by the test. # -# This method will be called when two or more selectors are found with -# the same ntagselector value. This method will use the expected value -# to filter the selctors and try to find the one that was used by the test. +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# expectedValue: The value that is expected to be found (value captured by the extension). +# expectedIndex: The index that is expected to contain the expected value. # +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. def parseValueSelector(selectors, expectedValue, expectedIndex, type): jsonObject = {} selectorIndexes = [] - counter = 0 + selectorIndex = 0 selectorsFound = 0 for selector in selectors: if(selector['value'] == expectedValue ): selectorsFound += 1 - selectorIndexes.append(counter) - counter+=1 - - jsonObject = processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") - - return jsonObject - + selectorIndexes.append(selectorIndex) + selectorIndex+=1 + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType, browserName): jsonObject = {} elements = [] @@ -210,19 +256,10 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty if os.path.exists(filename): try: - print("\n============= Step " + str(stepId) + "=============") - - print("Tag " + tag) - print("Search by " + searchType) - print("index " + str(index)) - print("action " + str(action)) - text = open(filename, 'r').read() soup = BeautifulSoup(text, 'html.parser') selectorsFound = soup.select(ntagselector) - #print("Selectors: " + str(selectorsFound)) numberSelectorsFound = len(selectorsFound) - print("Selectors found: " + str(numberSelectorsFound)) if(index > numberSelectorsFound): jsonObject["selectors"] = [] @@ -257,14 +294,21 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty element["index"] = index element["value"] = value element["selector"] = "original" + if(searchType == "value"): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href"): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text"): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc"): + element["value"] = selectorsFound[0]["src"] elements.append(element) jsonObject["selectors"] = elements jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound jsonObject["rc"] = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound - pprint.pprint(jsonObject) - print("==============================================") + except Exception as ex: print("Failed to open file " + ex) print (ex) @@ -272,14 +316,21 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty return jsonObject +# Description: +# This method will be call to execuete the Muuk Report analysis. +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. def createMuukReport(classname, browserName): - print("starting Muuk Report on " + str(browserName)) path = 'build/reports/' filename = path + classname + ".json" muukReport = {} steps = [] if(os.path.exists(filename)): - #try: + try: jsonFile = open(filename, 'r') elements = json.load(jsonFile) for element in elements['stepsFeedback']: @@ -287,8 +338,7 @@ def createMuukReport(classname, browserName): domInfo = obtainFeedbackFromDOM(classname, element.get("id"), element.get("selector"), valueData["value"], element.get("index"), element.get("tag"), - element.get("objectType"), - element.get("action"), + element.get("objectType"), element.get("action"), valueData["searchType"], browserName) if(domInfo): @@ -298,15 +348,18 @@ def createMuukReport(classname, browserName): element["selectors"] = domInfo["selectors"] steps.append(element) - #except Exception as ex: - #print("Exception found during DOM parsing. Exception = " + str(ex)) + except Exception as ex: + print("Exception found during DOM parsing. Exception = " + str(ex)) # Closing file - jsonFile.close() + jsonFile.close() else: - print("Muuk Report does not exists!") + print("Muuk Report was not found!") muukReport["steps"] = steps - pprint.pprint(steps) + + # Print report if touch file exists + if(os.path.exists("TOUCH_TRACE_REPORT")): + pprint.pprint(steps) return muukReport From 88da281f6d4f154ed920e41531cf53e5a001c1bd Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Thu, 24 Jun 2021 19:04:20 -0700 Subject: [PATCH 24/74] Fix problem with JS Snippet exception. --- domparser.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/domparser.py b/domparser.py index 813ebec..addeb5d 100644 --- a/domparser.py +++ b/domparser.py @@ -334,19 +334,23 @@ def createMuukReport(classname, browserName): jsonFile = open(filename, 'r') elements = json.load(jsonFile) for element in elements['stepsFeedback']: - valueData = json.loads(element.get("value")) - domInfo = obtainFeedbackFromDOM(classname, element.get("id"), - element.get("selector"), valueData["value"], - element.get("index"), element.get("tag"), - element.get("objectType"), element.get("action"), - valueData["searchType"], - browserName) - if(domInfo): - element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] - element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] - element["rc"] = domInfo["rc"] - element["selectors"] = domInfo["selectors"] - steps.append(element) + type = element.get("type") + if(type == "step"): + valueData = json.loads(element.get("value")) + domInfo = obtainFeedbackFromDOM(classname, element.get("id"), + element.get("selector"), valueData["value"], + element.get("index"), element.get("tag"), + element.get("objectType"), element.get("action"), + valueData["searchType"], + browserName) + if(domInfo): + element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] + element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] + element["rc"] = domInfo["rc"] + element["selectors"] = domInfo["selectors"] + steps.append(element) + else: + steps.append(element) except Exception as ex: print("Exception found during DOM parsing. Exception = " + str(ex)) From 4896c3976a4b9044a7b550158ceeaaa5ce2bd848 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 7 Jul 2021 10:56:24 -0700 Subject: [PATCH 25/74] Return indexes if expectedIndex is > number of selectors found. --- domparser.py | 58 ++++++++++++++++++++++++++-------------------------- mkcli.py | 7 +++++-- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/domparser.py b/domparser.py index addeb5d..610d644 100644 --- a/domparser.py +++ b/domparser.py @@ -44,29 +44,21 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele if(selectorsFound == 0): # No selectors were found with the expected value - element = {} - element["index"] = expectedIndex - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - element["selector"] = "found" - elements.append(element) - returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE - elif(selectorsFound == 1): - if(expectedIndex in selectorIndexes): - # The expected selector was found and it is the only selector. + if(expectedIndex <= selectorsFound): element = {} element["index"] = expectedIndex if(attribute == "text"): element["value"] = selectors[expectedIndex].text else: element["value"] = selectors[expectedIndex][attribute] - element["selector"] = "original" + element["selector"] = "found" elements.append(element) - returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE else: - # The incorrect selector was found and this is the only selector with the expected value + returnCode = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND + elif(selectorsFound == 1): + if(expectedIndex in selectorIndexes): + # The expected selector was found and it is the only selector. element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -75,6 +67,18 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele element["value"] = selectors[expectedIndex][attribute] element["selector"] = "original" elements.append(element) + returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + else: + # The incorrect selector was found and this is the only selector with the expected value + if(expectedIndex <= selectorsFound): + element = {} + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" + elements.append(element) element = {} element["index"] = selectorIndexes[selectorsFound -1] @@ -100,14 +104,15 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX else: # The expected element was NOT found on the selectors - element = {} - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - element["index"] = expectedIndex - element["selector"] = "original" - elements.append(element) + if(expectedIndex <= selectorsFound): + element = {} + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["index"] = expectedIndex + element["selector"] = "original" + elements.append(element) element = {} if(attribute == "text"): @@ -227,7 +232,6 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): # Returns: # jsonObject with the number of selectors found, the selctors and the return code. def parseValueSelector(selectors, expectedValue, expectedIndex, type): - jsonObject = {} selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 @@ -261,11 +265,7 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty selectorsFound = soup.select(ntagselector) numberSelectorsFound = len(selectorsFound) - if(index > numberSelectorsFound): - jsonObject["selectors"] = [] - jsonObject["rc"] = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - elif(action == "assignment" or action == "mouseover"): + if(action == "assignment" or action == "mouseover"): jsonObject["selectors"] = [] jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 diff --git a/mkcli.py b/mkcli.py index e7011d4..95bf440 100644 --- a/mkcli.py +++ b/mkcli.py @@ -15,7 +15,7 @@ def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) path = 'build/test-results/'+browserName - + feedbackData = [] if os.path.exists(path): for filename in os.listdir(path): @@ -44,8 +44,11 @@ def gatherFeedbackData(browserName): "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "", "systemerror": e.find('system-err').text if e.find('system-err') is not None else "", "failureMessage": failureMessage, + "muukReport": {}, } - testResult["muukReport"] = createMuukReport(testResult.get("className"), browserName) + if testSuccess == False : + testResult["muukReport"] = createMuukReport(testResult.get("className"), browserName) + feedbackData.append(testResult) else: print("gatherFeedbackData - path does not exists ") From fed3e6ddf63ebdc347871d870d8d52d252527596 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Tue, 13 Jul 2021 16:26:47 -0700 Subject: [PATCH 26/74] Rever back changes to execute on production --- mkcli.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/mkcli.py b/mkcli.py index 95bf440..e1cdb56 100644 --- a/mkcli.py +++ b/mkcli.py @@ -9,13 +9,13 @@ import xml.etree.ElementTree from time import strftime from mkcloud import gatherScreenshots, resizeImages -import ssl +#import ssl from domparser import createMuukReport def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) path = 'build/test-results/'+browserName - + feedbackData = [] if os.path.exists(path): for filename in os.listdir(path): @@ -63,6 +63,7 @@ def gatherFeedbackData(browserName): return(feedbackData) + def run(args): #Gets the value from the flags print("Starting process") @@ -82,12 +83,12 @@ def run(args): exitCode = 1 #Check if we received a browser and get the string for the gradlew task command browserName = getBrowserName(browser) - #muuktestRoute = 'https://portal.muuktest.com:8081/' - #supportRoute = 'https://testing.muuktest.com:8082/' + muuktestRoute = 'https://portal.muuktest.com:8081/' + supportRoute = 'https://testing.muuktest.com:8082/' - muuktestRoute = 'https://localhost:8081/' - supportRoute = 'https://localhost:8082/' + #muuktestRoute = 'https://localhost:8081/' + #supportRoute = 'https://localhost:8082/' @@ -108,8 +109,8 @@ def run(args): try: key_file = open(path,'r') key = key_file.read() - #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) - r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) + r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) + #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) responseObject = json.loads(r.content) token = responseObject["token"] userId = responseObject["userId"] @@ -146,14 +147,14 @@ def run(args): # This route downloads the scripts by the property. url = muuktestRoute+'download_byproperty/' - context = ssl._create_unverified_context() + #context = ssl._create_unverified_context() data = urllib.parse.urlencode(values, doseq=True).encode('UTF-8') #now using urlopen get the file and store it locally auth_request = request.Request(url,headers=auth, data=data) auth_request.add_header('Authorization', 'Bearer '+token) - #response = request.urlopen(auth_request) - response = request.urlopen(auth_request, context=context) + response = request.urlopen(auth_request) + #response = request.urlopen(auth_request, context=context) #response = request.urlopen(url,data) file = response.read() @@ -198,8 +199,8 @@ def run(args): } try: - #requests.post(supportRoute+"tracking_data", json=payload) - requests.post(supportRoute+"tracking_data", json=payload, verify=False) + requests.post(supportRoute+"tracking_data", json=payload) + # equests.post(supportRoute+"tracking_data", json=payload, verify=False) except Exception as e: print("No connection to support Data Base") @@ -221,12 +222,12 @@ def run(args): #CLOUD SCREENSHOTS STARTS # resizeImages(browserName) - cloudKey = getCloudKey() + #cloudKey = getCloudKey() filesData = gatherScreenshots(browserName) try: if filesData != {}: - #requests.post(muuktestRoute + 'upload_cloud_steps_images/', headers=hed, files = filesData) - requests.post(muuktestRoute + 'upload_cloud_steps_images/', data={'cloudKey': cloudKey}, headers=hed, files = filesData, verify=False) + requests.post(muuktestRoute + 'upload_cloud_steps_images/', headers=hed, files = filesData) + #requests.post(muuktestRoute + 'upload_cloud_steps_images/', data={'cloudKey': cloudKey}, headers=hed, files = filesData, verify=False) else: print ("filesData empty.. cannot send screenshots") except Exception as e: @@ -235,8 +236,8 @@ def run(args): #CLOUD SCREENSHOTS ENDS try: #Executions feedback - #requests.post(url, json=values, headers=hed) - requests.post(url, json=values, headers=hed, verify=False) + requests.post(url, json=values, headers=hed) + #requests.post(url, json=values, headers=hed, verify=False) #save the executed test entry to the database requests.post(supportRoute+"tracking_data", data={ @@ -266,9 +267,7 @@ def getBrowserName(browser): #select a browser from the list or return firefox as default return switcher.get(browser,"firefoxTest") -def getCloudKey(): - return('lkussv97qginpidrtoey-1ec3r3678pe7n0wulh1v9k-zthz8er1ukl59ux1k0epop-mioxpeadsu5iegbjhd4j4') - + def main(): parser=argparse.ArgumentParser(description="MuukTest cli to download tests from the cloud") parser.add_argument("-p",help="property to search the test for" ,dest="field", type=str, required=True) From 8ef8d32d36b1cd61b1ce793d6f8e7bf3bfcb7b21 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Thu, 5 Aug 2021 21:37:18 -0700 Subject: [PATCH 27/74] Check index for the case where only one selector is found --- domparser.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/domparser.py b/domparser.py index 610d644..6fb327c 100644 --- a/domparser.py +++ b/domparser.py @@ -303,9 +303,26 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty elif(searchType == "imgsrc"): element["value"] = selectorsFound[0]["src"] elements.append(element) + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + if(index > 0): + element = {} + element["index"] = 0 + element["selector"] = "found" + if(searchType == "value"): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href"): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text"): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc"): + element["value"] = selectorsFound[0]["src"] + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + jsonObject["selectors"] = elements jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound - jsonObject["rc"] = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + jsonObject["rc"] = returnCode jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound From a6ba7f41cbee4c1f1fcc8283ece7df56fd2aaccb Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Thu, 5 Aug 2021 21:41:46 -0700 Subject: [PATCH 28/74] re-check-in --- domparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domparser.py b/domparser.py index 6fb327c..820a8bf 100644 --- a/domparser.py +++ b/domparser.py @@ -383,4 +383,4 @@ def createMuukReport(classname, browserName): if(os.path.exists("TOUCH_TRACE_REPORT")): pprint.pprint(steps) - return muukReport + return muukReport \ No newline at end of file From c64b9d3e1d4358e65c07273d0830add79455ca54 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Fri, 6 Aug 2021 08:49:48 -0700 Subject: [PATCH 29/74] Check index for the case where only one selector is found --- domparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domparser.py b/domparser.py index 820a8bf..e20c1b7 100644 --- a/domparser.py +++ b/domparser.py @@ -379,7 +379,7 @@ def createMuukReport(classname, browserName): muukReport["steps"] = steps - # Print report if touch file exists + # Print report if touch file exists if(os.path.exists("TOUCH_TRACE_REPORT")): pprint.pprint(steps) From eff3e363b27f255efac8bae8ec4ff401eb30bd12 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 18 Aug 2021 19:14:33 -0700 Subject: [PATCH 30/74] Remove build and machine info from the failures so we just send the exception. --- mkcli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mkcli.py b/mkcli.py index e1cdb56..c86b3bf 100644 --- a/mkcli.py +++ b/mkcli.py @@ -32,7 +32,11 @@ def gatherFeedbackData(browserName): if e.find('testcase') is not None : if e.find('testcase').find('failure') is not None : error = e.find('testcase').find('failure').attrib['message'] + if(error.find("Build info") != -1): + error = error.split(sep = "Build", maxsplit=1)[0] failureMessage = e.find('testcase').find('failure').text + if(failureMessage.find("Build info") != -1): + failureMessage = failureMessage.split(sep = "Build", maxsplit=1)[0] testResult = { "className": e.attrib['name'] if e.attrib['name'] is not None else "", @@ -200,7 +204,7 @@ def run(args): try: requests.post(supportRoute+"tracking_data", json=payload) - # equests.post(supportRoute+"tracking_data", json=payload, verify=False) + #requests.post(supportRoute+"tracking_data", json=payload, verify=False) except Exception as e: print("No connection to support Data Base") From 72072ad1d79f6ee79a4653bfe4a3ec233164b224 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 13 Oct 2021 10:18:44 -0700 Subject: [PATCH 31/74] Do a deeper analysis usign link text so we can reduce the number of indixes. --- domparser.py | 85 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/domparser.py b/domparser.py index e20c1b7..f131fb8 100644 --- a/domparser.py +++ b/domparser.py @@ -44,7 +44,7 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele if(selectorsFound == 0): # No selectors were found with the expected value - if(expectedIndex <= selectorsFound): + if(expectedIndex <= len(selectors)): element = {} element["index"] = expectedIndex if(attribute == "text"): @@ -70,15 +70,17 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX else: # The incorrect selector was found and this is the only selector with the expected value - if(expectedIndex <= selectorsFound): - element = {} - element["index"] = expectedIndex + element = {} + element["index"] = expectedIndex + element["selector"] = "original" + if(expectedIndex <= len(selectors)): if(attribute == "text"): element["value"] = selectors[expectedIndex].text else: element["value"] = selectors[expectedIndex][attribute] - element["selector"] = "original" - elements.append(element) + else: + element["value"] = expectedValue + elements.append(element) element = {} element["index"] = selectorIndexes[selectorsFound -1] @@ -104,16 +106,18 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX else: # The expected element was NOT found on the selectors - if(expectedIndex <= selectorsFound): - element = {} + element = {} + element["index"] = expectedIndex + element["selector"] = "original" + if(expectedIndex <= len(selectors)): if(attribute == "text"): element["value"] = selectors[expectedIndex].text else: element["value"] = selectors[expectedIndex][attribute] - element["index"] = expectedIndex - element["selector"] = "original" - elements.append(element) - + else: + element["value"] = expectedValue + elements.append(element) + element = {} if(attribute == "text"): element["value"] = expectedValue @@ -137,16 +141,17 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele # # Parameters: # selectors: Array of selectors found with the same ntagselector. -# expectedValue: The value that is expected to be found (value captured by the extension). +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). # expectedIndex: The index that is expected to contain the expected value. # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseTextSelector(selectors, expectedValue, expectedIndex): +def parseTextSelector(selectors, searchInfo, expectedIndex): jsonObject = {} selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 + expectedValue = searchInfo["value"] if(expectedValue == ""): jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 @@ -155,7 +160,8 @@ def parseTextSelector(selectors, expectedValue, expectedIndex): return jsonObject for selector in selectors: - if((selector.text).strip() == expectedValue.strip()): + selectorText = selector.text.replace("'", "") + if(selectorText.strip() == expectedValue.strip()): selectorsFound += 1 selectorIndexes.append(selectorIndex) selectorIndex+=1 @@ -169,15 +175,17 @@ def parseTextSelector(selectors, expectedValue, expectedIndex): # # Parameters: # selectors: Array of selectors found with the same ntagselector. -# expectedValue: The value that is expected to be found (value captured by the extension). +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). # expectedIndex: The index that is expected to contain the expected value. # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseImageSelector(selectors, expectedValue, expectedIndex): +def parseImageSelector(selectors, searchInfo, expectedIndex): selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 + expectedValue = searchInfo["value"] + for selector in selectors: if(selector['src'] == expectedValue ): selectorsFound += 1 @@ -193,16 +201,19 @@ def parseImageSelector(selectors, expectedValue, expectedIndex): # # Parameters: # selectors: Array of selectors found with the same ntagselector. -# expectedValue: The value that is expected to be found (value captured by the extension). +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). # expectedIndex: The index that is expected to contain the expected value. # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseHypertextSelector(selectors, expectedValue, expectedIndex): +def parseHypertextSelector(selectors, searchInfo, expectedIndex): jsonObject = {} selectorIndexes = [] + filteredIndexes = [] selectorIndex = 0 selectorsFound = 0 + expectedValue = searchInfo["value"] + expectedText = searchInfo["text"] if(expectedValue == ""): jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 @@ -217,6 +228,18 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): selectorIndexes.append(selectorIndex) selectorIndex+=1 + # If more than 1 selector was found using the same value, lest's filter now by text and update + # the selectorIndexes with the new indexes (hopefully only one!). + if(selectorsFound > 1 and expectedText != ""): + for i in selectorIndexes: + if(str(selectors[i].string) == expectedText): + filteredIndexes.append(i) + if(len(filteredIndexes) > 0 ): + selectorIndexes = [] + selectorsFound = len(filteredIndexes) + for index in filteredIndexes: + selectorIndexes.append(index) + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") # Description: @@ -226,15 +249,16 @@ def parseHypertextSelector(selectors, expectedValue, expectedIndex): # # Parameters: # selectors: Array of selectors found with the same ntagselector. -# expectedValue: The value that is expected to be found (value captured by the extension). +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). # expectedIndex: The index that is expected to contain the expected value. # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseValueSelector(selectors, expectedValue, expectedIndex, type): +def parseValueSelector(selectors, searchInfo, expectedIndex, type): selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 + expectedValue = searchInfo["value"] for selector in selectors: if(selector['value'] == expectedValue ): @@ -252,7 +276,7 @@ def parseValueSelector(selectors, expectedValue, expectedIndex, type): # # Returns: # jsonObject with the number of selector information. -def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, type, action, searchType, browserName): +def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, action, searchInfo, browserName): jsonObject = {} elements = [] path = 'build/reports/geb/' + browserName + '/' @@ -260,6 +284,7 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty if os.path.exists(filename): try: + searchType = searchInfo["searchType"] text = open(filename, 'r').read() soup = BeautifulSoup(text, 'html.parser') selectorsFound = soup.select(ntagselector) @@ -276,13 +301,13 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR elif(numberSelectorsFound > 1 ): if(searchType == "value"): - jsonObject = parseValueSelector(selectorsFound, value, index, type) + jsonObject = parseValueSelector(selectorsFound, searchInfo, index, type) elif(searchType == "href"): - jsonObject = parseHypertextSelector(selectorsFound, value, index) + jsonObject = parseHypertextSelector(selectorsFound, searchInfo, index) elif(searchType == "text"): - jsonObject = parseTextSelector(selectorsFound, value, index) + jsonObject = parseTextSelector(selectorsFound, searchInfo, index) elif(searchType == "imgsrc"): - jsonObject = parseImageSelector(selectorsFound, value, index) + jsonObject = parseImageSelector(selectorsFound, searchInfo, index) else: # Backend sent an undef searchType, we will return no info jsonObject["selectors"] = [] @@ -292,7 +317,7 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, value, index, tag, ty elif(numberSelectorsFound == 1 ): element = {} element["index"] = index - element["value"] = value + element["value"] = searchInfo["value"] element["selector"] = "original" if(searchType == "value"): element["value"] = selectorsFound[0]["value"] @@ -353,12 +378,12 @@ def createMuukReport(classname, browserName): for element in elements['stepsFeedback']: type = element.get("type") if(type == "step"): - valueData = json.loads(element.get("value")) + valueInfo = json.loads(element.get("value")) domInfo = obtainFeedbackFromDOM(classname, element.get("id"), - element.get("selector"), valueData["value"], + element.get("selector"), element.get("index"), element.get("tag"), element.get("objectType"), element.get("action"), - valueData["searchType"], + valueInfo, browserName) if(domInfo): element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] From e5c4b0669ef73e5d084838acf69909619a4acf10 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 13 Oct 2021 10:23:48 -0700 Subject: [PATCH 32/74] Do a deeper analysis usign link text so we can reduce the number of indixes. --- domparser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/domparser.py b/domparser.py index f131fb8..881a18c 100644 --- a/domparser.py +++ b/domparser.py @@ -18,7 +18,6 @@ MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 - # Description: # This method will be called to handle the result of filter operation done # on the selectors found. From 09591da9950bd2faaf5abb2155ce3b0c1f8d8597 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 13 Oct 2021 10:40:11 -0700 Subject: [PATCH 33/74] Do a deeper analysis usign link text so we can reduce the number of indixes. --- domparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domparser.py b/domparser.py index 881a18c..fdc5a11 100644 --- a/domparser.py +++ b/domparser.py @@ -225,7 +225,7 @@ def parseHypertextSelector(selectors, searchInfo, expectedIndex): if(selector['href'] == expectedValue): selectorsFound += 1 selectorIndexes.append(selectorIndex) - selectorIndex+=1 + selectorIndex+=1 # If more than 1 selector was found using the same value, lest's filter now by text and update # the selectorIndexes with the new indexes (hopefully only one!). From fbecbeaf7bd688e853bf7eba3f6625c5946aff3d Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Tue, 7 Dec 2021 12:08:36 -0800 Subject: [PATCH 34/74] Improves to feedback analyzer for autohealing --- domparser.py | 274 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 232 insertions(+), 42 deletions(-) diff --git a/domparser.py b/domparser.py index fdc5a11..842caa9 100644 --- a/domparser.py +++ b/domparser.py @@ -3,6 +3,7 @@ import json import pprint from bs4 import BeautifulSoup +import traceback UNKNOWN_ERROR = 1 NO_TAG_PROVIDED_BY_BE = 2 @@ -17,6 +18,7 @@ MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 # Description: # This method will be called to handle the result of filter operation done @@ -47,9 +49,9 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele element = {} element["index"] = expectedIndex if(attribute == "text"): - element["value"] = selectors[expectedIndex].text + element["value"] = selectors[expectedIndex].text if (selectors[expectedIndex].text != "") else "" else: - element["value"] = selectors[expectedIndex][attribute] + element["value"] = selectors[expectedIndex][attribute] if (attribute in selectors[expectedIndex]) else "" element["selector"] = "found" elements.append(element) returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE @@ -145,7 +147,7 @@ def processResults(selectors, expectedIndex, expectedValue, selectorsFound, sele # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseTextSelector(selectors, searchInfo, expectedIndex): +def parseTextSelector(selectors, searchInfo, expectedIndex, tag): jsonObject = {} selectorIndexes = [] selectorIndex = 0 @@ -179,7 +181,7 @@ def parseTextSelector(selectors, searchInfo, expectedIndex): # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseImageSelector(selectors, searchInfo, expectedIndex): +def parseImageSelector(selectors, searchInfo, expectedIndex, tag): selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 @@ -205,14 +207,17 @@ def parseImageSelector(selectors, searchInfo, expectedIndex): # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseHypertextSelector(selectors, searchInfo, expectedIndex): +def parseHypertextSelector(selectors, searchInfo, expectedIndex, tag): jsonObject = {} selectorIndexes = [] filteredIndexes = [] selectorIndex = 0 selectorsFound = 0 expectedValue = searchInfo["value"] - expectedText = searchInfo["text"] + if hasattr(searchInfo, 'text'): + expectedText = searchInfo["text"] + else: + expectedText = "" if(expectedValue == ""): jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 @@ -225,7 +230,7 @@ def parseHypertextSelector(selectors, searchInfo, expectedIndex): if(selector['href'] == expectedValue): selectorsFound += 1 selectorIndexes.append(selectorIndex) - selectorIndex+=1 + selectorIndex+=1 # If more than 1 selector was found using the same value, lest's filter now by text and update # the selectorIndexes with the new indexes (hopefully only one!). @@ -253,20 +258,166 @@ def parseHypertextSelector(selectors, searchInfo, expectedIndex): # # Returns: # jsonObject with the number of selectors found, the selctors and the return code. -def parseValueSelector(selectors, searchInfo, expectedIndex, type): +def parseValueSelector(selectors, searchInfo, expectedIndex, type, tag): selectorIndexes = [] selectorIndex = 0 selectorsFound = 0 - expectedValue = searchInfo["value"] + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + expectedText = searchInfo["text"] if ('text' in searchInfo) else "" for selector in selectors: - if(selector['value'] == expectedValue ): + if(('value' in selector) and selector['value'] == expectedValue ): selectorsFound += 1 selectorIndexes.append(selectorIndex) selectorIndex+=1 + # If we have text information available and this is a select element, let's try to + # find the correct value using the text + if(selectorsFound == 0 and expectedText != "" and tag == "select"): + return handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag) + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") +def handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag): + jsonObject = {} + elements = [] + value = "" + # Let's first verify the selector with the expected index + selectElement = selectors[expectedIndex] + options = selectElement.find_all("option") + for option in options: + if(option.text.strip() == expectedText.strip()): + value = option.get("value") + break + + element = {} + element["index"] = expectedIndex + element["value"] = value + element["selector"] = "found" + elements.append(element) + + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + jsonObject["selectors"] = elements + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + + return jsonObject + +# Description: +# This method will . +# +# Returns: +# . +def verifyAttributesOnElements(selectors, attributes): + selectorsFound = [] + selectorIndexes = [] + index = 0 + + # First let's try to find any selector on the list with the ID frm the element attributes + index = 0 + for selector in selectors: + if(selector.has_attr('id')): + if(selector['id'] == attributes['id']): + selectorsFound.append(selector) + selectorIndexes.append(index) + return [selectorsFound, selectorIndexes] + index+=1 + + # If we get here it means no selector with ID was found, let's check with name + if(len(selectorsFound) == 0): + index = 0 + for selector in selectors: + if(selector.has_attr('name')): + if(selector['name'] == attributes['name']): + selectorsFound.append(selector) + selectorIndexes.append(index) + return [selectorsFound, selectorIndexes] + index+=1 + + # If we get here it means no ID and NAME were found, let's check with text + if(len(selectorsFound) == 0): + index = 0 + for selector in selectors: + if(selector.has_attr('text')): + if(selector['text'] == attributes['text']): + selectorsFound.append(selector) + selectorIndexes.append(index) + index+=1 + + # If we get here it means no ID and NAME were found, let's check with text + index = 0 + for selector in selectors: + if(selector.has_attr('type')): + if(selector['type'] == attributes['type']): + selectorsFound.append(selector) + selectorIndexes.append(index) + index+=1 + + return [selectorsFound, selectorIndexes] + + +# Description: +# This method will . +# +# Returns: +# . +def filterSelectorsByAttributes(selectors, attributes, searchInfo, stepId,index): + + print("Start analysis for step, " + str(stepId) + " for " + str(len(selectors)) + " Selectors") + print("Attributes, " + str(attributes)) + #print("Selectors:, " + str(selectors)) + selectorsFound = [] + selectorIndexes = [] + elements = [] + jsonObject = {} + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + searchType = searchInfo["searchType"] + + [selectorsFound,selectorIndexes] = verifyAttributesOnElements(selectors, attributes); + print("Returned from verifyAttributesOnElements") + print(selectorsFound) + + if(len(selectorsFound) == 1): + element = {} + element["index"] = index + element["selector"] = "original" + element["value"] = expectedValue + elements.append(element) + if(index != selectorIndexes[0]): + element = {} + element["index"] = selectorIndexes[0] + element["selector"] = "found" + if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): + print("Value in selector") + element["value"] = selectorsFound[0]['value'] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + else: + print("Value NOT in selector") + element["value"] = "undef" + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elements.append(element) + else: + element = {} + element["index"] = index + element["selector"] = "found" + if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): + print("Value in selector") + element["value"] = selectorsFound[0]['value'] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + else: + print("Value NOT in selector") + element["value"] = "undef" + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elements.append(element) + + jsonObject["selectors"] = elements + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + else: + jsonObject["selectors"] = selectors + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + return jsonObject + # Description: # This method will be call for each step and will parse the DOM files generated # for the test to find the selectors for this step. If more than one selector is found, @@ -275,7 +426,8 @@ def parseValueSelector(selectors, searchInfo, expectedIndex, type): # # Returns: # jsonObject with the number of selector information. -def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, action, searchInfo, browserName): +def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, action, searchInfo, browserName, attributes): + #os.chdir("/home/serrano/Development/executor/local/executor") jsonObject = {} elements = [] path = 'build/reports/geb/' + browserName + '/' @@ -288,8 +440,9 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act soup = BeautifulSoup(text, 'html.parser') selectorsFound = soup.select(ntagselector) numberSelectorsFound = len(selectorsFound) + print(numberSelectorsFound) - if(action == "assignment" or action == "mouseover"): + if(action == "mouseover"): jsonObject["selectors"] = [] jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 @@ -298,34 +451,22 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act jsonObject["selectors"] = [] jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR - elif(numberSelectorsFound > 1 ): - if(searchType == "value"): - jsonObject = parseValueSelector(selectorsFound, searchInfo, index, type) - elif(searchType == "href"): - jsonObject = parseHypertextSelector(selectorsFound, searchInfo, index) - elif(searchType == "text"): - jsonObject = parseTextSelector(selectorsFound, searchInfo, index) - elif(searchType == "imgsrc"): - jsonObject = parseImageSelector(selectorsFound, searchInfo, index) - else: - # Backend sent an undef searchType, we will return no info - jsonObject["selectors"] = [] - jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - + elif(numberSelectorsFound == 1 ): element = {} element["index"] = index element["value"] = searchInfo["value"] element["selector"] = "original" - if(searchType == "value"): + if(searchType == "value" and selectorsFound[0].has_attr('value')): element["value"] = selectorsFound[0]["value"] - elif(searchType == "href"): + elif(searchType == "href" and selectorsFound[0].has_attr('href')): element["value"] = selectorsFound[0]["href"] - elif(searchType == "text"): + elif(searchType == "text" and selectorsFound[0].has_attr('text')): element["value"] = selectorsFound[0].text - elif(searchType == "imgsrc"): + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] elements.append(element) returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR @@ -333,14 +474,16 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act element = {} element["index"] = 0 element["selector"] = "found" - if(searchType == "value"): + if(searchType == "value" and selectorsFound[0].has_attr('value')): element["value"] = selectorsFound[0]["value"] - elif(searchType == "href"): + elif(searchType == "href" and selectorsFound[0].has_attr('href')): element["value"] = selectorsFound[0]["href"] - elif(searchType == "text"): + elif(searchType == "text" and selectorsFound[0].has_attr('text')): element["value"] = selectorsFound[0].text - elif(searchType == "imgsrc"): + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] elements.append(element) returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX @@ -348,12 +491,44 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound jsonObject["rc"] = returnCode + elif(numberSelectorsFound > 1 ): + if(searchType == "value"): + jsonObject = parseValueSelector(selectorsFound, searchInfo, index, type, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(selectorsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(selectorsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(selectorsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + + # If we are done with the search and we were not ablt to find a selector. Let's try to use the attributes to find the element. + if(jsonObject["rc"] == NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE or + jsonObject["rc"] == NO_SEARCH_TYPE_PROVIDED_BY_BE): + print("Need to search using attributes now for step " + str(stepId)) + # This code is to do analysis on the element's attributes. This will be enable on a future sprint + #jsonObject = filterSelectorsByAttributes(selectorsFound, attributes, searchInfo, stepId, index) + #newNumberSelectorsFound = len(newSelectorsFound) + jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound except Exception as ex: - print("Failed to open file " + ex) + print("Failed to open file " + str(filename) + ex) print (ex) - + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + jsonObject = {} + return jsonObject @@ -366,6 +541,8 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act # Returns: # jsonObject with the number of selector information. def createMuukReport(classname, browserName): + #os.chdir("/home/serrano/Development/executor/local/executor") + print(os.getcwd()) path = 'build/reports/' filename = path + classname + ".json" muukReport = {} @@ -378,12 +555,15 @@ def createMuukReport(classname, browserName): type = element.get("type") if(type == "step"): valueInfo = json.loads(element.get("value")) + attributes = json.loads(element.get("attributes")) + attributes['value'] = valueInfo['value'] domInfo = obtainFeedbackFromDOM(classname, element.get("id"), element.get("selector"), element.get("index"), element.get("tag"), element.get("objectType"), element.get("action"), valueInfo, - browserName) + browserName, + attributes) if(domInfo): element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] @@ -394,17 +574,27 @@ def createMuukReport(classname, browserName): steps.append(element) except Exception as ex: - print("Exception found during DOM parsing. Exception = " + str(ex)) + print("Exception found during DOM parsing. Exception = ") + print(ex) + print(traceback.format_exc()) # Closing file jsonFile.close() else: print("Muuk Report was not found!") - - muukReport["steps"] = steps + + # Let's validate the data we generated is a valid json + try: + json.loads(json.dumps(steps)) + muukReport["steps"] = steps + except Exception as ex: + pprint.pprint("Invalid JSON format was found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + muukReport["steps"] = {} # Print report if touch file exists if(os.path.exists("TOUCH_TRACE_REPORT")): - pprint.pprint(steps) + pprint.pprint(muukReport["steps"]) return muukReport \ No newline at end of file From ba041e82827668420422ac7388ef2cfdf2edaa8a Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Tue, 4 Jan 2022 13:06:11 -0600 Subject: [PATCH 35/74] mkcli later internal version --- build.gradle | 3 + domparser.py | 600 +++++++++++++++++++++++++++++++++++++++++++++++++++ mkcli.py | 65 ++++-- 3 files changed, 654 insertions(+), 14 deletions(-) create mode 100644 domparser.py diff --git a/build.gradle b/build.gradle index afdc692..55d6a67 100644 --- a/build.gradle +++ b/build.gradle @@ -39,6 +39,9 @@ dependencies { testCompile "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion" compile group: 'org.seleniumhq.selenium', name: 'selenium-support', version: '3.14.0' + + // JSON simple for MuukReport + implementation group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1' } diff --git a/domparser.py b/domparser.py new file mode 100644 index 0000000..842caa9 --- /dev/null +++ b/domparser.py @@ -0,0 +1,600 @@ + +import os +import json +import pprint +from bs4 import BeautifulSoup +import traceback + +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 + +# Description: +# This method will be called to handle the result of filter operation done +# on the selectors found. +# +# There are 3 options for the result: +# 1) No selectors were found having the value we are expecting. On this case, +# information returned will be the element with the index that was expected. +# +# 2) We found only one selector, we have two options here: +# a) Found the correct selector: Return the original element. +# b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. +# +# 3) We found two or more selectors with the same src. We have two options here: +# a) The correct selector was found. Return the original element. +# b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +# +def processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, attribute): + jsonObject = {} + elements = [] + + if(selectorsFound == 0): + # No selectors were found with the expected value + if(expectedIndex <= len(selectors)): + element = {} + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text if (selectors[expectedIndex].text != "") else "" + else: + element["value"] = selectors[expectedIndex][attribute] if (attribute in selectors[expectedIndex]) else "" + element["selector"] = "found" + elements.append(element) + returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE + else: + returnCode = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND + elif(selectorsFound == 1): + if(expectedIndex in selectorIndexes): + # The expected selector was found and it is the only selector. + element = {} + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + else: + # The incorrect selector was found and this is the only selector with the expected value + element = {} + element["index"] = expectedIndex + element["selector"] = "original" + if(expectedIndex <= len(selectors)): + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + else: + element["value"] = expectedValue + elements.append(element) + + element = {} + element["index"] = selectorIndexes[selectorsFound -1] + if(attribute == "text"): + element["value"] = selectors[selectorIndexes[selectorsFound -1]].text + else: + element["value"] = selectors[selectorIndexes[selectorsFound -1]][attribute] + element["selector"] = "found" + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + elif(selectorsFound > 1): + # Several selectors were found with same value + if(expectedIndex in selectorIndexes): + # The expected element was found on the selectors + element = {} + element["index"] = expectedIndex + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + element["selector"] = "original" + elements.append(element) + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX + else: + # The expected element was NOT found on the selectors + element = {} + element["index"] = expectedIndex + element["selector"] = "original" + if(expectedIndex <= len(selectors)): + if(attribute == "text"): + element["value"] = selectors[expectedIndex].text + else: + element["value"] = selectors[expectedIndex][attribute] + else: + element["value"] = expectedValue + elements.append(element) + + element = {} + if(attribute == "text"): + element["value"] = expectedValue + else: + element["value"] = expectedValue + element["index"] = str(selectorIndexes) + element["selector"] = "found" + elements.append(element) + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX + + jsonObject["numberOfElementsWithSameSelectorAndValue"] = selectorsFound + jsonObject["selectors"] = elements + jsonObject["rc"] = returnCode + + return jsonObject + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the text value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseTextSelector(selectors, searchInfo, expectedIndex, tag): + jsonObject = {} + selectorIndexes = [] + selectorIndex = 0 + selectorsFound = 0 + expectedValue = searchInfo["value"] + + if(expectedValue == ""): + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for selector in selectors: + selectorText = selector.text.replace("'", "") + if(selectorText.strip() == expectedValue.strip()): + selectorsFound += 1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 + + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "text") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the src value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseImageSelector(selectors, searchInfo, expectedIndex, tag): + selectorIndexes = [] + selectorIndex = 0 + selectorsFound = 0 + expectedValue = searchInfo["value"] + + for selector in selectors: + if(selector['src'] == expectedValue ): + selectorsFound += 1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 + + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "src") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the href value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseHypertextSelector(selectors, searchInfo, expectedIndex, tag): + jsonObject = {} + selectorIndexes = [] + filteredIndexes = [] + selectorIndex = 0 + selectorsFound = 0 + expectedValue = searchInfo["value"] + if hasattr(searchInfo, 'text'): + expectedText = searchInfo["text"] + else: + expectedText = "" + + if(expectedValue == ""): + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for selector in selectors: + if(selector and selector.has_attr('href')): + if(selector['href'] == expectedValue): + selectorsFound += 1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 + + # If more than 1 selector was found using the same value, lest's filter now by text and update + # the selectorIndexes with the new indexes (hopefully only one!). + if(selectorsFound > 1 and expectedText != ""): + for i in selectorIndexes: + if(str(selectors[i].string) == expectedText): + filteredIndexes.append(i) + if(len(filteredIndexes) > 0 ): + selectorIndexes = [] + selectorsFound = len(filteredIndexes) + for index in filteredIndexes: + selectorIndexes.append(index) + + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selectors: Array of selectors found with the same ntagselector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseValueSelector(selectors, searchInfo, expectedIndex, type, tag): + selectorIndexes = [] + selectorIndex = 0 + selectorsFound = 0 + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + expectedText = searchInfo["text"] if ('text' in searchInfo) else "" + + for selector in selectors: + if(('value' in selector) and selector['value'] == expectedValue ): + selectorsFound += 1 + selectorIndexes.append(selectorIndex) + selectorIndex+=1 + + # If we have text information available and this is a select element, let's try to + # find the correct value using the text + if(selectorsFound == 0 and expectedText != "" and tag == "select"): + return handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag) + + return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") + +def handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag): + jsonObject = {} + elements = [] + value = "" + # Let's first verify the selector with the expected index + selectElement = selectors[expectedIndex] + options = selectElement.find_all("option") + for option in options: + if(option.text.strip() == expectedText.strip()): + value = option.get("value") + break + + element = {} + element["index"] = expectedIndex + element["value"] = value + element["selector"] = "found" + elements.append(element) + + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + jsonObject["selectors"] = elements + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + + return jsonObject + +# Description: +# This method will . +# +# Returns: +# . +def verifyAttributesOnElements(selectors, attributes): + selectorsFound = [] + selectorIndexes = [] + index = 0 + + # First let's try to find any selector on the list with the ID frm the element attributes + index = 0 + for selector in selectors: + if(selector.has_attr('id')): + if(selector['id'] == attributes['id']): + selectorsFound.append(selector) + selectorIndexes.append(index) + return [selectorsFound, selectorIndexes] + index+=1 + + # If we get here it means no selector with ID was found, let's check with name + if(len(selectorsFound) == 0): + index = 0 + for selector in selectors: + if(selector.has_attr('name')): + if(selector['name'] == attributes['name']): + selectorsFound.append(selector) + selectorIndexes.append(index) + return [selectorsFound, selectorIndexes] + index+=1 + + # If we get here it means no ID and NAME were found, let's check with text + if(len(selectorsFound) == 0): + index = 0 + for selector in selectors: + if(selector.has_attr('text')): + if(selector['text'] == attributes['text']): + selectorsFound.append(selector) + selectorIndexes.append(index) + index+=1 + + # If we get here it means no ID and NAME were found, let's check with text + index = 0 + for selector in selectors: + if(selector.has_attr('type')): + if(selector['type'] == attributes['type']): + selectorsFound.append(selector) + selectorIndexes.append(index) + index+=1 + + return [selectorsFound, selectorIndexes] + + +# Description: +# This method will . +# +# Returns: +# . +def filterSelectorsByAttributes(selectors, attributes, searchInfo, stepId,index): + + print("Start analysis for step, " + str(stepId) + " for " + str(len(selectors)) + " Selectors") + print("Attributes, " + str(attributes)) + #print("Selectors:, " + str(selectors)) + selectorsFound = [] + selectorIndexes = [] + elements = [] + jsonObject = {} + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + searchType = searchInfo["searchType"] + + [selectorsFound,selectorIndexes] = verifyAttributesOnElements(selectors, attributes); + print("Returned from verifyAttributesOnElements") + print(selectorsFound) + + if(len(selectorsFound) == 1): + element = {} + element["index"] = index + element["selector"] = "original" + element["value"] = expectedValue + elements.append(element) + if(index != selectorIndexes[0]): + element = {} + element["index"] = selectorIndexes[0] + element["selector"] = "found" + if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): + print("Value in selector") + element["value"] = selectorsFound[0]['value'] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + else: + print("Value NOT in selector") + element["value"] = "undef" + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elements.append(element) + else: + element = {} + element["index"] = index + element["selector"] = "found" + if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): + print("Value in selector") + element["value"] = selectorsFound[0]['value'] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 + else: + print("Value NOT in selector") + element["value"] = "undef" + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + elements.append(element) + + jsonObject["selectors"] = elements + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + else: + jsonObject["selectors"] = selectors + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + return jsonObject + +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, action, searchInfo, browserName, attributes): + #os.chdir("/home/serrano/Development/executor/local/executor") + jsonObject = {} + elements = [] + path = 'build/reports/geb/' + browserName + '/' + filename = path + classname + "_" + str(stepId) + ".html" + + if os.path.exists(filename): + try: + searchType = searchInfo["searchType"] + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + selectorsFound = soup.select(ntagselector) + numberSelectorsFound = len(selectorsFound) + print(numberSelectorsFound) + + if(action == "mouseover"): + jsonObject["selectors"] = [] + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + else: + if(numberSelectorsFound == 0 ): + jsonObject["selectors"] = [] + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + elif(numberSelectorsFound == 1 ): + element = {} + element["index"] = index + element["value"] = searchInfo["value"] + element["selector"] = "original" + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + elements.append(element) + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + if(index > 0): + element = {} + element["index"] = 0 + element["selector"] = "found" + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + elements.append(element) + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + + jsonObject["selectors"] = elements + jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + elif(numberSelectorsFound > 1 ): + if(searchType == "value"): + jsonObject = parseValueSelector(selectorsFound, searchInfo, index, type, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(selectorsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(selectorsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(selectorsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + jsonObject["selectors"] = [] + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 + + # If we are done with the search and we were not ablt to find a selector. Let's try to use the attributes to find the element. + if(jsonObject["rc"] == NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE or + jsonObject["rc"] == NO_SEARCH_TYPE_PROVIDED_BY_BE): + print("Need to search using attributes now for step " + str(stepId)) + # This code is to do analysis on the element's attributes. This will be enable on a future sprint + #jsonObject = filterSelectorsByAttributes(selectorsFound, attributes, searchInfo, stepId, index) + #newNumberSelectorsFound = len(newSelectorsFound) + + jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound + + except Exception as ex: + print("Failed to open file " + str(filename) + ex) + print (ex) + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + jsonObject = {} + + return jsonObject + + +# Description: +# This method will be call to execuete the Muuk Report analysis. +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def createMuukReport(classname, browserName): + #os.chdir("/home/serrano/Development/executor/local/executor") + print(os.getcwd()) + path = 'build/reports/' + filename = path + classname + ".json" + muukReport = {} + steps = [] + if(os.path.exists(filename)): + try: + jsonFile = open(filename, 'r') + elements = json.load(jsonFile) + for element in elements['stepsFeedback']: + type = element.get("type") + if(type == "step"): + valueInfo = json.loads(element.get("value")) + attributes = json.loads(element.get("attributes")) + attributes['value'] = valueInfo['value'] + domInfo = obtainFeedbackFromDOM(classname, element.get("id"), + element.get("selector"), + element.get("index"), element.get("tag"), + element.get("objectType"), element.get("action"), + valueInfo, + browserName, + attributes) + if(domInfo): + element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] + element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] + element["rc"] = domInfo["rc"] + element["selectors"] = domInfo["selectors"] + steps.append(element) + else: + steps.append(element) + + except Exception as ex: + print("Exception found during DOM parsing. Exception = ") + print(ex) + print(traceback.format_exc()) + + # Closing file + jsonFile.close() + else: + print("Muuk Report was not found!") + + # Let's validate the data we generated is a valid json + try: + json.loads(json.dumps(steps)) + muukReport["steps"] = steps + except Exception as ex: + pprint.pprint("Invalid JSON format was found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + muukReport["steps"] = {} + + # Print report if touch file exists + if(os.path.exists("TOUCH_TRACE_REPORT")): + pprint.pprint(muukReport["steps"]) + + return muukReport \ No newline at end of file diff --git a/mkcli.py b/mkcli.py index c24a245..80731f5 100644 --- a/mkcli.py +++ b/mkcli.py @@ -8,9 +8,10 @@ import urllib import xml.etree.ElementTree from time import strftime -from mkcloud import gatherScreenshots, resizeImages +from mkcloud import gatherScreenshots, resizeImages, getCloudKey from mkvideo import Video #import ssl +from domparser import createMuukReport def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) @@ -21,6 +22,7 @@ def gatherFeedbackData(browserName): for filename in os.listdir(path): testSuccess = True error = '' + failureMessage = '' if filename.endswith('.xml'): e = xml.etree.ElementTree.parse('build/test-results/'+browserName+'/' + filename).getroot() @@ -31,7 +33,12 @@ def gatherFeedbackData(browserName): if e.find('testcase') is not None : if e.find('testcase').find('failure') is not None : error = e.find('testcase').find('failure').attrib['message'] - + if(error.find("Build info") != -1): + error = error.split(sep = "Build", maxsplit=1)[0] + failureMessage = e.find('testcase').find('failure').text + if(failureMessage.find("Build info") != -1): + failureMessage = failureMessage.split(sep = "Build", maxsplit=1)[0] + testResult = { "className": e.attrib['name'] if e.attrib['name'] is not None else "", "success": testSuccess, @@ -39,12 +46,19 @@ def gatherFeedbackData(browserName): "hostname": e.attrib['hostname'] if e.attrib['hostname'] is not None else "", "executionTime": e.attrib['time'] if e.attrib['time'] is not None else "", "error": error, - "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "" + "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "", + "systemerror": e.find('system-err').text if e.find('system-err') is not None else "", + "failureMessage": failureMessage, + "muukReport": {}, } + if testSuccess == False : + testResult["muukReport"] = createMuukReport(testResult.get("className"), browserName) + feedbackData.append(testResult) else: print("gatherFeedbackData - path does not exists ") testResult = { + "muukReport": {}, "success" : False, #"executionAt": "", "error" : "Test failed during execution. This could be compilation error", @@ -63,13 +77,24 @@ def run(args): noexec = args.noexec route = 'src/test/groovy' browser = args.browser + # internal cloud only + executionNumber = args.executionnumber or None + scheduleExecutionNumber = args.scheduleexecutionnumber or None + + if scheduleExecutionNumber is not None: + scheduleExecutionNumber = int(scheduleExecutionNumber) + else: + scheduleExecutionNumber = 0 + + origin = args.origin or None + originid = args.originid or None + ######## dimensions = args.dimensions if dimensions is not None: checkDimensions = isinstance(dimensions[0], int) & isinstance(dimensions[1],int) else: checkDimensions = False - executionNumber = None #Exit code to report at circleci exitCode = 1 #Check if we received a browser and get the string for the gradlew task command @@ -99,7 +124,7 @@ def run(args): token='' try: key_file = open(path,'r') - key = key_file.read() + key = key_file.read().strip() r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) responseObject = json.loads(r.content) @@ -128,10 +153,16 @@ def run(args): print(dest) shutil.copytree(route, dest) shutil.copytree("build/", dest+"/build") + #internal cloud only + files = [f for f in os.listdir(".") if f.endswith('.mp4')] + shutil.move(files[0], dest) if len(files) == 1 else print("Not a video to backup") + ######### + shutil.rmtree(route, ignore_errors=True) os.makedirs(route) - values = {'property': field, 'value[]': valueArr, 'userId': userId} + #values = {'property': field, 'value[]': valueArr, 'userId': userId, 'executionnumber': executionNumber} + values = {'property': field, 'value[]': valueArr, 'userId': userId, 'executionnumber': executionNumber, 'origin': origin, 'originid': originid, 'scheduleExecutionNumber': scheduleExecutionNumber} # Add screen dimension data if it is set as an argument if checkDimensions == True: values['dimensions'] = [dimensions[0],dimensions[1]] @@ -191,7 +222,7 @@ def run(args): try: requests.post(supportRoute+"tracking_data", json=payload) - # equests.post(supportRoute+"tracking_data", json=payload, verify=False) + #requests.post(supportRoute+"tracking_data", json=payload, verify=False) except Exception as e: print("No connection to support Data Base") @@ -199,26 +230,28 @@ def run(args): if noexec == False : #Execute the test videoNameFile = str(organizationId) + "_" + str(executionNumber) + ".mp4" + v = Video() print("File name for video: " + videoNameFile) print("Executing test...") try: - v = Video() v.checkAndStartRecording(videoNameFile) - v.checkActiveSession() - v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ls -ltr | grep *.mp4") + #v.checkActiveSession() + #v.executeCmd("ps -ef | grep ffmpeg") + #v.executeCmd("ls -ltr | grep *.mp4") exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) - v.executeCmd("ps -ef | grep ffmpeg") + #v.executeCmd("ps -ef | grep ffmpeg") v.checkAndStopRecording() - v.executeCmd("ls -ltr | grep *.mp4") + #v.executeCmd("ls -ltr | grep *.mp4") del v testsExecuted = gatherFeedbackData(browserName) url = muuktestRoute+'feedback/' - values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber)} + #values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber)} + values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber), 'origin': origin, 'originid': originid, 'scheduleExecutionNumber': scheduleExecutionNumber} + hed = {'Authorization': 'Bearer ' + token} #CLOUD SCREENSHOTS STARTS # @@ -281,6 +314,10 @@ def main(): parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") parser.add_argument("-browser",help="(Optional). Select one of the available browsers to run the test (default firefox)", type=str, dest="browser") parser.add_argument("-dimensions",help="(Optional). Dimensions to execute the tests, a pair of values for width height, ex. -dimensions 1800 300", type=int, nargs=2, dest="dimensions") + parser.add_argument("-executionnumber",help="(Optional) this numbers contain the executionnumber from the cloud execution", type=str, dest="executionnumber") + parser.add_argument("-origin",help="Test origin, this is cloud only", type=str, dest="origin") + parser.add_argument("-originid",help="Test origin id (scheduling)", type=str, dest="originid") + parser.add_argument("-scheduleexecutionnumber",help="(Optional) this numbers contain the executionnumber (scheduling)", type=str, dest="scheduleexecutionnumber") parser.set_defaults(func=run) args=parser.parse_args() args.func(args) From 53368f797571ee93c26321c949d37b88294323e5 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 13:17:07 -0600 Subject: [PATCH 36/74] Added appspec file --- appspec.yml | 65 ++++++++++++++++++++++++++++++++++++ scripts/install_dependencies | 3 ++ scripts/start_services | 3 ++ 3 files changed, 71 insertions(+) create mode 100644 appspec.yml create mode 100644 scripts/install_dependencies create mode 100644 scripts/start_services diff --git a/appspec.yml b/appspec.yml new file mode 100644 index 0000000..d608cad --- /dev/null +++ b/appspec.yml @@ -0,0 +1,65 @@ +# This is an appspec.yml file for use with an EC2/On-Premises deployment in CodeDeploy. +# The lines in this file starting with the hashtag symbol are +# instructional comments and can be safely left in the file or +# ignored. +# For help completing this file, see the "AppSpec File Reference" in the +# "CodeDeploy User Guide" at +# https://docs.aws.amazon.com/codedeploy/latest/userguide/app-spec-ref.html +version: 0.0 +# Specify "os: linux" if this revision targets Amazon Linux, +# Red Hat Enterprise Linux (RHEL), or Ubuntu Server +# instances. +# Specify "os: windows" if this revision targets Windows Server instances. +# (You cannot specify both "os: linux" and "os: windows".) +os: linux +# During the Install deployment lifecycle event (which occurs between the +# BeforeInstall and AfterInstall events), copy the specified files +# in "source" starting from the root of the revision's file bundle +# to "destination" on the Amazon EC2 instance. +# Specify multiple "source" and "destination" pairs if you want to copy +# from multiple sources or to multiple destinations. +# If you are not copying any files to the Amazon EC2 instance, then remove the +# "files" section altogether. A blank or incomplete "files" section +# may cause associated deployments to fail. +files: + - source: / + destination: /projects/executor/ +hooks: +# For each deployment lifecycle event, specify multiple "location" entries +# if you want to run multiple scripts during that event. +# You can specify "timeout" as the number of seconds to wait until failing the deployment +# if the specified scripts do not run within the specified time limit for the +# specified event. For example, 900 seconds is 15 minutes. If not specified, +# the default is 1800 seconds (30 minutes). +# Note that the maximum amount of time that all scripts must finish executing +# for each individual deployment lifecycle event is 3600 seconds (1 hour). +# Otherwise, the deployment will stop and CodeDeploy will consider the deployment +# to have failed to the Amazon EC2 instance. Make sure that the total number of seconds +# that are specified in "timeout" for all scripts in each individual deployment +# lifecycle event does not exceed a combined 3600 seconds (1 hour). +# For deployments to Amazon Linux, Ubuntu Server, or RHEL instances, +# you can specify "runas" in an event to +# run as the specified user. For more information, see the documentation. +# If you are deploying to Windows Server instances, +# remove "runas" altogether. +# If you do not want to run any commands during a particular deployment +# lifecycle event, remove that event declaration altogether. Blank or +# incomplete event declarations may cause associated deployments to fail. + +# During the BeforeInstall deployment lifecycle event, run the commands +# in the script specified in "location". + BeforeInstall: + - location: scripts/install_dependencies + timeout: 300 + runas: root + - location: scripts/start_services + timeout: 300 + runas: root +# During the ApplicationStop deployment lifecycle event, run the commands +# in the script specified in "location" starting from the root of the +# revision's file bundle. +# ApplicationStop: +# - location: scripts/stop_services +# timeout: 300 +# runas: root + diff --git a/scripts/install_dependencies b/scripts/install_dependencies new file mode 100644 index 0000000..f278153 --- /dev/null +++ b/scripts/install_dependencies @@ -0,0 +1,3 @@ +#!/bin/bash +sudo apt install python3-pip +sudo pip3 install BeautifulSoup4 \ No newline at end of file diff --git a/scripts/start_services b/scripts/start_services new file mode 100644 index 0000000..b4250a4 --- /dev/null +++ b/scripts/start_services @@ -0,0 +1,3 @@ +#!/bin/bash +Xvfb :99 -screen 0 1366x768x16 & + From 76bd0582bb350ca6b60734febd4fce382df0086b Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 13:31:41 -0600 Subject: [PATCH 37/74] Testing installing dependencies --- scripts/install_dependencies | 4 +++- scripts/stop_services | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 scripts/stop_services diff --git a/scripts/install_dependencies b/scripts/install_dependencies index f278153..4f6947a 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -1,3 +1,5 @@ #!/bin/bash sudo apt install python3-pip -sudo pip3 install BeautifulSoup4 \ No newline at end of file +yes +sudo pip3 install BeautifulSoup4 +yes \ No newline at end of file diff --git a/scripts/stop_services b/scripts/stop_services new file mode 100644 index 0000000..490c8ca --- /dev/null +++ b/scripts/stop_services @@ -0,0 +1,6 @@ +#!/bin/bash +#isExistApp = `pgrep httpd` +#if [[ -n $isExistApp ]]; then +# service httpd stop +#fi + From c8970a9c771016a114ee04662a8807fea8b24925 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 13:44:26 -0600 Subject: [PATCH 38/74] Testing installing dependencies --- scripts/install_dependencies | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/install_dependencies b/scripts/install_dependencies index 4f6947a..3c2a432 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -1,5 +1,3 @@ #!/bin/bash -sudo apt install python3-pip -yes -sudo pip3 install BeautifulSoup4 -yes \ No newline at end of file +sudo apt -y install python3-pip +sudo pip3 -y install BeautifulSoup4 \ No newline at end of file From ad30315f3e6c3346e76ca1f345e25647bba1c258 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 13:47:17 -0600 Subject: [PATCH 39/74] Testing installing dependencies --- scripts/install_dependencies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_dependencies b/scripts/install_dependencies index 3c2a432..bdf7888 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -1,3 +1,3 @@ #!/bin/bash sudo apt -y install python3-pip -sudo pip3 -y install BeautifulSoup4 \ No newline at end of file +sudo pip3 install BeautifulSoup4 \ No newline at end of file From aa65aa433599a4e8e28de39fff157d605a93a8f9 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 13:58:45 -0600 Subject: [PATCH 40/74] Change location to home --- appspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appspec.yml b/appspec.yml index d608cad..d26a737 100644 --- a/appspec.yml +++ b/appspec.yml @@ -23,7 +23,7 @@ os: linux # may cause associated deployments to fail. files: - source: / - destination: /projects/executor/ + destination: /home/ubuntu/projects/executor/ hooks: # For each deployment lifecycle event, specify multiple "location" entries # if you want to run multiple scripts during that event. From adf1a820fa396ed7df8c93aac20e1feb98ce1e3a Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 14:40:46 -0600 Subject: [PATCH 41/74] Added AfterInstall --- appspec.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appspec.yml b/appspec.yml index d26a737..678089b 100644 --- a/appspec.yml +++ b/appspec.yml @@ -52,8 +52,11 @@ hooks: - location: scripts/install_dependencies timeout: 300 runas: root +# During the AfterInstall deployment lifecycle event, run the commands +# in the script specified in "location". + AfterInstall: - location: scripts/start_services - timeout: 300 + timeout: 180 runas: root # During the ApplicationStop deployment lifecycle event, run the commands # in the script specified in "location" starting from the root of the From 259e688987be9354e85f733e31899f32fa98e2b3 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 5 Jan 2022 17:45:37 -0600 Subject: [PATCH 42/74] Added xvfb into the dependencies --- scripts/install_dependencies | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install_dependencies b/scripts/install_dependencies index bdf7888..dc80e62 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -1,3 +1,4 @@ #!/bin/bash sudo apt -y install python3-pip -sudo pip3 install BeautifulSoup4 \ No newline at end of file +sudo pip3 install BeautifulSoup4 +sudo apt -y install xvfb \ No newline at end of file From 249d3cbccbfaaaa91cc809bfbaba830c04872500 Mon Sep 17 00:00:00 2001 From: arianafrancof <75339687+arianafrancof@users.noreply.github.com> Date: Wed, 5 Jan 2022 18:15:42 -0600 Subject: [PATCH 43/74] Update start_services --- scripts/start_services | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index b4250a4..a2057ee 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -1,3 +1,2 @@ #!/bin/bash Xvfb :99 -screen 0 1366x768x16 & - From bab63a546a2770e6b2f9442dc9df561c6ceed2e4 Mon Sep 17 00:00:00 2001 From: arianafrancof <75339687+arianafrancof@users.noreply.github.com> Date: Wed, 5 Jan 2022 18:27:16 -0600 Subject: [PATCH 44/74] Testing pipeline From 42ae5af28e53cea7ca8ac8fe825a1cfbe9643770 Mon Sep 17 00:00:00 2001 From: arianafrancof <75339687+arianafrancof@users.noreply.github.com> Date: Thu, 6 Jan 2022 09:44:34 -0600 Subject: [PATCH 45/74] Testing pipeline with new permissions --- mkcli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkcli.py b/mkcli.py index 80731f5..470fdf7 100644 --- a/mkcli.py +++ b/mkcli.py @@ -325,3 +325,6 @@ def main(): if __name__=="__main__": main() + + + From 49c1b407f81f5896de8c56ad345d2075818c3760 Mon Sep 17 00:00:00 2001 From: arianafrancof <75339687+arianafrancof@users.noreply.github.com> Date: Thu, 6 Jan 2022 09:53:42 -0600 Subject: [PATCH 46/74] Testing new pipeline --- mkcli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mkcli.py b/mkcli.py index 470fdf7..80731f5 100644 --- a/mkcli.py +++ b/mkcli.py @@ -325,6 +325,3 @@ def main(): if __name__=="__main__": main() - - - From 23d7dd4addc88bab8ed3dc9ab9310c2f63c29b9f Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Fri, 7 Jan 2022 16:02:43 -0600 Subject: [PATCH 47/74] Added bck script --- appspec.yml | 5 ++++- scripts/backup | 2 ++ scripts/start_services | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 scripts/backup diff --git a/appspec.yml b/appspec.yml index 678089b..d035474 100644 --- a/appspec.yml +++ b/appspec.yml @@ -49,6 +49,9 @@ hooks: # During the BeforeInstall deployment lifecycle event, run the commands # in the script specified in "location". BeforeInstall: + - location: scripts/backup + timeout: 300 + runas: root - location: scripts/install_dependencies timeout: 300 runas: root @@ -56,7 +59,7 @@ hooks: # in the script specified in "location". AfterInstall: - location: scripts/start_services - timeout: 180 + timeout: 300 runas: root # During the ApplicationStop deployment lifecycle event, run the commands # in the script specified in "location" starting from the root of the diff --git a/scripts/backup b/scripts/backup new file mode 100644 index 0000000..454b3de --- /dev/null +++ b/scripts/backup @@ -0,0 +1,2 @@ +#!/bin/bash +cp -r projects/executor /projects/executor_bck \ No newline at end of file diff --git a/scripts/start_services b/scripts/start_services index a2057ee..054256e 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -1,2 +1,3 @@ #!/bin/bash +sed -i 's/portal.muuktest.com/testing.muuktest.com/g' mkcli.py Xvfb :99 -screen 0 1366x768x16 & From 01d37f0f5019e65b3f43239da4a256532aa3390f Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Fri, 7 Jan 2022 16:12:56 -0600 Subject: [PATCH 48/74] Path to backup --- scripts/backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/backup b/scripts/backup index 454b3de..37920d7 100644 --- a/scripts/backup +++ b/scripts/backup @@ -1,2 +1,2 @@ #!/bin/bash -cp -r projects/executor /projects/executor_bck \ No newline at end of file +cp -r /home/ubuntu/projects/executor /home/ubuntu/projects/executor_bck \ No newline at end of file From d527233a16edbd73c6538a3ad19d5265ab91cff1 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Fri, 7 Jan 2022 17:03:53 -0600 Subject: [PATCH 49/74] Added absolute path to modify route when start services --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 054256e..101976e 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -1,3 +1,3 @@ #!/bin/bash -sed -i 's/portal.muuktest.com/testing.muuktest.com/g' mkcli.py +sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py Xvfb :99 -screen 0 1366x768x16 & From a7338870db0f8a24e5d7d8c62cf47e5c7da5b28d Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Mon, 10 Jan 2022 14:19:02 -0600 Subject: [PATCH 50/74] Added validation when creating backup --- scripts/backup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/backup b/scripts/backup index 37920d7..b57bdd8 100644 --- a/scripts/backup +++ b/scripts/backup @@ -1,2 +1,5 @@ #!/bin/bash -cp -r /home/ubuntu/projects/executor /home/ubuntu/projects/executor_bck \ No newline at end of file +if [ -d "/home/ubuntu/projects/executor" ]; then + # Control will enter here if $DIRECTORY exists. + cp -r /home/ubuntu/projects/executor /home/ubuntu/projects/executor_bck +fi \ No newline at end of file From 863c5be282dca9ca09f09cb01004070b5cdad21a Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 12 Jan 2022 13:04:12 -0600 Subject: [PATCH 51/74] Added IP's validations --- appspec.yml | 3 +++ scripts/config_files | 23 +++++++++++++++++++++++ scripts/install_dependencies | 3 +++ scripts/start_services | 1 - scripts/stop_services | 5 ----- 5 files changed, 29 insertions(+), 6 deletions(-) create mode 100755 scripts/config_files diff --git a/appspec.yml b/appspec.yml index d035474..3497f16 100644 --- a/appspec.yml +++ b/appspec.yml @@ -58,6 +58,9 @@ hooks: # During the AfterInstall deployment lifecycle event, run the commands # in the script specified in "location". AfterInstall: + - location: scripts/config_files + timeout: 300 + runas: root - location: scripts/start_services timeout: 300 runas: root diff --git a/scripts/config_files b/scripts/config_files new file mode 100755 index 0000000..493f7ef --- /dev/null +++ b/scripts/config_files @@ -0,0 +1,23 @@ +#!/bin/bash +#Updating the path to executor instance +path_exec='"path":"//"' +new_path_exec='"path":"/home/ubuntu/projects/executor/"' +sudo sed -i "s%$path_exec%$new_path_exec%g" /home/ubuntu/projects/support/config/default.json + +#Verifying the instance queue +#Finding the current ip +IP=$(hostname -I | awk '{print $1}') +#172.31.1.41 Ari Testing Instance +testingInstance="172.31.20.165" +stagingInstance="172.31.12.113" +testingInstanceTmp="172.31.1.41"#ari-testing + +if [[ "$IP" == $testingInstance || "$IP" == $testingInstanceTmp ]]; then + echo "It's the testing instance" + #Change queue name + sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py +elif [[ "$IP" == $stagingInstance ]]; then + echo "It's the staging instance" + #Change queue name + sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py +fi \ No newline at end of file diff --git a/scripts/install_dependencies b/scripts/install_dependencies index dc80e62..3db6f0c 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -1,4 +1,7 @@ #!/bin/bash +#Installing Pip3 sudo apt -y install python3-pip +#Installing BeautifulSoup4 sudo pip3 install BeautifulSoup4 +#Installing xvfb sudo apt -y install xvfb \ No newline at end of file diff --git a/scripts/start_services b/scripts/start_services index 101976e..a2057ee 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -1,3 +1,2 @@ #!/bin/bash -sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py Xvfb :99 -screen 0 1366x768x16 & diff --git a/scripts/stop_services b/scripts/stop_services index 490c8ca..a9bf588 100644 --- a/scripts/stop_services +++ b/scripts/stop_services @@ -1,6 +1 @@ #!/bin/bash -#isExistApp = `pgrep httpd` -#if [[ -n $isExistApp ]]; then -# service httpd stop -#fi - From 169ab341f3ad51a2752a3a5ed02762e1dba80733 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 12 Jan 2022 13:12:41 -0600 Subject: [PATCH 52/74] Fixing validation instances --- scripts/config_files | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/config_files b/scripts/config_files index 493f7ef..9c12a14 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -1,23 +1,21 @@ #!/bin/bash -#Updating the path to executor instance -path_exec='"path":"//"' -new_path_exec='"path":"/home/ubuntu/projects/executor/"' -sudo sed -i "s%$path_exec%$new_path_exec%g" /home/ubuntu/projects/support/config/default.json - -#Verifying the instance queue +#Verifying the instance #Finding the current ip IP=$(hostname -I | awk '{print $1}') -#172.31.1.41 Ari Testing Instance +#Testing Instance testingInstance="172.31.20.165" +#Staging Instance stagingInstance="172.31.12.113" -testingInstanceTmp="172.31.1.41"#ari-testing +#172.31.1.41 Ari Testing Instance +testingInstanceTmp="192.168.1.157" +echo $IP if [[ "$IP" == $testingInstance || "$IP" == $testingInstanceTmp ]]; then echo "It's the testing instance" - #Change queue name + #Change portal route sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py elif [[ "$IP" == $stagingInstance ]]; then echo "It's the staging instance" - #Change queue name + #Change portal route sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py fi \ No newline at end of file From aa0a3dc83c815eaeefd13d29d32316d92b51b45f Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 12 Jan 2022 13:14:44 -0600 Subject: [PATCH 53/74] Changing testing instance ip --- scripts/config_files | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config_files b/scripts/config_files index 9c12a14..1d4e34a 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -7,7 +7,7 @@ testingInstance="172.31.20.165" #Staging Instance stagingInstance="172.31.12.113" #172.31.1.41 Ari Testing Instance -testingInstanceTmp="192.168.1.157" +testingInstanceTmp="172.31.1.41" echo $IP if [[ "$IP" == $testingInstance || "$IP" == $testingInstanceTmp ]]; then From c0221dd5c6115310d31a627ce7fbf42bf0d8b5ae Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Fri, 14 Jan 2022 09:50:13 -0600 Subject: [PATCH 54/74] Modified path to backup --- scripts/backup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/backup b/scripts/backup index b57bdd8..c5a9c2a 100644 --- a/scripts/backup +++ b/scripts/backup @@ -1,5 +1,6 @@ #!/bin/bash if [ -d "/home/ubuntu/projects/executor" ]; then # Control will enter here if $DIRECTORY exists. - cp -r /home/ubuntu/projects/executor /home/ubuntu/projects/executor_bck + mkdir -p /home/ubuntu/backup + cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor fi \ No newline at end of file From eb1bb010e0a10bc609daf973092d79ff648ef3ca Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Tue, 18 Jan 2022 20:21:46 -0600 Subject: [PATCH 55/74] Commentign backup hook --- scripts/backup | 4 ++-- scripts/config_files | 7 ++++++- scripts/install_dependencies | 7 ++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/backup b/scripts/backup index c5a9c2a..2fe81f3 100644 --- a/scripts/backup +++ b/scripts/backup @@ -1,6 +1,6 @@ #!/bin/bash if [ -d "/home/ubuntu/projects/executor" ]; then # Control will enter here if $DIRECTORY exists. - mkdir -p /home/ubuntu/backup - cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor + #mkdir -p /home/ubuntu/backup + #cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor fi \ No newline at end of file diff --git a/scripts/config_files b/scripts/config_files index 1d4e34a..9038a2d 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -18,4 +18,9 @@ elif [[ "$IP" == $stagingInstance ]]; then echo "It's the staging instance" #Change portal route sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py -fi \ No newline at end of file +fi + +#Setting the cloud key +cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') +sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py +sudo sed -i "$ a \ \ $cloudKey" /home/ubuntu/projects/executor/mkcloud.py diff --git a/scripts/install_dependencies b/scripts/install_dependencies index 3db6f0c..a7acb7e 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -3,5 +3,10 @@ sudo apt -y install python3-pip #Installing BeautifulSoup4 sudo pip3 install BeautifulSoup4 +#sudo pip3 install bs4 #Installing xvfb -sudo apt -y install xvfb \ No newline at end of file +sudo apt -y install xvfb +#Installing google chrome +#sudo apt-get install google-chrome-stable +#Installing jq to read json files +sudo apt -y install jq \ No newline at end of file From 2bd4817a7e8e6271700793ea9e9809721039af74 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Tue, 18 Jan 2022 20:28:41 -0600 Subject: [PATCH 56/74] Commenting backup hook into the appspec file --- appspec.yml | 6 +++--- scripts/backup | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/appspec.yml b/appspec.yml index 3497f16..c40e22c 100644 --- a/appspec.yml +++ b/appspec.yml @@ -49,9 +49,9 @@ hooks: # During the BeforeInstall deployment lifecycle event, run the commands # in the script specified in "location". BeforeInstall: - - location: scripts/backup - timeout: 300 - runas: root + #- location: scripts/backup + # timeout: 300 + # runas: root - location: scripts/install_dependencies timeout: 300 runas: root diff --git a/scripts/backup b/scripts/backup index 2fe81f3..c5a9c2a 100644 --- a/scripts/backup +++ b/scripts/backup @@ -1,6 +1,6 @@ #!/bin/bash if [ -d "/home/ubuntu/projects/executor" ]; then # Control will enter here if $DIRECTORY exists. - #mkdir -p /home/ubuntu/backup - #cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor + mkdir -p /home/ubuntu/backup + cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor fi \ No newline at end of file From 42a4fda1aad491218ee625e43063294ac2a31f69 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 10:21:26 -0600 Subject: [PATCH 57/74] Added return into the getCloudKey function --- scripts/config_files | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/config_files b/scripts/config_files index 9038a2d..4a08260 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -21,6 +21,7 @@ elif [[ "$IP" == $stagingInstance ]]; then fi #Setting the cloud key +#The cloud key it's located in a external env.json file cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py -sudo sed -i "$ a \ \ $cloudKey" /home/ubuntu/projects/executor/mkcloud.py +sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py From df841bbb6f2566e54b7a8c79874b99ded15ca232 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 15:49:29 -0600 Subject: [PATCH 58/74] Added installing firefox and chrome in dependencies --- appspec.yml | 8 ++++---- build.gradle | 2 +- scripts/config_files | 2 +- scripts/install_dependencies | 10 +++++++++- scripts/start_services | 7 +++++++ scripts/stop_services | 6 ++++++ 6 files changed, 28 insertions(+), 7 deletions(-) diff --git a/appspec.yml b/appspec.yml index c40e22c..79f6ba0 100644 --- a/appspec.yml +++ b/appspec.yml @@ -67,8 +67,8 @@ hooks: # During the ApplicationStop deployment lifecycle event, run the commands # in the script specified in "location" starting from the root of the # revision's file bundle. -# ApplicationStop: -# - location: scripts/stop_services -# timeout: 300 -# runas: root + ApplicationStop: + - location: scripts/stop_services + timeout: 300 + runas: root diff --git a/build.gradle b/build.gradle index 55d6a67..9028dfe 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ ext { groovyVersion = '2.4.12' gebVersion = '2.2' seleniumVersion = '3.6.0' - chromeDriverVersion = '79.0.3945.36' + chromeDriverVersion = '97.0.4692.71' geckoDriverVersion = '0.23.0' } } diff --git a/scripts/config_files b/scripts/config_files index 4a08260..a2ae6e7 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -24,4 +24,4 @@ fi #The cloud key it's located in a external env.json file cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py -sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py +sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py \ No newline at end of file diff --git a/scripts/install_dependencies b/scripts/install_dependencies index a7acb7e..e0108d5 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -6,7 +6,15 @@ sudo pip3 install BeautifulSoup4 #sudo pip3 install bs4 #Installing xvfb sudo apt -y install xvfb +#Installing firefox +sudo apt -y install firefox #Installing google chrome -#sudo apt-get install google-chrome-stable +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +sudo apt -y install ./google-chrome-stable_current_amd64.deb +sudo apt-get install google-chrome-stable +#Installing ffmpeg +sudo apt -y install ffmpeg +#Installing request +pip3 install request #Installing jq to read json files sudo apt -y install jq \ No newline at end of file diff --git a/scripts/start_services b/scripts/start_services index a2057ee..8756d1b 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -1,2 +1,9 @@ #!/bin/bash +sudo chmod 777 /home/ubuntu/projects/executor/gradlew Xvfb :99 -screen 0 1366x768x16 & + +#Starting queue service in case it is needed +serviceStatus=`systemctl is-active muuktest-queued.service` +if [[ "$serviceStatus" != 'active' ]]; then + sudo systemctl start muuktest-queued.service +fi \ No newline at end of file diff --git a/scripts/stop_services b/scripts/stop_services index a9bf588..29cb840 100644 --- a/scripts/stop_services +++ b/scripts/stop_services @@ -1 +1,7 @@ #!/bin/bash +#Stoping queue service in case it is needed +serviceStatus=`systemctl is-active muuktest-queued.service` +if [[ "$serviceStatus" == 'active' ]]; then + echo "muuktest-queued.service is running" + sudo systemctl stop muuktest-queued.service +fi \ No newline at end of file From 2a5cab6dc8b26f573741d9de0ff9c5a1bc83603f Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 17:19:32 -0600 Subject: [PATCH 59/74] Added rm google.deb --- scripts/install_dependencies | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install_dependencies b/scripts/install_dependencies index e0108d5..7e6b874 100644 --- a/scripts/install_dependencies +++ b/scripts/install_dependencies @@ -12,6 +12,7 @@ sudo apt -y install firefox wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb sudo apt -y install ./google-chrome-stable_current_amd64.deb sudo apt-get install google-chrome-stable +sudo rm google-chrome-stable_current_amd64.deb #Installing ffmpeg sudo apt -y install ffmpeg #Installing request From cb165866a4520f4c83a57a1b74ea51c40644c896 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 17:28:26 -0600 Subject: [PATCH 60/74] Added validation to the file env.json --- scripts/config_files | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/config_files b/scripts/config_files index a2ae6e7..eb24a4b 100755 --- a/scripts/config_files +++ b/scripts/config_files @@ -11,17 +11,19 @@ testingInstanceTmp="172.31.1.41" echo $IP if [[ "$IP" == $testingInstance || "$IP" == $testingInstanceTmp ]]; then - echo "It's the testing instance" - #Change portal route - sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py + echo "It's the testing instance" + #Change portal route + sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py elif [[ "$IP" == $stagingInstance ]]; then - echo "It's the staging instance" - #Change portal route - sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py + echo "It's the staging instance" + #Change portal route + sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py fi #Setting the cloud key #The cloud key it's located in a external env.json file -cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') -sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py -sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py \ No newline at end of file +if [[ -f "/home/ubuntu/env.json" ]]; then + cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') + sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py + sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py +fi \ No newline at end of file From ccc7f6627297ff7a93f344c46785cbe44cac7e2d Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 17:39:02 -0600 Subject: [PATCH 61/74] Added validation to start queue service --- scripts/start_services | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 8756d1b..f9f5462 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -4,6 +4,7 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` -if [[ "$serviceStatus" != 'active' ]]; then +queueServiceFile=/lib/systemd/system/muuktest-queued.service +if [[ "$serviceStatus" != 'active' ]] && [[ -f "$queueServiceFile"]]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 21d928e60056862b55c848ecf76f5fa38af4b4e4 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 17:46:30 -0600 Subject: [PATCH 62/74] Modifyied verification queue service --- scripts/start_services | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/start_services b/scripts/start_services index f9f5462..2cc093f 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -4,7 +4,7 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` -queueServiceFile=/lib/systemd/system/muuktest-queued.service -if [[ "$serviceStatus" != 'active' ]] && [[ -f "$queueServiceFile"]]; then +queueServiceFile="/lib/systemd/system/muuktest-queued.service" +if [[ "$serviceStatus" != 'active' ]] && [ -f $queueServiceFile]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 61399e9a1bee9d8b23b2afe9f1720f456ac5759d Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 17:59:46 -0600 Subject: [PATCH 63/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 2cc093f..d54b7b7 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active' ]] && [ -f $queueServiceFile]; then +if [[ "$serviceStatus" != 'active' && -f $queueServiceFile]]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From aa0fcb58536aa420c0e313675eee6d14ab17fe78 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:15:41 -0600 Subject: [PATCH 64/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index d54b7b7..e18cd53 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active' && -f $queueServiceFile]]; then +if [[ "$serviceStatus" != 'active']] && -f "$queueServiceFile"; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From f4f2206a3f0b836986bf4de885023f00c0966ab2 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:16:58 -0600 Subject: [PATCH 65/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index e18cd53..892dccd 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active']] && -f "$queueServiceFile"; then +if [[ "$serviceStatus" != 'active']] && [ -f "$queueServiceFile" ]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From c4f17efdc58f3d4beb664cb9a8d5d8d866aee501 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:18:37 -0600 Subject: [PATCH 66/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 892dccd..19ef937 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active']] && [ -f "$queueServiceFile" ]; then +if [[ "$serviceStatus" != 'active']] && [ -f $queueServiceFile ]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 88e32e7691f86a742c486d1aa167d824dcf6ec1e Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:20:54 -0600 Subject: [PATCH 67/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 19ef937..892dccd 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active']] && [ -f $queueServiceFile ]; then +if [[ "$serviceStatus" != 'active']] && [ -f "$queueServiceFile" ]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From d3a107b6f93fcec5b4e1aac55d7d6d6c4a131a50 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:22:31 -0600 Subject: [PATCH 68/74] Modified verification queue service --- scripts/start_services | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/start_services b/scripts/start_services index 892dccd..a7bf8c0 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -4,7 +4,7 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` -queueServiceFile="/lib/systemd/system/muuktest-queued.service" -if [[ "$serviceStatus" != 'active']] && [ -f "$queueServiceFile" ]; then +queueServiceFile=/lib/systemd/system/muuktest-queued.service +if [[ "$serviceStatus" != 'active'] && -f "$queueServiceFile" ]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From b06479dd2968d7df7452992cc650b849a401323f Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:23:53 -0600 Subject: [PATCH 69/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index a7bf8c0..16be1b2 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile=/lib/systemd/system/muuktest-queued.service -if [[ "$serviceStatus" != 'active'] && -f "$queueServiceFile" ]; then +if [[ "$serviceStatus" != 'active'] && [-f "$queueServiceFile" ]]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 5288990f00b0ff5804ce1970ef8202b8238be230 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:25:31 -0600 Subject: [PATCH 70/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 16be1b2..650a2fa 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile=/lib/systemd/system/muuktest-queued.service -if [[ "$serviceStatus" != 'active'] && [-f "$queueServiceFile" ]]; then +if [[ "$serviceStatus" != 'active'] && [ -f "$queueServiceFile" ]]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 40ec8713bb5483560fae4dff7be521ef0c8f3c23 Mon Sep 17 00:00:00 2001 From: Ariana Franco Date: Wed, 19 Jan 2022 18:28:20 -0600 Subject: [PATCH 71/74] Modified verification queue service --- scripts/start_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_services b/scripts/start_services index 650a2fa..e2d58ea 100644 --- a/scripts/start_services +++ b/scripts/start_services @@ -5,6 +5,6 @@ Xvfb :99 -screen 0 1366x768x16 & #Starting queue service in case it is needed serviceStatus=`systemctl is-active muuktest-queued.service` queueServiceFile=/lib/systemd/system/muuktest-queued.service -if [[ "$serviceStatus" != 'active'] && [ -f "$queueServiceFile" ]]; then +if [[ "$serviceStatus" != 'active' ]] && [ -f "$queueServiceFile" ]; then sudo systemctl start muuktest-queued.service fi \ No newline at end of file From 5602126d7bfa70acac5fe3a11d074ba16045bd30 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Fri, 6 May 2022 18:46:43 -0700 Subject: [PATCH 72/74] Handle custom selectors. --- CSSAnalyzer.py | 593 ++++++++++++++++++++++++++++++++++++++++++++ XPATHAnalyzer.py | 245 ++++++++++++++++++ domparser.py | 632 +++++++---------------------------------------- 3 files changed, 927 insertions(+), 543 deletions(-) create mode 100644 CSSAnalyzer.py create mode 100644 XPATHAnalyzer.py diff --git a/CSSAnalyzer.py b/CSSAnalyzer.py new file mode 100644 index 0000000..bd99e70 --- /dev/null +++ b/CSSAnalyzer.py @@ -0,0 +1,593 @@ + +import os +import json +import pprint +from bs4 import BeautifulSoup +import traceback + +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 +SELECTOR_BUILD_FROM_ATTRIBUTES = 15 + +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 + +SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR ] + + +# Description: +# This method will be called to handle the result of filter (by value, text,etc) operation done +# on the selectors found. +# +# There are 3 options for the result: +# 1) No selectors were found having the value we are expecting. On this case, +# information returned will be the element with the index that was expected. +# +# 2) We found only one selector, we have two options here: +# a) Found the correct selector: Return the original element. +# b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. +# +# 3) We found two or more selectors with the same src. We have two options here: +# a) The correct selector was found. Return the original element. +# b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +# +def processResults(selector, htmlElements, expectedIndex, expectedValue, elementsFoundWithValue, indexesFound, attribute): + jsonObject = {} + elements = [] + + if(elementsFoundWithValue == 0): + # No selectors were found with the expected value + if(expectedIndex <= len(htmlElements)): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE + else: + element = {} + element["index"] = "-1" + element["selector"] = "" + returnCode = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND + elif(elementsFoundWithValue == 1): + if(expectedIndex in indexesFound): + # The expected selector was found and it is the only selector. + element = {} + element["index"] = expectedIndex + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[expectedIndex].text + else: + element["value"] = htmlElements[expectedIndex][attribute] + returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + else: + # The incorrect selector was found and this is the only selector with the expected value + element = {} + element["index"] = indexesFound[elementsFoundWithValue -1] + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[indexesFound[elementsFoundWithValue -1]].text + else: + element["value"] = htmlElements[indexesFound[elementsFoundWithValue -1]][attribute] + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + elif(elementsFoundWithValue > 1): + # Several selectors were found with same value + if(expectedIndex in indexesFound): + # The expected element was found on the selectors + element = {} + element["index"] = expectedIndex + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[expectedIndex].text + else: + element["value"] = htmlElements[expectedIndex][attribute] + elements.append(element) + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX + else: + # The expected element was NOT found on the selector + element = {} + element["index"] = str(indexesFound) + element["selector"] = selector + if(attribute == "text"): + element["value"] = expectedValue + else: + element["value"] = expectedValue + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX + + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = elementsFoundWithValue + jsonObject["selectors"] = element + jsonObject["rc"] = returnCode + + return jsonObject + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the text value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseTextSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + jsonObject = {} + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + + if(expectedValue == ""): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for htmlElment in htmlElements: + selectorText = htmlElment.text.replace("'", "") + if(selectorText.strip() == expectedValue.strip()): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "text") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the src value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseImageSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + + for selector in htmlElements: + if(selector['src'] == expectedValue ): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "src") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the href value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseHypertextSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + jsonObject = {} + indexesFound = [] + filteredIndexes = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + if hasattr(searchInfo, 'text'): + expectedText = searchInfo["text"] + else: + expectedText = "" + + if(expectedValue == ""): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for element in htmlElements: + if(element and element.has_attr('href')): + if(element['href'] == expectedValue): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + # If more than 1 element was found using the same value, lest's filter now by text and update + # the indexesFound with the new indexes (hopefully only one!). + if(numberElementsFoundWithValue > 1 and expectedText != ""): + for i in indexesFound: + if(str(htmlElements[i].string) == expectedText): + filteredIndexes.append(i) + if(len(filteredIndexes) > 0 ): + indexesFound = [] + numberElementsFoundWithValue = len(filteredIndexes) + for index in filteredIndexes: + indexesFound.append(index) + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "href") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseValueSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + expectedText = searchInfo["text"] if ('text' in searchInfo) else "" + + for element in htmlElements: + if(('value' in element) and element['value'] == expectedValue ): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + # If we have text information available and this is a select element, let's try to + # find the correct value using the text + if(numberElementsFoundWithValue == 0 and expectedText != "" and tag == "select"): + return handleSelectElement(selector, htmlElements, expectedText, expectedIndex, indexesFound, tag) + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "value") + +def handleSelectElement(selector, htmlElements, expectedText, expectedIndex, selectorIndexes, tag): + jsonObject = {} + value = "" + # Let's first verify the selector with the expected index + selectElement = htmlElements[expectedIndex] + options = selectElement.find_all("option") + for option in options: + if(option.text.strip() == expectedText.strip()): + value = option.get("value") + break + + element = {} + element["index"] = expectedIndex + element["value"] = value + element["selector"] = selector + + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + + return jsonObject + + +# Description: +# This function will be called when the selector string returns 0 elements. It means the selector that we have is not working and we need to +# build a new one. This function will will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def buildSelectorFromAttributes(tag, attributes, soup): + jsonObject = {} + + # Start building the CSS selector + selector = tag + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(soup.select(tag + "[id='"+attributes["id"]+"']")) > 0): + selector = selector + "[id='"+attributes["id"]+"']" + if(attributes["name"] != "undef"): + if(len(soup.select(tag + "[name='"+attributes["name"]+"']")) > 0): + selector = selector + "[name='"+attributes["name"]+"']" + if(attributes["type"] != "undef"): + if(len(soup.select(tag + "[type='"+attributes["type"]+"']")) > 0): + selector = selector + "[type='"+attributes["type"]+"']" + + selectorsFound = soup.select(selector) + numberSelectorsFound = len(selectorsFound) + index = 0 + selectorsWithTextIndex = 0 + + bFoundWithText = False + if(numberSelectorsFound > 1 and attributes["text"] != "undef"): + for sel in selectorsFound: + if(sel.string == attributes['text']): + selectorsWithTextIndex = index + bFoundWithText = True + break + index += 1 + + if(numberSelectorsFound == 0 ): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + elif(numberSelectorsFound == 1 or bFoundWithText ): + element = {} + element["selector"] = selector + element["index"] = selectorsWithTextIndex + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + + return jsonObject + + +# Description: +# This function will be called when we have more than one element as a result of the selector query and we DO NOT have +# a value to try to filter the attributes and find the correct one. So, using the object's attributes we will try to find the +# correct selector. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def findIndexFromAttributes(selector, tag, attributes, soup): + jsonObject = {} + idFound = False + if(attributes["id"] != "false" and attributes["id"] != "undef"): + objectID = attributes["id"] + if(attributes["name"] != "undef"): + objectName = attributes["name"] + if(attributes["type"] != "undef"): + objectType = attributes["type"] + + htmlElementsFound = soup.select(selector) + numberElementsFound = len(htmlElementsFound) + if(numberElementsFound > 1 ): + newIndex = 0 + for element in htmlElementsFound: + if(element.has_attr('id') and element['id'] == objectID and + element.has_attr('name') and element['name'] == objectName and + element.has_attr('type') and element['type'] == objectType): + idFound = True + break + newIndex = newIndex + 1 + + if(idFound): + element = {} + element["selector"] = selector + element["index"] = newIndex + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + else: + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + return jsonObject + +# Description: +# This function will be called when the object DOES NOT HAVE a selector string, so we need to build it from scratch. +# This function will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def buildCSSSelector(tag, attributes, searchInfo, index, soup): + jsonObject = {} + cssSelector = tag + searchType = searchInfo["searchType"] + value = searchInfo["value"] + + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(soup.select(tag + "[id='"+attributes["id"]+"']")) > 0): + cssSelector = cssSelector + "[id = '" + attributes["id"] + "']" + if(attributes["name"] != "undef"): + if(len(soup.select(tag + "[name='"+attributes["name"]+"']")) > 0): + cssSelector = cssSelector + "[name = '" + attributes["name"] + "']" + if(attributes["type"] != "undef"): + if(len(soup.select(tag + "[type='"+attributes["type"]+"']")) > 0): + cssSelector = cssSelector + "[type = '" + attributes["type"] + "']" + + # now that we have a selector, let's va;idate it returns elements. + htmlElementsFound = soup.select(cssSelector) + numberElementsFound = len(htmlElementsFound) + + if(numberElementsFound == 1): + element = {} + element["selector"] = cssSelector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + elif(numberElementsFound > 1): + if(searchType == "value"): + jsonObject = parseValueSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(cssSelector, numberElementsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + element = {} + element["selector"] = cssSelector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + element = {} + element["selector"] = "" + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + + return jsonObject + + +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def obtainCSSFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + jsonObject = {} + path = 'build/reports/geb/' + browserName + '/' + filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): + try: + searchType = searchInfo["searchType"] + value = searchInfo["value"] + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + if(selector is None): + if(selector_type == CUSTOM_CSS_SELECTOR): + print("NO CSS Selector, need to build it") + return buildCSSSelector(tag, attributes, searchInfo, index, soup) + else: + selectorsFound = soup.select(selector) + numberSelectorsFound = len(selectorsFound) + + if(action == "mouseover"): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + if(numberSelectorsFound == 0): + if(len(attributes) > 0 and selector_type == CUSTOM_CSS_SELECTOR ): + return buildSelectorFromAttributes(tag, attributes, soup) + else: + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + elif(numberSelectorsFound == 1 ): + element = {} + element["selector"] = selector + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + if(index > 0): + element["index"] = 0 + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + else: + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + elif(numberSelectorsFound > 1 ): + if(value != "undef"): + if(searchType == "value"): + jsonObject = parseValueSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(selector, selectorsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + # We do not have a value to try to identify the element, let's use the attributes + return findIndexFromAttributes(selector, tag, attributes, soup) + + jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound + + except Exception as ex: + print("Failed to open file " + str(filename) + ex) + print (ex) + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + jsonObject = {} + + return jsonObject \ No newline at end of file diff --git a/XPATHAnalyzer.py b/XPATHAnalyzer.py new file mode 100644 index 0000000..c118bcf --- /dev/null +++ b/XPATHAnalyzer.py @@ -0,0 +1,245 @@ + +from logging import root +import os +import json +import pprint +import lxml.etree +import lxml.html +from xml.sax import saxutils +from lxml import html +import traceback + +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 +SELECTOR_BUILD_FROM_ATTRIBUTES = 15 + +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 + +# Description: +# This method returns the inner HTML of the element received as a parameter. +# +# Parameters: +# element: HTML element +# +# Returns: +# inner HTML (string) +# +def inner_html(element): + return (saxutils.escape(element.text) if element.text else '') + \ + ''.join([html.tostring(child, encoding=str) for child in element.iterchildren()]) + +# Description: +# This function build an XPATH selector string. It will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# root: (LXML object to query HTML DOM) +# Returns: +# XPATH selector +# +def buildXPATHSelector(tag, attributes, root): + jsonObject = {} + elements = [] + textUsedOnSelector = False + + selector = "//" + tag + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(root.xpath("//" + tag + "[@id='"+attributes["id"]+"']")) > 0): + selector = selector + "[@id='"+attributes["id"]+"']" + + if(attributes["name"] != "undef"): + if(len(root.xpath("//" + tag + "[@name='"+attributes["name"]+"']")) > 0): + selector = selector + "[@name='"+attributes["name"]+"']" + + if(attributes["type"] != "undef"): + if(len(root.xpath("//" + tag + "[@type='"+attributes["type"]+"']")) > 0): + selector = selector + "[@type='"+attributes["type"]+"']" + + if( attributes["text"] and len(attributes["text"]) > 0 and attributes["text"] != "undef" ): + textUsedOnSelector = True + text = attributes["text"] + innerHTMLText = "" + + # As MuukTest sends the text that is obtained from innerText it does not contain any tag that the DOM will contian when searching + # for a XPATH expression. So, we will query all the tag elements and get the innerText so we can compare and then build the xpath + elements = root.xpath("//" + tag) + elementIndex = -1 + for element in elements: + if(element.text_content() == text): + elementIndex = elementIndex + 1 + text = element.text_content() + innerHTMLText = inner_html(element) + + splittedText = text.split("\n") + if(len(splittedText) > 1 or (innerHTMLText != "" and innerHTMLText != text ) ): + # if we are using normalize-space we need to use the complete text but we need to escape invalid characters. + + text = text.replace("\n", " ") + text = text.replace("'", "\'") + text = text.replace("$", '\\$') + text = text.strip() + + selector = selector + "[normalize-space() = \""+text+"\"]" + try: + htmlElementsFound = root.xpath(selector) + except Exception as ex: + # if we failed to obtain selectors, lets use only the tag and the index + selector = "//" + tag + element = {} + element["selector"] = selector + element["index"] = elementIndex + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + return jsonObject + + + else: + if(len(attributes["text"]) > 40): + text = attributes["text"][0:40] + + # Some characters will cause problems on the XPATH expression when using contains, we need to escape the next + # characters: + print(repr(text)) + #text = text.replace("$", '\$') // no need to escape + text = text.replace("'", "\'") + text = text.strip() + + selector = selector + "[contains(text(),\""+text+"\")]" + + try: + htmlElementsFound = root.xpath(selector) + except Exception as ex: + selector = "//" + tag + element = {} + element["selector"] = selector + element["index"] = elementIndex + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + return jsonObject + + htmlElementsFound = root.xpath(selector) + numberhtmlElementsFound = len(htmlElementsFound) + if(numberhtmlElementsFound == 0 ): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + elif(numberhtmlElementsFound == 1 ): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + elif(numberhtmlElementsFound > 1 and textUsedOnSelector): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = numberhtmlElementsFound + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberhtmlElementsFound + + return jsonObject + +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + jsonObject = {} + path = 'build/reports/geb/' + browserName + '/' + filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): + try: + searchType = searchInfo["searchType"] + value = searchInfo["value"] + text = open(filename, 'r').read() + root = lxml.html.fromstring(text) + if(selector is None): + return buildXPATHSelector(tag, attributes, root) + else: + htmlElementsFound = root.xpath(selector) + numberSelectorsFound = len(htmlElementsFound) + + if(action == "mouseover"): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + if(numberSelectorsFound == 0): + return buildXPATHSelector(tag, attributes, root) + + elif(numberSelectorsFound == 1 ): + element = {} + element["selector"] = selector + if(index > 0): + element["index"] = 0 + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + else: + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + + elif(numberSelectorsFound > 1 ): + element = {} + element["selector"] = selector + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound + + except Exception as ex: + print("Failed to open file " + str(filename) + ex) + print (ex) + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + print(ex) + print(traceback.format_exc()) + jsonObject = {} + + return jsonObject \ No newline at end of file diff --git a/domparser.py b/domparser.py index 842caa9..a9f5dcf 100644 --- a/domparser.py +++ b/domparser.py @@ -3,534 +3,16 @@ import json import pprint from bs4 import BeautifulSoup +from CSSAnalyzer import obtainCSSFeedbackFromDOM +from XPATHAnalyzer import obtainXPATHFeedbackFromDOM import traceback -UNKNOWN_ERROR = 1 -NO_TAG_PROVIDED_BY_BE = 2 -NO_VALUE_PROVIDED_BY_BE = 3 -NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 -ACTION_NOT_VALID_FOR_ANALYSIS = 5 -STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 -ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 -NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 -SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 -SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 -MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 -MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 -NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 -SELECT_ELEMENT_INCORRECT_VALUE = 14 - -# Description: -# This method will be called to handle the result of filter operation done -# on the selectors found. -# -# There are 3 options for the result: -# 1) No selectors were found having the value we are expecting. On this case, -# information returned will be the element with the index that was expected. -# -# 2) We found only one selector, we have two options here: -# a) Found the correct selector: Return the original element. -# b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. -# -# 3) We found two or more selectors with the same src. We have two options here: -# a) The correct selector was found. Return the original element. -# b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. -# -# Returns: -# jsonObject with the number of selectors found, the selctors and the return code. -# -def processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, attribute): - jsonObject = {} - elements = [] - - if(selectorsFound == 0): - # No selectors were found with the expected value - if(expectedIndex <= len(selectors)): - element = {} - element["index"] = expectedIndex - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text if (selectors[expectedIndex].text != "") else "" - else: - element["value"] = selectors[expectedIndex][attribute] if (attribute in selectors[expectedIndex]) else "" - element["selector"] = "found" - elements.append(element) - returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE - else: - returnCode = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND - elif(selectorsFound == 1): - if(expectedIndex in selectorIndexes): - # The expected selector was found and it is the only selector. - element = {} - element["index"] = expectedIndex - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - element["selector"] = "original" - elements.append(element) - returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX - else: - # The incorrect selector was found and this is the only selector with the expected value - element = {} - element["index"] = expectedIndex - element["selector"] = "original" - if(expectedIndex <= len(selectors)): - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - else: - element["value"] = expectedValue - elements.append(element) - - element = {} - element["index"] = selectorIndexes[selectorsFound -1] - if(attribute == "text"): - element["value"] = selectors[selectorIndexes[selectorsFound -1]].text - else: - element["value"] = selectors[selectorIndexes[selectorsFound -1]][attribute] - element["selector"] = "found" - elements.append(element) - returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX - elif(selectorsFound > 1): - # Several selectors were found with same value - if(expectedIndex in selectorIndexes): - # The expected element was found on the selectors - element = {} - element["index"] = expectedIndex - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - element["selector"] = "original" - elements.append(element) - returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX - else: - # The expected element was NOT found on the selectors - element = {} - element["index"] = expectedIndex - element["selector"] = "original" - if(expectedIndex <= len(selectors)): - if(attribute == "text"): - element["value"] = selectors[expectedIndex].text - else: - element["value"] = selectors[expectedIndex][attribute] - else: - element["value"] = expectedValue - elements.append(element) - - element = {} - if(attribute == "text"): - element["value"] = expectedValue - else: - element["value"] = expectedValue - element["index"] = str(selectorIndexes) - element["selector"] = "found" - elements.append(element) - returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX - - jsonObject["numberOfElementsWithSameSelectorAndValue"] = selectorsFound - jsonObject["selectors"] = elements - jsonObject["rc"] = returnCode - - return jsonObject - -# Description: -# This method will be called when two or more selectors were found with -# the same ntagselector value. This method will use the text value attribute -# to filter the selctors and try to find the one that was used by the test. -# -# Parameters: -# selectors: Array of selectors found with the same ntagselector. -# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). -# expectedIndex: The index that is expected to contain the expected value. -# -# Returns: -# jsonObject with the number of selectors found, the selctors and the return code. -def parseTextSelector(selectors, searchInfo, expectedIndex, tag): - jsonObject = {} - selectorIndexes = [] - selectorIndex = 0 - selectorsFound = 0 - expectedValue = searchInfo["value"] - - if(expectedValue == ""): - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - jsonObject["selectors"] = [] - jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE - return jsonObject - - for selector in selectors: - selectorText = selector.text.replace("'", "") - if(selectorText.strip() == expectedValue.strip()): - selectorsFound += 1 - selectorIndexes.append(selectorIndex) - selectorIndex+=1 - - return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "text") - -# Description: -# This method will be called when two or more selectors were found with -# the same ntagselector value. This method will use the src value attribute -# to filter the selctors and try to find the one that was used by the test. -# -# Parameters: -# selectors: Array of selectors found with the same ntagselector. -# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). -# expectedIndex: The index that is expected to contain the expected value. -# -# Returns: -# jsonObject with the number of selectors found, the selctors and the return code. -def parseImageSelector(selectors, searchInfo, expectedIndex, tag): - selectorIndexes = [] - selectorIndex = 0 - selectorsFound = 0 - expectedValue = searchInfo["value"] - - for selector in selectors: - if(selector['src'] == expectedValue ): - selectorsFound += 1 - selectorIndexes.append(selectorIndex) - selectorIndex+=1 - - return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "src") - -# Description: -# This method will be called when two or more selectors were found with -# the same ntagselector value. This method will use the href value attribute -# to filter the selctors and try to find the one that was used by the test. -# -# Parameters: -# selectors: Array of selectors found with the same ntagselector. -# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). -# expectedIndex: The index that is expected to contain the expected value. -# -# Returns: -# jsonObject with the number of selectors found, the selctors and the return code. -def parseHypertextSelector(selectors, searchInfo, expectedIndex, tag): - jsonObject = {} - selectorIndexes = [] - filteredIndexes = [] - selectorIndex = 0 - selectorsFound = 0 - expectedValue = searchInfo["value"] - if hasattr(searchInfo, 'text'): - expectedText = searchInfo["text"] - else: - expectedText = "" - - if(expectedValue == ""): - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - jsonObject["selectors"] = [] - jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE - return jsonObject - - for selector in selectors: - if(selector and selector.has_attr('href')): - if(selector['href'] == expectedValue): - selectorsFound += 1 - selectorIndexes.append(selectorIndex) - selectorIndex+=1 - - # If more than 1 selector was found using the same value, lest's filter now by text and update - # the selectorIndexes with the new indexes (hopefully only one!). - if(selectorsFound > 1 and expectedText != ""): - for i in selectorIndexes: - if(str(selectors[i].string) == expectedText): - filteredIndexes.append(i) - if(len(filteredIndexes) > 0 ): - selectorIndexes = [] - selectorsFound = len(filteredIndexes) - for index in filteredIndexes: - selectorIndexes.append(index) - - return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "href") - -# Description: -# This method will be called when two or more selectors were found with -# the same ntagselector value. This method will use the value attribute -# to filter the selctors and try to find the one that was used by the test. -# -# Parameters: -# selectors: Array of selectors found with the same ntagselector. -# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). -# expectedIndex: The index that is expected to contain the expected value. -# -# Returns: -# jsonObject with the number of selectors found, the selctors and the return code. -def parseValueSelector(selectors, searchInfo, expectedIndex, type, tag): - selectorIndexes = [] - selectorIndex = 0 - selectorsFound = 0 - expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" - expectedText = searchInfo["text"] if ('text' in searchInfo) else "" - - for selector in selectors: - if(('value' in selector) and selector['value'] == expectedValue ): - selectorsFound += 1 - selectorIndexes.append(selectorIndex) - selectorIndex+=1 - - # If we have text information available and this is a select element, let's try to - # find the correct value using the text - if(selectorsFound == 0 and expectedText != "" and tag == "select"): - return handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag) - - return processResults(selectors, expectedIndex, expectedValue, selectorsFound, selectorIndexes, "value") - -def handleSelectElement(selectors, expectedText, expectedIndex, selectorIndexes, tag): - jsonObject = {} - elements = [] - value = "" - # Let's first verify the selector with the expected index - selectElement = selectors[expectedIndex] - options = selectElement.find_all("option") - for option in options: - if(option.text.strip() == expectedText.strip()): - value = option.get("value") - break - - element = {} - element["index"] = expectedIndex - element["value"] = value - element["selector"] = "found" - elements.append(element) - - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 - jsonObject["selectors"] = elements - jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE - - return jsonObject - -# Description: -# This method will . -# -# Returns: -# . -def verifyAttributesOnElements(selectors, attributes): - selectorsFound = [] - selectorIndexes = [] - index = 0 - - # First let's try to find any selector on the list with the ID frm the element attributes - index = 0 - for selector in selectors: - if(selector.has_attr('id')): - if(selector['id'] == attributes['id']): - selectorsFound.append(selector) - selectorIndexes.append(index) - return [selectorsFound, selectorIndexes] - index+=1 - - # If we get here it means no selector with ID was found, let's check with name - if(len(selectorsFound) == 0): - index = 0 - for selector in selectors: - if(selector.has_attr('name')): - if(selector['name'] == attributes['name']): - selectorsFound.append(selector) - selectorIndexes.append(index) - return [selectorsFound, selectorIndexes] - index+=1 - - # If we get here it means no ID and NAME were found, let's check with text - if(len(selectorsFound) == 0): - index = 0 - for selector in selectors: - if(selector.has_attr('text')): - if(selector['text'] == attributes['text']): - selectorsFound.append(selector) - selectorIndexes.append(index) - index+=1 - - # If we get here it means no ID and NAME were found, let's check with text - index = 0 - for selector in selectors: - if(selector.has_attr('type')): - if(selector['type'] == attributes['type']): - selectorsFound.append(selector) - selectorIndexes.append(index) - index+=1 - - return [selectorsFound, selectorIndexes] - - -# Description: -# This method will . -# -# Returns: -# . -def filterSelectorsByAttributes(selectors, attributes, searchInfo, stepId,index): - - print("Start analysis for step, " + str(stepId) + " for " + str(len(selectors)) + " Selectors") - print("Attributes, " + str(attributes)) - #print("Selectors:, " + str(selectors)) - selectorsFound = [] - selectorIndexes = [] - elements = [] - jsonObject = {} - expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" - searchType = searchInfo["searchType"] - - [selectorsFound,selectorIndexes] = verifyAttributesOnElements(selectors, attributes); - print("Returned from verifyAttributesOnElements") - print(selectorsFound) - - if(len(selectorsFound) == 1): - element = {} - element["index"] = index - element["selector"] = "original" - element["value"] = expectedValue - elements.append(element) - if(index != selectorIndexes[0]): - element = {} - element["index"] = selectorIndexes[0] - element["selector"] = "found" - if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): - print("Value in selector") - element["value"] = selectorsFound[0]['value'] - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 - else: - print("Value NOT in selector") - element["value"] = "undef" - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - elements.append(element) - else: - element = {} - element["index"] = index - element["selector"] = "found" - if (selectorsFound[0].has_attr('value') and selectorsFound[0]['value'] != ""): - print("Value in selector") - element["value"] = selectorsFound[0]['value'] - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 1 - else: - print("Value NOT in selector") - element["value"] = "undef" - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - elements.append(element) - - jsonObject["selectors"] = elements - jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE - else: - jsonObject["selectors"] = selectors - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR - - return jsonObject - -# Description: -# This method will be call for each step and will parse the DOM files generated -# for the test to find the selectors for this step. If more than one selector is found, -# this method makes another search on the DOM using the value to filter the -# selectors found. -# -# Returns: -# jsonObject with the number of selector information. -def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, action, searchInfo, browserName, attributes): - #os.chdir("/home/serrano/Development/executor/local/executor") - jsonObject = {} - elements = [] - path = 'build/reports/geb/' + browserName + '/' - filename = path + classname + "_" + str(stepId) + ".html" - - if os.path.exists(filename): - try: - searchType = searchInfo["searchType"] - text = open(filename, 'r').read() - soup = BeautifulSoup(text, 'html.parser') - selectorsFound = soup.select(ntagselector) - numberSelectorsFound = len(selectorsFound) - print(numberSelectorsFound) - - if(action == "mouseover"): - jsonObject["selectors"] = [] - jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - else: - if(numberSelectorsFound == 0 ): - jsonObject["selectors"] = [] - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR - - elif(numberSelectorsFound == 1 ): - element = {} - element["index"] = index - element["value"] = searchInfo["value"] - element["selector"] = "original" - if(searchType == "value" and selectorsFound[0].has_attr('value')): - element["value"] = selectorsFound[0]["value"] - elif(searchType == "href" and selectorsFound[0].has_attr('href')): - element["value"] = selectorsFound[0]["href"] - elif(searchType == "text" and selectorsFound[0].has_attr('text')): - element["value"] = selectorsFound[0].text - elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): - element["value"] = selectorsFound[0]["src"] - else: - element["value"] = searchInfo["value"] - elements.append(element) - returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR - - if(index > 0): - element = {} - element["index"] = 0 - element["selector"] = "found" - if(searchType == "value" and selectorsFound[0].has_attr('value')): - element["value"] = selectorsFound[0]["value"] - elif(searchType == "href" and selectorsFound[0].has_attr('href')): - element["value"] = selectorsFound[0]["href"] - elif(searchType == "text" and selectorsFound[0].has_attr('text')): - element["value"] = selectorsFound[0].text - elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): - element["value"] = selectorsFound[0]["src"] - else: - element["value"] = searchInfo["value"] - elements.append(element) - returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX - - jsonObject["selectors"] = elements - jsonObject["numberOfElementsWithSameSelectorAndValue"] = numberSelectorsFound - jsonObject["rc"] = returnCode - - elif(numberSelectorsFound > 1 ): - if(searchType == "value"): - jsonObject = parseValueSelector(selectorsFound, searchInfo, index, type, tag) - elif(searchType == "href"): - jsonObject = parseHypertextSelector(selectorsFound, searchInfo, index, tag) - elif(searchType == "text"): - jsonObject = parseTextSelector(selectorsFound, searchInfo, index, tag) - elif(searchType == "imgsrc"): - jsonObject = parseImageSelector(selectorsFound, searchInfo, index, tag) - else: - # Backend sent an undef searchType, we will return no info - jsonObject["selectors"] = [] - jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE - jsonObject["numberOfElementsWithSameSelectorAndValue"] = 0 - - # If we are done with the search and we were not ablt to find a selector. Let's try to use the attributes to find the element. - if(jsonObject["rc"] == NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE or - jsonObject["rc"] == NO_SEARCH_TYPE_PROVIDED_BY_BE): - print("Need to search using attributes now for step " + str(stepId)) - # This code is to do analysis on the element's attributes. This will be enable on a future sprint - #jsonObject = filterSelectorsByAttributes(selectorsFound, attributes, searchInfo, stepId, index) - #newNumberSelectorsFound = len(newSelectorsFound) - - jsonObject["numberOfElementsWithSameSelector"] = numberSelectorsFound - - except Exception as ex: - print("Failed to open file " + str(filename) + ex) - print (ex) - - # Let's validate the data we generated is a valid json for this step - try: - json.loads(json.dumps(jsonObject)) - except Exception as ex: - pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") - print(ex) - print(traceback.format_exc()) - jsonObject = {} - - return jsonObject +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 +SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR, XPATH_SELECTOR ] # Description: # This method will be call to execuete the Muuk Report analysis. @@ -542,7 +24,6 @@ def obtainFeedbackFromDOM(classname, stepId, ntagselector, index, tag, type, act # jsonObject with the number of selector information. def createMuukReport(classname, browserName): #os.chdir("/home/serrano/Development/executor/local/executor") - print(os.getcwd()) path = 'build/reports/' filename = path + classname + ".json" muukReport = {} @@ -552,25 +33,58 @@ def createMuukReport(classname, browserName): jsonFile = open(filename, 'r') elements = json.load(jsonFile) for element in elements['stepsFeedback']: - type = element.get("type") - if(type == "step"): + type = element.get("type") + if(type == "step"): + element["feedback"] = [] + selectors = json.loads(element.get("selectors").replace('\$', '\\\$')) + selectorToUse = element.get("selectorToUse") valueInfo = json.loads(element.get("value")) attributes = json.loads(element.get("attributes")) attributes['value'] = valueInfo['value'] - domInfo = obtainFeedbackFromDOM(classname, element.get("id"), - element.get("selector"), - element.get("index"), element.get("tag"), - element.get("objectType"), element.get("action"), - valueInfo, - browserName, - attributes) - if(domInfo): - element["numberOfElementsWithSameSelector"] = domInfo["numberOfElementsWithSameSelector"] - element["numberOfElementsWithSameSelectorAndValue"] = domInfo["numberOfElementsWithSameSelectorAndValue"] - element["rc"] = domInfo["rc"] - element["selectors"] = domInfo["selectors"] - steps.append(element) - else: + + for i in SELECTORS_ARRAY: + + if(i < len(selectors)): + selector = selectors[i]['selector'] + if(selector == ""): + selector = None + index = selectors[i]['index'] + else: + selector = None + index = None + + if(i < 3): + domInfo = obtainCSSFeedbackFromDOM(classname, element.get("id"), + selector, + index, + element.get("tag"), + element.get("objectType"), element.get("action"), + valueInfo, + browserName, + attributes, + SELECTORS_ARRAY[i]) + else: + # XPATH library is case sensitive and MuukTest creates the tag as upper case, we need to fix this. + selector = selector.replace(element.get("tag").upper(), element.get("tag"), 1) + domInfo = obtainXPATHFeedbackFromDOM(classname, element.get("id"), + selector, + index, + element.get("tag"), + element.get("objectType"), element.get("action"), + valueInfo, + browserName, + attributes, + SELECTORS_ARRAY[i]) + + if(domInfo): + element["feedback"].append(domInfo) + steps.append(element) + + # Now that we have the 4 selector arrays, let's define which one we should use + selectorToUse = findBetterSelectorToUse(element["feedback"], attributes) + element["feebackSelectorToUse"] = selectorToUse + + else: steps.append(element) except Exception as ex: @@ -597,4 +111,36 @@ def createMuukReport(classname, browserName): if(os.path.exists("TOUCH_TRACE_REPORT")): pprint.pprint(muukReport["steps"]) - return muukReport \ No newline at end of file + return muukReport + +# if the Object has text, we will use XPATH selector. +# else if Object has the next attributes ['id', 'name', 'type', 'role', 'title'], use custom CSS +# else if ntagselector has dynamic classes, use dynamic selector +# else use clasic selector. +def findBetterSelectorToUse(selectors, attributes): + + selectorToUse = -1 + classic = selectors[0] if len(selectors) > 0 else [] + dynamic = selectors[1] if len(selectors) > 1 else [] + customeCSS = selectors[2] if len(selectors) > 2 else [] + xpath = selectors[3] if len(selectors) > 3 else [] + + if(xpath and xpath["numberOfElementsFoundWithSelectorAndValue"] > 0 and attributes["text"] != "undef" and + ("contains" in xpath["selectors"]["selector"] or "normalize-space" in xpath["selectors"]["selector"] )): + selectorToUse = 3 + elif(customeCSS and (attributes["id"] != "undef" or attributes["name"] != "undef" or attributes["type"] != "undef" ) and customeCSS["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 2 + elif(classic and classic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 0 + elif(dynamic and dynamic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 1 + + # If we were not able to choose a selector with values, check if we have one that return any element at least. + if(selectorToUse == -1): + if(classic and classic["numberOfElementsFoundWithSelector"] > 0): + selectorToUse = 0 + elif(dynamic and dynamic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 1 + + return selectorToUse + \ No newline at end of file From 7eacccaf313ee0975a8efaf993a731101c850456 Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Wed, 11 May 2022 16:16:43 -0700 Subject: [PATCH 73/74] Handle exception when XPATH selector does not exist. --- domparser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/domparser.py b/domparser.py index a9f5dcf..40572ef 100644 --- a/domparser.py +++ b/domparser.py @@ -55,7 +55,7 @@ def createMuukReport(classname, browserName): if(i < 3): domInfo = obtainCSSFeedbackFromDOM(classname, element.get("id"), - selector, + selector, index, element.get("tag"), element.get("objectType"), element.get("action"), @@ -64,8 +64,9 @@ def createMuukReport(classname, browserName): attributes, SELECTORS_ARRAY[i]) else: - # XPATH library is case sensitive and MuukTest creates the tag as upper case, we need to fix this. - selector = selector.replace(element.get("tag").upper(), element.get("tag"), 1) + if(selector != None): + # XPATH library is case sensitive and MuukTest creates the tag as upper case, we need to fix this. + selector = selector.replace(element.get("tag").upper(), element.get("tag"), 1) domInfo = obtainXPATHFeedbackFromDOM(classname, element.get("id"), selector, index, From 2e670d6eb17fdaa85ba2dbd2f51ea1d50c64b20e Mon Sep 17 00:00:00 2001 From: Jesus Serrano Date: Tue, 31 May 2022 16:55:39 -0700 Subject: [PATCH 74/74] Fix problems on DOM parser. --- CSSAnalyzer.py | 34 +++++++++++++++++++++++----------- XPATHAnalyzer.py | 26 ++++++++++++++++++-------- domparser.py | 43 +++++++++++++++++++++++++++++++++---------- 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/CSSAnalyzer.py b/CSSAnalyzer.py index bd99e70..df53960 100644 --- a/CSSAnalyzer.py +++ b/CSSAnalyzer.py @@ -1,9 +1,9 @@ import os import json -import pprint from bs4 import BeautifulSoup import traceback +import logging UNKNOWN_ERROR = 1 NO_TAG_PROVIDED_BY_BE = 2 @@ -27,7 +27,9 @@ XPATH_SELECTOR = 3 SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR ] +SELECTORS_ARRAY_NAMES = ["CLASSIC_SELECTOR", "DYMANIC_SELECTOR", "CUSTOM_CSS_SELECTOR" ] +logger = logging.getLogger(__name__) # Description: # This method will be called to handle the result of filter (by value, text,etc) operation done @@ -436,6 +438,7 @@ def buildCSSSelector(tag, attributes, searchInfo, index, soup): # now that we have a selector, let's va;idate it returns elements. htmlElementsFound = soup.select(cssSelector) numberElementsFound = len(htmlElementsFound) + logger.info("buildCSSSelector - Found " + str(numberElementsFound) + " with selector " + cssSelector) if(numberElementsFound == 1): element = {} @@ -453,7 +456,7 @@ def buildCSSSelector(tag, attributes, searchInfo, index, soup): elif(searchType == "text"): jsonObject = parseTextSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) elif(searchType == "imgsrc"): - jsonObject = parseImageSelector(cssSelector, numberElementsFound, searchInfo, index, tag) + jsonObject = parseImageSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) else: # Backend sent an undef searchType, we will return no info element = {} @@ -483,6 +486,7 @@ def buildCSSSelector(tag, attributes, searchInfo, index, soup): # Returns: # jsonObject with the number of selector information. def obtainCSSFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + logging.info("Starting CSS analysis for "+ SELECTORS_ARRAY_NAMES[selector_type] + " selector " + str(selector) + " witn index " + str(index) + " on step " + str(stepId)) jsonObject = {} path = 'build/reports/geb/' + browserName + '/' filename = path + classname + "_" + str(stepId) + ".html" @@ -494,8 +498,8 @@ def obtainCSSFeedbackFromDOM(classname, stepId, selector, index, tag, type, acti soup = BeautifulSoup(text, 'html.parser') if(selector is None): if(selector_type == CUSTOM_CSS_SELECTOR): - print("NO CSS Selector, need to build it") - return buildCSSSelector(tag, attributes, searchInfo, index, soup) + logging.info("NO CSS Selector, need to build it") + return buildCSSSelector(tag, attributes, searchInfo, 0, soup) else: selectorsFound = soup.select(selector) numberSelectorsFound = len(selectorsFound) @@ -573,21 +577,29 @@ def obtainCSSFeedbackFromDOM(classname, stepId, selector, index, tag, type, acti jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 else: # We do not have a value to try to identify the element, let's use the attributes - return findIndexFromAttributes(selector, tag, attributes, soup) + if(len(attributes) > 0 and selector_type == CUSTOM_CSS_SELECTOR ): + logging.info("Found "+ str(numberSelectorsFound) + " selectors and no value to filter, let's build from attributes") + return findIndexFromAttributes(selector, tag, attributes, soup) + else: + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound except Exception as ex: - print("Failed to open file " + str(filename) + ex) - print (ex) + logging.error (ex) # Let's validate the data we generated is a valid json for this step try: json.loads(json.dumps(jsonObject)) except Exception as ex: - pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") - print(ex) - print(traceback.format_exc()) + logging.error("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) jsonObject = {} - return jsonObject \ No newline at end of file + return jsonObject diff --git a/XPATHAnalyzer.py b/XPATHAnalyzer.py index c118bcf..b85e86e 100644 --- a/XPATHAnalyzer.py +++ b/XPATHAnalyzer.py @@ -8,6 +8,7 @@ from xml.sax import saxutils from lxml import html import traceback +import logging UNKNOWN_ERROR = 1 NO_TAG_PROVIDED_BY_BE = 2 @@ -30,6 +31,8 @@ CUSTOM_CSS_SELECTOR = 2 XPATH_SELECTOR = 3 +logger = logging.getLogger(__name__) + # Description: # This method returns the inner HTML of the element received as a parameter. # @@ -55,6 +58,7 @@ def inner_html(element): # XPATH selector # def buildXPATHSelector(tag, attributes, root): + logger.info("buildXPATHSelector - attributes = " + str(attributes)) jsonObject = {} elements = [] textUsedOnSelector = False @@ -118,7 +122,7 @@ def buildXPATHSelector(tag, attributes, root): # Some characters will cause problems on the XPATH expression when using contains, we need to escape the next # characters: - print(repr(text)) + #logging.info(repr(text)) #text = text.replace("$", '\$') // no need to escape text = text.replace("'", "\'") text = text.strip() @@ -138,8 +142,11 @@ def buildXPATHSelector(tag, attributes, root): jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR return jsonObject + logger.info("buildXPATHSelector - Selector build from attributes = " + str(selector)) htmlElementsFound = root.xpath(selector) numberhtmlElementsFound = len(htmlElementsFound) + logger.info("buildXPATHSelector - numberhtmlElementsFound = " + str(numberhtmlElementsFound)) + if(numberhtmlElementsFound == 0 ): element = {} element["selector"] = selector @@ -156,14 +163,15 @@ def buildXPATHSelector(tag, attributes, root): jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES jsonObject["numberOfElementsFoundWithSelector"] = 1 jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 - elif(numberhtmlElementsFound > 1 and textUsedOnSelector): + elif(numberhtmlElementsFound > 1 or textUsedOnSelector): element = {} element["selector"] = selector element["index"] = 0 jsonObject["selectors"] = element jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES jsonObject["numberOfElementsFoundWithSelector"] = numberhtmlElementsFound - jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberhtmlElementsFound + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberhtmlElementsFound + return jsonObject @@ -176,6 +184,7 @@ def buildXPATHSelector(tag, attributes, root): # Returns: # jsonObject with the number of selector information. def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + logging.info("Starting XPATH analysis for selector " + str(selector) + " witn index " + str(index) + " on step " + str(stepId)) jsonObject = {} path = 'build/reports/geb/' + browserName + '/' filename = path + classname + "_" + str(stepId) + ".html" @@ -186,6 +195,7 @@ def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, ac text = open(filename, 'r').read() root = lxml.html.fromstring(text) if(selector is None): + logging.info("No XPATH selector found, let's build from attributes") return buildXPATHSelector(tag, attributes, root) else: htmlElementsFound = root.xpath(selector) @@ -200,6 +210,7 @@ def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, ac jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 else: if(numberSelectorsFound == 0): + logging.info("Found "+ str(numberSelectorsFound) + " selectors and no value to filter, let's build from attributes") return buildXPATHSelector(tag, attributes, root) elif(numberSelectorsFound == 1 ): @@ -230,16 +241,15 @@ def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, ac jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound except Exception as ex: - print("Failed to open file " + str(filename) + ex) - print (ex) + logging.error(ex) # Let's validate the data we generated is a valid json for this step try: json.loads(json.dumps(jsonObject)) except Exception as ex: - pprint.pprint("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") - print(ex) - print(traceback.format_exc()) + logging.error("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) jsonObject = {} return jsonObject \ No newline at end of file diff --git a/domparser.py b/domparser.py index 40572ef..c1aacc2 100644 --- a/domparser.py +++ b/domparser.py @@ -6,6 +6,10 @@ from CSSAnalyzer import obtainCSSFeedbackFromDOM from XPATHAnalyzer import obtainXPATHFeedbackFromDOM import traceback +import logging +import shutil +import pathlib + CLASSIC_SELECTOR = 0 DYMANIC_SELECTOR = 1 @@ -13,6 +17,14 @@ XPATH_SELECTOR = 3 SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR, XPATH_SELECTOR ] +LOG_FILE_NAME = "DOMParser.log" + +logging.basicConfig(filename = LOG_FILE_NAME, + filemode = "w", + format = "%(levelname)s %(asctime)s - %(message)s", + level = logging.INFO) +logger = logging.getLogger(__name__) + # Description: # This method will be call to execuete the Muuk Report analysis. @@ -23,7 +35,7 @@ # Returns: # jsonObject with the number of selector information. def createMuukReport(classname, browserName): - #os.chdir("/home/serrano/Development/executor/local/executor") + logging.info("Starting parsing analysis") path = 'build/reports/' filename = path + classname + ".json" muukReport = {} @@ -76,7 +88,7 @@ def createMuukReport(classname, browserName): browserName, attributes, SELECTORS_ARRAY[i]) - + logging.info("Object = " + json.dumps(domInfo,sort_keys=True, indent=4)) if(domInfo): element["feedback"].append(domInfo) steps.append(element) @@ -89,28 +101,39 @@ def createMuukReport(classname, browserName): steps.append(element) except Exception as ex: - print("Exception found during DOM parsing. Exception = ") - print(ex) - print(traceback.format_exc()) + logging.error("Exception found during DOM parsing. Exception = ") + logging.error(ex) + logging.error(traceback.format_exc()) # Closing file jsonFile.close() else: - print("Muuk Report was not found!") + logging.error("Muuk Report was not found!") # Let's validate the data we generated is a valid json try: json.loads(json.dumps(steps)) muukReport["steps"] = steps except Exception as ex: - pprint.pprint("Invalid JSON format was found, will not send feedback to BE") - print(ex) - print(traceback.format_exc()) + logging.error("Invalid JSON format was found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) muukReport["steps"] = {} # Print report if touch file exists if(os.path.exists("TOUCH_TRACE_REPORT")): - pprint.pprint(muukReport["steps"]) + pprint.pprint((muukReport["steps"])) + + logging.info("Final Feedback Object:") + logging.info(json.dumps(muukReport["steps"],sort_keys=True, indent=4)) + + # Last step is to copy the log file to build folder + try: + source = str(pathlib.Path(__file__).parent.resolve()) + "/" + LOG_FILE_NAME + destination = str(pathlib.Path(__file__).parent.resolve()) + "/build/" +LOG_FILE_NAME; + shutil.copyfile(source, destination) + except Exception as ex: + print(ex) return muukReport