From c001108f4d9af9b2330c5dd355d8fa27afc12a20 Mon Sep 17 00:00:00 2001 From: napoles Date: Tue, 5 Apr 2022 16:00:00 -0500 Subject: [PATCH 1/3] Apply success criteria to return exit code --- mkcli.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mkcli.py b/mkcli.py index 5dcf2b0..c6a3ffb 100644 --- a/mkcli.py +++ b/mkcli.py @@ -11,6 +11,12 @@ from mkcloud import gatherScreenshots, resizeImages #import ssl +def getExitCodeFromSuccessCriteria(successCriteria, results): + #Applies a success criteria on results in order to get accurate exitCode + numOfSuccess = sum(result['success'] for result in results) + successRate = numOfSuccess / len(results) + return 0 if successRate >= (successCriteria / 100) else 1 + def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) path = 'build/test-results/'+browserName @@ -62,6 +68,7 @@ def run(args): noexec = args.noexec route = 'src/test/groovy' browser = args.browser + successCriteria = 80 if args.criteria is None else args.criteria dimensions = args.dimensions if dimensions is not None: checkDimensions = isinstance(dimensions[0], int) & isinstance(dimensions[1],int) @@ -240,6 +247,10 @@ def run(args): else: print(field+': is not an allowed property') + #If exitCode is 0 everything is OK, no need to check criteria. + if exitCode != 0 and len(testsExecuted) > 0: + exitCode = getExitCodeFromSuccessCriteria(successCriteria, testsExecuted) + print("exiting script with exitcode: " + str(exitCode)) exit(exitCode) @@ -264,6 +275,7 @@ def main(): parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") parser.add_argument("-browser",help="(Optional). Select one of the available browsers to run the test (default firefox)", type=str, dest="browser") parser.add_argument("-dimensions",help="(Optional). Dimensions to execute the tests, a pair of values for width height, ex. -dimensions 1800 300", type=int, nargs=2, dest="dimensions") + parser.add_argument("-criteria",help="(Optional). Minimal rate of passed tests to return success code. Value from 0 to 100. Default 80", type=int, dest="criteria") parser.set_defaults(func=run) args=parser.parse_args() args.func(args) From 6616629c0b71b8a7f45189f5e75604cff4de7e38 Mon Sep 17 00:00:00 2001 From: napoles Date: Wed, 6 Apr 2022 12:32:32 -0500 Subject: [PATCH 2/3] Read criteria from a file --- criteria.json | 5 +++++ mkcli.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 criteria.json diff --git a/criteria.json b/criteria.json new file mode 100644 index 0000000..e9d9e9a --- /dev/null +++ b/criteria.json @@ -0,0 +1,5 @@ +{ + "high": 90, + "medium": 80, + "low": 60 +} \ No newline at end of file diff --git a/mkcli.py b/mkcli.py index c6a3ffb..7d98d6c 100644 --- a/mkcli.py +++ b/mkcli.py @@ -68,8 +68,8 @@ def run(args): noexec = args.noexec route = 'src/test/groovy' browser = args.browser - successCriteria = 80 if args.criteria is None else args.criteria dimensions = args.dimensions + successCriteria = args.criteria if dimensions is not None: checkDimensions = isinstance(dimensions[0], int) & isinstance(dimensions[1],int) else: @@ -95,6 +95,13 @@ def run(args): userId = '' if field == "hashtag": + if successCriteria is None: + #if not a specific criteria was set, look for this hashtag on criteria file + criteria_file = open('criteria.json') + if criteria_file is not None: + criteria = json.load(criteria_file) + successCriteria = criteria.get(value) + criteria_file.close() value = "#"+value valueArr = [] @@ -248,7 +255,7 @@ def run(args): print(field+': is not an allowed property') #If exitCode is 0 everything is OK, no need to check criteria. - if exitCode != 0 and len(testsExecuted) > 0: + if exitCode != 0 and successCriteria is not None and len(testsExecuted) > 0: exitCode = getExitCodeFromSuccessCriteria(successCriteria, testsExecuted) print("exiting script with exitcode: " + str(exitCode)) @@ -275,7 +282,7 @@ def main(): parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") parser.add_argument("-browser",help="(Optional). Select one of the available browsers to run the test (default firefox)", type=str, dest="browser") parser.add_argument("-dimensions",help="(Optional). Dimensions to execute the tests, a pair of values for width height, ex. -dimensions 1800 300", type=int, nargs=2, dest="dimensions") - parser.add_argument("-criteria",help="(Optional). Minimal rate of passed tests to return success code. Value from 0 to 100. Default 80", type=int, dest="criteria") + parser.add_argument("-criteria",help="(Optional). Minimal rate of passed tests to return success code. Value from 0 to 100.", type=int, dest="criteria") parser.set_defaults(func=run) args=parser.parse_args() args.func(args) From 27908d0847bac07e3e43cfcab75583bfe6a5d6fb Mon Sep 17 00:00:00 2001 From: napoles Date: Wed, 6 Apr 2022 14:03:18 -0500 Subject: [PATCH 3/3] Use suitable path for criteria file --- mkcli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/mkcli.py b/mkcli.py index 7d98d6c..46af5f1 100644 --- a/mkcli.py +++ b/mkcli.py @@ -97,11 +97,16 @@ def run(args): if field == "hashtag": if successCriteria is None: #if not a specific criteria was set, look for this hashtag on criteria file - criteria_file = open('criteria.json') - if criteria_file is not None: - criteria = json.load(criteria_file) - successCriteria = criteria.get(value) - criteria_file.close() + path = dirname + '/criteria.json' + try: + with open(path, 'r') as criteria_file: + criteria = json.load(criteria_file) + successCriteria = criteria.get(value) + if successCriteria is not None: + print("A success rate criteria of:", successCriteria, "% is being applied to this execution.") + except IOError: + #couldn't read the file, likely it does not exist. Nothing wrong + pass value = "#"+value valueArr = []