From cc7da7b7f81d549ff1a34605d48bbd75a69597ae Mon Sep 17 00:00:00 2001 From: Roman Plevka Date: Tue, 22 May 2018 09:56:49 +0200 Subject: [PATCH 01/66] modified claiming log message not to be so verbose --- claims.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims.py b/claims.py index 5a06429..29e82f4 100755 --- a/claims.py +++ b/claims.py @@ -190,7 +190,7 @@ def claim(test, reason, sticky=False, propagate=False): :param sticky: whether to make the claim sticky (False by default) ''' - logging.info('claiming {0} with reason: {1}'.format(test['url'], reason)) + logging.info('claiming {0} with reason: {1}'.format(test["className"]+"::"+test["name"], reason)) claim_req = requests.post( u'{0}/claim/claim'.format(test['url']), auth=requests.auth.HTTPBasicAuth( From 4653a9708a26281a4459067be1de3d824841e2a9 Mon Sep 17 00:00:00 2001 From: Roman Plevka Date: Tue, 22 May 2018 10:28:05 +0200 Subject: [PATCH 02/66] claim_by_rules now recognizes field to match on --- claims.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/claims.py b/claims.py index 29e82f4..dceb720 100755 --- a/claims.py +++ b/claims.py @@ -209,7 +209,8 @@ def claim(test, reason, sticky=False, propagate=False): def claim_by_rules(fails, rules, dryrun=False): for rule in rules: - for fail in [i for i in fails if re.search(rule['pattern'], i['errorDetails'])]: - logging.debug(u'{0} matching pattern: {1} url: {2}'.format(fail['name'], rule['pattern'], fail['url'])) + field = rule.get('field') or 'errorDetails' + for fail in [i for i in fails if re.search(rule['pattern'], i[field])]: + logging.info(u'{0} matching pattern: {1} url: {2}'.format(fail['name'], rule['pattern'], fail['url'])) if not dryrun: claim(fail, rule['reason']) From 510f3347f970f613db4f876d3bcb5e713bc00908 Mon Sep 17 00:00:00 2001 From: Roman Plevka Date: Tue, 22 May 2018 17:05:38 +0200 Subject: [PATCH 03/66] added duration and stdout to the attributes to fetch --- claims.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims.py b/claims.py index dceb720..7b64fdf 100755 --- a/claims.py +++ b/claims.py @@ -33,7 +33,7 @@ FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") PARAMS = { - u'tree': u'suites[cases[className,name,status,errorDetails,errorStackTrace,testActions[reason]]]{0}' + u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' } ep = [u'ui', u'api', u'cli'] From 49137f7e89e1be57faf4e234ccf56693d3a43058 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 6 Jun 2018 22:51:28 +0200 Subject: [PATCH 04/66] Show some stats --- claims.py | 7 ++++++ claimstats.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 claimstats.py diff --git a/claims.py b/claims.py index 7b64fdf..832b0d2 100755 --- a/claims.py +++ b/claims.py @@ -133,6 +133,13 @@ def filter_fails(bld): # bld = fetch_test_report(config['url'], config['job'], config['build']) +def filter_claimed(reports): + """ + Only return results which do not have claim/waiver + """ + return [i for i in reports if i['testActions'][0]['reason']] + + def filter_not_claimed(reports): """ Only return results which do not have claim/waiver diff --git a/claimstats.py b/claimstats.py new file mode 100755 index 0000000..db11d40 --- /dev/null +++ b/claimstats.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import claims +import tabulate + +reports = claims.fetch_all_reports() +reports = claims.flatten_reports(reports) +stat_all = len(reports) +reports_fails = claims.filter_fails(reports) +stat_failed = len(reports_fails) +reports_claimed = claims.filter_claimed(reports_fails) +stat_claimed = len(reports_claimed) + +print("\nOverall stats") +print(tabulate.tabulate( + [[stat_all, stat_failed, stat_claimed]], + headers=['all reports', 'failures', 'claimed failures'])) + +reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} +for report in reports_claimed: + reason = report['testActions'][0]['reason'] + if reason not in reports_per_reason: + reports_per_reason[reason] = 0 + reports_per_reason[reason] += 1 + +print("\nHow various reasons for claims are used") +print(tabulate.tabulate( + sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True), + headers=['claim reason', 'number of times'])) + +reports_per_class = {} +for report in reports: + class_name = report['className'] + if class_name not in reports_per_class: + reports_per_class[class_name] = {'all': 0, 'failed': 0} + reports_per_class[class_name]['all'] += 1 + if report in reports_fails: + reports_per_class[class_name]['failed'] += 1 + +print("\nHow many failures are there per class") +print(tabulate.tabulate( + sorted([(c, r['all'], r['failed'], float(r['failed'])/r['all']) for c,r in reports_per_class.items()], + key=lambda x: x[3], reverse=True), + headers=['class name', 'number of reports', 'number of failures', 'failures ratio'], + floatfmt=".3f")) + +reports_per_method = {} +for report in reports: + method = report['className'].split('.')[2] + if method not in reports_per_method: + reports_per_method[method] = {'all': 0, 'failed': 0} + reports_per_method[method]['all'] += 1 + if report in reports_fails: + reports_per_method[method]['failed'] += 1 + +print("\nHow many failures are there per method (CLI vs. API vs. UI)") +print(tabulate.tabulate( + sorted([(c, r['all'], r['failed'], float(r['failed'])/r['all']) for c,r in reports_per_method.items()], + key=lambda x: x[3], reverse=True), + headers=['method', 'number of reports', 'number of failures', 'failures ratio'], + floatfmt=".3f")) From a92068d05f4d13b1d34ed358f41f403c16da67ef Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 7 Jun 2018 10:24:23 +0200 Subject: [PATCH 05/66] Also indicate which rules are in the KB --- claimstats.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/claimstats.py b/claimstats.py index db11d40..154a37b 100755 --- a/claimstats.py +++ b/claimstats.py @@ -17,7 +17,10 @@ [[stat_all, stat_failed, stat_claimed]], headers=['all reports', 'failures', 'claimed failures'])) +rules = claims.load_rules() +rules_reasons = [r['reason'] for r in rules] reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} +reports_per_reason.update({r:0 for r in rules_reasons}) for report in reports_claimed: reason = report['testActions'][0]['reason'] if reason not in reports_per_reason: @@ -25,9 +28,11 @@ reports_per_reason[reason] += 1 print("\nHow various reasons for claims are used") +reports_per_reason = sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True) +reports_per_reason = [(r, c, r in rules_reasons) for r, c in reports_per_reason] print(tabulate.tabulate( - sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True), - headers=['claim reason', 'number of times'])) + reports_per_reason, + headers=['claim reason', 'number of times', 'is it in current knowleadgebase?'])) reports_per_class = {} for report in reports: From fc03bb01dc4b39c9785a367163fd2032ebd9b4d4 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 13 Jun 2018 15:32:16 +0200 Subject: [PATCH 06/66] Implement possibility to add some logick into rules. Add some basic tests --- test_claims.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 test_claims.py diff --git a/test_claims.py b/test_claims.py new file mode 100755 index 0000000..899fb12 --- /dev/null +++ b/test_claims.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import claims + +checkme = { + 'greeting': 'Hello world', + 'area': 'IT Crowd', +} + +assert claims.rule_matches(checkme, {'field': 'greeting', 'pattern': 'Hel+o'}) == True +assert claims.rule_matches(checkme, {'field': 'greeting', 'pattern': 'This is not there'}) == False +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == False +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True +assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'This is not there'}]}]}) == False +assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False +assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True +assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True From 7f1aa7c0cacf9717e3b258a13cbeb2109c7c1120 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 13 Jun 2018 21:44:29 +0200 Subject: [PATCH 07/66] Implement possibility to add some logick into rules. Add some basic tests --- claims.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/claims.py b/claims.py index 832b0d2..2218548 100755 --- a/claims.py +++ b/claims.py @@ -214,10 +214,39 @@ def claim(test, reason, sticky=False, propagate=False): return(claim_req) +def rule_matches(data, rule, indentation=0): + """ + Returns True id data matches to rule, orhervise returns False + """ + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, data, rule, indentation)) + if 'field' in rule and 'pattern' in rule: + # This is simple rule, we can just check regexp against given field and we are done + assert rule['field'] in data + out = re.search(rule['pattern'], data[rule['field']]) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + elif 'AND' in rule: + # We need to check if all sub-rules in list of rules rule['AND'] matches + out = None + for r in rule['AND']: + r_out = rule_matches(data, r, indentation+4) + out = r_out if out is None else out and r_out + if not out: + break + return out + elif 'OR' in rule: + # We need to check if at least one sub-rule in list of rules rule['OR'] matches + for r in rule['OR']: + if rule_matches(data, r, indentation+4): + return True + return False + else: + raise Exception('Rule %s not formatted correctly' % rule) + + def claim_by_rules(fails, rules, dryrun=False): for rule in rules: - field = rule.get('field') or 'errorDetails' - for fail in [i for i in fails if re.search(rule['pattern'], i[field])]: - logging.info(u'{0} matching pattern: {1} url: {2}'.format(fail['name'], rule['pattern'], fail['url'])) + for fail in [i for i in fails if rule_matches(i, rule)]: + logging.info(u'{0} matching pattern: {1} url: {2}'.format(fail['name'], rule['reason'], fail['url'])) if not dryrun: claim(fail, rule['reason']) From a8557b2e39a1bfb654f82c57d38544824c9bad6d Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 13 Jun 2018 22:52:58 +0200 Subject: [PATCH 08/66] Log via logging only, no print here and there --- claims.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/claims.py b/claims.py index 2218548..00aa59b 100755 --- a/claims.py +++ b/claims.py @@ -1,5 +1,6 @@ from __future__ import division import os +import sys import json import logging import re @@ -13,10 +14,7 @@ config = {} # load config with open("config.yaml", "r") as file: - try: - config = yaml.load(file) - except yaml.YAMLERROR as exc: - print(exc) + config = yaml.load(file) # get the jenkins crumb (csrf protection) crumb_request = requests.get( '{0}/crumbIssuer/api/json'.format(config['url']), @@ -75,11 +73,11 @@ def fetch_test_report(url=None, job=None, build=None, build_url=None): def fetch_all_reports(job=None, build=None): if 'DEBUG_CLAIMS_CACHE' in os.environ: if os.path.isfile(os.environ['DEBUG_CLAIMS_CACHE']): - print("DEBUG: Because environment variable DEBUG_CLAIMS_CACHE is set to '{0}', loading data from there".format( + logging.debug("Because environment variable DEBUG_CLAIMS_CACHE is set to '{0}', loading data from there".format( os.environ['DEBUG_CLAIMS_CACHE'])) return pickle.load(open(os.environ['DEBUG_CLAIMS_CACHE'], 'r')) else: - print("DEBUG: Environment variable DEBUG_CLAIMS_CACHE set to '{0}' but that file does not exist, creating one".format( + loading.debug("Environment variable DEBUG_CLAIMS_CACHE set to '{0}' but that file does not exist, creating one".format( os.environ['DEBUG_CLAIMS_CACHE'])) if job is None: From dbc2516ebe9513be8670afba9a1aded01b8fd406 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 07:05:54 +0200 Subject: [PATCH 09/66] Typo --- claims.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims.py b/claims.py index 00aa59b..a46138c 100755 --- a/claims.py +++ b/claims.py @@ -77,7 +77,7 @@ def fetch_all_reports(job=None, build=None): os.environ['DEBUG_CLAIMS_CACHE'])) return pickle.load(open(os.environ['DEBUG_CLAIMS_CACHE'], 'r')) else: - loading.debug("Environment variable DEBUG_CLAIMS_CACHE set to '{0}' but that file does not exist, creating one".format( + logging.debug("Environment variable DEBUG_CLAIMS_CACHE set to '{0}' but that file does not exist, creating one".format( os.environ['DEBUG_CLAIMS_CACHE'])) if job is None: From af0eff83c3676eaeb00b38e128fd5ec75153af64 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 08:35:07 +0200 Subject: [PATCH 10/66] Rewrote to classes --- claimable.py | 14 +- claims.py | 429 ++++++++++++++++++++++++-------------------------- claimstats.py | 15 +- unclaimed.py | 9 +- 4 files changed, 221 insertions(+), 246 deletions(-) diff --git a/claimable.py b/claimable.py index c9de83e..b4d83f8 100755 --- a/claimable.py +++ b/claimable.py @@ -1,13 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: UTF-8 -*- import claims -reports = claims.fetch_all_reports() -reports = claims.flatten_reports(reports) -reports = claims.filter_fails(reports) -reports = claims.filter_not_claimed(reports) +config = claims.Config() +jenkins = claims.Jenkins(config) +results = claims.Results(config, jenkins).get_failed().get_unclaimed() +rules = claims.Rules() -rules = claims.load_rules() - -claims.claim_by_rules(reports, rules, dryrun=True) +results.claim_by_rules(rules, dryrun=False) diff --git a/claims.py b/claims.py index a46138c..430e714 100755 --- a/claims.py +++ b/claims.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from __future__ import division import os import sys @@ -7,61 +9,74 @@ import requests import yaml import pickle +import collections logging.basicConfig(level=logging.INFO) -requests.packages.urllib3.disable_warnings() -config = {} -# load config -with open("config.yaml", "r") as file: - config = yaml.load(file) -# get the jenkins crumb (csrf protection) -crumb_request = requests.get( - '{0}/crumbIssuer/api/json'.format(config['url']), - auth=requests.auth.HTTPBasicAuth(config['usr'], config['pwd']), - verify=False - ) -if crumb_request.status_code != 200: - raise requests.HTTPError( - 'failed to obtain crumb: {0}'.format(crumb_request.reason) - ) -else: - crumb = json.loads(crumb_request.text) - headers = {crumb['crumbRequestField']: crumb['crumb']} - -FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") -PARAMS = { - u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' +class Config(collections.UserDict): + def __init__(self): + with open("config.yaml", "r") as file: + self.data = yaml.load(file) + + # If cache is configured, save it into configuration + if 'DEBUG_CLAIMS_CACHE' in os.environ: + self.data['cache'] = os.environ['DEBUG_CLAIMS_CACHE'] + else: + self.data['cache'] = None + + +class Jenkins(object): + + PULL_PARAMS = { + u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' } -ep = [u'ui', u'api', u'cli'] - - -def fetch_test_report(url=None, job=None, build=None, build_url=None): - '''fetches the test report for a given jenkins url, job and build - or a given complete build url - Usage: - fetch_test_report(url=jenkins_url, job=job_name, build=build_id) - or - fetch_test_report(build_url=fullbuildurl) - Returns: - a List of test dicts - ''' - if build_url is None: - if not(url and job and build): - raise TypeError('fetch_test_report requires either url+job+build or build_url params') - build_url = u'{0}/job/{1}/{2}'.format(url, job, build) - bld_req = requests.get( - build_url+'/testReport/api/json', - auth=requests.auth.HTTPBasicAuth( - config['usr'], - config['pwd'] - ), - params=PARAMS, - verify=False - ) - - if bld_req.status_code == 200: + + def __init__(self, config): + self.config = config + self.headers = None + + def _init_headers(self): + requests.packages.urllib3.disable_warnings() + + # Get the Jenkins crumb (csrf protection) + crumb_request = requests.get( + '{0}/crumbIssuer/api/json'.format(self.config['url']), + auth=requests.auth.HTTPBasicAuth(self.config['usr'], self.config['pwd']), + verify=False + ) + + if crumb_request.status_code != 200: + raise requests.HTTPError( + 'Failed to obtain crumb: {0}'.format(crumb_request.reason)) + + crumb = json.loads(crumb_request.text) + self.headers = {crumb['crumbRequestField']: crumb['crumb']} + + def pull_reports(self, job, build): + """ + Fetches the test report for a given job and build + """ + build_url = '{0}/job/{1}/{2}'.format( + self.config['url'], job, build) + + logging.debug("Getting {}".format(build_url)) + bld_req = requests.get( + build_url + '/testReport/api/json', + auth=requests.auth.HTTPBasicAuth( + self.config['usr'], self.config['pwd']), + params=self.PULL_PARAMS, + verify=False + ) + + if bld_req.status_code == 404: + return [] + if bld_req.status_code != 200: + raise requests.HTTPError( + 'Failed to obtain: {0}'.format(bld_req)) + cases = json.loads(bld_req.text)['suites'][0]['cases'] + + # Enritch individual reports with URL for c in cases: className = c['className'].split('.')[-1] testPath = '.'.join(c['className'].split('.')[:-1]) @@ -69,182 +84,144 @@ def fetch_test_report(url=None, job=None, build=None, build_url=None): return(cases) - -def fetch_all_reports(job=None, build=None): - if 'DEBUG_CLAIMS_CACHE' in os.environ: - if os.path.isfile(os.environ['DEBUG_CLAIMS_CACHE']): - logging.debug("Because environment variable DEBUG_CLAIMS_CACHE is set to '{0}', loading data from there".format( - os.environ['DEBUG_CLAIMS_CACHE'])) - return pickle.load(open(os.environ['DEBUG_CLAIMS_CACHE'], 'r')) - else: - logging.debug("Environment variable DEBUG_CLAIMS_CACHE set to '{0}' but that file does not exist, creating one".format( - os.environ['DEBUG_CLAIMS_CACHE'])) - - if job is None: - job = config['job'] - if build is None: - build = config['bld'] - results = {} - for i in list(reversed(range(1, 5))): - results['t{}'.format(i)] = {} - for j in [6, 7]: - job1 = job.format(i, j) - tr = fetch_test_report(config['url'], job1, build) - fails = tr #parse_fails(tr) - results['t{}'.format(i)]['el{}'.format(j)] = fails - - if 'DEBUG_CLAIMS_CACHE' in os.environ: - pickle.dump(results, open(os.environ['DEBUG_CLAIMS_CACHE'], 'w')) - - return(results) - - -def flatten_reports(reports): - """ - From tree dict like this: - { - 'tier1': { - 'el7': [...] - }, - ... - } - create a flat list of test results with distro and tier added. - """ - reports_flat = [] - for tier in reports.keys(): - for distro in reports[tier].keys(): - if reports[tier][distro] is not None: - for report in reports[tier][distro]: - report['distro'] = distro - report['tier'] = tier - reports_flat.append(report) - return reports_flat - - -def filter_fails(bld): - if not bld: - bld = [] - return([i for i in bld if i.get('status') in FAIL_STATUSES]) - -# fetch_test_report(config['url'], config['job'], config['bld']) -# fetch the failed tests with claim reasons -# bld = fetch_test_report(config['url'], config['job'], config['build']) - - -def filter_claimed(reports): - """ - Only return results which do not have claim/waiver - """ - return [i for i in reports if i['testActions'][0]['reason']] - - -def filter_not_claimed(reports): - """ - Only return results which do not have claim/waiver - """ - return [i for i in reports if not i['testActions'][0]['reason']] - - -def load_rules(): - with open('kb.json', 'r') as file: - return json.loads(file.read()) - file.close() - - -def parse_reasons(fails, fallback=False): - ''' parses the claim reasons from the given list of tests - - :param fails: An input list of tests - :param fallback: Whether to replace the reason by - the 'ErrorDetails' field if 'reason' is None - ''' - reasons = {} - for f in fails: - if(fallback and not f['testActions'][0]['reason']): - reason = f.get('errorDetails') - else: - reason = f['testActions'][0]['reason'] - if reasons.get(reason): - reasons[reason] += 1 + def push_claim(self, test, reason, sticky=False, propagate=False): + '''Claims a given test with a given reason + + :param test: a dict test representation (need to contain the 'url' key) + + :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) + + :param sticky: whether to make the claim sticky (False by default) + + :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) + ''' + logging.info('claiming {0} with reason: {1}'.format(test["className"]+"::"+test["name"], reason)) + + if self.headers is None: + self._init_headers() + + claim_req = requests.post( + u'{0}/claim/claim'.format(test['url']), + auth=requests.auth.HTTPBasicAuth( + self.config['usr'], + self.config['pwd'] + ), + data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, + headers=self.headers, + allow_redirects=False, + verify=False + ) + + if bld_req.status_code != 302: + raise requests.HTTPError( + 'Failed to claim: {0}'.format(claim_req)) + + test['testActions'][0]['reason'] = reason + return(claim_req) + + + +class Results(collections.UserList): + + TIERS = [1, 2, 3, 4] + RHELS = [6, 7] + FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") + + def __init__(self, config, jenkins): + self.config = config + self.jenkins = jenkins + + # If cache is configured, load data from it + if config['cache']: + if os.path.isfile(config['cache']): + logging.debug("Because cache is set to '{0}', loading data from there".format( + config['cache'])) + self.data = pickle.load(open(config['cache'], 'rb')) + return + else: + logging.debug("Cache set to '{0}' but that file does not exist, creating one".format( + config['cache'])) + + self.data = [] + for i in self.TIERS: + for j in self.RHELS: + for report in jenkins.pull_reports( + config['job'].format(i, j), + config['bld']): + report['tier'] = 't{}'.format(i) + report['distro'] = 'el{}'.format(j) + self.data.append(report) + + if config['cache']: + pickle.dump(self.data, open(config['cache'], 'wb')) + + def copy(self): + return self.__class__(self.config, self.jenkins) + + def rule_matches(self, result, rule, indentation=0): + """ + Returns True id result matches to rule, orhervise returns False + """ + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, result, rule, indentation)) + if 'field' in rule and 'pattern' in rule: + # This is simple rule, we can just check regexp against given field and we are done + assert rule['field'] in result + out = re.search(rule['pattern'], result[rule['field']]) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + elif 'AND' in rule: + # We need to check if all sub-rules in list of rules rule['AND'] matches + out = None + for r in rule['AND']: + r_out = self.rule_matches(result, r, indentation+4) + out = r_out if out is None else out and r_out + if not out: + break + return out + elif 'OR' in rule: + # We need to check if at least one sub-rule in list of rules rule['OR'] matches + for r in rule['OR']: + if self.rule_matches(result, r, indentation+4): + return True + return False else: - reasons[reason] = 1 - return(reasons) - - -def get_endpoints_ratio(tests, ep): - endpoints = {i: 0 for i in ep} - for t in tests: - for e in endpoints.keys(): - if u'tests.foreman.{}'.format(e) in t['className']: - endpoints[e] += 1 - return endpoints - - -def get_endpoints_failure_ratio(total, fails): - f = get_endpoints_ratio(fails, ep) - t = get_endpoints_ratio(total, ep) - return {i: (f[i] / t[i]) * 100 for i in ep} - - -def claim(test, reason, sticky=False, propagate=False): - '''Claims a given test with a given reason - - :param test: a dict test representation (need to contain the 'url' key) - - :param reason: a string - reason - - :param sticky: whether to make the claim sticky (False by default) - ''' - logging.info('claiming {0} with reason: {1}'.format(test["className"]+"::"+test["name"], reason)) - claim_req = requests.post( - u'{0}/claim/claim'.format(test['url']), - auth=requests.auth.HTTPBasicAuth( - config['usr'], - config['pwd'] - ), - data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, - headers=headers, - allow_redirects=False, - verify=False - ) - # fixme: do a request result verification - test['testActions'][0]['reason'] = reason - return(claim_req) - - -def rule_matches(data, rule, indentation=0): - """ - Returns True id data matches to rule, orhervise returns False - """ - logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, data, rule, indentation)) - if 'field' in rule and 'pattern' in rule: - # This is simple rule, we can just check regexp against given field and we are done - assert rule['field'] in data - out = re.search(rule['pattern'], data[rule['field']]) is not None - logging.debug("%s=> %s" % (" "*indentation, out)) + raise Exception('Rule %s not formatted correctly' % rule) + + def claim_by_rules(self, rules, dryrun=False): + for rule in rules: + for result in self.get_failed(): + if self.rule_matches(result, rule): + logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(result['className'], result['name'], rule['reason'], result['url'])) + if not dryrun: + self.jenkins.push_claim(result, rule['reason']) + + def get_failed(self): + """ + Return only failed results + """ + out = self.copy() + out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES] return out - elif 'AND' in rule: - # We need to check if all sub-rules in list of rules rule['AND'] matches - out = None - for r in rule['AND']: - r_out = rule_matches(data, r, indentation+4) - out = r_out if out is None else out and r_out - if not out: - break + + def get_claimed(self): + """ + Only return failed results which do not have claim/waiver + """ + out = self.copy() + out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES and i['testActions'][0]['reason']] return out - elif 'OR' in rule: - # We need to check if at least one sub-rule in list of rules rule['OR'] matches - for r in rule['OR']: - if rule_matches(data, r, indentation+4): - return True - return False - else: - raise Exception('Rule %s not formatted correctly' % rule) - - -def claim_by_rules(fails, rules, dryrun=False): - for rule in rules: - for fail in [i for i in fails if rule_matches(i, rule)]: - logging.info(u'{0} matching pattern: {1} url: {2}'.format(fail['name'], rule['reason'], fail['url'])) - if not dryrun: - claim(fail, rule['reason']) + + def get_unclaimed(self): + """ + Only return results which do not have claim/waiver + """ + out = self.copy() + out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES and not i['testActions'][0]['reason']] + return out + + +class Rules(collections.UserList): + + def __init__(self): + with open('kb.json', 'r') as fp: + self.data = json.loads(fp.read()) diff --git a/claimstats.py b/claimstats.py index 154a37b..cf3bddf 100755 --- a/claimstats.py +++ b/claimstats.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- +#!/usr/bin/env python3 import claims import tabulate -reports = claims.fetch_all_reports() -reports = claims.flatten_reports(reports) +config = claims.Config() +jenkins = claims.Jenkins(config) +reports = claims.Results(config, jenkins) + stat_all = len(reports) -reports_fails = claims.filter_fails(reports) +reports_fails = reports.get_failed() stat_failed = len(reports_fails) -reports_claimed = claims.filter_claimed(reports_fails) +reports_claimed = reports.get_claimed() stat_claimed = len(reports_claimed) print("\nOverall stats") @@ -17,7 +18,7 @@ [[stat_all, stat_failed, stat_claimed]], headers=['all reports', 'failures', 'claimed failures'])) -rules = claims.load_rules() +rules = claims.Rules() rules_reasons = [r['reason'] for r in rules] reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} reports_per_reason.update({r:0 for r in rules_reasons}) diff --git a/unclaimed.py b/unclaimed.py index f68880f..5ca7a01 100755 --- a/unclaimed.py +++ b/unclaimed.py @@ -1,11 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/python3 import claims -reports = claims.fetch_all_reports() -reports = claims.flatten_reports(reports) -reports = claims.filter_fails(reports) -reports = claims.filter_not_claimed(reports) +config = claims.Config() +jenkins = claims.Jenkins(config) +reports = claims.Results(config, jenkins).get_failed().get_unclaimed() for r in reports: print(u'{0} {1} {2}'.format(r['distro'], r['className'], r['name'])) From 664e5ad355151a89b69c4348e9642dc56e660016 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 12:26:50 +0200 Subject: [PATCH 11/66] Rename Results and Rules to singular --- claimable.py | 4 ++-- claims.py | 4 ++-- claimstats.py | 4 ++-- unclaimed.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/claimable.py b/claimable.py index b4d83f8..ebc7a6f 100755 --- a/claimable.py +++ b/claimable.py @@ -5,7 +5,7 @@ config = claims.Config() jenkins = claims.Jenkins(config) -results = claims.Results(config, jenkins).get_failed().get_unclaimed() -rules = claims.Rules() +results = claims.Report(config, jenkins).get_failed().get_unclaimed() +rules = claims.Ruleset() results.claim_by_rules(rules, dryrun=False) diff --git a/claims.py b/claims.py index 430e714..bc158fa 100755 --- a/claims.py +++ b/claims.py @@ -121,7 +121,7 @@ def push_claim(self, test, reason, sticky=False, propagate=False): -class Results(collections.UserList): +class Report(collections.UserList): TIERS = [1, 2, 3, 4] RHELS = [6, 7] @@ -220,7 +220,7 @@ def get_unclaimed(self): return out -class Rules(collections.UserList): +class Ruleset(collections.UserList): def __init__(self): with open('kb.json', 'r') as fp: diff --git a/claimstats.py b/claimstats.py index cf3bddf..c3273ae 100755 --- a/claimstats.py +++ b/claimstats.py @@ -5,7 +5,7 @@ config = claims.Config() jenkins = claims.Jenkins(config) -reports = claims.Results(config, jenkins) +reports = claims.Report(config, jenkins) stat_all = len(reports) reports_fails = reports.get_failed() @@ -18,7 +18,7 @@ [[stat_all, stat_failed, stat_claimed]], headers=['all reports', 'failures', 'claimed failures'])) -rules = claims.Rules() +rules = claims.Ruleset() rules_reasons = [r['reason'] for r in rules] reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} reports_per_reason.update({r:0 for r in rules_reasons}) diff --git a/unclaimed.py b/unclaimed.py index 5ca7a01..08c8e6d 100755 --- a/unclaimed.py +++ b/unclaimed.py @@ -4,7 +4,7 @@ config = claims.Config() jenkins = claims.Jenkins(config) -reports = claims.Results(config, jenkins).get_failed().get_unclaimed() +reports = claims.Report(config, jenkins).get_failed().get_unclaimed() for r in reports: print(u'{0} {1} {2}'.format(r['distro'], r['className'], r['name'])) From db09bac1756e850a9815cb063f4255b0af865a76 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 13:17:33 +0200 Subject: [PATCH 12/66] Move some code here and there as we discussed with Roman --- claimable.py | 6 +- claims.py | 216 ++++++++++++++++++++++++++------------------------ claimstats.py | 4 +- unclaimed.py | 4 +- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/claimable.py b/claimable.py index ebc7a6f..e4c94f4 100755 --- a/claimable.py +++ b/claimable.py @@ -3,9 +3,7 @@ import claims -config = claims.Config() -jenkins = claims.Jenkins(config) -results = claims.Report(config, jenkins).get_failed().get_unclaimed() +results = claims.Report() rules = claims.Ruleset() -results.claim_by_rules(rules, dryrun=False) +claims.claim_by_rules(results, rules, dryrun=False) diff --git a/claims.py b/claims.py index bc158fa..24cd691 100755 --- a/claims.py +++ b/claims.py @@ -24,24 +24,19 @@ def __init__(self): else: self.data['cache'] = None + # Additional params when talking to Jenkins + self['headers'] = None + self['pull_params'] = { + u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' + } -class Jenkins(object): - - PULL_PARAMS = { - u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' - } - - def __init__(self, config): - self.config = config - self.headers = None - - def _init_headers(self): + def init_headers(self): requests.packages.urllib3.disable_warnings() # Get the Jenkins crumb (csrf protection) crumb_request = requests.get( - '{0}/crumbIssuer/api/json'.format(self.config['url']), - auth=requests.auth.HTTPBasicAuth(self.config['usr'], self.config['pwd']), + '{0}/crumbIssuer/api/json'.format(self['url']), + auth=requests.auth.HTTPBasicAuth(self['usr'], self['pwd']), verify=False ) @@ -50,44 +45,62 @@ def _init_headers(self): 'Failed to obtain crumb: {0}'.format(crumb_request.reason)) crumb = json.loads(crumb_request.text) - self.headers = {crumb['crumbRequestField']: crumb['crumb']} + self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} - def pull_reports(self, job, build): - """ - Fetches the test report for a given job and build - """ - build_url = '{0}/job/{1}/{2}'.format( - self.config['url'], job, build) - logging.debug("Getting {}".format(build_url)) - bld_req = requests.get( - build_url + '/testReport/api/json', - auth=requests.auth.HTTPBasicAuth( - self.config['usr'], self.config['pwd']), - params=self.PULL_PARAMS, - verify=False - ) +config = Config() - if bld_req.status_code == 404: - return [] - if bld_req.status_code != 200: - raise requests.HTTPError( - 'Failed to obtain: {0}'.format(bld_req)) - cases = json.loads(bld_req.text)['suites'][0]['cases'] +class Case(collections.UserDict): + """ + Result of one test case + """ - # Enritch individual reports with URL - for c in cases: - className = c['className'].split('.')[-1] - testPath = '.'.join(c['className'].split('.')[:-1]) - c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) + FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") - return(cases) + def __init__(self, data): + self.data = data - def push_claim(self, test, reason, sticky=False, propagate=False): - '''Claims a given test with a given reason + def is_failed(self): + return self['status'] in self.FAIL_STATUSES + + def is_claimed(self): + return self['status'] in self.FAIL_STATUSES and self['testActions'][0]['reason'] + + def is_unclaimed(self): + return self['status'] in self.FAIL_STATUSES and not self['testActions'][0]['reason'] + + def matches_to_rule(self, rule, indentation=0): + """ + Returns True if result matches to rule, orhervise returns False + """ + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self, rule, indentation)) + if 'field' in rule and 'pattern' in rule: + # This is simple rule, we can just check regexp against given field and we are done + assert rule['field'] in self + out = re.search(rule['pattern'], self[rule['field']]) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + elif 'AND' in rule: + # We need to check if all sub-rules in list of rules rule['AND'] matches + out = None + for r in rule['AND']: + r_out = self.matches_to_rule(r, indentation+4) + out = r_out if out is None else out and r_out + if not out: + break + return out + elif 'OR' in rule: + # We need to check if at least one sub-rule in list of rules rule['OR'] matches + for r in rule['OR']: + if self.matches_to_rule(r, indentation+4): + return True + return False + else: + raise Exception('Rule %s not formatted correctly' % rule) - :param test: a dict test representation (need to contain the 'url' key) + def push_claim(self, reason, sticky=False, propagate=False): + '''Claims a given test with a given reason :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) @@ -95,42 +108,40 @@ def push_claim(self, test, reason, sticky=False, propagate=False): :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) ''' - logging.info('claiming {0} with reason: {1}'.format(test["className"]+"::"+test["name"], reason)) + logging.info('claiming {0}::{1} with reason: {2}'.format(self["className"], self["name"], reason)) - if self.headers is None: - self._init_headers() + if config['headers'] is None: + config.init_headers() claim_req = requests.post( - u'{0}/claim/claim'.format(test['url']), + u'{0}/claim/claim'.format(self['url']), auth=requests.auth.HTTPBasicAuth( - self.config['usr'], - self.config['pwd'] + config['usr'], + config['pwd'] ), data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, - headers=self.headers, + headers=config['headers'], allow_redirects=False, verify=False ) - if bld_req.status_code != 302: + if claim_req.status_code != 302: raise requests.HTTPError( 'Failed to claim: {0}'.format(claim_req)) - test['testActions'][0]['reason'] = reason + self['testActions'][0]['reason'] = reason return(claim_req) - class Report(collections.UserList): + """ + Report is a list of Cases (i.e. test results) + """ TIERS = [1, 2, 3, 4] RHELS = [6, 7] - FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") - - def __init__(self, config, jenkins): - self.config = config - self.jenkins = jenkins + def __init__(self): # If cache is configured, load data from it if config['cache']: if os.path.isfile(config['cache']): @@ -145,79 +156,65 @@ def __init__(self, config, jenkins): self.data = [] for i in self.TIERS: for j in self.RHELS: - for report in jenkins.pull_reports( + for report in self.pull_reports( config['job'].format(i, j), config['bld']): report['tier'] = 't{}'.format(i) report['distro'] = 'el{}'.format(j) - self.data.append(report) + self.data.append(Case(report)) if config['cache']: pickle.dump(self.data, open(config['cache'], 'wb')) - def copy(self): - return self.__class__(self.config, self.jenkins) - - def rule_matches(self, result, rule, indentation=0): + def pull_reports(self, job, build): """ - Returns True id result matches to rule, orhervise returns False + Fetches the test report for a given job and build """ - logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, result, rule, indentation)) - if 'field' in rule and 'pattern' in rule: - # This is simple rule, we can just check regexp against given field and we are done - assert rule['field'] in result - out = re.search(rule['pattern'], result[rule['field']]) is not None - logging.debug("%s=> %s" % (" "*indentation, out)) - return out - elif 'AND' in rule: - # We need to check if all sub-rules in list of rules rule['AND'] matches - out = None - for r in rule['AND']: - r_out = self.rule_matches(result, r, indentation+4) - out = r_out if out is None else out and r_out - if not out: - break - return out - elif 'OR' in rule: - # We need to check if at least one sub-rule in list of rules rule['OR'] matches - for r in rule['OR']: - if self.rule_matches(result, r, indentation+4): - return True - return False - else: - raise Exception('Rule %s not formatted correctly' % rule) + build_url = '{0}/job/{1}/{2}'.format( + config['url'], job, build) - def claim_by_rules(self, rules, dryrun=False): - for rule in rules: - for result in self.get_failed(): - if self.rule_matches(result, rule): - logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(result['className'], result['name'], rule['reason'], result['url'])) - if not dryrun: - self.jenkins.push_claim(result, rule['reason']) + logging.debug("Getting {}".format(build_url)) + bld_req = requests.get( + build_url + '/testReport/api/json', + auth=requests.auth.HTTPBasicAuth( + config['usr'], config['pwd']), + params=config['pull_params'], + verify=False + ) + + if bld_req.status_code == 404: + return [] + if bld_req.status_code != 200: + raise requests.HTTPError( + 'Failed to obtain: {0}'.format(bld_req)) + + cases = json.loads(bld_req.text)['suites'][0]['cases'] + + # Enrich individual reports with URL + for c in cases: + className = c['className'].split('.')[-1] + testPath = '.'.join(c['className'].split('.')[:-1]) + c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) + + return(cases) def get_failed(self): """ Return only failed results """ - out = self.copy() - out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES] - return out + return [i for i in self.data if i.is_failed()] def get_claimed(self): """ Only return failed results which do not have claim/waiver """ - out = self.copy() - out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES and i['testActions'][0]['reason']] - return out + return [i for i in self.data if i.is_claimed()] def get_unclaimed(self): """ Only return results which do not have claim/waiver """ - out = self.copy() - out.data = [i for i in self.data if i.get('status') in self.FAIL_STATUSES and not i['testActions'][0]['reason']] - return out + return [i for i in self.data if i.is_unclaimed()] class Ruleset(collections.UserList): @@ -225,3 +222,12 @@ class Ruleset(collections.UserList): def __init__(self): with open('kb.json', 'r') as fp: self.data = json.loads(fp.read()) + + +def claim_by_rules(report, rules, dryrun=False): + for rule in rules: + for case in report.get_unclaimed(): + if case.matches_to_rule(rule): + logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) + if not dryrun: + case.push_claim(rule['reason']) diff --git a/claimstats.py b/claimstats.py index c3273ae..402ebb3 100755 --- a/claimstats.py +++ b/claimstats.py @@ -3,9 +3,7 @@ import claims import tabulate -config = claims.Config() -jenkins = claims.Jenkins(config) -reports = claims.Report(config, jenkins) +reports = claims.Report() stat_all = len(reports) reports_fails = reports.get_failed() diff --git a/unclaimed.py b/unclaimed.py index 08c8e6d..7360461 100755 --- a/unclaimed.py +++ b/unclaimed.py @@ -2,9 +2,7 @@ import claims -config = claims.Config() -jenkins = claims.Jenkins(config) -reports = claims.Report(config, jenkins).get_failed().get_unclaimed() +reports = claims.Report().get_unclaimed() for r in reports: print(u'{0} {1} {2}'.format(r['distro'], r['className'], r['name'])) From ecab09f209a0f1d7e27d205d70f0e287785214e5 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 15:14:47 +0200 Subject: [PATCH 13/66] Fix is_claimed and is_unclaimed --- claims.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/claims.py b/claims.py index 24cd691..f3b9836 100755 --- a/claims.py +++ b/claims.py @@ -65,10 +65,13 @@ def is_failed(self): return self['status'] in self.FAIL_STATUSES def is_claimed(self): - return self['status'] in self.FAIL_STATUSES and self['testActions'][0]['reason'] + return self['status'] in self.FAIL_STATUSES \ + and 'reason' in self['testActions'][0] \ + and self['testActions'][0]['reason'] def is_unclaimed(self): - return self['status'] in self.FAIL_STATUSES and not self['testActions'][0]['reason'] + return self['status'] in self.FAIL_STATUSES \ + and 'reason' not in self['testActions'][0] def matches_to_rule(self, rule, indentation=0): """ From d235f2dcc53f1aa239b4ecefa34771cdc0afd8fe Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 15:56:19 +0200 Subject: [PATCH 14/66] Simplify, thanks rplevka --- claims.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/claims.py b/claims.py index f3b9836..3a2954e 100755 --- a/claims.py +++ b/claims.py @@ -66,12 +66,11 @@ def is_failed(self): def is_claimed(self): return self['status'] in self.FAIL_STATUSES \ - and 'reason' in self['testActions'][0] \ - and self['testActions'][0]['reason'] + and self['testActions'][0].get('reason']) def is_unclaimed(self): return self['status'] in self.FAIL_STATUSES \ - and 'reason' not in self['testActions'][0] + and not self['testActions'][0].get('reason']) def matches_to_rule(self, rule, indentation=0): """ From 33707e61b6469c7061360baf089c2aa15e81b595 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 16:10:57 +0200 Subject: [PATCH 15/66] Discussed with rplevka and we do not want these is_* and get_* methods --- claimable.py | 4 ++-- claims.py | 31 +------------------------------ claimstats.py | 4 ++-- unclaimed.py | 2 +- 4 files changed, 6 insertions(+), 35 deletions(-) diff --git a/claimable.py b/claimable.py index e4c94f4..1226065 100755 --- a/claimable.py +++ b/claimable.py @@ -3,7 +3,7 @@ import claims -results = claims.Report() +report = claims.Report() rules = claims.Ruleset() -claims.claim_by_rules(results, rules, dryrun=False) +claims.claim_by_rules(report, rules, dryrun=True) diff --git a/claims.py b/claims.py index 3a2954e..e1cbd57 100755 --- a/claims.py +++ b/claims.py @@ -61,17 +61,6 @@ class Case(collections.UserDict): def __init__(self, data): self.data = data - def is_failed(self): - return self['status'] in self.FAIL_STATUSES - - def is_claimed(self): - return self['status'] in self.FAIL_STATUSES \ - and self['testActions'][0].get('reason']) - - def is_unclaimed(self): - return self['status'] in self.FAIL_STATUSES \ - and not self['testActions'][0].get('reason']) - def matches_to_rule(self, rule, indentation=0): """ Returns True if result matches to rule, orhervise returns False @@ -200,24 +189,6 @@ def pull_reports(self, job, build): return(cases) - def get_failed(self): - """ - Return only failed results - """ - return [i for i in self.data if i.is_failed()] - - def get_claimed(self): - """ - Only return failed results which do not have claim/waiver - """ - return [i for i in self.data if i.is_claimed()] - - def get_unclaimed(self): - """ - Only return results which do not have claim/waiver - """ - return [i for i in self.data if i.is_unclaimed()] - class Ruleset(collections.UserList): @@ -228,7 +199,7 @@ def __init__(self): def claim_by_rules(report, rules, dryrun=False): for rule in rules: - for case in report.get_unclaimed(): + for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: if case.matches_to_rule(rule): logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) if not dryrun: diff --git a/claimstats.py b/claimstats.py index 402ebb3..e9679c0 100755 --- a/claimstats.py +++ b/claimstats.py @@ -6,9 +6,9 @@ reports = claims.Report() stat_all = len(reports) -reports_fails = reports.get_failed() +reports_fails = [i for i in reports if i['status'] in claims.Case.FAIL_STATUSES] stat_failed = len(reports_fails) -reports_claimed = reports.get_claimed() +reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] stat_claimed = len(reports_claimed) print("\nOverall stats") diff --git a/unclaimed.py b/unclaimed.py index 7360461..61b8879 100755 --- a/unclaimed.py +++ b/unclaimed.py @@ -2,7 +2,7 @@ import claims -reports = claims.Report().get_unclaimed() +reports = [i for i in claims.Report() if i['status'] in claims.Case.FAIL_STATUSES and not i['testActions'][0].get('reason')] for r in reports: print(u'{0} {1} {2}'.format(r['distro'], r['className'], r['name'])) From a7153ebccb022b3201ef5317a81be29fcd8c527c Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 14 Jun 2018 23:28:39 +0200 Subject: [PATCH 16/66] Add possibility to learn tests start and end time --- claims.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/claims.py b/claims.py index e1cbd57..cc3e345 100755 --- a/claims.py +++ b/claims.py @@ -10,6 +10,7 @@ import yaml import pickle import collections +import datetime logging.basicConfig(level=logging.INFO) @@ -57,13 +58,21 @@ class Case(collections.UserDict): """ FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") + LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') + LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' def __init__(self, data): self.data = data + def __getitem__(self, name): + if name in ('start', 'end') and \ + ('start' not in self.data or 'end' not in self.data): + self.load_timings() + return self.data[name] + def matches_to_rule(self, rule, indentation=0): """ - Returns True if result matches to rule, orhervise returns False + Returns True if result matches to rule, otherwise returns False """ logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self, rule, indentation)) if 'field' in rule and 'pattern' in rule: @@ -123,6 +132,35 @@ def push_claim(self, reason, sticky=False, propagate=False): self['testActions'][0]['reason'] = reason return(claim_req) + def load_timings(self): + log = self['stdout'].split("\n") + log_size = len(log) + log_used = 0 + start = None + end = None + counter = 0 + while start is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + start = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter += 1 + log_used += counter + counter = -1 + while end is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + end = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter -= 1 + log_used -= counter + assert log_used <= log_size, \ + "Make sure detected start date is not below end date and vice versa" + self['start'] = start + self['end'] = end + class Report(collections.UserList): """ From 316ea0d0a6142a21641aa3405158fdff93bd529c Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Sat, 16 Jun 2018 22:29:17 +0200 Subject: [PATCH 17/66] When we can not parse start/end time, return KeyError --- claims.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/claims.py b/claims.py index cc3e345..38656a3 100755 --- a/claims.py +++ b/claims.py @@ -133,6 +133,8 @@ def push_claim(self, reason, sticky=False, propagate=False): return(claim_req) def load_timings(self): + if self['stdout'] is None: + return log = self['stdout'].split("\n") log_size = len(log) log_used = 0 From d524206e5e09b4f7cd4ab8c9f2473e65b189c62f Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Sat, 16 Jun 2018 22:30:03 +0200 Subject: [PATCH 18/66] Tool to generate timeline of tests run --- rungraph.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 rungraph.py diff --git a/rungraph.py b/rungraph.py new file mode 100755 index 0000000..f946f94 --- /dev/null +++ b/rungraph.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import logging +import datetime +import svgwrite +import claims + + +STATUS_COLOR = { +'FAILED': 'red', +'FIXED': 'blue', +'PASSED': 'green', +'REGRESSION': 'purple', +'SKIPPED': 'fuchsia', +} +LANE_HEIGHT = 10 +LANES_START = LANE_HEIGHT # we will place a timeline into the first lane +HOUR = 3600 +X_CONTRACTION = 0.1 + + +def overlaps(a, b): + """ + Return true if two intervals overlap: + overlaps((1, 3), (2, 10)) => True + overlaps((1, 3), (5, 10)) => False + """ + if b[0] <= a[0] <= b[1] or b[0] <= a[1] <= b[1]: + return True + else: + return False + + +def scale(a): + return (a[0] * X_CONTRACTION, a[1]) + + +reports = [i for i in claims.Report() if i['tier'] == 't4'] + +# Load all the reports and sort them in lanes +###counter = 0 +lanes = [] +start = None +end = None +for r in reports: + # Get start and end time. If unknown, skip the result + try: + r_start = r['start'].timestamp() + r_end = r['end'].timestamp() + except KeyError: + logging.info("No start time for %s::%s" % (r['className'], r['name'])) + continue + # Find overal widtht of time line + if start is None or r_start < start: + logging.debug("Test %s started before current minimum of %s" % (r['name'], start)) + start = r_start + if end is None or r_end > end: + end = r_end + r['interval'] = (r_start, r_end) + # Check if there is a free lane for us, if not, create a new one + lane_found = False + for lane in lanes: + lane_found = True + for interval in lane: + if overlaps(r['interval'], interval['interval']): + lane_found = False + break + if lane_found: + break + if not lane_found: + logging.debug("Adding lane %s" % (len(lanes)+1)) + lane = [] + lanes.append(lane) + lane.append(r) + ###counter += 1 + ###if counter > 10: break + +# Create a drawing with timeline +dwg = svgwrite.Drawing('/tmp/rungraph.svg', + size=scale((end-start, LANE_HEIGHT*(len(lanes)+1)))) +dwg.add(dwg.line( + scale((0, LANE_HEIGHT)), + scale((end-start, LANE_HEIGHT)), + style="stroke: black; stroke-width: 1;" +)) +start_full_hour = int(start / HOUR) * HOUR +timeline = start_full_hour - start +while start + timeline <= end: + if timeline >= 0: + dwg.add(dwg.line( + scale((timeline, LANE_HEIGHT)), + scale((timeline, 2*LANE_HEIGHT/3)), + style="stroke: black; stroke-width: 1;" + )) + dwg.add(dwg.text( + datetime.datetime.fromtimestamp(start+timeline) \ + .strftime('%Y-%m-%d %H:%M:%S'), + insert=scale((timeline, 2*LANE_HEIGHT/3)), + style="fill: black; font-size: 3pt;" + )) + timeline += HOUR/4 + +# Draw tests +for lane_no in range(len(lanes)): + for r in lanes[lane_no]: + logging.debug("In lane %s adding %s::%s %s" \ + % (lane_no, r['className'], r['name'], r['interval'])) + s, e = r['interval'] + dwg.add(dwg.rect( + insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + size=scale((e - s, LANE_HEIGHT/2)), + style="fill: %s; stroke: %s; stroke-width: 0;" \ + % (STATUS_COLOR[r['status']], STATUS_COLOR[r['status']]) + )) + dwg.add(dwg.text( + "%s::%s" % (r['className'], r['name']), + insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + transform="rotate(-30, %s, %s)" \ + % scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + style="fill: gray; font-size: 2pt;" + )) +dwg.save() From 5a17e1b8c2e34862494d4811ea3e98d2e101d2ab Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Sun, 17 Jun 2018 00:03:30 +0200 Subject: [PATCH 19/66] Utility to see stability of the tests --- tests-stability.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 tests-stability.py diff --git a/tests-stability.py b/tests-stability.py new file mode 100755 index 0000000..5ee274d --- /dev/null +++ b/tests-stability.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import logging +import datetime +import collections +import statistics +import tabulate +import csv +import claims + +BUILDS = [14, 13, 12, 10, 9, 8, 7, 6] +matrix = collections.OrderedDict() + + +def sanitize_state(state): + if state == 'REGRESSION': + state = 'FAILED' + if state == 'FIXED': + state = 'PASSED' + if state == 'PASSED': + return 0 + if state == 'FAILED': + return 1 + raise KeyError("Do not know how to handle state %s" % state) + + +for build_id in range(len(BUILDS)): + build = BUILDS[build_id] + claims.config['bld'] = build + claims.config['cache'] = 'cache-%s-%s.pickle' \ + % (datetime.datetime.now().strftime('%Y%m%d'), build) + logging.info("Initializing report for build %s with cache in %s" \ + % (build, claims.config['cache'])) + #report = [i for i in claims.Report() if i['tier'] == 't4'] + report = claims.Report() + for r in report: + t = "%s::%s@%s" % (r['className'], r['name'], r['distro']) + if t not in matrix: + matrix[t] = [None for i in BUILDS] + try: + state = sanitize_state(r['status']) + except KeyError: + continue + matrix[t][build_id] = state + +# Count statistical measure of the results +for k,v in matrix.items(): + try: + stdev = statistics.pstdev([i for i in v if i is not None]) + except statistics.StatisticsError: + stdev = None + v.append(stdev) + try: + stdev = statistics.pstdev([i for i in v[:3] if i is not None]) + except statistics.StatisticsError: + stdev = None + v.append(stdev) + +print("Legend:\n 0 ... PASSED or FIXED\n 1 ... FAILED or REGRESSION\n Population standard deviation, 0 is best (stable), 0.5 is worst (unstable)\n Same but only for newest 3 builds") +matrix_flat = [[k]+v for k,v in matrix.items()] +headers = ['test']+BUILDS+['pstdev (all)', 'pstdev (3 newest)'] +print(tabulate.tabulate( + matrix_flat, + headers=headers, + floatfmt=".3f" +)) + +filename = "/tmp/tests-stability.csv" +print("Writing data to %s" % filename) +with open(filename, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerows([headers]) + writer.writerows(matrix_flat) From d0c9bd877fd96638d022e064b59d4a13114b0c9a Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 18 Jun 2018 08:29:00 +0200 Subject: [PATCH 20/66] Add possibility to get production log for each case. Problem he is tests run in parallel so it is hard/impossible (?) to tell which test caused what --- claims.py | 89 +++++++++++++++++++++++++++++++++++++++++++++- tests-stability.py | 2 +- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/claims.py b/claims.py index 38656a3..13a8176 100755 --- a/claims.py +++ b/claims.py @@ -49,7 +49,83 @@ def init_headers(self): self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} -config = Config() +class ForemanDebug(object): + + def __init__(self, tier, rhel): + self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], config['job'].format(tier, rhel), config['bld']) + self._extracted = None + + @property + def extracted(self): + if self._extracted is None: + logging.debug('Going to download %s' % self._url) + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as localfile: + logging.debug('Going to save to %s' % localfile.name) + self._download_file(localfile, self._url) + self._tmpdir = tempfile.TemporaryDirectory() + subprocess.call(['tar', '-xf', localfile.name, '--directory', self._tmpdir.name]) + logging.info('Extracted to %s' % self._tmpdir.name) + self._extracted = os.path.join(self._tmpdir.name, 'foreman-debug') + return self._extracted + + def _download_file(self, localfile, url): + r = requests.get(url, stream=True) + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + localfile.write(chunk) + localfile.close() + logging.info('File %s saved to %s' % (url, localfile.name)) + + +class ProductionLog(object): + + FILE_ENCODING = 'ISO-8859-1' # guessed, that wile contains ugly binary mess as well + DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 + DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 + + def __init__(self, tier, rhel): + self._foreman_debug = ForemanDebug(tier, rhel) + self._log = None + + @property + def log(self): + if self._log is None: + logfile = os.path.join(self._foreman_debug.extracted, 'var', 'log', 'foreman', 'production.log') + self._log = [] + buf = [] + last = None + with open(logfile, 'r', encoding=self.FILE_ENCODING) as fp: + for line in fp: + + # This line starts with date - denotes first line of new log record + if re.search(self.DATE_REGEXP, line): + + # This is a new log record, so firs save previous one + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + last = datetime.datetime.strptime(line[:19], self.DATE_FMT) + buf = [] + buf.append(re.sub(self.DATE_REGEXP, '', line, count=1)) + + # This line does not start with line - comtains continuation of a log recorder started before + else: + buf.append(line) + + # Save last line + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + + logging.info("File %s parsed into memory and deleted" % logfile) + return self._log + + def from_to(self, from_time, to_time): + out = [] + for i in self.log: + if from_time <= i['time'] <= to_time: + out.append(i) + if i['time'] > to_time: + break + return out class Case(collections.UserDict): @@ -173,6 +249,13 @@ class Report(collections.UserList): RHELS = [6, 7] def __init__(self): + # Initialize production.log instance + self.production_log = {} + for tier in self.TIERS: + self.production_log[tier] = {} + for rhel in self.RHELS: + self.production_log[tier][rhel] = ProductionLog(tier, rhel) + self.production_log = ProductionLog(config['job'], config) # If cache is configured, load data from it if config['cache']: if os.path.isfile(config['cache']): @@ -192,6 +275,7 @@ def __init__(self): config['bld']): report['tier'] = 't{}'.format(i) report['distro'] = 'el{}'.format(j) + report['production.log'] = self.production_log[i][j] self.data.append(Case(report)) if config['cache']: @@ -237,6 +321,9 @@ def __init__(self): self.data = json.loads(fp.read()) +# Create shared config file +config = Config() + def claim_by_rules(report, rules, dryrun=False): for rule in rules: for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: diff --git a/tests-stability.py b/tests-stability.py index 5ee274d..f77c9aa 100755 --- a/tests-stability.py +++ b/tests-stability.py @@ -9,7 +9,7 @@ import csv import claims -BUILDS = [14, 13, 12, 10, 9, 8, 7, 6] +BUILDS = [22, 21, 19, 18, 17, 14, 13, 12, 10, 9, 8, 7, 6] matrix = collections.OrderedDict() From 262c29516302087119cc86df92d90d5c3f8348a0 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 18 Jun 2018 10:50:03 +0200 Subject: [PATCH 21/66] Make relevant piece of production.log part of every test result --- claims.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/claims.py b/claims.py index 13a8176..706b011 100755 --- a/claims.py +++ b/claims.py @@ -11,6 +11,8 @@ import pickle import collections import datetime +import tempfile +import subprocess logging.basicConfig(level=logging.INFO) @@ -140,10 +142,18 @@ class Case(collections.UserDict): def __init__(self, data): self.data = data + def __contains__(self, name): + return name in self.data or name in ('start', 'end', 'production.log') + def __getitem__(self, name): if name in ('start', 'end') and \ ('start' not in self.data or 'end' not in self.data): self.load_timings() + if name == 'production.log': + return "\n".join( + ["\n".join(i['data']) for i in + self.data['production.log'].from_to( + self['start'], self['end'])]) return self.data[name] def matches_to_rule(self, rule, indentation=0): @@ -153,10 +163,13 @@ def matches_to_rule(self, rule, indentation=0): logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self, rule, indentation)) if 'field' in rule and 'pattern' in rule: # This is simple rule, we can just check regexp against given field and we are done - assert rule['field'] in self - out = re.search(rule['pattern'], self[rule['field']]) is not None - logging.debug("%s=> %s" % (" "*indentation, out)) - return out + try: + out = re.search(rule['pattern'], self[rule['field']]) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + except KeyError: + logging.debug("%s=> Failed to get field %s from case" % (" "*indentation, rule['field'])) + return None elif 'AND' in rule: # We need to check if all sub-rules in list of rules rule['AND'] matches out = None @@ -250,12 +263,11 @@ class Report(collections.UserList): def __init__(self): # Initialize production.log instance - self.production_log = {} + self.production_logs = {} for tier in self.TIERS: - self.production_log[tier] = {} + self.production_logs[tier] = {} for rhel in self.RHELS: - self.production_log[tier][rhel] = ProductionLog(tier, rhel) - self.production_log = ProductionLog(config['job'], config) + self.production_logs[tier][rhel] = ProductionLog(tier, rhel) # If cache is configured, load data from it if config['cache']: if os.path.isfile(config['cache']): @@ -275,7 +287,7 @@ def __init__(self): config['bld']): report['tier'] = 't{}'.format(i) report['distro'] = 'el{}'.format(j) - report['production.log'] = self.production_log[i][j] + report['production.log'] = self.production_logs[i][j] self.data.append(Case(report)) if config['cache']: From bfb36c0c8ca462cb8633b072368ada705e3be8f8 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 18 Jun 2018 15:32:41 +0200 Subject: [PATCH 22/66] Add caching to production.log functionality --- claims.py | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/claims.py b/claims.py index 706b011..fcbfac5 100755 --- a/claims.py +++ b/claims.py @@ -13,6 +13,7 @@ import datetime import tempfile import subprocess +import shutil logging.basicConfig(level=logging.INFO) @@ -66,17 +67,19 @@ def extracted(self): self._download_file(localfile, self._url) self._tmpdir = tempfile.TemporaryDirectory() subprocess.call(['tar', '-xf', localfile.name, '--directory', self._tmpdir.name]) - logging.info('Extracted to %s' % self._tmpdir.name) + logging.debug('Extracted to %s' % self._tmpdir.name) self._extracted = os.path.join(self._tmpdir.name, 'foreman-debug') return self._extracted def _download_file(self, localfile, url): r = requests.get(url, stream=True) - for chunk in r.iter_content(chunk_size=1024): + for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks localfile.write(chunk) + if r.status_code != 200: + raise requests.HTTPError("Failed to get foreman-debug %s" % url) localfile.close() - logging.info('File %s saved to %s' % (url, localfile.name)) + logging.debug('File %s saved to %s' % (url, localfile.name)) class ProductionLog(object): @@ -86,17 +89,32 @@ class ProductionLog(object): DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 def __init__(self, tier, rhel): - self._foreman_debug = ForemanDebug(tier, rhel) self._log = None + self._logfile = None + self._cache = None + + if 'cache' in config: + self._cache = '%s-t%s-el%s-production.log' \ + % (config['cache'].replace('.pickle', ''), tier, rhel) + if self._cache and os.path.isfile(self._cache): + self._logfile = self._cache + logging.debug("Loading production.log from cached %s" % self._logfile) + return None + else: + logging.debug("Cache for production.log (%s) set, but not available. Will create it if we have a chance" % self._cache) + + self._foreman_debug = ForemanDebug(tier, rhel) @property def log(self): if self._log is None: - logfile = os.path.join(self._foreman_debug.extracted, 'var', 'log', 'foreman', 'production.log') + if self._logfile is None: + self._logfile = os.path.join(self._foreman_debug.extracted, + 'var', 'log', 'foreman', 'production.log') self._log = [] buf = [] last = None - with open(logfile, 'r', encoding=self.FILE_ENCODING) as fp: + with open(self._logfile, 'r', encoding=self.FILE_ENCODING) as fp: for line in fp: # This line starts with date - denotes first line of new log record @@ -117,7 +135,12 @@ def log(self): if len(buf) != 0: self._log.append({'time': last, 'data': buf}) - logging.info("File %s parsed into memory and deleted" % logfile) + # Cache file we have downloaded + if self._cache and not os.path.isfile(self._cache): + logging.debug("Caching production.log %s to %s" % (self._logfile, self._cache)) + shutil.copyfile(self._logfile, self._cache) + + logging.debug("File %s parsed into memory and deleted" % self._logfile) return self._log def from_to(self, from_time, to_time): @@ -150,9 +173,9 @@ def __getitem__(self, name): ('start' not in self.data or 'end' not in self.data): self.load_timings() if name == 'production.log': - return "\n".join( + self['production.log'] = "\n".join( ["\n".join(i['data']) for i in - self.data['production.log'].from_to( + self.data['OBJECT:production.log'].from_to( self['start'], self['end'])]) return self.data[name] @@ -287,7 +310,7 @@ def __init__(self): config['bld']): report['tier'] = 't{}'.format(i) report['distro'] = 'el{}'.format(j) - report['production.log'] = self.production_logs[i][j] + report['OBJECT:production.log'] = self.production_logs[i][j] self.data.append(Case(report)) if config['cache']: From 053e6f4585e5630c083481ad01ac39df64be7230 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 19 Jun 2018 00:39:51 +0200 Subject: [PATCH 23/66] Time is seriously wrong in the log :-/ --- claims.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/claims.py b/claims.py index fcbfac5..2e441f7 100755 --- a/claims.py +++ b/claims.py @@ -148,8 +148,17 @@ def from_to(self, from_time, to_time): for i in self.log: if from_time <= i['time'] <= to_time: out.append(i) - if i['time'] > to_time: - break + # Do not do following as time is not sequentional in the log (or maybe some workers are off or with different TZ?): + # TODO: Fix ordering of the log and uncomment this + # + # E.g.: + # 2018-06-17T17:29:44 [I|dyn|] start terminating clock... + # 2018-06-17T21:34:49 [I|app|] Current user: foreman_admin (administrator) + # 2018-06-17T21:37:21 [...] + # 2018-06-17T17:41:38 [I|app|] Started POST "/katello/api/v2/organizations"... + # + #if i['time'] > to_time: + # break return out From 994d128e8e55d97ad67249af8fa5a1843338452d Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 25 Jun 2018 13:54:57 +0200 Subject: [PATCH 24/66] Format it in a same way as elsewhere --- unclaimed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unclaimed.py b/unclaimed.py index 61b8879..9e4b5e9 100755 --- a/unclaimed.py +++ b/unclaimed.py @@ -5,4 +5,4 @@ reports = [i for i in claims.Report() if i['status'] in claims.Case.FAIL_STATUSES and not i['testActions'][0].get('reason')] for r in reports: - print(u'{0} {1} {2}'.format(r['distro'], r['className'], r['name'])) + print(u'{0} {1}::{2}'.format(r['distro'], r['className'], r['name'])) From 683c8e08f1225b575d0e4a5e991009db02e2a6a0 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Fri, 29 Jun 2018 14:36:14 +0200 Subject: [PATCH 25/66] More useful debug message and workaround one traceback --- claims.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/claims.py b/claims.py index 2e441f7..b630f72 100755 --- a/claims.py +++ b/claims.py @@ -192,11 +192,14 @@ def matches_to_rule(self, rule, indentation=0): """ Returns True if result matches to rule, otherwise returns False """ - logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self, rule, indentation)) + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self['name'], rule, indentation)) if 'field' in rule and 'pattern' in rule: # This is simple rule, we can just check regexp against given field and we are done try: - out = re.search(rule['pattern'], self[rule['field']]) is not None + data = self[rule['field']] + if data is None: + data = '' + out = re.search(rule['pattern'], data) is not None logging.debug("%s=> %s" % (" "*indentation, out)) return out except KeyError: From 8b0a1c892dae8a0690d28a0318738e279af4fee0 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 5 Jul 2018 22:45:53 +0200 Subject: [PATCH 26/66] Base stats per tier --- claimstats.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/claimstats.py b/claimstats.py index e9679c0..09a302f 100755 --- a/claimstats.py +++ b/claimstats.py @@ -13,8 +13,25 @@ print("\nOverall stats") print(tabulate.tabulate( - [[stat_all, stat_failed, stat_claimed]], - headers=['all reports', 'failures', 'claimed failures'])) + [[stat_all, stat_failed, stat_failed/stat_all*100, stat_claimed, stat_claimed/stat_failed*100]], + headers=['all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], + floatfmt=".0f")) + +stats = [] +for t in ["t%s" % i for i in claims.Report.TIERS]: + filtered = [r for r in reports if r['tier'] == t] + stat_all = len(filtered) + reports_fails = [i for i in filtered if i['status'] in claims.Case.FAIL_STATUSES] + stat_failed = len(reports_fails) + reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] + stat_claimed = len(reports_claimed) + stats.append([t, stat_all, stat_failed, stat_failed/stat_all*100, stat_claimed, stat_claimed/stat_failed*100]) + +print("\nStats per tier") +print(tabulate.tabulate( + stats, + headers=['tier', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], + floatfmt=".0f")) rules = claims.Ruleset() rules_reasons = [r['reason'] for r in rules] From ec9469e521ab4bd39b5cd98c54511515fa3a24c8 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Sat, 7 Jul 2018 22:52:28 +0200 Subject: [PATCH 27/66] Make it possible to specify more job groups --- claims.py | 46 ++++++++++++++++++++++++++++------------------ config.yaml.sample | 18 ++++++++++++++++-- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/claims.py b/claims.py index b630f72..3c4f2f5 100755 --- a/claims.py +++ b/claims.py @@ -18,6 +18,9 @@ logging.basicConfig(level=logging.INFO) class Config(collections.UserDict): + + LATEST = 'latest' # how do we call latest job group in the config? + def __init__(self): with open("config.yaml", "r") as file: self.data = yaml.load(file) @@ -34,6 +37,15 @@ def __init__(self): u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' } + def get_builds(self, job_group=''): + if job_group == '': + job_group = self.LATEST + out = collections.OrderedDict() + for job in self.data['job_groups'][job_group]['jobs']: + key = self.data['job_groups'][job_group]['template'].format(**job) + out[key] = job + return out + def init_headers(self): requests.packages.urllib3.disable_warnings() @@ -293,16 +305,12 @@ class Report(collections.UserList): Report is a list of Cases (i.e. test results) """ - TIERS = [1, 2, 3, 4] - RHELS = [6, 7] + def __init__(self, job_group=''): + # If job group is not specified, we want latest one + if job_group == '': + job_group = config.LATEST + self.job_group = job_group - def __init__(self): - # Initialize production.log instance - self.production_logs = {} - for tier in self.TIERS: - self.production_logs[tier] = {} - for rhel in self.RHELS: - self.production_logs[tier][rhel] = ProductionLog(tier, rhel) # If cache is configured, load data from it if config['cache']: if os.path.isfile(config['cache']): @@ -314,16 +322,18 @@ def __init__(self): logging.debug("Cache set to '{0}' but that file does not exist, creating one".format( config['cache'])) + # Load the actual data self.data = [] - for i in self.TIERS: - for j in self.RHELS: - for report in self.pull_reports( - config['job'].format(i, j), - config['bld']): - report['tier'] = 't{}'.format(i) - report['distro'] = 'el{}'.format(j) - report['OBJECT:production.log'] = self.production_logs[i][j] - self.data.append(Case(report)) + for name, meta in config.get_builds(self.job_group).items: + build = meta['build'] + rhel = meta['rhel'] + tier = meta['tier'] + production_log = ProductionLog(tier, rhel) # FIXME should we provide build as well? + for report in self.pull_reports(name, build): + report['tier'] = tier + report['distro'] = rhel + report['OBJECT:production.log'] = production_log + self.data.append(Case(report)) if config['cache']: pickle.dump(self.data, open(config['cache'], 'wb')) diff --git a/config.yaml.sample b/config.yaml.sample index c9b400a..6de2875 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -1,5 +1,19 @@ usr: jenkins_username pwd: jenkins_password url: https://jenkins.url -job: automation-6.2-tier{0}-rhel{1} -bld: lastCompletedBuild +job_groups: + latest: + template: automation-6.4-tier{tier}-rhel{rhel} + jobs: + - build: lastCompletedBuild + rhel: 7 + tier: 1 + - build: lastCompletedBuild + rhel: 7 + tier: 2 + - build: lastCompletedBuild + rhel: 7 + tier: 3 + - build: lastCompletedBuild + rhel: 7 + tier: 4 From c3e367e175240541476be1e65f79a76470c43305 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Sat, 7 Jul 2018 22:59:05 +0200 Subject: [PATCH 28/66] Use job/build to identify production.log --- claims.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/claims.py b/claims.py index 3c4f2f5..6382bd9 100755 --- a/claims.py +++ b/claims.py @@ -66,8 +66,8 @@ def init_headers(self): class ForemanDebug(object): - def __init__(self, tier, rhel): - self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], config['job'].format(tier, rhel), config['bld']) + def __init__(self, job, build): + self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) self._extracted = None @property @@ -100,14 +100,14 @@ class ProductionLog(object): DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 - def __init__(self, tier, rhel): + def __init__(self, job, build): self._log = None self._logfile = None self._cache = None if 'cache' in config: self._cache = '%s-t%s-el%s-production.log' \ - % (config['cache'].replace('.pickle', ''), tier, rhel) + % (config['cache'].replace('.pickle', ''), job, build) if self._cache and os.path.isfile(self._cache): self._logfile = self._cache logging.debug("Loading production.log from cached %s" % self._logfile) @@ -115,7 +115,7 @@ def __init__(self, tier, rhel): else: logging.debug("Cache for production.log (%s) set, but not available. Will create it if we have a chance" % self._cache) - self._foreman_debug = ForemanDebug(tier, rhel) + self._foreman_debug = ForemanDebug(job, build) @property def log(self): @@ -328,7 +328,7 @@ def __init__(self, job_group=''): build = meta['build'] rhel = meta['rhel'] tier = meta['tier'] - production_log = ProductionLog(tier, rhel) # FIXME should we provide build as well? + production_log = ProductionLog(name, build) for report in self.pull_reports(name, build): report['tier'] = tier report['distro'] = rhel From 4d352a03aa46e199b7c24d7591e730cc6109aa17 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 9 Jul 2018 10:40:12 +0200 Subject: [PATCH 29/66] Some useful code --- claimable.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/claimable.py b/claimable.py index 1226065..bf35332 100755 --- a/claimable.py +++ b/claimable.py @@ -6,4 +6,10 @@ report = claims.Report() rules = claims.Ruleset() +#report = [r for r in report if r['name'] == 'test_positive_create_with_puppet_class_id'] +#rules = [r for r in rules if r['reason'] == 'https://github.com/SatelliteQE/robottelo/issues/6115'] + claims.claim_by_rules(report, rules, dryrun=True) + +#for case in [i for i in report if i['status'] in claims.Case.FAIL_STATUSES]: +# print(case['url']) From d8ae238f388f85cec5b3704e0267b5f5e0a8eba4 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 9 Jul 2018 14:49:26 +0200 Subject: [PATCH 30/66] Fix script (some variables were overwritten by per tier values and tiers are not numbers) --- claimable.py | 4 ++-- claims.py | 2 +- claimstats.py | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/claimable.py b/claimable.py index bf35332..4a1d6dd 100755 --- a/claimable.py +++ b/claimable.py @@ -11,5 +11,5 @@ claims.claim_by_rules(report, rules, dryrun=True) -#for case in [i for i in report if i['status'] in claims.Case.FAIL_STATUSES]: -# print(case['url']) +for case in [i for i in report if i['status'] in claims.Case.FAIL_STATUSES]: + print(case['url']) diff --git a/claims.py b/claims.py index 6382bd9..2780099 100755 --- a/claims.py +++ b/claims.py @@ -324,7 +324,7 @@ def __init__(self, job_group=''): # Load the actual data self.data = [] - for name, meta in config.get_builds(self.job_group).items: + for name, meta in config.get_builds(self.job_group).items(): build = meta['build'] rhel = meta['rhel'] tier = meta['tier'] diff --git a/claimstats.py b/claimstats.py index 09a302f..1026f0a 100755 --- a/claimstats.py +++ b/claimstats.py @@ -18,14 +18,14 @@ floatfmt=".0f")) stats = [] -for t in ["t%s" % i for i in claims.Report.TIERS]: +for t in [i['tier'] for i in claims.config.get_builds().values()]: filtered = [r for r in reports if r['tier'] == t] - stat_all = len(filtered) - reports_fails = [i for i in filtered if i['status'] in claims.Case.FAIL_STATUSES] - stat_failed = len(reports_fails) - reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] - stat_claimed = len(reports_claimed) - stats.append([t, stat_all, stat_failed, stat_failed/stat_all*100, stat_claimed, stat_claimed/stat_failed*100]) + stat_all_tiered = len(filtered) + reports_fails_tiered = [i for i in filtered if i['status'] in claims.Case.FAIL_STATUSES] + stat_failed_tiered = len(reports_fails_tiered) + reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] + stat_claimed_tiered = len(reports_claimed_tiered) + stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, stat_failed_tiered/stat_all_tiered*100, stat_claimed_tiered, stat_claimed_tiered/stat_failed_tiered*100]) print("\nStats per tier") print(tabulate.tabulate( From 2ea66fcde3c5e793f51fc11d964fc695d012d8c3 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 10 Jul 2018 21:40:20 +0200 Subject: [PATCH 31/66] Fixed tests --- test_claims.py | 77 ++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/test_claims.py b/test_claims.py index 899fb12..2e7d5b5 100755 --- a/test_claims.py +++ b/test_claims.py @@ -3,41 +3,44 @@ import claims -checkme = { - 'greeting': 'Hello world', - 'area': 'IT Crowd', -} +def test_rule_matches(): + checkme = { + 'name': 'test', + 'greeting': 'Hello world', + 'area': 'IT Crowd', + } + result = claims.Case(checkme) -assert claims.rule_matches(checkme, {'field': 'greeting', 'pattern': 'Hel+o'}) == True -assert claims.rule_matches(checkme, {'field': 'greeting', 'pattern': 'This is not there'}) == False -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == False -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True -assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'This is not there'}]}]}) == False -assert claims.rule_matches(checkme, {'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False -assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True -assert claims.rule_matches(checkme, {'AND': [{'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'field': 'greeting', 'pattern': 'Hel+o'}) == True + assert result.matches_to_rule({'field': 'greeting', 'pattern': 'This is not there'}) == False + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}]}]}) == True + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == True + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == True + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}]}]}) == False + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}]}) == False + assert result.matches_to_rule({'AND': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}]}) == False + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'area', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'OR': [{'AND': [{'field': 'greeting', 'pattern': 'Hel+o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True + assert result.matches_to_rule({'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'IT'}]}]}) == True + assert result.matches_to_rule({'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'AND': [{'field': 'area', 'pattern': 'This is not there'}]}]}) == False + assert result.matches_to_rule({'OR': [{'AND': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'This is not there'}]}) == False + assert result.matches_to_rule({'AND': [{'OR': [{'field': 'greeting', 'pattern': 'Hel*o'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True + assert result.matches_to_rule({'AND': [{'OR': [{'field': 'greeting', 'pattern': 'This is not there'}, {'field': 'greeting', 'pattern': 'world'}]}, {'field': 'area', 'pattern': 'IT'}]}) == True From 235215ebbd60cca1079b5bd4fdee81f9e3a800d4 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 10 Jul 2018 22:44:20 +0200 Subject: [PATCH 32/66] Move tests and module --- claims.py => claims/__init__.py | 0 test/conftest.py | 6 ++++++ test_claims.py => test/test_claims.py | 0 3 files changed, 6 insertions(+) rename claims.py => claims/__init__.py (100%) create mode 100644 test/conftest.py rename test_claims.py => test/test_claims.py (100%) diff --git a/claims.py b/claims/__init__.py similarity index 100% rename from claims.py rename to claims/__init__.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..9dec13d --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/test_claims.py b/test/test_claims.py similarity index 100% rename from test_claims.py rename to test/test_claims.py From 9f4fdba3968d076f8bcaf725134fca6674e13c50 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 10 Jul 2018 23:05:10 +0200 Subject: [PATCH 33/66] Tabulate is needed --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6c9fdba..8a2c822 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ PyYAML requests +tabulate From df5b6acd2c4725259375905827ff759807a8c005 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 10 Jul 2018 23:55:35 +0200 Subject: [PATCH 34/66] Started on cli tool --- claims/__init__.py | 390 ---------------------------------------- claims/cmd.py | 135 ++++++++++++++ claims/lib.py | 392 +++++++++++++++++++++++++++++++++++++++++ test/test_claimscmd.py | 22 +++ 4 files changed, 549 insertions(+), 390 deletions(-) create mode 100755 claims/cmd.py create mode 100755 claims/lib.py create mode 100644 test/test_claimscmd.py diff --git a/claims/__init__.py b/claims/__init__.py index 2780099..e69de29 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -1,390 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import division -import os -import sys -import json -import logging -import re -import requests -import yaml -import pickle -import collections -import datetime -import tempfile -import subprocess -import shutil - -logging.basicConfig(level=logging.INFO) - -class Config(collections.UserDict): - - LATEST = 'latest' # how do we call latest job group in the config? - - def __init__(self): - with open("config.yaml", "r") as file: - self.data = yaml.load(file) - - # If cache is configured, save it into configuration - if 'DEBUG_CLAIMS_CACHE' in os.environ: - self.data['cache'] = os.environ['DEBUG_CLAIMS_CACHE'] - else: - self.data['cache'] = None - - # Additional params when talking to Jenkins - self['headers'] = None - self['pull_params'] = { - u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' - } - - def get_builds(self, job_group=''): - if job_group == '': - job_group = self.LATEST - out = collections.OrderedDict() - for job in self.data['job_groups'][job_group]['jobs']: - key = self.data['job_groups'][job_group]['template'].format(**job) - out[key] = job - return out - - def init_headers(self): - requests.packages.urllib3.disable_warnings() - - # Get the Jenkins crumb (csrf protection) - crumb_request = requests.get( - '{0}/crumbIssuer/api/json'.format(self['url']), - auth=requests.auth.HTTPBasicAuth(self['usr'], self['pwd']), - verify=False - ) - - if crumb_request.status_code != 200: - raise requests.HTTPError( - 'Failed to obtain crumb: {0}'.format(crumb_request.reason)) - - crumb = json.loads(crumb_request.text) - self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} - - -class ForemanDebug(object): - - def __init__(self, job, build): - self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) - self._extracted = None - - @property - def extracted(self): - if self._extracted is None: - logging.debug('Going to download %s' % self._url) - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as localfile: - logging.debug('Going to save to %s' % localfile.name) - self._download_file(localfile, self._url) - self._tmpdir = tempfile.TemporaryDirectory() - subprocess.call(['tar', '-xf', localfile.name, '--directory', self._tmpdir.name]) - logging.debug('Extracted to %s' % self._tmpdir.name) - self._extracted = os.path.join(self._tmpdir.name, 'foreman-debug') - return self._extracted - - def _download_file(self, localfile, url): - r = requests.get(url, stream=True) - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - localfile.write(chunk) - if r.status_code != 200: - raise requests.HTTPError("Failed to get foreman-debug %s" % url) - localfile.close() - logging.debug('File %s saved to %s' % (url, localfile.name)) - - -class ProductionLog(object): - - FILE_ENCODING = 'ISO-8859-1' # guessed, that wile contains ugly binary mess as well - DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 - DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 - - def __init__(self, job, build): - self._log = None - self._logfile = None - self._cache = None - - if 'cache' in config: - self._cache = '%s-t%s-el%s-production.log' \ - % (config['cache'].replace('.pickle', ''), job, build) - if self._cache and os.path.isfile(self._cache): - self._logfile = self._cache - logging.debug("Loading production.log from cached %s" % self._logfile) - return None - else: - logging.debug("Cache for production.log (%s) set, but not available. Will create it if we have a chance" % self._cache) - - self._foreman_debug = ForemanDebug(job, build) - - @property - def log(self): - if self._log is None: - if self._logfile is None: - self._logfile = os.path.join(self._foreman_debug.extracted, - 'var', 'log', 'foreman', 'production.log') - self._log = [] - buf = [] - last = None - with open(self._logfile, 'r', encoding=self.FILE_ENCODING) as fp: - for line in fp: - - # This line starts with date - denotes first line of new log record - if re.search(self.DATE_REGEXP, line): - - # This is a new log record, so firs save previous one - if len(buf) != 0: - self._log.append({'time': last, 'data': buf}) - last = datetime.datetime.strptime(line[:19], self.DATE_FMT) - buf = [] - buf.append(re.sub(self.DATE_REGEXP, '', line, count=1)) - - # This line does not start with line - comtains continuation of a log recorder started before - else: - buf.append(line) - - # Save last line - if len(buf) != 0: - self._log.append({'time': last, 'data': buf}) - - # Cache file we have downloaded - if self._cache and not os.path.isfile(self._cache): - logging.debug("Caching production.log %s to %s" % (self._logfile, self._cache)) - shutil.copyfile(self._logfile, self._cache) - - logging.debug("File %s parsed into memory and deleted" % self._logfile) - return self._log - - def from_to(self, from_time, to_time): - out = [] - for i in self.log: - if from_time <= i['time'] <= to_time: - out.append(i) - # Do not do following as time is not sequentional in the log (or maybe some workers are off or with different TZ?): - # TODO: Fix ordering of the log and uncomment this - # - # E.g.: - # 2018-06-17T17:29:44 [I|dyn|] start terminating clock... - # 2018-06-17T21:34:49 [I|app|] Current user: foreman_admin (administrator) - # 2018-06-17T21:37:21 [...] - # 2018-06-17T17:41:38 [I|app|] Started POST "/katello/api/v2/organizations"... - # - #if i['time'] > to_time: - # break - return out - - -class Case(collections.UserDict): - """ - Result of one test case - """ - - FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") - LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') - LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' - - def __init__(self, data): - self.data = data - - def __contains__(self, name): - return name in self.data or name in ('start', 'end', 'production.log') - - def __getitem__(self, name): - if name in ('start', 'end') and \ - ('start' not in self.data or 'end' not in self.data): - self.load_timings() - if name == 'production.log': - self['production.log'] = "\n".join( - ["\n".join(i['data']) for i in - self.data['OBJECT:production.log'].from_to( - self['start'], self['end'])]) - return self.data[name] - - def matches_to_rule(self, rule, indentation=0): - """ - Returns True if result matches to rule, otherwise returns False - """ - logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self['name'], rule, indentation)) - if 'field' in rule and 'pattern' in rule: - # This is simple rule, we can just check regexp against given field and we are done - try: - data = self[rule['field']] - if data is None: - data = '' - out = re.search(rule['pattern'], data) is not None - logging.debug("%s=> %s" % (" "*indentation, out)) - return out - except KeyError: - logging.debug("%s=> Failed to get field %s from case" % (" "*indentation, rule['field'])) - return None - elif 'AND' in rule: - # We need to check if all sub-rules in list of rules rule['AND'] matches - out = None - for r in rule['AND']: - r_out = self.matches_to_rule(r, indentation+4) - out = r_out if out is None else out and r_out - if not out: - break - return out - elif 'OR' in rule: - # We need to check if at least one sub-rule in list of rules rule['OR'] matches - for r in rule['OR']: - if self.matches_to_rule(r, indentation+4): - return True - return False - else: - raise Exception('Rule %s not formatted correctly' % rule) - - def push_claim(self, reason, sticky=False, propagate=False): - '''Claims a given test with a given reason - - :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) - - :param sticky: whether to make the claim sticky (False by default) - - :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) - ''' - logging.info('claiming {0}::{1} with reason: {2}'.format(self["className"], self["name"], reason)) - - if config['headers'] is None: - config.init_headers() - - claim_req = requests.post( - u'{0}/claim/claim'.format(self['url']), - auth=requests.auth.HTTPBasicAuth( - config['usr'], - config['pwd'] - ), - data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, - headers=config['headers'], - allow_redirects=False, - verify=False - ) - - if claim_req.status_code != 302: - raise requests.HTTPError( - 'Failed to claim: {0}'.format(claim_req)) - - self['testActions'][0]['reason'] = reason - return(claim_req) - - def load_timings(self): - if self['stdout'] is None: - return - log = self['stdout'].split("\n") - log_size = len(log) - log_used = 0 - start = None - end = None - counter = 0 - while start is None: - match = self.LOG_DATE_REGEXP.match(log[counter]) - if match: - start = datetime.datetime.strptime(match.group(1), - self.LOG_DATE_FORMAT) - break - counter += 1 - log_used += counter - counter = -1 - while end is None: - match = self.LOG_DATE_REGEXP.match(log[counter]) - if match: - end = datetime.datetime.strptime(match.group(1), - self.LOG_DATE_FORMAT) - break - counter -= 1 - log_used -= counter - assert log_used <= log_size, \ - "Make sure detected start date is not below end date and vice versa" - self['start'] = start - self['end'] = end - - -class Report(collections.UserList): - """ - Report is a list of Cases (i.e. test results) - """ - - def __init__(self, job_group=''): - # If job group is not specified, we want latest one - if job_group == '': - job_group = config.LATEST - self.job_group = job_group - - # If cache is configured, load data from it - if config['cache']: - if os.path.isfile(config['cache']): - logging.debug("Because cache is set to '{0}', loading data from there".format( - config['cache'])) - self.data = pickle.load(open(config['cache'], 'rb')) - return - else: - logging.debug("Cache set to '{0}' but that file does not exist, creating one".format( - config['cache'])) - - # Load the actual data - self.data = [] - for name, meta in config.get_builds(self.job_group).items(): - build = meta['build'] - rhel = meta['rhel'] - tier = meta['tier'] - production_log = ProductionLog(name, build) - for report in self.pull_reports(name, build): - report['tier'] = tier - report['distro'] = rhel - report['OBJECT:production.log'] = production_log - self.data.append(Case(report)) - - if config['cache']: - pickle.dump(self.data, open(config['cache'], 'wb')) - - def pull_reports(self, job, build): - """ - Fetches the test report for a given job and build - """ - build_url = '{0}/job/{1}/{2}'.format( - config['url'], job, build) - - logging.debug("Getting {}".format(build_url)) - bld_req = requests.get( - build_url + '/testReport/api/json', - auth=requests.auth.HTTPBasicAuth( - config['usr'], config['pwd']), - params=config['pull_params'], - verify=False - ) - - if bld_req.status_code == 404: - return [] - if bld_req.status_code != 200: - raise requests.HTTPError( - 'Failed to obtain: {0}'.format(bld_req)) - - cases = json.loads(bld_req.text)['suites'][0]['cases'] - - # Enrich individual reports with URL - for c in cases: - className = c['className'].split('.')[-1] - testPath = '.'.join(c['className'].split('.')[:-1]) - c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) - - return(cases) - - -class Ruleset(collections.UserList): - - def __init__(self): - with open('kb.json', 'r') as fp: - self.data = json.loads(fp.read()) - - -# Create shared config file -config = Config() - -def claim_by_rules(report, rules, dryrun=False): - for rule in rules: - for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: - if case.matches_to_rule(rule): - logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) - if not dryrun: - case.push_claim(rule['reason']) diff --git a/claims/cmd.py b/claims/cmd.py new file mode 100755 index 0000000..e58e4fd --- /dev/null +++ b/claims/cmd.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import logging +import argparse +import re +import tabulate + +import lib + +logging.basicConfig(level=logging.INFO) + +class ClaimsCli(object): + + LATEST = 'latest' + + def __init__(self): + self.job_group = self.LATEST + self.grep_results = None + self.grep_rules = None + self._results = None + self._rules = None + + @property + def results(self): + if not self._results: + self._results = lib.Report() + if self.grep_results: + self._results = [r for r in self._results if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] + return self._results + + def show_failed(self): + print(tabulate.tabulate( + [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES], + headers=['failed test name'], tablefmt=self.output)) + + def show_claimed(self): + print(tabulate.tabulate( + [[r['testName'], r['testActions'][0].get('reason')] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], + headers=['claimed test name', 'claim reason'], tablefmt=self.output)) + + def show_unclaimed(self): + print(tabulate.tabulate( + [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], + headers=['unclaimed test name'], tablefmt=self.output)) + + def handle_args(self): + parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') + + # Actions + parser.add_argument('--clean-cache', action='store_true', + help='Cleans cache for latest job group. If you want to clean cache for older job group, use rm in .cache directory') + parser.add_argument('--show-failed', action='store_true', + help='Show all failed tests') + parser.add_argument('--show-claimed', action='store_true', + help='Show claimed tests') + parser.add_argument('--show-unclaimed', action='store_true', + help='Show failed and not yet claimed tests') + parser.add_argument('--show-claimable', action='store_true', + help='Show failed, not yet claimed but claimable tests') + parser.add_argument('--show', action='store', + help='Show detailed info about given test case') + parser.add_argument('--claim', action='store_true', + help='Claim claimable tests') + parser.add_argument('--stats', action='store_true', + help='Show stats for selected job group') + parser.add_argument('--history', action='store_true', + help='Show how tests results and duration evolved') + parser.add_argument('--timegraph', action='store_true', + help='Generate time graph') + + # Modifiers + parser.add_argument('--job-group', action='store', + help='Specify group of jobs to perform the action with (default: latest)') + parser.add_argument('--grep-results', action='store', metavar='REGEXP', + help='Only work with tests, whose "className+name" matches the regexp') + parser.add_argument('--grep-rules', action='store', metavar='REGEXP', + help='Only work with rules, whose reason matches the regexp') + parser.add_argument('--output', action='store', choices=['simple', 'csv', 'html'], default='simple', + help='Format tables as plain, csv or html (default: simple)') + parser.add_argument('-d', '--debug', action='store_true', + help='Show also debug messages') + + args = parser.parse_args() + print(args) + + # Handle "--debug" + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + logging.debug("Debug mode enabled") + + # Handle "--job-group something" + if args.job_group: + self.job_group = args.job_group + logging.debug("Job group we are going to work with is %s" % self.job_group) + + # Handle "--grep-results something" + if args.grep_results: + self.grep_results = args.grep_results + logging.debug("Going to consider only results matching %s" % self.grep_results) + + # Handle "--grep-rules something" + if args.grep_rules: + self.grep_rules = args.grep_rules + logging.debug("Going to consider only rules matching %s" % self.grep_rules) + + # Handle "--output something" + self.output = args.output + logging.debug("Using output type %s" % self.output) + + # Actions + + # Show failed + if args.show_failed: + self.show_failed() + return 0 + + # Show claimed + if args.show_claimed: + self.show_claimed() + return 0 + + # Show unclaimed + if args.show_unclaimed: + self.show_unclaimed() + return 0 + + return 0 + +def main(): + """Main program""" + return ClaimsCli().handle_args() + +if __name__ == "__main__": + main() diff --git a/claims/lib.py b/claims/lib.py new file mode 100755 index 0000000..e647f8d --- /dev/null +++ b/claims/lib.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 + +from __future__ import division +import os +import sys +import json +import logging +import re +import requests +import yaml +import pickle +import collections +import datetime +import tempfile +import subprocess +import shutil + +logging.basicConfig(level=logging.INFO) + +class Config(collections.UserDict): + + LATEST = 'latest' # how do we call latest job group in the config? + + def __init__(self): + with open("config.yaml", "r") as file: + self.data = yaml.load(file) + + # If cache is configured, save it into configuration + if 'DEBUG_CLAIMS_CACHE' in os.environ: + self.data['cache'] = os.environ['DEBUG_CLAIMS_CACHE'] + else: + self.data['cache'] = None + + # Additional params when talking to Jenkins + self['headers'] = None + self['pull_params'] = { + u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' + } + + def get_builds(self, job_group=''): + if job_group == '': + job_group = self.LATEST + out = collections.OrderedDict() + for job in self.data['job_groups'][job_group]['jobs']: + key = self.data['job_groups'][job_group]['template'].format(**job) + out[key] = job + return out + + def init_headers(self): + requests.packages.urllib3.disable_warnings() + + # Get the Jenkins crumb (csrf protection) + crumb_request = requests.get( + '{0}/crumbIssuer/api/json'.format(self['url']), + auth=requests.auth.HTTPBasicAuth(self['usr'], self['pwd']), + verify=False + ) + + if crumb_request.status_code != 200: + raise requests.HTTPError( + 'Failed to obtain crumb: {0}'.format(crumb_request.reason)) + + crumb = json.loads(crumb_request.text) + self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} + + +class ForemanDebug(object): + + def __init__(self, job, build): + self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) + self._extracted = None + + @property + def extracted(self): + if self._extracted is None: + logging.debug('Going to download %s' % self._url) + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as localfile: + logging.debug('Going to save to %s' % localfile.name) + self._download_file(localfile, self._url) + self._tmpdir = tempfile.TemporaryDirectory() + subprocess.call(['tar', '-xf', localfile.name, '--directory', self._tmpdir.name]) + logging.debug('Extracted to %s' % self._tmpdir.name) + self._extracted = os.path.join(self._tmpdir.name, 'foreman-debug') + return self._extracted + + def _download_file(self, localfile, url): + r = requests.get(url, stream=True) + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + localfile.write(chunk) + if r.status_code != 200: + raise requests.HTTPError("Failed to get foreman-debug %s" % url) + localfile.close() + logging.debug('File %s saved to %s' % (url, localfile.name)) + + +class ProductionLog(object): + + FILE_ENCODING = 'ISO-8859-1' # guessed, that wile contains ugly binary mess as well + DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 + DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 + + def __init__(self, job, build): + self._log = None + self._logfile = None + self._cache = None + + if 'cache' in config: + self._cache = '%s-t%s-el%s-production.log' \ + % (config['cache'].replace('.pickle', ''), job, build) + if self._cache and os.path.isfile(self._cache): + self._logfile = self._cache + logging.debug("Loading production.log from cached %s" % self._logfile) + return None + else: + logging.debug("Cache for production.log (%s) set, but not available. Will create it if we have a chance" % self._cache) + + self._foreman_debug = ForemanDebug(job, build) + + @property + def log(self): + if self._log is None: + if self._logfile is None: + self._logfile = os.path.join(self._foreman_debug.extracted, + 'var', 'log', 'foreman', 'production.log') + self._log = [] + buf = [] + last = None + with open(self._logfile, 'r', encoding=self.FILE_ENCODING) as fp: + for line in fp: + + # This line starts with date - denotes first line of new log record + if re.search(self.DATE_REGEXP, line): + + # This is a new log record, so firs save previous one + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + last = datetime.datetime.strptime(line[:19], self.DATE_FMT) + buf = [] + buf.append(re.sub(self.DATE_REGEXP, '', line, count=1)) + + # This line does not start with line - comtains continuation of a log recorder started before + else: + buf.append(line) + + # Save last line + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + + # Cache file we have downloaded + if self._cache and not os.path.isfile(self._cache): + logging.debug("Caching production.log %s to %s" % (self._logfile, self._cache)) + shutil.copyfile(self._logfile, self._cache) + + logging.debug("File %s parsed into memory and deleted" % self._logfile) + return self._log + + def from_to(self, from_time, to_time): + out = [] + for i in self.log: + if from_time <= i['time'] <= to_time: + out.append(i) + # Do not do following as time is not sequentional in the log (or maybe some workers are off or with different TZ?): + # TODO: Fix ordering of the log and uncomment this + # + # E.g.: + # 2018-06-17T17:29:44 [I|dyn|] start terminating clock... + # 2018-06-17T21:34:49 [I|app|] Current user: foreman_admin (administrator) + # 2018-06-17T21:37:21 [...] + # 2018-06-17T17:41:38 [I|app|] Started POST "/katello/api/v2/organizations"... + # + #if i['time'] > to_time: + # break + return out + + +class Case(collections.UserDict): + """ + Result of one test case + """ + + FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") + LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') + LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' + + def __init__(self, data): + self.data = data + + def __contains__(self, name): + return name in self.data or name in ('start', 'end', 'production.log') + + def __getitem__(self, name): + if name == 'testName': + self['testName'] = "%s.%s" % (self['className'], self['name']) + if name in ('start', 'end') and \ + ('start' not in self.data or 'end' not in self.data): + self.load_timings() + if name == 'production.log': + self['production.log'] = "\n".join( + ["\n".join(i['data']) for i in + self.data['OBJECT:production.log'].from_to( + self['start'], self['end'])]) + return self.data[name] + + def matches_to_rule(self, rule, indentation=0): + """ + Returns True if result matches to rule, otherwise returns False + """ + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self['name'], rule, indentation)) + if 'field' in rule and 'pattern' in rule: + # This is simple rule, we can just check regexp against given field and we are done + try: + data = self[rule['field']] + if data is None: + data = '' + out = re.search(rule['pattern'], data) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + except KeyError: + logging.debug("%s=> Failed to get field %s from case" % (" "*indentation, rule['field'])) + return None + elif 'AND' in rule: + # We need to check if all sub-rules in list of rules rule['AND'] matches + out = None + for r in rule['AND']: + r_out = self.matches_to_rule(r, indentation+4) + out = r_out if out is None else out and r_out + if not out: + break + return out + elif 'OR' in rule: + # We need to check if at least one sub-rule in list of rules rule['OR'] matches + for r in rule['OR']: + if self.matches_to_rule(r, indentation+4): + return True + return False + else: + raise Exception('Rule %s not formatted correctly' % rule) + + def push_claim(self, reason, sticky=False, propagate=False): + '''Claims a given test with a given reason + + :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) + + :param sticky: whether to make the claim sticky (False by default) + + :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) + ''' + logging.info('claiming {0}::{1} with reason: {2}'.format(self["className"], self["name"], reason)) + + if config['headers'] is None: + config.init_headers() + + claim_req = requests.post( + u'{0}/claim/claim'.format(self['url']), + auth=requests.auth.HTTPBasicAuth( + config['usr'], + config['pwd'] + ), + data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, + headers=config['headers'], + allow_redirects=False, + verify=False + ) + + if claim_req.status_code != 302: + raise requests.HTTPError( + 'Failed to claim: {0}'.format(claim_req)) + + self['testActions'][0]['reason'] = reason + return(claim_req) + + def load_timings(self): + if self['stdout'] is None: + return + log = self['stdout'].split("\n") + log_size = len(log) + log_used = 0 + start = None + end = None + counter = 0 + while start is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + start = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter += 1 + log_used += counter + counter = -1 + while end is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + end = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter -= 1 + log_used -= counter + assert log_used <= log_size, \ + "Make sure detected start date is not below end date and vice versa" + self['start'] = start + self['end'] = end + + +class Report(collections.UserList): + """ + Report is a list of Cases (i.e. test results) + """ + + def __init__(self, job_group=''): + # If job group is not specified, we want latest one + if job_group == '': + job_group = config.LATEST + self.job_group = job_group + + # If cache is configured, load data from it + if config['cache']: + if os.path.isfile(config['cache']): + logging.debug("Because cache is set to '{0}', loading data from there".format( + config['cache'])) + self.data = pickle.load(open(config['cache'], 'rb')) + return + else: + logging.debug("Cache set to '{0}' but that file does not exist, creating one".format( + config['cache'])) + + # Load the actual data + self.data = [] + for name, meta in config.get_builds(self.job_group).items(): + build = meta['build'] + rhel = meta['rhel'] + tier = meta['tier'] + production_log = ProductionLog(name, build) + for report in self.pull_reports(name, build): + report['tier'] = tier + report['distro'] = rhel + report['OBJECT:production.log'] = production_log + self.data.append(Case(report)) + + if config['cache']: + pickle.dump(self.data, open(config['cache'], 'wb')) + + def pull_reports(self, job, build): + """ + Fetches the test report for a given job and build + """ + build_url = '{0}/job/{1}/{2}'.format( + config['url'], job, build) + + logging.debug("Getting {}".format(build_url)) + bld_req = requests.get( + build_url + '/testReport/api/json', + auth=requests.auth.HTTPBasicAuth( + config['usr'], config['pwd']), + params=config['pull_params'], + verify=False + ) + + if bld_req.status_code == 404: + return [] + if bld_req.status_code != 200: + raise requests.HTTPError( + 'Failed to obtain: {0}'.format(bld_req)) + + cases = json.loads(bld_req.text)['suites'][0]['cases'] + + # Enrich individual reports with URL + for c in cases: + className = c['className'].split('.')[-1] + testPath = '.'.join(c['className'].split('.')[:-1]) + c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) + + return(cases) + + +class Ruleset(collections.UserList): + + def __init__(self): + with open('kb.json', 'r') as fp: + self.data = json.loads(fp.read()) + + +# Create shared config file +config = Config() + +def claim_by_rules(report, rules, dryrun=False): + for rule in rules: + for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: + if case.matches_to_rule(rule): + logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) + if not dryrun: + case.push_claim(rule['reason']) diff --git a/test/test_claimscmd.py b/test/test_claimscmd.py new file mode 100644 index 0000000..3f58fbf --- /dev/null +++ b/test/test_claimscmd.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import sys +import pytest + +import io +from contextlib import redirect_stdout + +import claims.cmd + +class TestClaimsCli(object): + + def test_help(self): + sys.argv = ['./something.py', '--help'] + f = io.StringIO() + with pytest.raises(SystemExit) as e: + with redirect_stdout(f): + claims.cmd.main() + assert e.value.code == 0 + assert 'Manipulate Jenkins claims with grace' in f.getvalue() + assert 'optional arguments:' in f.getvalue() From 61a252ddca12ebcd16ad27ec497cd4d109955062 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 11 Jul 2018 19:06:38 +0200 Subject: [PATCH 35/66] Pass job group to Report object --- claims/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index e58e4fd..bde2a82 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -24,7 +24,7 @@ def __init__(self): @property def results(self): if not self._results: - self._results = lib.Report() + self._results = lib.Report(self.job_group) if self.grep_results: self._results = [r for r in self._results if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] return self._results From e0bbec95742fc6314c756eb88763535fcdb8cea4 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 11 Jul 2018 20:32:42 +0200 Subject: [PATCH 36/66] Create a helper function for GET requests --- claims/lib.py | 97 ++++++++++++++++++++++--------------------- test/test_requests.py | 23 ++++++++++ 2 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 test/test_requests.py diff --git a/claims/lib.py b/claims/lib.py index e647f8d..02ce230 100755 --- a/claims/lib.py +++ b/claims/lib.py @@ -6,6 +6,7 @@ import json import logging import re +import urllib3 import requests import yaml import pickle @@ -15,8 +16,39 @@ import subprocess import shutil +CACHEDIR = '.cache/' + logging.basicConfig(level=logging.INFO) +def request_get(url, params=None, expected_codes=[200], cached=True): + # If available, read it from cache + if cached and os.path.isfile(cached): + with open(cached, 'r') as fp: + return fp.read() + + # Get the response from the server + urllib3.disable_warnings() + response = requests.get( + url, + auth=requests.auth.HTTPBasicAuth( + config['usr'], config['pwd']), + params=params, + verify=False + ) + if response.status_code not in expected_codes: + raise requests.HTTPError("Failed to get %s with %s" % (url, response.status_code)) + if response.status_code == 404: + return [] + + # If cache was configured, dump data in there + if cached: + os.makedirs(os.path.dirname(cached), exist_ok=True) + with open(cached, 'w') as fp: + fp.write(response.text) + + return response.text + + class Config(collections.UserDict): LATEST = 'latest' # how do we call latest job group in the config? @@ -25,12 +57,6 @@ def __init__(self): with open("config.yaml", "r") as file: self.data = yaml.load(file) - # If cache is configured, save it into configuration - if 'DEBUG_CLAIMS_CACHE' in os.environ: - self.data['cache'] = os.environ['DEBUG_CLAIMS_CACHE'] - else: - self.data['cache'] = None - # Additional params when talking to Jenkins self['headers'] = None self['pull_params'] = { @@ -47,20 +73,9 @@ def get_builds(self, job_group=''): return out def init_headers(self): - requests.packages.urllib3.disable_warnings() - - # Get the Jenkins crumb (csrf protection) - crumb_request = requests.get( - '{0}/crumbIssuer/api/json'.format(self['url']), - auth=requests.auth.HTTPBasicAuth(self['usr'], self['pwd']), - verify=False - ) - - if crumb_request.status_code != 200: - raise requests.HTTPError( - 'Failed to obtain crumb: {0}'.format(crumb_request.reason)) - - crumb = json.loads(crumb_request.text) + url = '{0}/crumbIssuer/api/json'.format(self['url']) + crumb_data = request_get(url, params=None, expected_codes=[200], cached=False) + crumb = json.loads(crumb_data) self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} @@ -302,6 +317,8 @@ def load_timings(self): self['end'] = end + + class Report(collections.UserList): """ Report is a list of Cases (i.e. test results) @@ -312,17 +329,12 @@ def __init__(self, job_group=''): if job_group == '': job_group = config.LATEST self.job_group = job_group + self.cache = os.path.join(CACHEDIR, self.job_group, 'main.pickle') - # If cache is configured, load data from it - if config['cache']: - if os.path.isfile(config['cache']): - logging.debug("Because cache is set to '{0}', loading data from there".format( - config['cache'])) - self.data = pickle.load(open(config['cache'], 'rb')) - return - else: - logging.debug("Cache set to '{0}' but that file does not exist, creating one".format( - config['cache'])) + # Attempt to load data from cache + if os.path.isfile(self.cache): + self.data = pickle.load(open(self.cache, 'rb')) + return # Load the actual data self.data = [] @@ -337,8 +349,8 @@ def __init__(self, job_group=''): report['OBJECT:production.log'] = production_log self.data.append(Case(report)) - if config['cache']: - pickle.dump(self.data, open(config['cache'], 'wb')) + # Dump parsed data into cache + pickle.dump(self.data, open(self.cache, 'wb')) def pull_reports(self, job, build): """ @@ -346,23 +358,12 @@ def pull_reports(self, job, build): """ build_url = '{0}/job/{1}/{2}'.format( config['url'], job, build) - - logging.debug("Getting {}".format(build_url)) - bld_req = requests.get( - build_url + '/testReport/api/json', - auth=requests.auth.HTTPBasicAuth( - config['usr'], config['pwd']), + build_data = request_get( + build_url+'/testReport/api/json', params=config['pull_params'], - verify=False - ) - - if bld_req.status_code == 404: - return [] - if bld_req.status_code != 200: - raise requests.HTTPError( - 'Failed to obtain: {0}'.format(bld_req)) - - cases = json.loads(bld_req.text)['suites'][0]['cases'] + expected_codes=[200, 404], + cached=os.path.join(CACHEDIR, self.job_group, job, 'main.json')) + cases = json.loads(build_data)['suites'][0]['cases'] # Enrich individual reports with URL for c in cases: diff --git a/test/test_requests.py b/test/test_requests.py new file mode 100644 index 0000000..8c3200e --- /dev/null +++ b/test/test_requests.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import requests +import tempfile +import pytest + +import claims.lib + +class TestClaimsRequestWrapper(): + + def test_get_sanity(self): + a = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=False) + b = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[200], cached=False) + assert a == b + with pytest.raises(requests.HTTPError) as e: + claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[404], cached=False) + + def test_get_caching(self): + fp, fname = tempfile.mkstemp() + a = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) + b = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) + assert a == b From 5ba22f09fdc537c157a560c1249411a7bdf2f129 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 11 Jul 2018 21:41:56 +0200 Subject: [PATCH 37/66] Now handles foreman-debug/production.log --- claims/lib.py | 92 +++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/claims/lib.py b/claims/lib.py index 02ce230..1647469 100755 --- a/claims/lib.py +++ b/claims/lib.py @@ -20,9 +20,9 @@ logging.basicConfig(level=logging.INFO) -def request_get(url, params=None, expected_codes=[200], cached=True): +def request_get(url, params=None, expected_codes=[200], cached=True, stream=False): # If available, read it from cache - if cached and os.path.isfile(cached): + if cached and not stream and os.path.isfile(cached): with open(cached, 'r') as fp: return fp.read() @@ -35,10 +35,23 @@ def request_get(url, params=None, expected_codes=[200], cached=True): params=params, verify=False ) + + # Check we got expected exit code if response.status_code not in expected_codes: raise requests.HTTPError("Failed to get %s with %s" % (url, response.status_code)) + + # If we were streaming file + if stream: + with open(cached, 'w+b') as fp: + for chunk in response.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + fp.write(chunk) + fp.close() + return + + # In some cases 404 just means "we have nothing" if response.status_code == 404: - return [] + return '' # If cache was configured, dump data in there if cached: @@ -81,63 +94,45 @@ def init_headers(self): class ForemanDebug(object): - def __init__(self, job, build): + def __init__(self, job_group, job, build): self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) self._extracted = None @property def extracted(self): if self._extracted is None: - logging.debug('Going to download %s' % self._url) - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as localfile: - logging.debug('Going to save to %s' % localfile.name) - self._download_file(localfile, self._url) - self._tmpdir = tempfile.TemporaryDirectory() - subprocess.call(['tar', '-xf', localfile.name, '--directory', self._tmpdir.name]) - logging.debug('Extracted to %s' % self._tmpdir.name) - self._extracted = os.path.join(self._tmpdir.name, 'foreman-debug') + fp, fname = tempfile.mkstemp() + print(fname) + request_get(self._url, cached=fname, stream=True) + tmpdir = tempfile.mkdtemp() + subprocess.call(['tar', '-xf', fname, '--directory', tmpdir]) + logging.debug('Extracted to %s' % tmpdir) + self._extracted = os.path.join(tmpdir, 'foreman-debug') return self._extracted - def _download_file(self, localfile, url): - r = requests.get(url, stream=True) - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - localfile.write(chunk) - if r.status_code != 200: - raise requests.HTTPError("Failed to get foreman-debug %s" % url) - localfile.close() - logging.debug('File %s saved to %s' % (url, localfile.name)) - class ProductionLog(object): - FILE_ENCODING = 'ISO-8859-1' # guessed, that wile contains ugly binary mess as well + FILE_ENCODING = 'ISO-8859-1' # guessed it only, that file contains ugly binary mess as well DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 - def __init__(self, job, build): + def __init__(self, job_group, job, build): self._log = None - self._logfile = None - self._cache = None - - if 'cache' in config: - self._cache = '%s-t%s-el%s-production.log' \ - % (config['cache'].replace('.pickle', ''), job, build) - if self._cache and os.path.isfile(self._cache): - self._logfile = self._cache - logging.debug("Loading production.log from cached %s" % self._logfile) - return None - else: - logging.debug("Cache for production.log (%s) set, but not available. Will create it if we have a chance" % self._cache) + self._logfile = os.path.join(CACHEDIR, job_group, job, 'production.log') + self._foreman_debug = None - self._foreman_debug = ForemanDebug(job, build) + # If we do not have logfile downloaded already, we will need foreman-debug + if not os.path.isfile(self._logfile): + self._foreman_debug = ForemanDebug(job_group, job, build) @property def log(self): if self._log is None: - if self._logfile is None: - self._logfile = os.path.join(self._foreman_debug.extracted, - 'var', 'log', 'foreman', 'production.log') + if self._foreman_debug is not None: + a = os.path.join(self._foreman_debug.extracted, + 'var', 'log', 'foreman', 'production.log') + shutil.copy2(a, self._logfile) self._log = [] buf = [] last = None @@ -162,12 +157,7 @@ def log(self): if len(buf) != 0: self._log.append({'time': last, 'data': buf}) - # Cache file we have downloaded - if self._cache and not os.path.isfile(self._cache): - logging.debug("Caching production.log %s to %s" % (self._logfile, self._cache)) - shutil.copyfile(self._logfile, self._cache) - - logging.debug("File %s parsed into memory and deleted" % self._logfile) + logging.debug("File %s parsed into memory" % self._logfile) return self._log def from_to(self, from_time, to_time): @@ -329,11 +319,11 @@ def __init__(self, job_group=''): if job_group == '': job_group = config.LATEST self.job_group = job_group - self.cache = os.path.join(CACHEDIR, self.job_group, 'main.pickle') + self._cache = os.path.join(CACHEDIR, self.job_group, 'main.pickle') # Attempt to load data from cache - if os.path.isfile(self.cache): - self.data = pickle.load(open(self.cache, 'rb')) + if os.path.isfile(self._cache): + self.data = pickle.load(open(self._cache, 'rb')) return # Load the actual data @@ -342,7 +332,7 @@ def __init__(self, job_group=''): build = meta['build'] rhel = meta['rhel'] tier = meta['tier'] - production_log = ProductionLog(name, build) + production_log = ProductionLog(self.job_group, name, build) for report in self.pull_reports(name, build): report['tier'] = tier report['distro'] = rhel @@ -350,7 +340,7 @@ def __init__(self, job_group=''): self.data.append(Case(report)) # Dump parsed data into cache - pickle.dump(self.data, open(self.cache, 'wb')) + pickle.dump(self.data, open(self._cache, 'wb')) def pull_reports(self, job, build): """ From 237498dbba7fe3f22bd39d4a3fa9aba8b1bf1137 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 11 Jul 2018 21:59:21 +0200 Subject: [PATCH 38/66] Added possibility to print test details --- claims/cmd.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/claims/cmd.py b/claims/cmd.py index bde2a82..e3044dd 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -44,6 +44,32 @@ def show_unclaimed(self): [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output)) + def show(self, test_class, test_name): + MAXWIDTH = 100 + FIELDS_EXTRA = ['start', 'end', 'production.log'] + FIELDS_SKIP = ['OBJECT:production.log'] + data = [] + for r in self.results: + if r['className'] == test_class and r['name'] == test_name: + for k in sorted(r.keys()) + FIELDS_EXTRA: + if k in FIELDS_SKIP: + continue + v = r[k] + print("%s:" % k) + if isinstance(v, str): + for row in v.split("\n"): + if k == 'url': + print(" "*len(k), row) + if k == 'production.log' and len(row) == 0: + continue + width = len(row) + printed = MAXWIDTH + print(" "*len(k), row[0:MAXWIDTH]) + while printed < width: + print(" "*(len(k)+4), row[printed:printed+MAXWIDTH-4]) + printed += len(row[printed:printed+MAXWIDTH-4]) + break + def handle_args(self): parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') @@ -125,6 +151,14 @@ def handle_args(self): self.show_unclaimed() return 0 + # Show test details + if args.show: + self.grep_results = None # to be sure we will not be missing the test because of filtering + class_name = '.'.join(args.show.split('.')[:-1]) + name = args.show.split('.')[-1] + self.show(class_name, name) + return 0 + return 0 def main(): From bdaeb27ad0eb9f0e53f7154c0c2fcd9ec2199e0f Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 17 Jul 2018 11:14:58 +0200 Subject: [PATCH 39/66] Added stats functionality --- claims/cmd.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index e3044dd..1429c8f 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -29,6 +29,14 @@ def results(self): self._results = [r for r in self._results if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] return self._results + @property + def rules(self): + if not self._rules: + self._rules = lib.Ruleset() + if self.grep_rules: + self._rules = [r for r in self._rules if re.search(self.grep_rules, r['reason'])] + return self._rules + def show_failed(self): print(tabulate.tabulate( [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES], @@ -70,6 +78,89 @@ def show(self, test_class, test_name): printed += len(row[printed:printed+MAXWIDTH-4]) break + def claim(self): + pass + + def stats(self): + def _perc(perc_from, perc_sum): + """Just a shortcur to safely count percentage""" + try: + return float(perc_from)/perc_sum*100 + except ZeroDivisionError: + return None + + stat_all = len(self.results) + reports_fails = [i for i in self.results if i['status'] in lib.Case.FAIL_STATUSES] + stat_failed = len(reports_fails) + reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] + stat_claimed = len(reports_claimed) + + stats_all = ['TOTAL', stat_all, stat_failed, _perc(stat_failed, stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] + + stats = [] + for t in [i['tier'] for i in lib.config.get_builds().values()]: + filtered = [r for r in self.results if r['tier'] == t] + stat_all_tiered = len(filtered) + reports_fails_tiered = [i for i in filtered if i['status'] in lib.Case.FAIL_STATUSES] + stat_failed_tiered = len(reports_fails_tiered) + reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] + stat_claimed_tiered = len(reports_claimed_tiered) + stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, _perc(stat_failed_tiered, stat_all_tiered), stat_claimed_tiered, _perc(stat_claimed_tiered, stat_failed_tiered)]) + + print("\nOverall stats") + print(tabulate.tabulate( + stats + [stats_all], + headers=['tier', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], + floatfmt=".01f")) + + reports_per_method = {} + for report in self.results: + method = report['className'].split('.')[2] + if method not in reports_per_method: + reports_per_method[method] = {'all': 0, 'failed': 0} + reports_per_method[method]['all'] += 1 + if report in reports_fails: + reports_per_method[method]['failed'] += 1 + + print("\nHow many failures are there per endpoint") + print(tabulate.tabulate( + sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_method.items()], + key=lambda x: x[3], reverse=True), + headers=['method', 'number of reports', 'number of failures', 'failures ratio'], + floatfmt=".1f")) + + rules_reasons = [r['reason'] for r in self.rules] + reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} + reports_per_reason.update({r:0 for r in rules_reasons}) + for report in reports_claimed: + reason = report['testActions'][0]['reason'] + if reason not in reports_per_reason: + reports_per_reason[reason] = 0 + reports_per_reason[reason] += 1 + + print("\nHow various reasons for claims are used") + reports_per_reason = sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True) + reports_per_reason = [(r, c, r in rules_reasons) for r, c in reports_per_reason] + print(tabulate.tabulate( + reports_per_reason, + headers=['claim reason', 'claimed times', 'claiming automated?'])) + + reports_per_class = {} + for report in self.results: + class_name = report['className'] + if class_name not in reports_per_class: + reports_per_class[class_name] = {'all': 0, 'failed': 0} + reports_per_class[class_name]['all'] += 1 + if report in reports_fails: + reports_per_class[class_name]['failed'] += 1 + + print("\nHow many failures are there per class") + print(tabulate.tabulate( + sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_class.items()], + key=lambda x: x[3], reverse=True), + headers=['class name', 'number of reports', 'number of failures', 'failures ratio'], + floatfmt=".1f")) + def handle_args(self): parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') @@ -139,25 +230,29 @@ def handle_args(self): # Show failed if args.show_failed: self.show_failed() - return 0 # Show claimed - if args.show_claimed: + elif args.show_claimed: self.show_claimed() - return 0 # Show unclaimed - if args.show_unclaimed: + elif args.show_unclaimed: self.show_unclaimed() - return 0 # Show test details - if args.show: + elif args.show: self.grep_results = None # to be sure we will not be missing the test because of filtering class_name = '.'.join(args.show.split('.')[:-1]) name = args.show.split('.')[-1] self.show(class_name, name) - return 0 + + # Do a claim work + elif args.claim: + pass + + # Show statistics + elif args.stats: + self.stats() return 0 From 69d477d4e4c5e6bc44c597f48a0af29c057ef18c Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 17 Jul 2018 11:15:29 +0200 Subject: [PATCH 40/66] Make '--output csv' actually work --- claims/cmd.py | 49 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 1429c8f..6f2bd17 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 # -*- coding: UTF-8 -*- +import sys import logging import argparse import re import tabulate +import csv import lib logging.basicConfig(level=logging.INFO) + class ClaimsCli(object): LATEST = 'latest' @@ -37,20 +40,34 @@ def rules(self): self._rules = [r for r in self._rules if re.search(self.grep_rules, r['reason'])] return self._rules + def _table(self, data, headers=[], tablefmt=None, floatfmt=None): + if self.output == 'csv': + writer = csv.writer(sys.stdout) + if headers: + writer.writerow(headers) + for row in data: + writer.writerow(row) + else: + print(tabulate.tabulate( + data, + headers=headers, + floatfmt=floatfmt, + tablefmt=self.output)) + def show_failed(self): - print(tabulate.tabulate( + self._table( [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES], - headers=['failed test name'], tablefmt=self.output)) + headers=['failed test name'], tablefmt=self.output) def show_claimed(self): - print(tabulate.tabulate( + self._table( [[r['testName'], r['testActions'][0].get('reason')] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], - headers=['claimed test name', 'claim reason'], tablefmt=self.output)) + headers=['claimed test name', 'claim reason'], tablefmt=self.output) def show_unclaimed(self): - print(tabulate.tabulate( + self._table( [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], - headers=['unclaimed test name'], tablefmt=self.output)) + headers=['unclaimed test name'], tablefmt=self.output) def show(self, test_class, test_name): MAXWIDTH = 100 @@ -108,10 +125,11 @@ def _perc(perc_from, perc_sum): stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, _perc(stat_failed_tiered, stat_all_tiered), stat_claimed_tiered, _perc(stat_claimed_tiered, stat_failed_tiered)]) print("\nOverall stats") - print(tabulate.tabulate( + self._table( stats + [stats_all], headers=['tier', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], - floatfmt=".01f")) + floatfmt=".01f", + tablefmt=self.output) reports_per_method = {} for report in self.results: @@ -123,11 +141,12 @@ def _perc(perc_from, perc_sum): reports_per_method[method]['failed'] += 1 print("\nHow many failures are there per endpoint") - print(tabulate.tabulate( + self._table( sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_method.items()], key=lambda x: x[3], reverse=True), headers=['method', 'number of reports', 'number of failures', 'failures ratio'], - floatfmt=".1f")) + floatfmt=".1f", + tablefmt=self.output) rules_reasons = [r['reason'] for r in self.rules] reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} @@ -141,9 +160,10 @@ def _perc(perc_from, perc_sum): print("\nHow various reasons for claims are used") reports_per_reason = sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True) reports_per_reason = [(r, c, r in rules_reasons) for r, c in reports_per_reason] - print(tabulate.tabulate( + self._table( reports_per_reason, - headers=['claim reason', 'claimed times', 'claiming automated?'])) + headers=['claim reason', 'claimed times', 'claiming automated?'], + tablefmt=self.output) reports_per_class = {} for report in self.results: @@ -155,11 +175,12 @@ def _perc(perc_from, perc_sum): reports_per_class[class_name]['failed'] += 1 print("\nHow many failures are there per class") - print(tabulate.tabulate( + self._table( sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_class.items()], key=lambda x: x[3], reverse=True), headers=['class name', 'number of reports', 'number of failures', 'failures ratio'], - floatfmt=".1f")) + floatfmt=".1f", + tablefmt=self.output) def handle_args(self): parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') From 1b76bc43400db07f9b340d9c711a1951a8ca459b Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 18 Jul 2018 14:46:58 +0200 Subject: [PATCH 41/66] Way to show history --- claims/cmd.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 6f2bd17..df0efde 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -7,6 +7,8 @@ import re import tabulate import csv +import collections +import statistics import lib @@ -27,10 +29,12 @@ def __init__(self): @property def results(self): if not self._results: - self._results = lib.Report(self.job_group) + self._results = {} + if self.job_group not in self._results: + self._results[self.job_group] = lib.Report(self.job_group) if self.grep_results: - self._results = [r for r in self._results if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] - return self._results + self._results[self.job_group] = [r for r in self._results[self.job_group] if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] + return self._results[self.job_group] @property def rules(self): @@ -182,6 +186,62 @@ def _perc(perc_from, perc_sum): floatfmt=".1f", tablefmt=self.output) + def history(self): + + def sanitize_state(state): + if state == 'REGRESSION': + state = 'FAILED' + if state == 'FIXED': + state = 'PASSED' + if state == 'PASSED': + return 0 + if state == 'FAILED': + return 1 + raise KeyError("Do not know how to handle state %s" % state) + + matrix = collections.OrderedDict() + + # Load tests results + job_groups = lib.config['job_groups'].keys() + for job_group in job_groups: + logging.info('Loading job group %s' % job_group) + self.job_group = job_group + report = self.results + for r in report: + t = "%s::%s@%s" % (r['className'], r['name'], r['distro']) + if t not in matrix: + matrix[t] = dict.fromkeys(job_group) + try: + state = sanitize_state(r['status']) + except KeyError: + continue # e.g. state "SKIPPED" + matrix[t][job_group] = state + + # Count statistical measure of the results + for k,v in matrix.items(): + try: + stdev = statistics.pstdev([i for i in v.values() if i is not None]) + except statistics.StatisticsError: + stdev = None + v['stdev'] = stdev + + print("Legend:\n 0 ... PASSED or FIXED\n 1 ... FAILED or REGRESSION\n Population standard deviation, 0 is best (stable), 0.5 is worst (unstable)") + headers = ['test'] + list(job_groups) + ['pstdev (all)'] + matrix_flat = [] + for k,v in matrix.items(): + v_list = [] + for job_group in job_groups: + if job_group in v: + v_list.append(v[job_group]) + else: + v_list.append(None) + matrix_flat.append([k]+v_list+[v['stdev']]) + self._table( + matrix_flat, + headers=headers, + floatfmt=".3f" + ) + def handle_args(self): parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') @@ -275,6 +335,10 @@ def handle_args(self): elif args.stats: self.stats() + # Show tests history + elif args.history: + self.history() + return 0 def main(): From da498769c0fe04dd67eebbcf36e875b5cca6a8d8 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 18 Jul 2018 14:47:11 +0200 Subject: [PATCH 42/66] Provide some default --- claims/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index df0efde..4cdf61c 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -44,7 +44,7 @@ def rules(self): self._rules = [r for r in self._rules if re.search(self.grep_rules, r['reason'])] return self._rules - def _table(self, data, headers=[], tablefmt=None, floatfmt=None): + def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): if self.output == 'csv': writer = csv.writer(sys.stdout) if headers: From d116b288bcda6546e2d9268f6f6c1b7317828d59 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 23 Jul 2018 23:21:03 +0200 Subject: [PATCH 43/66] Added possibility to claim --- claims/cmd.py | 17 +++++++++++++++-- claims/lib.py | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 4cdf61c..98ab5eb 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -73,6 +73,12 @@ def show_unclaimed(self): [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output) + def show_claimable(self): + claimable = lib.claim_by_rules(self.results, self.rules, dryrun=True) + self._table( + [[r['testName']] for r in claimable], + headers=['claimable test name'], tablefmt=self.output) + def show(self, test_class, test_name): MAXWIDTH = 100 FIELDS_EXTRA = ['start', 'end', 'production.log'] @@ -100,7 +106,10 @@ def show(self, test_class, test_name): break def claim(self): - pass + claimed = lib.claim_by_rules(self.results, self.rules, dryrun=False) + self._table( + [[r['testName']] for r in claimed], + headers=['claimed test name'], tablefmt=self.output) def stats(self): def _perc(perc_from, perc_sum): @@ -320,6 +329,10 @@ def handle_args(self): elif args.show_unclaimed: self.show_unclaimed() + # Show claimable + elif args.show_claimable: + self.show_claimable() + # Show test details elif args.show: self.grep_results = None # to be sure we will not be missing the test because of filtering @@ -329,7 +342,7 @@ def handle_args(self): # Do a claim work elif args.claim: - pass + self.claim() # Show statistics elif args.stats: diff --git a/claims/lib.py b/claims/lib.py index 1647469..537b65f 100755 --- a/claims/lib.py +++ b/claims/lib.py @@ -375,9 +375,12 @@ def __init__(self): config = Config() def claim_by_rules(report, rules, dryrun=False): + claimed = [] for rule in rules: for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: if case.matches_to_rule(rule): - logging.info(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) + logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) if not dryrun: case.push_claim(rule['reason']) + claimed.append(case) + return claimed From 4a56f23736444c672997f090b4965d14c250c562 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 23 Jul 2018 23:30:15 +0200 Subject: [PATCH 44/66] Added possibility to clean cache for job groups --- claims/cmd.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index 98ab5eb..8b2e4dd 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: UTF-8 -*- +import os.path import sys import logging import argparse @@ -9,6 +10,7 @@ import csv import collections import statistics +import shutil import lib @@ -58,6 +60,14 @@ def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): floatfmt=floatfmt, tablefmt=self.output)) + def clean_cache(self): + d = os.path.join(lib.CACHEDIR, self.job_group) + try: + shutil.rmtree(d) + logging.info("Removed %s" % d) + except FileNotFoundError: + pass + def show_failed(self): self._table( [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES], @@ -256,7 +266,7 @@ def handle_args(self): # Actions parser.add_argument('--clean-cache', action='store_true', - help='Cleans cache for latest job group. If you want to clean cache for older job group, use rm in .cache directory') + help='Cleans cache for job group provided by "--job-group" option (default: latest)') parser.add_argument('--show-failed', action='store_true', help='Show all failed tests') parser.add_argument('--show-claimed', action='store_true', @@ -317,6 +327,10 @@ def handle_args(self): # Actions + # Clean cache + if args.clean_cache: + self.clean_cache() + # Show failed if args.show_failed: self.show_failed() From 56311ee32c7beff8a619e75b02ae7f48b495de50 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 23 Jul 2018 23:34:31 +0200 Subject: [PATCH 45/66] Get stats for correct job groupu --- claims/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index 8b2e4dd..eb9aab8 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -138,7 +138,7 @@ def _perc(perc_from, perc_sum): stats_all = ['TOTAL', stat_all, stat_failed, _perc(stat_failed, stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] stats = [] - for t in [i['tier'] for i in lib.config.get_builds().values()]: + for t in [i['tier'] for i in lib.config.get_builds(self.job_group).values()]: filtered = [r for r in self.results if r['tier'] == t] stat_all_tiered = len(filtered) reports_fails_tiered = [i for i in filtered if i['status'] in lib.Case.FAIL_STATUSES] From b55085081f2f2e56bb3767def73557f2c57dcf69 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 23 Jul 2018 23:44:44 +0200 Subject: [PATCH 46/66] Added two more columns into 'How many failures are there per endpoint' statistic --- claims/cmd.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index eb9aab8..6f95861 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -158,16 +158,18 @@ def _perc(perc_from, perc_sum): for report in self.results: method = report['className'].split('.')[2] if method not in reports_per_method: - reports_per_method[method] = {'all': 0, 'failed': 0} + reports_per_method[method] = {'all': 0, 'failed': 0, 'claimed': 0} reports_per_method[method]['all'] += 1 if report in reports_fails: reports_per_method[method]['failed'] += 1 + if report in reports_claimed: + reports_per_method[method]['claimed'] += 1 print("\nHow many failures are there per endpoint") self._table( - sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_method.items()], - key=lambda x: x[3], reverse=True), - headers=['method', 'number of reports', 'number of failures', 'failures ratio'], + sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all']), r['claimed'], _perc(r['claimed'], r['failed'])) for c,r in reports_per_method.items()], + key=lambda x: x[3], reverse=True) + [stats_all], + headers=['method', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], floatfmt=".1f", tablefmt=self.output) From ad69d7bc591eaf3d92c11dd4cf699c6b1bd4f5de Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 24 Jul 2018 07:55:54 +0200 Subject: [PATCH 47/66] Sameflake8 fixes --- claims/cmd.py | 118 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 38 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 6f95861..31ca33f 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -35,7 +35,9 @@ def results(self): if self.job_group not in self._results: self._results[self.job_group] = lib.Report(self.job_group) if self.grep_results: - self._results[self.job_group] = [r for r in self._results[self.job_group] if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] + self._results[self.job_group] \ + = [r for r in self._results[self.job_group] + if re.search(self.grep_results, "%s.%s" % (r['className'], r['name']))] return self._results[self.job_group] @property @@ -43,7 +45,8 @@ def rules(self): if not self._rules: self._rules = lib.Ruleset() if self.grep_rules: - self._rules = [r for r in self._rules if re.search(self.grep_rules, r['reason'])] + self._rules = [r for r in self._rules + if re.search(self.grep_rules, r['reason'])] return self._rules def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): @@ -70,17 +73,20 @@ def clean_cache(self): def show_failed(self): self._table( - [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES], + [[r['testName']] for r in self.results + if r['status'] in lib.Case.FAIL_STATUSES], headers=['failed test name'], tablefmt=self.output) def show_claimed(self): self._table( - [[r['testName'], r['testActions'][0].get('reason')] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], + [[r['testName'], r['testActions'][0].get('reason')] for r in self.results + if r['status'] in lib.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], headers=['claimed test name', 'claim reason'], tablefmt=self.output) def show_unclaimed(self): self._table( - [[r['testName']] for r in self.results if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], + [[r['testName']] for r in self.results + if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output) def show_claimable(self): @@ -93,7 +99,6 @@ def show(self, test_class, test_name): MAXWIDTH = 100 FIELDS_EXTRA = ['start', 'end', 'production.log'] FIELDS_SKIP = ['OBJECT:production.log'] - data = [] for r in self.results: if r['className'] == test_class and r['name'] == test_name: for k in sorted(r.keys()) + FIELDS_EXTRA: @@ -111,8 +116,9 @@ def show(self, test_class, test_name): printed = MAXWIDTH print(" "*len(k), row[0:MAXWIDTH]) while printed < width: - print(" "*(len(k)+4), row[printed:printed+MAXWIDTH-4]) - printed += len(row[printed:printed+MAXWIDTH-4]) + printed_new = printed+MAXWIDTH-4 + print(" "*(len(k)+4), row[printed:printed_new]) + printed += len(row[printed:printed_new]) break def claim(self): @@ -130,27 +136,37 @@ def _perc(perc_from, perc_sum): return None stat_all = len(self.results) - reports_fails = [i for i in self.results if i['status'] in lib.Case.FAIL_STATUSES] + reports_fails = [i for i in self.results + if i['status'] in lib.Case.FAIL_STATUSES] stat_failed = len(reports_fails) - reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] + reports_claimed = [i for i in reports_fails + if i['testActions'][0].get('reason')] stat_claimed = len(reports_claimed) - stats_all = ['TOTAL', stat_all, stat_failed, _perc(stat_failed, stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] + stats_all = ['TOTAL', stat_all, stat_failed, _perc(stat_failed, + stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] stats = [] - for t in [i['tier'] for i in lib.config.get_builds(self.job_group).values()]: + builds = lib.config.get_builds(self.job_group).values() + for t in [i['tier'] for i in builds]: filtered = [r for r in self.results if r['tier'] == t] stat_all_tiered = len(filtered) - reports_fails_tiered = [i for i in filtered if i['status'] in lib.Case.FAIL_STATUSES] + reports_fails_tiered = [i for i in filtered + if i['status'] in lib.Case.FAIL_STATUSES] stat_failed_tiered = len(reports_fails_tiered) - reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] + reports_claimed_tiered = [i for i in reports_fails_tiered + if i['testActions'][0].get('reason')] stat_claimed_tiered = len(reports_claimed_tiered) - stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, _perc(stat_failed_tiered, stat_all_tiered), stat_claimed_tiered, _perc(stat_claimed_tiered, stat_failed_tiered)]) + stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, + _perc(stat_failed_tiered, stat_all_tiered), + stat_claimed_tiered, _perc(stat_claimed_tiered, + stat_failed_tiered)]) print("\nOverall stats") self._table( stats + [stats_all], - headers=['tier', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], + headers=['tier', 'all reports', 'failures', 'failures [%]', + 'claimed failures', 'claimed failures [%]'], floatfmt=".01f", tablefmt=self.output) @@ -167,15 +183,18 @@ def _perc(perc_from, perc_sum): print("\nHow many failures are there per endpoint") self._table( - sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all']), r['claimed'], _perc(r['claimed'], r['failed'])) for c,r in reports_per_method.items()], + sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all']), + r['claimed'], _perc(r['claimed'], r['failed'])) + for c, r in reports_per_method.items()], key=lambda x: x[3], reverse=True) + [stats_all], - headers=['method', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], + headers=['method', 'all reports', 'failures', 'failures [%]', + 'claimed failures', 'claimed failures [%]'], floatfmt=".1f", tablefmt=self.output) rules_reasons = [r['reason'] for r in self.rules] reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} - reports_per_reason.update({r:0 for r in rules_reasons}) + reports_per_reason.update({r: 0 for r in rules_reasons}) for report in reports_claimed: reason = report['testActions'][0]['reason'] if reason not in reports_per_reason: @@ -183,8 +202,10 @@ def _perc(perc_from, perc_sum): reports_per_reason[reason] += 1 print("\nHow various reasons for claims are used") - reports_per_reason = sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True) - reports_per_reason = [(r, c, r in rules_reasons) for r, c in reports_per_reason] + reports_per_reason = sorted(reports_per_reason.items(), + key=lambda x: x[1], reverse=True) + reports_per_reason = [(r, c, r in rules_reasons) for r, c in + reports_per_reason] self._table( reports_per_reason, headers=['claim reason', 'claimed times', 'claiming automated?'], @@ -201,9 +222,11 @@ def _perc(perc_from, perc_sum): print("\nHow many failures are there per class") self._table( - sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) for c,r in reports_per_class.items()], + sorted([(c, r['all'], r['failed'], _perc(r['failed'], r['all'])) + for c, r in reports_per_class.items()], key=lambda x: x[3], reverse=True), - headers=['class name', 'number of reports', 'number of failures', 'failures ratio'], + headers=['class name', 'number of reports', 'number of failures', + 'failures ratio'], floatfmt=".1f", tablefmt=self.output) @@ -239,17 +262,21 @@ def sanitize_state(state): matrix[t][job_group] = state # Count statistical measure of the results - for k,v in matrix.items(): + for k, v in matrix.items(): try: stdev = statistics.pstdev([i for i in v.values() if i is not None]) except statistics.StatisticsError: stdev = None v['stdev'] = stdev - print("Legend:\n 0 ... PASSED or FIXED\n 1 ... FAILED or REGRESSION\n Population standard deviation, 0 is best (stable), 0.5 is worst (unstable)") + print("Legend:\n" + " 0 ... PASSED or FIXED\n" + " 1 ... FAILED or REGRESSION\n" + " Population standard deviation, 0 is best (stable)," + " 0.5 is worst (unstable)") headers = ['test'] + list(job_groups) + ['pstdev (all)'] matrix_flat = [] - for k,v in matrix.items(): + for k, v in matrix.items(): v_list = [] for job_group in job_groups: if job_group in v: @@ -264,11 +291,13 @@ def sanitize_state(state): ) def handle_args(self): - parser = argparse.ArgumentParser(description='Manipulate Jenkins claims with grace') + parser = argparse.ArgumentParser( + description='Manipulate Jenkins claims with grace') # Actions parser.add_argument('--clean-cache', action='store_true', - help='Cleans cache for job group provided by "--job-group" option (default: latest)') + help='Cleans cache for job group provided by' + ' "--job-group" option (default: latest)') parser.add_argument('--show-failed', action='store_true', help='Show all failed tests') parser.add_argument('--show-claimed', action='store_true', @@ -276,7 +305,8 @@ def handle_args(self): parser.add_argument('--show-unclaimed', action='store_true', help='Show failed and not yet claimed tests') parser.add_argument('--show-claimable', action='store_true', - help='Show failed, not yet claimed but claimable tests') + help='Show failed, not yet claimed but' + ' claimable tests') parser.add_argument('--show', action='store', help='Show detailed info about given test case') parser.add_argument('--claim', action='store_true', @@ -290,13 +320,18 @@ def handle_args(self): # Modifiers parser.add_argument('--job-group', action='store', - help='Specify group of jobs to perform the action with (default: latest)') + help='Specify group of jobs to perform the action' + ' with (default: latest)') parser.add_argument('--grep-results', action='store', metavar='REGEXP', - help='Only work with tests, whose "className+name" matches the regexp') + help='Only work with tests, whose' + ' "className+name" matches the regexp') parser.add_argument('--grep-rules', action='store', metavar='REGEXP', - help='Only work with rules, whose reason matches the regexp') - parser.add_argument('--output', action='store', choices=['simple', 'csv', 'html'], default='simple', - help='Format tables as plain, csv or html (default: simple)') + help='Only work with rules, whose reason matches' + ' the regexp') + parser.add_argument('--output', action='store', default='simple', + choices=['simple', 'csv', 'html'], + help='Format tables as plain, csv or html' + ' (default: simple)') parser.add_argument('-d', '--debug', action='store_true', help='Show also debug messages') @@ -311,17 +346,20 @@ def handle_args(self): # Handle "--job-group something" if args.job_group: self.job_group = args.job_group - logging.debug("Job group we are going to work with is %s" % self.job_group) + logging.debug("Job group we are going to work with is %s" + % self.job_group) # Handle "--grep-results something" if args.grep_results: self.grep_results = args.grep_results - logging.debug("Going to consider only results matching %s" % self.grep_results) + logging.debug("Going to consider only results matching %s" + % self.grep_results) # Handle "--grep-rules something" if args.grep_rules: self.grep_rules = args.grep_rules - logging.debug("Going to consider only rules matching %s" % self.grep_rules) + logging.debug("Going to consider only rules matching %s" + % self.grep_rules) # Handle "--output something" self.output = args.output @@ -351,7 +389,9 @@ def handle_args(self): # Show test details elif args.show: - self.grep_results = None # to be sure we will not be missing the test because of filtering + # To be sure we will not be missing the test because of filtering, + # erase grep_result filter first + self.grep_results = None class_name = '.'.join(args.show.split('.')[:-1]) name = args.show.split('.')[-1] self.show(class_name, name) @@ -370,9 +410,11 @@ def handle_args(self): return 0 + def main(): """Main program""" return ClaimsCli().handle_args() + if __name__ == "__main__": main() From 60bdf37ff2ee0d062b3fe907ae51f034fa91beb9 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 24 Jul 2018 08:12:50 +0200 Subject: [PATCH 48/66] Added function to draw timegraphs --- claims/cmd.py | 11 ++++ claims/timegraph.py | 125 +++++++++++++++++++++++++++++++++++++++++ test/test_timegraph.py | 13 +++++ 3 files changed, 149 insertions(+) create mode 100644 claims/timegraph.py create mode 100644 test/test_timegraph.py diff --git a/claims/cmd.py b/claims/cmd.py index 31ca33f..743b669 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -13,6 +13,7 @@ import shutil import lib +import timegraph logging.basicConfig(level=logging.INFO) @@ -290,6 +291,12 @@ def sanitize_state(state): floatfmt=".3f" ) + def timegraph(self): + for n, b in lib.config.get_builds(self.job_group).items(): + f = "/tmp/timegraph-%s-build%s.svg" % (n, b['build']) + timegraph.draw(self.results, f, b['tier']) + logging.info("Generated %s" % f) + def handle_args(self): parser = argparse.ArgumentParser( description='Manipulate Jenkins claims with grace') @@ -408,6 +415,10 @@ def handle_args(self): elif args.history: self.history() + # Generate time graphs per tier + elif args.timegraph: + self.timegraph() + return 0 diff --git a/claims/timegraph.py b/claims/timegraph.py new file mode 100644 index 0000000..f2ddcca --- /dev/null +++ b/claims/timegraph.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import logging +import datetime +import svgwrite + +import lib + + +STATUS_COLOR = { +'FAILED': 'red', +'FIXED': 'blue', +'PASSED': 'green', +'REGRESSION': 'purple', +'SKIPPED': 'fuchsia', +} +LANE_HEIGHT = 10 +LANES_START = LANE_HEIGHT # we will place a timeline into the first lane +HOUR = 3600 +X_CONTRACTION = 0.1 + + +def overlaps(a, b): + """ + Return true if two intervals overlap: + overlaps((1, 3), (2, 10)) => True + overlaps((1, 3), (5, 10)) => False + """ + if b[0] <= a[0] <= b[1] or b[0] <= a[1] <= b[1]: + return True + else: + return False + + +def scale(a): + return (a[0] * X_CONTRACTION, a[1]) + + +def draw(reports, filename, tier): + reports = [i for i in reports if i['tier'] == tier] + + # Load all the reports and sort them in lanes + ###counter = 0 + lanes = [] + start = None + end = None + for r in reports: + # Get start and end time. If unknown, skip the result + try: + r_start = r['start'].timestamp() + r_end = r['end'].timestamp() + except KeyError: + logging.info("No start time for %s::%s" % (r['className'], r['name'])) + continue + # Find overal widtht of time line + if start is None or r_start < start: + logging.debug("Test %s started before current minimum of %s" % (r['name'], start)) + start = r_start + if end is None or r_end > end: + end = r_end + r['interval'] = (r_start, r_end) + # Check if there is a free lane for us, if not, create a new one + lane_found = False + for lane in lanes: + lane_found = True + for interval in lane: + if overlaps(r['interval'], interval['interval']): + lane_found = False + break + if lane_found: + break + if not lane_found: + logging.debug("Adding lane %s" % (len(lanes)+1)) + lane = [] + lanes.append(lane) + lane.append(r) + ###counter += 1 + ###if counter > 10: break + + # Create a drawing with timeline + dwg = svgwrite.Drawing(filename, + size=scale((end-start, LANE_HEIGHT*(len(lanes)+1)))) + dwg.add(dwg.line( + scale((0, LANE_HEIGHT)), + scale((end-start, LANE_HEIGHT)), + style="stroke: black; stroke-width: 1;" + )) + start_full_hour = int(start / HOUR) * HOUR + timeline = start_full_hour - start + while start + timeline <= end: + if timeline >= 0: + dwg.add(dwg.line( + scale((timeline, LANE_HEIGHT)), + scale((timeline, 2*LANE_HEIGHT/3)), + style="stroke: black; stroke-width: 1;" + )) + dwg.add(dwg.text( + datetime.datetime.fromtimestamp(start+timeline) \ + .strftime('%Y-%m-%d %H:%M:%S'), + insert=scale((timeline, 2*LANE_HEIGHT/3)), + style="fill: black; font-size: 3pt;" + )) + timeline += HOUR/4 + + # Draw tests + for lane_no in range(len(lanes)): + for r in lanes[lane_no]: + logging.debug("In lane %s adding %s::%s %s" \ + % (lane_no, r['className'], r['name'], r['interval'])) + s, e = r['interval'] + dwg.add(dwg.rect( + insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + size=scale((e - s, LANE_HEIGHT/2)), + style="fill: %s; stroke: %s; stroke-width: 0;" \ + % (STATUS_COLOR[r['status']], STATUS_COLOR[r['status']]) + )) + dwg.add(dwg.text( + "%s::%s" % (r['className'], r['name']), + insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + transform="rotate(-30, %s, %s)" \ + % scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), + style="fill: gray; font-size: 2pt;" + )) + dwg.save() diff --git a/test/test_timegraph.py b/test/test_timegraph.py new file mode 100644 index 0000000..3261c3a --- /dev/null +++ b/test/test_timegraph.py @@ -0,0 +1,13 @@ + +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import pytest + +import claims.timegraph + +class TestTimegraph(): + + def test_overlaps(self): + assert claims.timegraph.overlaps((1, 3), (2, 10)) == True + assert claims.timegraph.overlaps((1, 3), (5, 10)) == False From 63aca59aacca3b528e7a17fd687e8ec233b4a05e Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 24 Jul 2018 08:16:49 +0200 Subject: [PATCH 49/66] Not needed now I hope --- claim_by_rules.py | 16 ------ claimable.py | 15 ------ claimstats.py | 83 ------------------------------ rungraph.py | 123 --------------------------------------------- tests-stability.py | 74 --------------------------- unclaimed.py | 8 --- 6 files changed, 319 deletions(-) delete mode 100755 claim_by_rules.py delete mode 100755 claimable.py delete mode 100755 claimstats.py delete mode 100755 rungraph.py delete mode 100755 tests-stability.py delete mode 100755 unclaimed.py diff --git a/claim_by_rules.py b/claim_by_rules.py deleted file mode 100755 index 1663409..0000000 --- a/claim_by_rules.py +++ /dev/null @@ -1,16 +0,0 @@ -import claims -import sys - -try: - target_url = sys.argv[1] -except IndexError: - raise ValueError('The targe url is supposed to be passed as an argument ' - 'to this script') - -rules = claims.load_rules() -r = claims.fetch_test_report(build_url=target_url) -f = claims.filter_fails(r) -# let's only take the unclaimed tests -u = claims.filter_not_claimed(f) - -claims.claim_by_rules(u, rules) diff --git a/claimable.py b/claimable.py deleted file mode 100755 index 4a1d6dd..0000000 --- a/claimable.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- - -import claims - -report = claims.Report() -rules = claims.Ruleset() - -#report = [r for r in report if r['name'] == 'test_positive_create_with_puppet_class_id'] -#rules = [r for r in rules if r['reason'] == 'https://github.com/SatelliteQE/robottelo/issues/6115'] - -claims.claim_by_rules(report, rules, dryrun=True) - -for case in [i for i in report if i['status'] in claims.Case.FAIL_STATUSES]: - print(case['url']) diff --git a/claimstats.py b/claimstats.py deleted file mode 100755 index 1026f0a..0000000 --- a/claimstats.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 - -import claims -import tabulate - -reports = claims.Report() - -stat_all = len(reports) -reports_fails = [i for i in reports if i['status'] in claims.Case.FAIL_STATUSES] -stat_failed = len(reports_fails) -reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] -stat_claimed = len(reports_claimed) - -print("\nOverall stats") -print(tabulate.tabulate( - [[stat_all, stat_failed, stat_failed/stat_all*100, stat_claimed, stat_claimed/stat_failed*100]], - headers=['all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], - floatfmt=".0f")) - -stats = [] -for t in [i['tier'] for i in claims.config.get_builds().values()]: - filtered = [r for r in reports if r['tier'] == t] - stat_all_tiered = len(filtered) - reports_fails_tiered = [i for i in filtered if i['status'] in claims.Case.FAIL_STATUSES] - stat_failed_tiered = len(reports_fails_tiered) - reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] - stat_claimed_tiered = len(reports_claimed_tiered) - stats.append(["t%s" % t, stat_all_tiered, stat_failed_tiered, stat_failed_tiered/stat_all_tiered*100, stat_claimed_tiered, stat_claimed_tiered/stat_failed_tiered*100]) - -print("\nStats per tier") -print(tabulate.tabulate( - stats, - headers=['tier', 'all reports', 'failures', 'failures [%]', 'claimed failures', 'claimed failures [%]'], - floatfmt=".0f")) - -rules = claims.Ruleset() -rules_reasons = [r['reason'] for r in rules] -reports_per_reason = {'UNKNOWN': stat_failed-stat_claimed} -reports_per_reason.update({r:0 for r in rules_reasons}) -for report in reports_claimed: - reason = report['testActions'][0]['reason'] - if reason not in reports_per_reason: - reports_per_reason[reason] = 0 - reports_per_reason[reason] += 1 - -print("\nHow various reasons for claims are used") -reports_per_reason = sorted(reports_per_reason.items(), key=lambda x: x[1], reverse=True) -reports_per_reason = [(r, c, r in rules_reasons) for r, c in reports_per_reason] -print(tabulate.tabulate( - reports_per_reason, - headers=['claim reason', 'number of times', 'is it in current knowleadgebase?'])) - -reports_per_class = {} -for report in reports: - class_name = report['className'] - if class_name not in reports_per_class: - reports_per_class[class_name] = {'all': 0, 'failed': 0} - reports_per_class[class_name]['all'] += 1 - if report in reports_fails: - reports_per_class[class_name]['failed'] += 1 - -print("\nHow many failures are there per class") -print(tabulate.tabulate( - sorted([(c, r['all'], r['failed'], float(r['failed'])/r['all']) for c,r in reports_per_class.items()], - key=lambda x: x[3], reverse=True), - headers=['class name', 'number of reports', 'number of failures', 'failures ratio'], - floatfmt=".3f")) - -reports_per_method = {} -for report in reports: - method = report['className'].split('.')[2] - if method not in reports_per_method: - reports_per_method[method] = {'all': 0, 'failed': 0} - reports_per_method[method]['all'] += 1 - if report in reports_fails: - reports_per_method[method]['failed'] += 1 - -print("\nHow many failures are there per method (CLI vs. API vs. UI)") -print(tabulate.tabulate( - sorted([(c, r['all'], r['failed'], float(r['failed'])/r['all']) for c,r in reports_per_method.items()], - key=lambda x: x[3], reverse=True), - headers=['method', 'number of reports', 'number of failures', 'failures ratio'], - floatfmt=".3f")) diff --git a/rungraph.py b/rungraph.py deleted file mode 100755 index f946f94..0000000 --- a/rungraph.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- - -import logging -import datetime -import svgwrite -import claims - - -STATUS_COLOR = { -'FAILED': 'red', -'FIXED': 'blue', -'PASSED': 'green', -'REGRESSION': 'purple', -'SKIPPED': 'fuchsia', -} -LANE_HEIGHT = 10 -LANES_START = LANE_HEIGHT # we will place a timeline into the first lane -HOUR = 3600 -X_CONTRACTION = 0.1 - - -def overlaps(a, b): - """ - Return true if two intervals overlap: - overlaps((1, 3), (2, 10)) => True - overlaps((1, 3), (5, 10)) => False - """ - if b[0] <= a[0] <= b[1] or b[0] <= a[1] <= b[1]: - return True - else: - return False - - -def scale(a): - return (a[0] * X_CONTRACTION, a[1]) - - -reports = [i for i in claims.Report() if i['tier'] == 't4'] - -# Load all the reports and sort them in lanes -###counter = 0 -lanes = [] -start = None -end = None -for r in reports: - # Get start and end time. If unknown, skip the result - try: - r_start = r['start'].timestamp() - r_end = r['end'].timestamp() - except KeyError: - logging.info("No start time for %s::%s" % (r['className'], r['name'])) - continue - # Find overal widtht of time line - if start is None or r_start < start: - logging.debug("Test %s started before current minimum of %s" % (r['name'], start)) - start = r_start - if end is None or r_end > end: - end = r_end - r['interval'] = (r_start, r_end) - # Check if there is a free lane for us, if not, create a new one - lane_found = False - for lane in lanes: - lane_found = True - for interval in lane: - if overlaps(r['interval'], interval['interval']): - lane_found = False - break - if lane_found: - break - if not lane_found: - logging.debug("Adding lane %s" % (len(lanes)+1)) - lane = [] - lanes.append(lane) - lane.append(r) - ###counter += 1 - ###if counter > 10: break - -# Create a drawing with timeline -dwg = svgwrite.Drawing('/tmp/rungraph.svg', - size=scale((end-start, LANE_HEIGHT*(len(lanes)+1)))) -dwg.add(dwg.line( - scale((0, LANE_HEIGHT)), - scale((end-start, LANE_HEIGHT)), - style="stroke: black; stroke-width: 1;" -)) -start_full_hour = int(start / HOUR) * HOUR -timeline = start_full_hour - start -while start + timeline <= end: - if timeline >= 0: - dwg.add(dwg.line( - scale((timeline, LANE_HEIGHT)), - scale((timeline, 2*LANE_HEIGHT/3)), - style="stroke: black; stroke-width: 1;" - )) - dwg.add(dwg.text( - datetime.datetime.fromtimestamp(start+timeline) \ - .strftime('%Y-%m-%d %H:%M:%S'), - insert=scale((timeline, 2*LANE_HEIGHT/3)), - style="fill: black; font-size: 3pt;" - )) - timeline += HOUR/4 - -# Draw tests -for lane_no in range(len(lanes)): - for r in lanes[lane_no]: - logging.debug("In lane %s adding %s::%s %s" \ - % (lane_no, r['className'], r['name'], r['interval'])) - s, e = r['interval'] - dwg.add(dwg.rect( - insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), - size=scale((e - s, LANE_HEIGHT/2)), - style="fill: %s; stroke: %s; stroke-width: 0;" \ - % (STATUS_COLOR[r['status']], STATUS_COLOR[r['status']]) - )) - dwg.add(dwg.text( - "%s::%s" % (r['className'], r['name']), - insert=scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), - transform="rotate(-30, %s, %s)" \ - % scale((s - start, LANES_START + LANE_HEIGHT*lane_no + LANE_HEIGHT/2)), - style="fill: gray; font-size: 2pt;" - )) -dwg.save() diff --git a/tests-stability.py b/tests-stability.py deleted file mode 100755 index f77c9aa..0000000 --- a/tests-stability.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- - -import logging -import datetime -import collections -import statistics -import tabulate -import csv -import claims - -BUILDS = [22, 21, 19, 18, 17, 14, 13, 12, 10, 9, 8, 7, 6] -matrix = collections.OrderedDict() - - -def sanitize_state(state): - if state == 'REGRESSION': - state = 'FAILED' - if state == 'FIXED': - state = 'PASSED' - if state == 'PASSED': - return 0 - if state == 'FAILED': - return 1 - raise KeyError("Do not know how to handle state %s" % state) - - -for build_id in range(len(BUILDS)): - build = BUILDS[build_id] - claims.config['bld'] = build - claims.config['cache'] = 'cache-%s-%s.pickle' \ - % (datetime.datetime.now().strftime('%Y%m%d'), build) - logging.info("Initializing report for build %s with cache in %s" \ - % (build, claims.config['cache'])) - #report = [i for i in claims.Report() if i['tier'] == 't4'] - report = claims.Report() - for r in report: - t = "%s::%s@%s" % (r['className'], r['name'], r['distro']) - if t not in matrix: - matrix[t] = [None for i in BUILDS] - try: - state = sanitize_state(r['status']) - except KeyError: - continue - matrix[t][build_id] = state - -# Count statistical measure of the results -for k,v in matrix.items(): - try: - stdev = statistics.pstdev([i for i in v if i is not None]) - except statistics.StatisticsError: - stdev = None - v.append(stdev) - try: - stdev = statistics.pstdev([i for i in v[:3] if i is not None]) - except statistics.StatisticsError: - stdev = None - v.append(stdev) - -print("Legend:\n 0 ... PASSED or FIXED\n 1 ... FAILED or REGRESSION\n Population standard deviation, 0 is best (stable), 0.5 is worst (unstable)\n Same but only for newest 3 builds") -matrix_flat = [[k]+v for k,v in matrix.items()] -headers = ['test']+BUILDS+['pstdev (all)', 'pstdev (3 newest)'] -print(tabulate.tabulate( - matrix_flat, - headers=headers, - floatfmt=".3f" -)) - -filename = "/tmp/tests-stability.csv" -print("Writing data to %s" % filename) -with open(filename, 'w', newline='') as f: - writer = csv.writer(f) - writer.writerows([headers]) - writer.writerows(matrix_flat) diff --git a/unclaimed.py b/unclaimed.py deleted file mode 100755 index 9e4b5e9..0000000 --- a/unclaimed.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/python3 - -import claims - -reports = [i for i in claims.Report() if i['status'] in claims.Case.FAIL_STATUSES and not i['testActions'][0].get('reason')] - -for r in reports: - print(u'{0} {1}::{2}'.format(r['distro'], r['className'], r['name'])) From 1c9583eccf90811784e4d20afe2ac6f7d04a083e Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 24 Jul 2018 08:17:03 +0200 Subject: [PATCH 50/66] This is needed for rungraphs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8a2c822..a652fac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ PyYAML requests tabulate +svgwrite From 3a118682fbcced9a7494c136bdf02e8a5ce57ac8 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 26 Jul 2018 22:13:57 +0200 Subject: [PATCH 51/66] Restructured claims module --- claims-cmd.py | 5 + claims/__init__.py | 10 ++ claims/build_logs.py | 91 ++++++++++ claims/case.py | 130 +++++++++++++++ claims/cmd.py | 38 ++--- claims/config.py | 39 +++++ claims/lib.py | 386 ------------------------------------------- claims/report.py | 64 +++++++ claims/ruleset.py | 9 + claims/timegraph.py | 2 - claims/utils.py | 61 +++++++ 11 files changed, 423 insertions(+), 412 deletions(-) create mode 100755 claims-cmd.py create mode 100644 claims/build_logs.py create mode 100644 claims/case.py create mode 100644 claims/config.py delete mode 100755 claims/lib.py create mode 100644 claims/report.py create mode 100644 claims/ruleset.py create mode 100755 claims/utils.py diff --git a/claims-cmd.py b/claims-cmd.py new file mode 100755 index 0000000..c81dae1 --- /dev/null +++ b/claims-cmd.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +import claims +claims.ClaimsCli().handle_args() diff --git a/claims/__init__.py b/claims/__init__.py index e69de29..9471715 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -0,0 +1,10 @@ +from .config import Config +from .build_logs import ForemanDebug, ProductionLog +from .case import Case +from .report import Report +from .ruleset import Ruleset +from .utils import request_get, claim_by_rules +from .cmd import ClaimsCli + +# Create shared config file +config = Config() diff --git a/claims/build_logs.py b/claims/build_logs.py new file mode 100644 index 0000000..d477173 --- /dev/null +++ b/claims/build_logs.py @@ -0,0 +1,91 @@ +import os +import re + +from .config import config + + +class ForemanDebug(object): + + def __init__(self, job_group, job, build): + self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) + self._extracted = None + + @property + def extracted(self): + if self._extracted is None: + fp, fname = tempfile.mkstemp() + print(fname) + request_get(self._url, cached=fname, stream=True) + tmpdir = tempfile.mkdtemp() + subprocess.call(['tar', '-xf', fname, '--directory', tmpdir]) + logging.debug('Extracted to %s' % tmpdir) + self._extracted = os.path.join(tmpdir, 'foreman-debug') + return self._extracted + + +class ProductionLog(object): + + FILE_ENCODING = 'ISO-8859-1' # guessed it only, that file contains ugly binary mess as well + DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 + DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 + + def __init__(self, job_group, job, build): + self._log = None + self._logfile = os.path.join(config.CACHEDIR, job_group, job, 'production.log') + self._foreman_debug = None + + # If we do not have logfile downloaded already, we will need foreman-debug + if not os.path.isfile(self._logfile): + self._foreman_debug = ForemanDebug(job_group, job, build) + + @property + def log(self): + if self._log is None: + if self._foreman_debug is not None: + a = os.path.join(self._foreman_debug.extracted, + 'var', 'log', 'foreman', 'production.log') + shutil.copy2(a, self._logfile) + self._log = [] + buf = [] + last = None + with open(self._logfile, 'r', encoding=self.FILE_ENCODING) as fp: + for line in fp: + + # This line starts with date - denotes first line of new log record + if re.search(self.DATE_REGEXP, line): + + # This is a new log record, so firs save previous one + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + last = datetime.datetime.strptime(line[:19], self.DATE_FMT) + buf = [] + buf.append(re.sub(self.DATE_REGEXP, '', line, count=1)) + + # This line does not start with line - comtains continuation of a log recorder started before + else: + buf.append(line) + + # Save last line + if len(buf) != 0: + self._log.append({'time': last, 'data': buf}) + + logging.debug("File %s parsed into memory" % self._logfile) + return self._log + + def from_to(self, from_time, to_time): + out = [] + for i in self.log: + if from_time <= i['time'] <= to_time: + out.append(i) + # Do not do following as time is not sequentional in the log (or maybe some workers are off or with different TZ?): + # TODO: Fix ordering of the log and uncomment this + # + # E.g.: + # 2018-06-17T17:29:44 [I|dyn|] start terminating clock... + # 2018-06-17T21:34:49 [I|app|] Current user: foreman_admin (administrator) + # 2018-06-17T21:37:21 [...] + # 2018-06-17T17:41:38 [I|app|] Started POST "/katello/api/v2/organizations"... + # + #if i['time'] > to_time: + # break + return out diff --git a/claims/case.py b/claims/case.py new file mode 100644 index 0000000..e21b0e0 --- /dev/null +++ b/claims/case.py @@ -0,0 +1,130 @@ +import collections +import re + + +class Case(collections.UserDict): + """ + Result of one test case + """ + + FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") + LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') + LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' + + def __init__(self, data): + self.data = data + + def __contains__(self, name): + return name in self.data or name in ('start', 'end', 'production.log') + + def __getitem__(self, name): + if name == 'testName': + self['testName'] = "%s.%s" % (self['className'], self['name']) + if name in ('start', 'end') and \ + ('start' not in self.data or 'end' not in self.data): + self.load_timings() + if name == 'production.log': + self['production.log'] = "\n".join( + ["\n".join(i['data']) for i in + self.data['OBJECT:production.log'].from_to( + self['start'], self['end'])]) + return self.data[name] + + def matches_to_rule(self, rule, indentation=0): + """ + Returns True if result matches to rule, otherwise returns False + """ + logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self['name'], rule, indentation)) + if 'field' in rule and 'pattern' in rule: + # This is simple rule, we can just check regexp against given field and we are done + try: + data = self[rule['field']] + if data is None: + data = '' + out = re.search(rule['pattern'], data) is not None + logging.debug("%s=> %s" % (" "*indentation, out)) + return out + except KeyError: + logging.debug("%s=> Failed to get field %s from case" % (" "*indentation, rule['field'])) + return None + elif 'AND' in rule: + # We need to check if all sub-rules in list of rules rule['AND'] matches + out = None + for r in rule['AND']: + r_out = self.matches_to_rule(r, indentation+4) + out = r_out if out is None else out and r_out + if not out: + break + return out + elif 'OR' in rule: + # We need to check if at least one sub-rule in list of rules rule['OR'] matches + for r in rule['OR']: + if self.matches_to_rule(r, indentation+4): + return True + return False + else: + raise Exception('Rule %s not formatted correctly' % rule) + + def push_claim(self, reason, sticky=False, propagate=False): + '''Claims a given test with a given reason + + :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) + + :param sticky: whether to make the claim sticky (False by default) + + :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) + ''' + logging.info('claiming {0}::{1} with reason: {2}'.format(self["className"], self["name"], reason)) + + if config['headers'] is None: + config.init_headers() + + claim_req = requests.post( + u'{0}/claim/claim'.format(self['url']), + auth=requests.auth.HTTPBasicAuth( + config['usr'], + config['pwd'] + ), + data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, + headers=config['headers'], + allow_redirects=False, + verify=False + ) + + if claim_req.status_code != 302: + raise requests.HTTPError( + 'Failed to claim: {0}'.format(claim_req)) + + self['testActions'][0]['reason'] = reason + return(claim_req) + + def load_timings(self): + if self['stdout'] is None: + return + log = self['stdout'].split("\n") + log_size = len(log) + log_used = 0 + start = None + end = None + counter = 0 + while start is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + start = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter += 1 + log_used += counter + counter = -1 + while end is None: + match = self.LOG_DATE_REGEXP.match(log[counter]) + if match: + end = datetime.datetime.strptime(match.group(1), + self.LOG_DATE_FORMAT) + break + counter -= 1 + log_used -= counter + assert log_used <= log_size, \ + "Make sure detected start date is not below end date and vice versa" + self['start'] = start + self['end'] = end diff --git a/claims/cmd.py b/claims/cmd.py index 743b669..9908efd 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -12,8 +12,7 @@ import statistics import shutil -import lib -import timegraph +import claims logging.basicConfig(level=logging.INFO) @@ -34,7 +33,7 @@ def results(self): if not self._results: self._results = {} if self.job_group not in self._results: - self._results[self.job_group] = lib.Report(self.job_group) + self._results[self.job_group] = claims.Report(self.job_group) if self.grep_results: self._results[self.job_group] \ = [r for r in self._results[self.job_group] @@ -44,7 +43,7 @@ def results(self): @property def rules(self): if not self._rules: - self._rules = lib.Ruleset() + self._rules = claims.Ruleset() if self.grep_rules: self._rules = [r for r in self._rules if re.search(self.grep_rules, r['reason'])] @@ -65,7 +64,7 @@ def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): tablefmt=self.output)) def clean_cache(self): - d = os.path.join(lib.CACHEDIR, self.job_group) + d = os.path.join(claims.CACHEDIR, self.job_group) try: shutil.rmtree(d) logging.info("Removed %s" % d) @@ -75,23 +74,23 @@ def clean_cache(self): def show_failed(self): self._table( [[r['testName']] for r in self.results - if r['status'] in lib.Case.FAIL_STATUSES], + if r['status'] in claims.Case.FAIL_STATUSES], headers=['failed test name'], tablefmt=self.output) def show_claimed(self): self._table( [[r['testName'], r['testActions'][0].get('reason')] for r in self.results - if r['status'] in lib.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], + if r['status'] in claims.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], headers=['claimed test name', 'claim reason'], tablefmt=self.output) def show_unclaimed(self): self._table( [[r['testName']] for r in self.results - if r['status'] in lib.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], + if r['status'] in claims.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output) def show_claimable(self): - claimable = lib.claim_by_rules(self.results, self.rules, dryrun=True) + claimable = claims.claim_by_rules(self.results, self.rules, dryrun=True) self._table( [[r['testName']] for r in claimable], headers=['claimable test name'], tablefmt=self.output) @@ -123,7 +122,7 @@ def show(self, test_class, test_name): break def claim(self): - claimed = lib.claim_by_rules(self.results, self.rules, dryrun=False) + claimed = claims.claim_by_rules(self.results, self.rules, dryrun=False) self._table( [[r['testName']] for r in claimed], headers=['claimed test name'], tablefmt=self.output) @@ -138,7 +137,7 @@ def _perc(perc_from, perc_sum): stat_all = len(self.results) reports_fails = [i for i in self.results - if i['status'] in lib.Case.FAIL_STATUSES] + if i['status'] in claims.Case.FAIL_STATUSES] stat_failed = len(reports_fails) reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] @@ -148,12 +147,12 @@ def _perc(perc_from, perc_sum): stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] stats = [] - builds = lib.config.get_builds(self.job_group).values() + builds = claims.config.get_builds(self.job_group).values() for t in [i['tier'] for i in builds]: filtered = [r for r in self.results if r['tier'] == t] stat_all_tiered = len(filtered) reports_fails_tiered = [i for i in filtered - if i['status'] in lib.Case.FAIL_STATUSES] + if i['status'] in claims.Case.FAIL_STATUSES] stat_failed_tiered = len(reports_fails_tiered) reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] @@ -247,7 +246,7 @@ def sanitize_state(state): matrix = collections.OrderedDict() # Load tests results - job_groups = lib.config['job_groups'].keys() + job_groups = claims.config['job_groups'].keys() for job_group in job_groups: logging.info('Loading job group %s' % job_group) self.job_group = job_group @@ -292,7 +291,7 @@ def sanitize_state(state): ) def timegraph(self): - for n, b in lib.config.get_builds(self.job_group).items(): + for n, b in claims.config.get_builds(self.job_group).items(): f = "/tmp/timegraph-%s-build%s.svg" % (n, b['build']) timegraph.draw(self.results, f, b['tier']) logging.info("Generated %s" % f) @@ -420,12 +419,3 @@ def handle_args(self): self.timegraph() return 0 - - -def main(): - """Main program""" - return ClaimsCli().handle_args() - - -if __name__ == "__main__": - main() diff --git a/claims/config.py b/claims/config.py new file mode 100644 index 0000000..49be52b --- /dev/null +++ b/claims/config.py @@ -0,0 +1,39 @@ +import collections +import yaml +import logging + + +class Config(collections.UserDict): + + LATEST = 'latest' # how do we call latest job group in the config? + CACHEDIR = '.cache/' # where is the cache stored + + def __init__(self): + with open("config.yaml", "r") as file: + self.data = yaml.load(file) + + # Additional params when talking to Jenkins + self['headers'] = None + self['pull_params'] = { + u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' + } + + def get_builds(self, job_group=''): + if job_group == '': + job_group = self.LATEST + out = collections.OrderedDict() + for job in self.data['job_groups'][job_group]['jobs']: + key = self.data['job_groups'][job_group]['template'].format(**job) + out[key] = job + return out + + def init_headers(self): + url = '{0}/crumbIssuer/api/json'.format(self['url']) + crumb_data = request_get(url, params=None, expected_codes=[200], cached=False) + crumb = json.loads(crumb_data) + self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} + + +logging.basicConfig(level=logging.INFO) + +config = Config() diff --git a/claims/lib.py b/claims/lib.py deleted file mode 100755 index 537b65f..0000000 --- a/claims/lib.py +++ /dev/null @@ -1,386 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import division -import os -import sys -import json -import logging -import re -import urllib3 -import requests -import yaml -import pickle -import collections -import datetime -import tempfile -import subprocess -import shutil - -CACHEDIR = '.cache/' - -logging.basicConfig(level=logging.INFO) - -def request_get(url, params=None, expected_codes=[200], cached=True, stream=False): - # If available, read it from cache - if cached and not stream and os.path.isfile(cached): - with open(cached, 'r') as fp: - return fp.read() - - # Get the response from the server - urllib3.disable_warnings() - response = requests.get( - url, - auth=requests.auth.HTTPBasicAuth( - config['usr'], config['pwd']), - params=params, - verify=False - ) - - # Check we got expected exit code - if response.status_code not in expected_codes: - raise requests.HTTPError("Failed to get %s with %s" % (url, response.status_code)) - - # If we were streaming file - if stream: - with open(cached, 'w+b') as fp: - for chunk in response.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - fp.write(chunk) - fp.close() - return - - # In some cases 404 just means "we have nothing" - if response.status_code == 404: - return '' - - # If cache was configured, dump data in there - if cached: - os.makedirs(os.path.dirname(cached), exist_ok=True) - with open(cached, 'w') as fp: - fp.write(response.text) - - return response.text - - -class Config(collections.UserDict): - - LATEST = 'latest' # how do we call latest job group in the config? - - def __init__(self): - with open("config.yaml", "r") as file: - self.data = yaml.load(file) - - # Additional params when talking to Jenkins - self['headers'] = None - self['pull_params'] = { - u'tree': u'suites[cases[className,duration,name,status,stdout,errorDetails,errorStackTrace,testActions[reason]]]{0}' - } - - def get_builds(self, job_group=''): - if job_group == '': - job_group = self.LATEST - out = collections.OrderedDict() - for job in self.data['job_groups'][job_group]['jobs']: - key = self.data['job_groups'][job_group]['template'].format(**job) - out[key] = job - return out - - def init_headers(self): - url = '{0}/crumbIssuer/api/json'.format(self['url']) - crumb_data = request_get(url, params=None, expected_codes=[200], cached=False) - crumb = json.loads(crumb_data) - self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} - - -class ForemanDebug(object): - - def __init__(self, job_group, job, build): - self._url = "%s/job/%s/%s/artifact/foreman-debug.tar.xz" % (config['url'], job, build) - self._extracted = None - - @property - def extracted(self): - if self._extracted is None: - fp, fname = tempfile.mkstemp() - print(fname) - request_get(self._url, cached=fname, stream=True) - tmpdir = tempfile.mkdtemp() - subprocess.call(['tar', '-xf', fname, '--directory', tmpdir]) - logging.debug('Extracted to %s' % tmpdir) - self._extracted = os.path.join(tmpdir, 'foreman-debug') - return self._extracted - - -class ProductionLog(object): - - FILE_ENCODING = 'ISO-8859-1' # guessed it only, that file contains ugly binary mess as well - DATE_REGEXP = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} ') # 2018-06-13T07:37:26 - DATE_FMT = '%Y-%m-%dT%H:%M:%S' # 2018-06-13T07:37:26 - - def __init__(self, job_group, job, build): - self._log = None - self._logfile = os.path.join(CACHEDIR, job_group, job, 'production.log') - self._foreman_debug = None - - # If we do not have logfile downloaded already, we will need foreman-debug - if not os.path.isfile(self._logfile): - self._foreman_debug = ForemanDebug(job_group, job, build) - - @property - def log(self): - if self._log is None: - if self._foreman_debug is not None: - a = os.path.join(self._foreman_debug.extracted, - 'var', 'log', 'foreman', 'production.log') - shutil.copy2(a, self._logfile) - self._log = [] - buf = [] - last = None - with open(self._logfile, 'r', encoding=self.FILE_ENCODING) as fp: - for line in fp: - - # This line starts with date - denotes first line of new log record - if re.search(self.DATE_REGEXP, line): - - # This is a new log record, so firs save previous one - if len(buf) != 0: - self._log.append({'time': last, 'data': buf}) - last = datetime.datetime.strptime(line[:19], self.DATE_FMT) - buf = [] - buf.append(re.sub(self.DATE_REGEXP, '', line, count=1)) - - # This line does not start with line - comtains continuation of a log recorder started before - else: - buf.append(line) - - # Save last line - if len(buf) != 0: - self._log.append({'time': last, 'data': buf}) - - logging.debug("File %s parsed into memory" % self._logfile) - return self._log - - def from_to(self, from_time, to_time): - out = [] - for i in self.log: - if from_time <= i['time'] <= to_time: - out.append(i) - # Do not do following as time is not sequentional in the log (or maybe some workers are off or with different TZ?): - # TODO: Fix ordering of the log and uncomment this - # - # E.g.: - # 2018-06-17T17:29:44 [I|dyn|] start terminating clock... - # 2018-06-17T21:34:49 [I|app|] Current user: foreman_admin (administrator) - # 2018-06-17T21:37:21 [...] - # 2018-06-17T17:41:38 [I|app|] Started POST "/katello/api/v2/organizations"... - # - #if i['time'] > to_time: - # break - return out - - -class Case(collections.UserDict): - """ - Result of one test case - """ - - FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") - LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') - LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' - - def __init__(self, data): - self.data = data - - def __contains__(self, name): - return name in self.data or name in ('start', 'end', 'production.log') - - def __getitem__(self, name): - if name == 'testName': - self['testName'] = "%s.%s" % (self['className'], self['name']) - if name in ('start', 'end') and \ - ('start' not in self.data or 'end' not in self.data): - self.load_timings() - if name == 'production.log': - self['production.log'] = "\n".join( - ["\n".join(i['data']) for i in - self.data['OBJECT:production.log'].from_to( - self['start'], self['end'])]) - return self.data[name] - - def matches_to_rule(self, rule, indentation=0): - """ - Returns True if result matches to rule, otherwise returns False - """ - logging.debug("%srule_matches(%s, %s, %s)" % (" "*indentation, self['name'], rule, indentation)) - if 'field' in rule and 'pattern' in rule: - # This is simple rule, we can just check regexp against given field and we are done - try: - data = self[rule['field']] - if data is None: - data = '' - out = re.search(rule['pattern'], data) is not None - logging.debug("%s=> %s" % (" "*indentation, out)) - return out - except KeyError: - logging.debug("%s=> Failed to get field %s from case" % (" "*indentation, rule['field'])) - return None - elif 'AND' in rule: - # We need to check if all sub-rules in list of rules rule['AND'] matches - out = None - for r in rule['AND']: - r_out = self.matches_to_rule(r, indentation+4) - out = r_out if out is None else out and r_out - if not out: - break - return out - elif 'OR' in rule: - # We need to check if at least one sub-rule in list of rules rule['OR'] matches - for r in rule['OR']: - if self.matches_to_rule(r, indentation+4): - return True - return False - else: - raise Exception('Rule %s not formatted correctly' % rule) - - def push_claim(self, reason, sticky=False, propagate=False): - '''Claims a given test with a given reason - - :param reason: string with a comment added to a claim (ideally this is a link to a bug or issue) - - :param sticky: whether to make the claim sticky (False by default) - - :param propagate: should jenkins auto-claim next time if same test fails again? (False by default) - ''' - logging.info('claiming {0}::{1} with reason: {2}'.format(self["className"], self["name"], reason)) - - if config['headers'] is None: - config.init_headers() - - claim_req = requests.post( - u'{0}/claim/claim'.format(self['url']), - auth=requests.auth.HTTPBasicAuth( - config['usr'], - config['pwd'] - ), - data={u'json': u'{{"assignee": "", "reason": "{0}", "sticky": {1}, "propagateToFollowingBuilds": {2}}}'.format(reason, sticky, propagate)}, - headers=config['headers'], - allow_redirects=False, - verify=False - ) - - if claim_req.status_code != 302: - raise requests.HTTPError( - 'Failed to claim: {0}'.format(claim_req)) - - self['testActions'][0]['reason'] = reason - return(claim_req) - - def load_timings(self): - if self['stdout'] is None: - return - log = self['stdout'].split("\n") - log_size = len(log) - log_used = 0 - start = None - end = None - counter = 0 - while start is None: - match = self.LOG_DATE_REGEXP.match(log[counter]) - if match: - start = datetime.datetime.strptime(match.group(1), - self.LOG_DATE_FORMAT) - break - counter += 1 - log_used += counter - counter = -1 - while end is None: - match = self.LOG_DATE_REGEXP.match(log[counter]) - if match: - end = datetime.datetime.strptime(match.group(1), - self.LOG_DATE_FORMAT) - break - counter -= 1 - log_used -= counter - assert log_used <= log_size, \ - "Make sure detected start date is not below end date and vice versa" - self['start'] = start - self['end'] = end - - - - -class Report(collections.UserList): - """ - Report is a list of Cases (i.e. test results) - """ - - def __init__(self, job_group=''): - # If job group is not specified, we want latest one - if job_group == '': - job_group = config.LATEST - self.job_group = job_group - self._cache = os.path.join(CACHEDIR, self.job_group, 'main.pickle') - - # Attempt to load data from cache - if os.path.isfile(self._cache): - self.data = pickle.load(open(self._cache, 'rb')) - return - - # Load the actual data - self.data = [] - for name, meta in config.get_builds(self.job_group).items(): - build = meta['build'] - rhel = meta['rhel'] - tier = meta['tier'] - production_log = ProductionLog(self.job_group, name, build) - for report in self.pull_reports(name, build): - report['tier'] = tier - report['distro'] = rhel - report['OBJECT:production.log'] = production_log - self.data.append(Case(report)) - - # Dump parsed data into cache - pickle.dump(self.data, open(self._cache, 'wb')) - - def pull_reports(self, job, build): - """ - Fetches the test report for a given job and build - """ - build_url = '{0}/job/{1}/{2}'.format( - config['url'], job, build) - build_data = request_get( - build_url+'/testReport/api/json', - params=config['pull_params'], - expected_codes=[200, 404], - cached=os.path.join(CACHEDIR, self.job_group, job, 'main.json')) - cases = json.loads(build_data)['suites'][0]['cases'] - - # Enrich individual reports with URL - for c in cases: - className = c['className'].split('.')[-1] - testPath = '.'.join(c['className'].split('.')[:-1]) - c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) - - return(cases) - - -class Ruleset(collections.UserList): - - def __init__(self): - with open('kb.json', 'r') as fp: - self.data = json.loads(fp.read()) - - -# Create shared config file -config = Config() - -def claim_by_rules(report, rules, dryrun=False): - claimed = [] - for rule in rules: - for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: - if case.matches_to_rule(rule): - logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) - if not dryrun: - case.push_claim(rule['reason']) - claimed.append(case) - return claimed diff --git a/claims/report.py b/claims/report.py new file mode 100644 index 0000000..4aa480b --- /dev/null +++ b/claims/report.py @@ -0,0 +1,64 @@ +import os +import collections +import pickle +import json + +from .config import config +from .build_logs import ProductionLog +from .utils import request_get +from .case import Case + + +class Report(collections.UserList): + """ + Report is a list of Cases (i.e. test results) + """ + + def __init__(self, job_group=''): + # If job group is not specified, we want latest one + if job_group == '': + job_group = config.LATEST + self.job_group = job_group + self._cache = os.path.join(config.CACHEDIR, self.job_group, 'main.pickle') + + # Attempt to load data from cache + if os.path.isfile(self._cache): + self.data = pickle.load(open(self._cache, 'rb')) + return + + # Load the actual data + self.data = [] + for name, meta in config.get_builds(self.job_group).items(): + build = meta['build'] + rhel = meta['rhel'] + tier = meta['tier'] + production_log = ProductionLog(self.job_group, name, build) + for report in self.pull_reports(name, build): + report['tier'] = tier + report['distro'] = rhel + report['OBJECT:production.log'] = production_log + self.data.append(Case(report)) + + # Dump parsed data into cache + pickle.dump(self.data, open(self._cache, 'wb')) + + def pull_reports(self, job, build): + """ + Fetches the test report for a given job and build + """ + build_url = '{0}/job/{1}/{2}'.format( + config['url'], job, build) + build_data = request_get( + build_url+'/testReport/api/json', + params=config['pull_params'], + expected_codes=[200, 404], + cached=os.path.join(config.CACHEDIR, self.job_group, job, 'main.json')) + cases = json.loads(build_data)['suites'][0]['cases'] + + # Enrich individual reports with URL + for c in cases: + className = c['className'].split('.')[-1] + testPath = '.'.join(c['className'].split('.')[:-1]) + c['url'] = u'{0}/testReport/junit/{1}/{2}/{3}'.format(build_url, testPath, className, c['name']) + + return(cases) diff --git a/claims/ruleset.py b/claims/ruleset.py new file mode 100644 index 0000000..974c524 --- /dev/null +++ b/claims/ruleset.py @@ -0,0 +1,9 @@ +import collections +import json + + +class Ruleset(collections.UserList): + + def __init__(self): + with open('kb.json', 'r') as fp: + self.data = json.loads(fp.read()) diff --git a/claims/timegraph.py b/claims/timegraph.py index f2ddcca..275dc71 100644 --- a/claims/timegraph.py +++ b/claims/timegraph.py @@ -5,8 +5,6 @@ import datetime import svgwrite -import lib - STATUS_COLOR = { 'FAILED': 'red', diff --git a/claims/utils.py b/claims/utils.py new file mode 100755 index 0000000..849a12c --- /dev/null +++ b/claims/utils.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +from __future__ import division +import os +import logging +import urllib3 +import requests + + +def request_get(url, params=None, expected_codes=[200], cached=True, stream=False): + # If available, read it from cache + if cached and not stream and os.path.isfile(cached): + with open(cached, 'r') as fp: + return fp.read() + + # Get the response from the server + urllib3.disable_warnings() + response = requests.get( + url, + auth=requests.auth.HTTPBasicAuth( + config['usr'], config['pwd']), + params=params, + verify=False + ) + + # Check we got expected exit code + if response.status_code not in expected_codes: + raise requests.HTTPError("Failed to get %s with %s" % (url, response.status_code)) + + # If we were streaming file + if stream: + with open(cached, 'w+b') as fp: + for chunk in response.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + fp.write(chunk) + fp.close() + return + + # In some cases 404 just means "we have nothing" + if response.status_code == 404: + return '' + + # If cache was configured, dump data in there + if cached: + os.makedirs(os.path.dirname(cached), exist_ok=True) + with open(cached, 'w') as fp: + fp.write(response.text) + + return response.text + + +def claim_by_rules(report, rules, dryrun=False): + claimed = [] + for rule in rules: + for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: + if case.matches_to_rule(rule): + logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) + if not dryrun: + case.push_claim(rule['reason']) + claimed.append(case) + return claimed From 4b73050d591ab98d18a6e788cdbaa64e65700a04 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 26 Jul 2018 22:21:38 +0200 Subject: [PATCH 52/66] Fixing tests --- claims/case.py | 1 + claims/utils.py | 3 +++ test/test_claims.py | 4 ++-- test/test_claimscmd.py | 2 +- test/test_requests.py | 12 ++++++------ 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/claims/case.py b/claims/case.py index e21b0e0..d2d3611 100644 --- a/claims/case.py +++ b/claims/case.py @@ -1,5 +1,6 @@ import collections import re +import logging class Case(collections.UserDict): diff --git a/claims/utils.py b/claims/utils.py index 849a12c..176a9ef 100755 --- a/claims/utils.py +++ b/claims/utils.py @@ -6,6 +6,9 @@ import urllib3 import requests +from .config import config +from .case import Case + def request_get(url, params=None, expected_codes=[200], cached=True, stream=False): # If available, read it from cache diff --git a/test/test_claims.py b/test/test_claims.py index 2e7d5b5..344e8fd 100755 --- a/test/test_claims.py +++ b/test/test_claims.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: UTF-8 -*- -import claims +import claims.case def test_rule_matches(): checkme = { @@ -9,7 +9,7 @@ def test_rule_matches(): 'greeting': 'Hello world', 'area': 'IT Crowd', } - result = claims.Case(checkme) + result = claims.case.Case(checkme) assert result.matches_to_rule({'field': 'greeting', 'pattern': 'Hel+o'}) == True assert result.matches_to_rule({'field': 'greeting', 'pattern': 'This is not there'}) == False diff --git a/test/test_claimscmd.py b/test/test_claimscmd.py index 3f58fbf..eaefd62 100644 --- a/test/test_claimscmd.py +++ b/test/test_claimscmd.py @@ -16,7 +16,7 @@ def test_help(self): f = io.StringIO() with pytest.raises(SystemExit) as e: with redirect_stdout(f): - claims.cmd.main() + claims.cmd.ClaimsCli().handle_args() assert e.value.code == 0 assert 'Manipulate Jenkins claims with grace' in f.getvalue() assert 'optional arguments:' in f.getvalue() diff --git a/test/test_requests.py b/test/test_requests.py index 8c3200e..fa93ad0 100644 --- a/test/test_requests.py +++ b/test/test_requests.py @@ -5,19 +5,19 @@ import tempfile import pytest -import claims.lib +import claims.utils class TestClaimsRequestWrapper(): def test_get_sanity(self): - a = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=False) - b = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[200], cached=False) + a = claims.utils.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=False) + b = claims.utils.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[200], cached=False) assert a == b with pytest.raises(requests.HTTPError) as e: - claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[404], cached=False) + claims.utils.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', params=None, expected_codes=[404], cached=False) def test_get_caching(self): fp, fname = tempfile.mkstemp() - a = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) - b = claims.lib.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) + a = claims.utils.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) + b = claims.utils.request_get('http://inecas.fedorapeople.org/fakerepos/zoo3/repodata/repomd.xml', cached=fname) assert a == b From 50c92ec963dfe7789d0f2a07c51bcc78886747e5 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 26 Jul 2018 22:27:51 +0200 Subject: [PATCH 53/66] Updated docs --- README.md | 138 ++++++++++++++----------------------------------- kb.json.sample | 1 + 2 files changed, 40 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 8d16aca..ebdadea 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,40 @@ -A set of scripts for easier parsing and batch processing of the Jenkins test reports. - -Usage Examples: - -Parse all tiers for the last completed builds of the job -``` -In [1]: import claims - -In [2]: claims.config -Out[2]: -{'bld': 'lastCompletedBuild', - 'job': 'automation-6.2-tier{0}-rhel{1}', - 'pwd': 'nbusr123', - 'url': 'https://jenkins.server.com', - 'usr': 'uradnik1'} - -In [5]: reports -Out[5]: -['t1': - [ - 'el6': [ - { - u'className': u'tests.foreman.cli.test_syncplan.SyncPlanTestCase', - u'errorDetails': None, - u'errorStackTrace': None, - u'name': u'test_negative_synchronize_custom_product_past_sync_date', - u'status': u'PASSED', - u'testActions': [], - 'url': u'https://jenkins.server.com/job/automation-6.2-tier4-rhel7/lastCompletedBuild/testReport/junit/tests.foreman.cli.test_syncplan/SyncPlanTestCase/test_negative_synchronize_custom_product_past_sync_date' - } - ], - 'el7': [ - {...} - ] - ], -'t2': [...], -... -] -``` - -Get a flat list of all failed tests: -``` -In [6]: failures = [] - -In [7]: for i in reports.keys(): - ...: for j in reports[i].keys(): - ...: failures += claims.parse_fails(reports[i][j]) - ...: - -In [8]: len(failures) -Out[8]: 324 -In [9]: failures -Out[9]: -[ - {u'className': u'tests.foreman.cli.test_syncplan.SyncPlanTestCase', - u'errorDetails': u'AssertionError: Repository contains invalid number of content entities', - u'errorStackTrace': u'self = Date: Thu, 26 Jul 2018 22:31:44 +0200 Subject: [PATCH 54/66] Do not import it all in too difficult manner --- claims/__init__.py | 1 + test/test_claims.py | 5 +++-- test/test_claimscmd.py | 5 +++-- test/test_requests.py | 3 ++- test/test_timegraph.py | 1 + 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/claims/__init__.py b/claims/__init__.py index 9471715..ba6ccde 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -6,5 +6,6 @@ from .utils import request_get, claim_by_rules from .cmd import ClaimsCli + # Create shared config file config = Config() diff --git a/test/test_claims.py b/test/test_claims.py index 344e8fd..05d2d8c 100755 --- a/test/test_claims.py +++ b/test/test_claims.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: UTF-8 -*- -import claims.case +import claims + def test_rule_matches(): checkme = { @@ -9,7 +10,7 @@ def test_rule_matches(): 'greeting': 'Hello world', 'area': 'IT Crowd', } - result = claims.case.Case(checkme) + result = claims.Case(checkme) assert result.matches_to_rule({'field': 'greeting', 'pattern': 'Hel+o'}) == True assert result.matches_to_rule({'field': 'greeting', 'pattern': 'This is not there'}) == False diff --git a/test/test_claimscmd.py b/test/test_claimscmd.py index eaefd62..11f765b 100644 --- a/test/test_claimscmd.py +++ b/test/test_claimscmd.py @@ -7,7 +7,8 @@ import io from contextlib import redirect_stdout -import claims.cmd +import claims + class TestClaimsCli(object): @@ -16,7 +17,7 @@ def test_help(self): f = io.StringIO() with pytest.raises(SystemExit) as e: with redirect_stdout(f): - claims.cmd.ClaimsCli().handle_args() + claims.ClaimsCli().handle_args() assert e.value.code == 0 assert 'Manipulate Jenkins claims with grace' in f.getvalue() assert 'optional arguments:' in f.getvalue() diff --git a/test/test_requests.py b/test/test_requests.py index fa93ad0..d30bc56 100644 --- a/test/test_requests.py +++ b/test/test_requests.py @@ -5,7 +5,8 @@ import tempfile import pytest -import claims.utils +import claims + class TestClaimsRequestWrapper(): diff --git a/test/test_timegraph.py b/test/test_timegraph.py index 3261c3a..997b181 100644 --- a/test/test_timegraph.py +++ b/test/test_timegraph.py @@ -6,6 +6,7 @@ import claims.timegraph + class TestTimegraph(): def test_overlaps(self): From d55c671caf13638d38ca0c6b03380c099dc2c5f2 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 26 Jul 2018 22:42:50 +0200 Subject: [PATCH 55/66] Mask more files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 43a5546..2829503 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc config.yaml kb.json +*.swp +.cache/ +.pytest_cache/ From cd4c1c147778dbc50c15ed5216b4cf1e4e136134 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 26 Jul 2018 22:48:33 +0200 Subject: [PATCH 56/66] We do not need to create a config instance here --- claims/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/claims/__init__.py b/claims/__init__.py index ba6ccde..4281b8a 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -5,7 +5,3 @@ from .ruleset import Ruleset from .utils import request_get, claim_by_rules from .cmd import ClaimsCli - - -# Create shared config file -config = Config() From 08405dbbe54f178ab12b93d3874cbab4791e9aed Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 30 Jul 2018 10:43:39 +0200 Subject: [PATCH 57/66] That constant is in the config object --- claims/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index 9908efd..3ef0798 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -64,7 +64,7 @@ def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): tablefmt=self.output)) def clean_cache(self): - d = os.path.join(claims.CACHEDIR, self.job_group) + d = os.path.join(claims.config.config.CACHEDIR, self.job_group) try: shutil.rmtree(d) logging.info("Removed %s" % d) From 04debbe8a5d55372c4dc46da036f395eace27531 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 30 Jul 2018 12:15:39 +0200 Subject: [PATCH 58/66] Add missing imports, avoid circular imports --- claims/__init__.py | 4 ++-- claims/build_logs.py | 9 ++++++++- claims/case.py | 17 ++++++++++++++++- claims/cmd.py | 10 +++++----- claims/config.py | 7 ++++++- claims/report.py | 2 ++ claims/utils.py | 19 ++----------------- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/claims/__init__.py b/claims/__init__.py index 4281b8a..2c9e4b2 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -1,7 +1,7 @@ from .config import Config from .build_logs import ForemanDebug, ProductionLog -from .case import Case +from .case import Case, claim_by_rules from .report import Report from .ruleset import Ruleset -from .utils import request_get, claim_by_rules +from .utils import request_get from .cmd import ClaimsCli diff --git a/claims/build_logs.py b/claims/build_logs.py index d477173..9e615e0 100644 --- a/claims/build_logs.py +++ b/claims/build_logs.py @@ -1,7 +1,13 @@ import os import re +import tempfile +import subprocess +import logging +import shutil +import datetime from .config import config +from .utils import request_get class ForemanDebug(object): @@ -15,7 +21,8 @@ def extracted(self): if self._extracted is None: fp, fname = tempfile.mkstemp() print(fname) - request_get(self._url, cached=fname, stream=True) + request_get(self._url, config['usr'], config['pwd'], + cached=fname, stream=True) tmpdir = tempfile.mkdtemp() subprocess.call(['tar', '-xf', fname, '--directory', tmpdir]) logging.debug('Extracted to %s' % tmpdir) diff --git a/claims/case.py b/claims/case.py index d2d3611..a071355 100644 --- a/claims/case.py +++ b/claims/case.py @@ -1,6 +1,10 @@ import collections import re import logging +import datetime +import requests + +from .config import config class Case(collections.UserDict): @@ -8,7 +12,6 @@ class Case(collections.UserDict): Result of one test case """ - FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") LOG_DATE_REGEXP = re.compile('^([0-9]{4}-[01][0-9]-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) -') LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' @@ -129,3 +132,15 @@ def load_timings(self): "Make sure detected start date is not below end date and vice versa" self['start'] = start self['end'] = end + + +def claim_by_rules(report, rules, dryrun=False): + claimed = [] + for rule in rules: + for case in [i for i in report if i['status'] in config.FAIL_STATUSES and not i['testActions'][0].get('reason')]: + if case.matches_to_rule(rule): + logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) + if not dryrun: + case.push_claim(rule['reason']) + claimed.append(case) + return claimed diff --git a/claims/cmd.py b/claims/cmd.py index 3ef0798..381bdf1 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -74,19 +74,19 @@ def clean_cache(self): def show_failed(self): self._table( [[r['testName']] for r in self.results - if r['status'] in claims.Case.FAIL_STATUSES], + if r['status'] in config.FAIL_STATUSES], headers=['failed test name'], tablefmt=self.output) def show_claimed(self): self._table( [[r['testName'], r['testActions'][0].get('reason')] for r in self.results - if r['status'] in claims.Case.FAIL_STATUSES and r['testActions'][0].get('reason')], + if r['status'] in config.FAIL_STATUSES and r['testActions'][0].get('reason')], headers=['claimed test name', 'claim reason'], tablefmt=self.output) def show_unclaimed(self): self._table( [[r['testName']] for r in self.results - if r['status'] in claims.Case.FAIL_STATUSES and not r['testActions'][0].get('reason')], + if r['status'] in config.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output) def show_claimable(self): @@ -137,7 +137,7 @@ def _perc(perc_from, perc_sum): stat_all = len(self.results) reports_fails = [i for i in self.results - if i['status'] in claims.Case.FAIL_STATUSES] + if i['status'] in config.FAIL_STATUSES] stat_failed = len(reports_fails) reports_claimed = [i for i in reports_fails if i['testActions'][0].get('reason')] @@ -152,7 +152,7 @@ def _perc(perc_from, perc_sum): filtered = [r for r in self.results if r['tier'] == t] stat_all_tiered = len(filtered) reports_fails_tiered = [i for i in filtered - if i['status'] in claims.Case.FAIL_STATUSES] + if i['status'] in config.FAIL_STATUSES] stat_failed_tiered = len(reports_fails_tiered) reports_claimed_tiered = [i for i in reports_fails_tiered if i['testActions'][0].get('reason')] diff --git a/claims/config.py b/claims/config.py index 49be52b..61fadfd 100644 --- a/claims/config.py +++ b/claims/config.py @@ -1,10 +1,14 @@ import collections import yaml +import json import logging +from .utils import request_get + class Config(collections.UserDict): + FAIL_STATUSES = ("FAILED", "ERROR", "REGRESSION") LATEST = 'latest' # how do we call latest job group in the config? CACHEDIR = '.cache/' # where is the cache stored @@ -29,7 +33,8 @@ def get_builds(self, job_group=''): def init_headers(self): url = '{0}/crumbIssuer/api/json'.format(self['url']) - crumb_data = request_get(url, params=None, expected_codes=[200], cached=False) + crumb_data = request_get(url, self['usr'], self['pwd'], + params=None, expected_codes=[200], cached=False) crumb = json.loads(crumb_data) self['headers'] = {crumb['crumbRequestField']: crumb['crumb']} diff --git a/claims/report.py b/claims/report.py index 4aa480b..dea4f14 100644 --- a/claims/report.py +++ b/claims/report.py @@ -50,6 +50,8 @@ def pull_reports(self, job, build): config['url'], job, build) build_data = request_get( build_url+'/testReport/api/json', + user=conf['usr'], + password=conf['pwd'], params=config['pull_params'], expected_codes=[200, 404], cached=os.path.join(config.CACHEDIR, self.job_group, job, 'main.json')) diff --git a/claims/utils.py b/claims/utils.py index 176a9ef..a01fd6a 100755 --- a/claims/utils.py +++ b/claims/utils.py @@ -6,11 +6,8 @@ import urllib3 import requests -from .config import config -from .case import Case - -def request_get(url, params=None, expected_codes=[200], cached=True, stream=False): +def request_get(url, user, password, params=None, expected_codes=[200], cached=True, stream=False): # If available, read it from cache if cached and not stream and os.path.isfile(cached): with open(cached, 'r') as fp: @@ -21,7 +18,7 @@ def request_get(url, params=None, expected_codes=[200], cached=True, stream=Fals response = requests.get( url, auth=requests.auth.HTTPBasicAuth( - config['usr'], config['pwd']), + user, password), params=params, verify=False ) @@ -50,15 +47,3 @@ def request_get(url, params=None, expected_codes=[200], cached=True, stream=Fals fp.write(response.text) return response.text - - -def claim_by_rules(report, rules, dryrun=False): - claimed = [] - for rule in rules: - for case in [i for i in report if i['status'] in Case.FAIL_STATUSES and not i['testActions'][0].get('reason')]: - if case.matches_to_rule(rule): - logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) - if not dryrun: - case.push_claim(rule['reason']) - claimed.append(case) - return claimed From ef2ac8185e10ece28b56b0c8ce029bb8a9879559 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 30 Jul 2018 12:17:16 +0200 Subject: [PATCH 59/66] Typo --- claims/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/claims/report.py b/claims/report.py index dea4f14..e3e9f19 100644 --- a/claims/report.py +++ b/claims/report.py @@ -50,8 +50,8 @@ def pull_reports(self, job, build): config['url'], job, build) build_data = request_get( build_url+'/testReport/api/json', - user=conf['usr'], - password=conf['pwd'], + user=config['usr'], + password=config['pwd'], params=config['pull_params'], expected_codes=[200, 404], cached=os.path.join(config.CACHEDIR, self.job_group, job, 'main.json')) From 15ac2cb6dd83cd39becd92294515944e7325a726 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 30 Jul 2018 12:19:35 +0200 Subject: [PATCH 60/66] More small fixes --- claims/cmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/claims/cmd.py b/claims/cmd.py index 381bdf1..693e257 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -13,6 +13,7 @@ import shutil import claims +from claims.config import config logging.basicConfig(level=logging.INFO) @@ -64,7 +65,7 @@ def _table(self, data, headers=[], tablefmt=None, floatfmt='%.01f'): tablefmt=self.output)) def clean_cache(self): - d = os.path.join(claims.config.config.CACHEDIR, self.job_group) + d = os.path.join(config.CACHEDIR, self.job_group) try: shutil.rmtree(d) logging.info("Removed %s" % d) From 820ee772d60418a1bca41b2bbfbc3f952c205ae0 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 31 Jul 2018 21:06:20 +0200 Subject: [PATCH 61/66] More of imports fixing --- claims/__init__.py | 1 + claims/cmd.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/claims/__init__.py b/claims/__init__.py index 2c9e4b2..87a8afd 100755 --- a/claims/__init__.py +++ b/claims/__init__.py @@ -5,3 +5,4 @@ from .ruleset import Ruleset from .utils import request_get from .cmd import ClaimsCli +from . import timegraph diff --git a/claims/cmd.py b/claims/cmd.py index 693e257..9350b62 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -13,7 +13,7 @@ import shutil import claims -from claims.config import config +from .config import config logging.basicConfig(level=logging.INFO) @@ -148,7 +148,7 @@ def _perc(perc_from, perc_sum): stat_all), stat_claimed, _perc(stat_claimed, stat_failed)] stats = [] - builds = claims.config.get_builds(self.job_group).values() + builds = config.get_builds(self.job_group).values() for t in [i['tier'] for i in builds]: filtered = [r for r in self.results if r['tier'] == t] stat_all_tiered = len(filtered) @@ -247,7 +247,7 @@ def sanitize_state(state): matrix = collections.OrderedDict() # Load tests results - job_groups = claims.config['job_groups'].keys() + job_groups = config['job_groups'].keys() for job_group in job_groups: logging.info('Loading job group %s' % job_group) self.job_group = job_group @@ -292,9 +292,9 @@ def sanitize_state(state): ) def timegraph(self): - for n, b in claims.config.get_builds(self.job_group).items(): + for n, b in config.get_builds(self.job_group).items(): f = "/tmp/timegraph-%s-build%s.svg" % (n, b['build']) - timegraph.draw(self.results, f, b['tier']) + claims.timegraph.draw(self.results, f, b['tier']) logging.info("Generated %s" % f) def handle_args(self): From 3684a4c6459d4409d50fda90a4f1d32df7f7d62b Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 15 Aug 2018 12:59:50 +0200 Subject: [PATCH 62/66] Remove some debug prints --- claims/build_logs.py | 1 - claims/cmd.py | 1 - 2 files changed, 2 deletions(-) diff --git a/claims/build_logs.py b/claims/build_logs.py index 9e615e0..6306b18 100644 --- a/claims/build_logs.py +++ b/claims/build_logs.py @@ -20,7 +20,6 @@ def __init__(self, job_group, job, build): def extracted(self): if self._extracted is None: fp, fname = tempfile.mkstemp() - print(fname) request_get(self._url, config['usr'], config['pwd'], cached=fname, stream=True) tmpdir = tempfile.mkdtemp() diff --git a/claims/cmd.py b/claims/cmd.py index 9350b62..9a49562 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -343,7 +343,6 @@ def handle_args(self): help='Show also debug messages') args = parser.parse_args() - print(args) # Handle "--debug" if args.debug: From 7a465173397768522ea676b3f6c61c4d31f69e14 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 15 Aug 2018 13:05:22 +0200 Subject: [PATCH 63/66] Also show rule used --- claims/case.py | 2 +- claims/cmd.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/claims/case.py b/claims/case.py index a071355..9c13238 100644 --- a/claims/case.py +++ b/claims/case.py @@ -142,5 +142,5 @@ def claim_by_rules(report, rules, dryrun=False): logging.debug(u"{0}::{1} matching pattern for '{2}' on {3}".format(case['className'], case['name'], rule['reason'], case['url'])) if not dryrun: case.push_claim(rule['reason']) - claimed.append(case) + claimed.append((case, rule)) return claimed diff --git a/claims/cmd.py b/claims/cmd.py index 9a49562..47ecb4d 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -93,8 +93,9 @@ def show_unclaimed(self): def show_claimable(self): claimable = claims.claim_by_rules(self.results, self.rules, dryrun=True) self._table( - [[r['testName']] for r in claimable], - headers=['claimable test name'], tablefmt=self.output) + [[i[0]['testName'], i[1]['reason']] for i in claimable], + headers=['claimable test name', 'claimable with reason'], + tablefmt=self.output) def show(self, test_class, test_name): MAXWIDTH = 100 @@ -125,8 +126,9 @@ def show(self, test_class, test_name): def claim(self): claimed = claims.claim_by_rules(self.results, self.rules, dryrun=False) self._table( - [[r['testName']] for r in claimed], - headers=['claimed test name'], tablefmt=self.output) + [[i[0]['testName'], i[1]['reason']] for i in claimed], + headers=['claimed test name', 'claimed with reason'], + tablefmt=self.output) def stats(self): def _perc(perc_from, perc_sum): From 51daf7d7c58ab8c890272c9a7b10620180ef05cf Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 16 Aug 2018 16:25:42 +0200 Subject: [PATCH 64/66] Show also tier and rhel --- claims/case.py | 4 +++- claims/cmd.py | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/claims/case.py b/claims/case.py index 9c13238..214f562 100644 --- a/claims/case.py +++ b/claims/case.py @@ -19,11 +19,13 @@ def __init__(self, data): self.data = data def __contains__(self, name): - return name in self.data or name in ('start', 'end', 'production.log') + return name in self.data or name in ('testName', 'testId', 'start', 'end', 'production.log') def __getitem__(self, name): if name == 'testName': self['testName'] = "%s.%s" % (self['className'], self['name']) + elif name == 'testId': + self['testId'] = "%s (tier%s, el%s)" % (self['testName'], self['tier'], self['distro']) if name in ('start', 'end') and \ ('start' not in self.data or 'end' not in self.data): self.load_timings() diff --git a/claims/cmd.py b/claims/cmd.py index 47ecb4d..5d6ee2b 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -74,26 +74,26 @@ def clean_cache(self): def show_failed(self): self._table( - [[r['testName']] for r in self.results + [[r['testId']] for r in self.results if r['status'] in config.FAIL_STATUSES], headers=['failed test name'], tablefmt=self.output) def show_claimed(self): self._table( - [[r['testName'], r['testActions'][0].get('reason')] for r in self.results + [[r['testId'], r['testActions'][0].get('reason')] for r in self.results if r['status'] in config.FAIL_STATUSES and r['testActions'][0].get('reason')], headers=['claimed test name', 'claim reason'], tablefmt=self.output) def show_unclaimed(self): self._table( - [[r['testName']] for r in self.results + [[r['testId']] for r in self.results if r['status'] in config.FAIL_STATUSES and not r['testActions'][0].get('reason')], headers=['unclaimed test name'], tablefmt=self.output) def show_claimable(self): claimable = claims.claim_by_rules(self.results, self.rules, dryrun=True) self._table( - [[i[0]['testName'], i[1]['reason']] for i in claimable], + [[i[0]['testId'], i[1]['reason']] for i in claimable], headers=['claimable test name', 'claimable with reason'], tablefmt=self.output) @@ -126,7 +126,7 @@ def show(self, test_class, test_name): def claim(self): claimed = claims.claim_by_rules(self.results, self.rules, dryrun=False) self._table( - [[i[0]['testName'], i[1]['reason']] for i in claimed], + [[i[0]['testId'], i[1]['reason']] for i in claimed], headers=['claimed test name', 'claimed with reason'], tablefmt=self.output) @@ -255,7 +255,7 @@ def sanitize_state(state): self.job_group = job_group report = self.results for r in report: - t = "%s::%s@%s" % (r['className'], r['name'], r['distro']) + t = r['testId'] if t not in matrix: matrix[t] = dict.fromkeys(job_group) try: From b3742fe2d00728e274442cb7a963e5712bc8c29b Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 16 Aug 2018 21:47:53 +0200 Subject: [PATCH 65/66] Way to diff two runs --- claims/cmd.py | 95 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 5d6ee2b..4e2b95e 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -24,6 +24,7 @@ class ClaimsCli(object): def __init__(self): self.job_group = self.LATEST + self.job_group_old = None self.grep_results = None self.grep_rules = None self._results = None @@ -233,18 +234,18 @@ def _perc(perc_from, perc_sum): floatfmt=".1f", tablefmt=self.output) - def history(self): + def _sanitize_state(self, state): + if state == 'REGRESSION': + state = 'FAILED' + if state == 'FIXED': + state = 'PASSED' + if state == 'PASSED': + return 0 + if state == 'FAILED': + return 1 + raise KeyError("Do not know how to handle state %s" % state) - def sanitize_state(state): - if state == 'REGRESSION': - state = 'FAILED' - if state == 'FIXED': - state = 'PASSED' - if state == 'PASSED': - return 0 - if state == 'FAILED': - return 1 - raise KeyError("Do not know how to handle state %s" % state) + def history(self): matrix = collections.OrderedDict() @@ -259,7 +260,7 @@ def sanitize_state(state): if t not in matrix: matrix[t] = dict.fromkeys(job_group) try: - state = sanitize_state(r['status']) + state = self._sanitize_state(r['status']) except KeyError: continue # e.g. state "SKIPPED" matrix[t][job_group] = state @@ -293,6 +294,62 @@ def sanitize_state(state): floatfmt=".3f" ) + def diff(self): + assert self.job_group_old, 'When using --diff, also specify --job-group-old' + logging.info('Diffing %s to %s' % (self.job_group_old, self.job_group)) + + matrix = collections.OrderedDict() + + # Load tests results + state_good = 'GOOD' + state_bad = 'BAD' + states = { + 0: state_good, + 1: state_bad, + } + job_groups = (self.job_group_old, self.job_group) + for job_group in job_groups: + logging.info('Loading job group %s' % job_group) + self.job_group = job_group + for r in self.results: + t = r['testId'] + if t not in matrix: + matrix[t] = dict.fromkeys(job_groups) + try: + state = states[self._sanitize_state(r['status'])] + except KeyError: + state = r['status'] + matrix[t][job_group] = state + + good = collections.OrderedDict() + bad = collections.OrderedDict() + stable = 0 + + # Find tests that got better and tests that got worse + for test, jgs in matrix.items(): + if jgs[self.job_group_old] != jgs[self.job_group]: + if jgs[self.job_group_old] == state_good: + bad[test] = (jgs[self.job_group_old], jgs[self.job_group]) + if jgs[self.job_group] == state_good: + good[test] = (jgs[self.job_group_old], jgs[self.job_group]) + else: + stable += 1 + + # Print diff findings + print("\nBad tests") + self._table( + [[k, "%s -> %s" % v] for k,v in bad.items()], + headers=['test', 'state change'], + tablefmt=self.output) + print("\nGood tests") + self._table( + [[k, "%s -> %s" % v] for k,v in good.items()], + headers=['test', 'state change'], + tablefmt=self.output) + print("\nCount of tests that stayed same") + print(stable) + + def timegraph(self): for n, b in config.get_builds(self.job_group).items(): f = "/tmp/timegraph-%s-build%s.svg" % (n, b['build']) @@ -324,6 +381,10 @@ def handle_args(self): help='Show stats for selected job group') parser.add_argument('--history', action='store_true', help='Show how tests results and duration evolved') + parser.add_argument('--diff', action='store_true', + help='Show which test result changed between two' + ' job groups. You will need --job-group' + ' and --job-group-old options set') parser.add_argument('--timegraph', action='store_true', help='Generate time graph') @@ -331,6 +392,8 @@ def handle_args(self): parser.add_argument('--job-group', action='store', help='Specify group of jobs to perform the action' ' with (default: latest)') + parser.add_argument('--job-group-old', action='store', + help='Only used with --diff') parser.add_argument('--grep-results', action='store', metavar='REGEXP', help='Only work with tests, whose' ' "className+name" matches the regexp') @@ -356,6 +419,10 @@ def handle_args(self): self.job_group = args.job_group logging.debug("Job group we are going to work with is %s" % self.job_group) + if args.job_group_old: + self.job_group_old = args.job_group_old + logging.debug("Old job group we are going to work with is %s" + % self.job_group_old) # Handle "--grep-results something" if args.grep_results: @@ -416,6 +483,10 @@ def handle_args(self): elif args.history: self.history() + # Show tests diff across two job groups + elif args.diff: + self.diff() + # Generate time graphs per tier elif args.timegraph: self.timegraph() From 1da10fe621778c4b3cd414cbd6be995c7999c6de Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Thu, 16 Aug 2018 22:04:02 +0200 Subject: [PATCH 66/66] Add counts --- claims/cmd.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/claims/cmd.py b/claims/cmd.py index 4e2b95e..4a60859 100755 --- a/claims/cmd.py +++ b/claims/cmd.py @@ -296,7 +296,6 @@ def history(self): def diff(self): assert self.job_group_old, 'When using --diff, also specify --job-group-old' - logging.info('Diffing %s to %s' % (self.job_group_old, self.job_group)) matrix = collections.OrderedDict() @@ -336,18 +335,17 @@ def diff(self): stable += 1 # Print diff findings - print("\nBad tests") + print("\nBad tests (%s)" % len(bad)) self._table( [[k, "%s -> %s" % v] for k,v in bad.items()], headers=['test', 'state change'], tablefmt=self.output) - print("\nGood tests") + print("\nGood tests (%s)" % len(good)) self._table( [[k, "%s -> %s" % v] for k,v in good.items()], headers=['test', 'state change'], tablefmt=self.output) - print("\nCount of tests that stayed same") - print(stable) + print("\nRest of the tests stayed same (%s)" % stable) def timegraph(self):