From 731bd01e1bd7866e62b28cdf678ed1f468d23e41 Mon Sep 17 00:00:00 2001 From: shahidash Date: Sat, 19 Dec 2015 12:29:28 +0530 Subject: [PATCH 1/4] added parse code for sar -n DEV section --- sar/__init__.py | 22 ++++++++++++++++- sar/parser.py | 64 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/sar/__init__.py b/sar/__init__.py index a661865..3a1ae36 100644 --- a/sar/__init__.py +++ b/sar/__init__.py @@ -13,6 +13,9 @@ """I/O usage part of SAR file""" PART_IO = 3 +"""IFACE usage part of SAR file""" +PART_IFACE = 4 + """CPU regexp pattern for detecting SAR section header""" PATTERN_CPU = ".*CPU.*(usr|user).*nice.*sys.*" @@ -71,6 +74,23 @@ } +"""IFACE pattern for detecting the IFACE section""" +#10:18:01 AM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s +PATTERN_IFACE = ".*IFACE.*rxpck\/s.*txpck\/s.*rxkB\/s.*txkB\/s.*rxcmp\/s.*txcmp\/s.*rxmcst\/s.*" + +"""Regexp terms for finding fields in SAR parts for IFACE""" +FIELDS_IFACE = [ + 'rxpck\/s', 'txpck\/s','rxkB\/s','txkB\/s','rxcmp/s','txcmp\/s','rxmcst\/s' +] + +"""Pair regexp terms with field names in IFACE output dictionary""" +FIELD_PAIRS_IFACE = { + + "rxpcks":FIELDS_IFACE[0], "txpcks":FIELDS_IFACE[1], 'rxkbs':FIELDS_IFACE[2], + 'txkbs':FIELDS_IFACE[3], 'rxcmps':FIELDS_IFACE[4], 'txcmps':FIELDS_IFACE[5], + 'rxmcsts':FIELDS_IFACE[6] +} + """Restart time regexp pattern for detecting SAR restart notices""" PATTERN_RESTART = ".*LINUX\ RESTART.*" @@ -83,5 +103,5 @@ __all__ = [ "PART_CPU", "PART_MEM", "PART_SWP", "PART_IO", "PATTERN_CPU", "PATTERN_MEM", "PATTERN_SWP", "PATTERN_IO", - "PATTERN_RESTART", "PATTERN_MULTISPLIT", "PATTERN_DATE" + "PATTERN_RESTART", "PATTERN_MULTISPLIT", "PATTERN_DATE","PATTERN_IFACE" ] diff --git a/sar/parser.py b/sar/parser.py index e0ec85d..c34e89f 100644 --- a/sar/parser.py +++ b/sar/parser.py @@ -6,10 +6,11 @@ Parses SAR ASCII output only, not binary files! ''' -from sar import PART_CPU, PART_MEM, PART_SWP, PART_IO, \ +from __init__ import PART_CPU, PART_MEM, PART_SWP, PART_IO,PART_IFACE, \ PATTERN_CPU, PATTERN_MEM, PATTERN_SWP, PATTERN_IO, PATTERN_RESTART, \ + PATTERN_IFACE,\ FIELDS_CPU, FIELD_PAIRS_CPU, FIELDS_MEM, FIELD_PAIRS_MEM, FIELDS_SWP, \ - FIELD_PAIRS_SWP, FIELDS_IO, FIELD_PAIRS_IO + FIELD_PAIRS_SWP, FIELDS_IO, FIELD_PAIRS_IO,FIELDS_IFACE,FIELD_PAIRS_IFACE import mmap import os import re @@ -46,10 +47,12 @@ def __init__(self, filename=''): self.__io_fields = None '''I/O usage indexes''' + self.__iface_fields = None + return None def load_file(self): - ''' + print ''' Loads SAR format logfile in ASCII format (sarXX). :return: ``True`` if loading and parsing of file went fine, \ ``False`` if it failed (at any point) @@ -57,11 +60,12 @@ def load_file(self): # We first split file into pieces searchunks = self._split_file() + # print len(searchunks) if (searchunks): # And then we parse pieces into meaningful data - cpu_usage, mem_usage, swp_usage, io_usage = \ + cpu_usage, mem_usage, swp_usage, io_usage, iface_usage = \ self._parse_file(searchunks) if (cpu_usage is False): @@ -71,12 +75,14 @@ def load_file(self): "cpu": cpu_usage, "mem": mem_usage, "swap": swp_usage, - "io": io_usage + "io": io_usage, + "iface":iface_usage } del(cpu_usage) del(mem_usage) del(swp_usage) del(io_usage) + del(iface_usage) return True @@ -99,6 +105,7 @@ def get_sar_info(self): Returns parsed sar info :return: ``Dictionary``-style list of SAR data ''' + print "get_sar_info" try: test = self._sarinfo["cpu"] @@ -128,7 +135,7 @@ def _split_file(self, data=''): the type of info they contain (SAR file sections) without parsing what is exactly what at this point ''' - + print "_split_file" # Filename passed checks through __init__ if ((self.__filename and os.access(self.__filename, os.R_OK)) or data != ''): @@ -242,10 +249,12 @@ def _parse_file(self, sar_parts): :return: ``Dictionary``-style info (but still non-parsed) \ from SAR file, split into sections we want to check ''' + print "_parse_file" cpu_usage = '' mem_usage = '' swp_usage = '' io_usage = '' + iface_usage = '' # If sar_parts is a list if (type(sar_parts) is ListType): @@ -255,6 +264,7 @@ def _parse_file(self, sar_parts): mem_pattern = re.compile(PATTERN_MEM) swp_pattern = re.compile(PATTERN_SWP) io_pattern = re.compile(PATTERN_IO) + iface_pattern = re.compile(PATTERN_IFACE) restart_pattern = re.compile(PATTERN_RESTART) ''' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''' @@ -329,21 +339,39 @@ def _parse_file(self, sar_parts): self.__restart_times.append(pieces[0]) del(pieces) + if (iface_pattern.search(part)): + if (iface_usage == ''): + iface_usage = part + try: + first_line = part.split("\n")[0] + except IndexError: + first_line = part + + self.__iface_fields = \ + self.__find_column(FIELDS_IFACE, first_line) + + else: + iface_usage += "\n" + part + + del(sar_parts) # Now we have parts pulled out and combined, do further # processing. cpu_output = self.__split_info(cpu_usage, PART_CPU) + mem_output = self.__split_info(mem_usage, PART_MEM) swp_output = self.__split_info(swp_usage, PART_SWP) io_output = self.__split_info(io_usage, PART_IO) + iface_output = self.__split_info(iface_usage,PART_IFACE) del(cpu_usage) del(mem_usage) del(swp_usage) del(io_usage) + del(iface_usage) - return (cpu_output, mem_output, swp_output, io_output) + return (cpu_output, mem_output, swp_output, io_output,iface_output) return (False, False, False) @@ -403,12 +431,14 @@ def __split_info(self, info_part, part_type=PART_CPU): pattern = PATTERN_SWP elif (part_type == PART_IO): pattern = PATTERN_IO + elif(part_type == PART_IFACE): + pattern = PATTERN_IFACE if (pattern == ''): return False return_dict = {} - + print pattern pattern_re = re.compile(pattern) for part_line in info_part.split("\n"): @@ -477,9 +507,13 @@ def __split_info(self, info_part, part_type=PART_CPU): elif part_type == PART_IO: fields = self.__io_fields pairs = FIELD_PAIRS_IO + elif part_type == PART_IFACE: + fields = self.__iface_fields + pairs = FIELD_PAIRS_IFACE + # print pairs for sectionname in pairs.iterkeys(): - + # print sectionname value = elems[fields[pairs[sectionname]]] if sectionname == 'membuffer' or \ @@ -504,6 +538,18 @@ def __split_info(self, info_part, part_type=PART_CPU): else: return_dict[full_time][sectionname] = value + if part_type == PART_IFACE: + cpuid = elems[(1 if is_24hr is True else 2)] + try: + blah = return_dict[full_time][cpuid] + del(blah) + except KeyError: + return_dict[full_time][cpuid] = {} + return_dict[full_time][cpuid][sectionname] = \ + value + else: + return_dict[full_time][sectionname] = value + return (return_dict) def __get_filedate(self): From 4169fb1c59b0188245b8a63205468cdc175fde39 Mon Sep 17 00:00:00 2001 From: shahidash Date: Sat, 19 Dec 2015 12:56:10 +0530 Subject: [PATCH 2/4] added sar -n DEV parser changes to multiparser --- sar/multiparser.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sar/multiparser.py b/sar/multiparser.py index 6098011..dfc27fb 100644 --- a/sar/multiparser.py +++ b/sar/multiparser.py @@ -14,7 +14,7 @@ ''' -import sar.parser as sarparse +import parser as sarparse from sar import PATTERN_MULTISPLIT import mmap import os @@ -61,19 +61,21 @@ def load_file(self): chunk = self.__get_chunk(start, end) parser = sarparse.Parser() - cpu_usage, mem_usage, swp_usage, io_usage = \ + cpu_usage, mem_usage, swp_usage, io_usage,iface_usage = \ parser._parse_file(parser._split_file(chunk)) self.__sarinfos[self.__get_part_date(chunk)] = { "cpu": cpu_usage, "mem": mem_usage, "swap": swp_usage, - "io": io_usage + "io": io_usage, + "iface":iface_usage } del(cpu_usage) del(mem_usage) del(swp_usage) del(io_usage) + del(iface_usage) del(parser) return(True) From 2cb89439a5989eecd76197578b58e337608e2ea4 Mon Sep 17 00:00:00 2001 From: shahidash Date: Sat, 19 Dec 2015 13:08:21 +0530 Subject: [PATCH 3/4] removed printfs and some import path changes.. --- sar/multiparser.py | 2 +- sar/parser.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/sar/multiparser.py b/sar/multiparser.py index dfc27fb..ef2d475 100644 --- a/sar/multiparser.py +++ b/sar/multiparser.py @@ -14,7 +14,7 @@ ''' -import parser as sarparse +import sar.parser as sarparse from sar import PATTERN_MULTISPLIT import mmap import os diff --git a/sar/parser.py b/sar/parser.py index c34e89f..46dcca0 100644 --- a/sar/parser.py +++ b/sar/parser.py @@ -52,7 +52,7 @@ def __init__(self, filename=''): return None def load_file(self): - print ''' + ''' Loads SAR format logfile in ASCII format (sarXX). :return: ``True`` if loading and parsing of file went fine, \ ``False`` if it failed (at any point) @@ -105,7 +105,6 @@ def get_sar_info(self): Returns parsed sar info :return: ``Dictionary``-style list of SAR data ''' - print "get_sar_info" try: test = self._sarinfo["cpu"] @@ -135,7 +134,6 @@ def _split_file(self, data=''): the type of info they contain (SAR file sections) without parsing what is exactly what at this point ''' - print "_split_file" # Filename passed checks through __init__ if ((self.__filename and os.access(self.__filename, os.R_OK)) or data != ''): @@ -249,7 +247,6 @@ def _parse_file(self, sar_parts): :return: ``Dictionary``-style info (but still non-parsed) \ from SAR file, split into sections we want to check ''' - print "_parse_file" cpu_usage = '' mem_usage = '' swp_usage = '' @@ -438,7 +435,6 @@ def __split_info(self, info_part, part_type=PART_CPU): return False return_dict = {} - print pattern pattern_re = re.compile(pattern) for part_line in info_part.split("\n"): @@ -511,9 +507,8 @@ def __split_info(self, info_part, part_type=PART_CPU): fields = self.__iface_fields pairs = FIELD_PAIRS_IFACE - # print pairs for sectionname in pairs.iterkeys(): - # print sectionname + value = elems[fields[pairs[sectionname]]] if sectionname == 'membuffer' or \ From 3733803b804509b7405d7c2d99ea56954806fd39 Mon Sep 17 00:00:00 2001 From: shahidash Date: Tue, 19 Jan 2016 12:24:07 +0530 Subject: [PATCH 4/4] cpu parsing handler.. --- sar/parser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sar/parser.py b/sar/parser.py index 46dcca0..d591a7c 100644 --- a/sar/parser.py +++ b/sar/parser.py @@ -509,7 +509,11 @@ def __split_info(self, info_part, part_type=PART_CPU): for sectionname in pairs.iterkeys(): - value = elems[fields[pairs[sectionname]]] + try : + value = elems[fields[pairs[sectionname]]] + except Exception as e: + if sectionname == 'idle' and len(elems) < 12: + value = elems[-1] if sectionname == 'membuffer' or \ sectionname == 'memcache' or \