CollectMetrics.py

#!/usr/bin/env python

# CollectMetrics.py

from datetime import timedelta, datetime
# from dateutil.tz import tzutc
from os import path
from datetime import datetime
from TimeIt import *
# from dateutil.tz import tzutc

import csv 
import datetime 
import json 
import os
import os.path
import socket 
import sys 
import time
import uuid

class CollectMetrics: 
    def __init__( self,
                metricsPath,
                rulesFile,
                LogSizeLimit = 100 * 1024 * 1024, # 100 MB size limit for metrics log 
                prefix='gauge',
                suffix=None,
                ext=None
            ):
        ts = time.time()
        today = datetime.datetime.fromtimestamp (float(ts)).strftime('%Y-%m-%d')

        self.alarmWeight = 6
        self.alarmMetrics = []
        self.rules = []
        self.metricsPath = metricsPath
        self.rulesFile = rulesFile
        self.logSizeLimit = LogSizeLimit
        self.prefix = prefix+'-' if prefix else ''
        self.suffix = '-'+suffix if suffix else ''
        self.ext = '.'+ext if ext else ''

        try:
            with open(self.rulesFile) as f:
                self.rules = json.load (f)
        except Exception as e:
            print('JSON FILE LOAD ERROR FOR:', self.rulesFile)
            print(e)

        if len(self.rules)>0:
            for r in self.rules:
                self.alarmWeight = self.alarmWeight+r.get('Of', 0)

    def getRules (self):
        return self.rules

    def getTs (self):
        return time.time()

    def getAlarmMetrics (self): 
        return self.alarmMetrics
    
    def getAlarmWeight(self):
        return self.alarmWeight
    
    def getAlarmStates (self): 
        return [
            'OK', 
            'ALARM',
            'INSUFFICIENT_DATA',
        ]
    
    def getStatistics (self, rs, action='Average'):
        actions = [
        'Minimum',
        'Maximum',
        'Sum',
        'Average', 
        'SampleCount',
        # 'Precentile',
        ]
                
        if action not in actions: 
            return None
        elif action == 'Minimum': 
            return min(rs)
        elif action == 'Maximum': 
            return max(rs)
        elif action == 'Sum': 
            return sum(rs)
        elif action == 'Average': 
            return sum(rs)/len(rs)
        elif action == 'SampleCount':
            return len(rs)
        elif action == 'Precentile': 
            return None
        
    def getComparisonOperator(self): 
        return [
            'GreaterThanOrEqualToThreshold',
            'GreaterThanThreshold',
            'LessThanOrEqualToThreshold',
            'LessThanThreshold',
            # 'InRangeToThreshold',         # GreaterThanOrEqualToThreshold & LessThanOrEqualToThreshold
            # 'NotInRangeToThreshold',      # not (GreaterThanOrEqualToThreshold & LessThanOrEqualToThreshold)
            'EqualToThreshold',
            'NotEqualToThreshold',
        ]
    
    def compare(self, value, threshold, operation): 
        value = float(value)
        threshold = float(threshold)
        if operation. Lower() == 'GreaterThanOrEqualToThreshold'.lower(): 
            return True if value >= threshold else False
        if operation. Lower() == 'GreaterThanThreshold'.lower(): 
            return True if value > threshold else False
        if operation. lower() == 'LessThanThreshold'.lower(): 
            return True if value < threshold else False 
        if operation. Lower() == 'LessThanOrEqualToThreshold'.lower(): 
            return True if value <= threshold else False 
        if operation. Lower() == 'EqualToThreshold'.lower(): 
            return True if value==threshold else False 
        if operation. Lower() == 'NotEqualToThreshold'.lower(): 
            return True if value != threshold else False
        
        return False
    
    def hasAlarm(self, debug=False):
        timer = TimeIt()

        ts = self.getTs()
        rules = self.getRules()

        res = {}
        res ['Alarm'] = 0
        res['Items'] = []
        res['Details'] = []
        res ['Timestamp'] = ts
        for rule in rules:
            alarm = self.processRule(rule, debug) 
            timer.set('hasAlarm, processRule '+rule['Metric']) 
            if alarm:
                res['Alarm'] = res ['Alarm'] + 1
                res['Items'].append(rule['Metric']) 
                message = 'ALARM'
            else:
                message = 'OK'

            res['Details'].append({
                    'Metric': rule['Metric'], 
                    'Name': rule['Name'], 
                    'Alarm': alarm, 
                    'Message': message
                })
            
        res['Count'] = len(res['Items'])
        res['Duration'] = self.getTs() - ts

        if debug:
            timer.get(True)
        return res
    
    def avgMetrics (self, metric, count): 
        res = []
        rs = self.getTail(metric, count)

        for r in rs:
            d = r.strip('\n').split(',') 
            try:
                float(d[1])
            except Exception as e: 
                d[1] = 0

            res.append(float(d[1]))
            
        return float(sum(res)/len (res)) if len(res) > 0 else 0

def processRule(self, rule, debug=False):
    # @requires: Comparison, For, Of, Metric, Threshold alarm = True
    alarm = True

    res = {}
    res['Rule'] = rule
    res['Data'] = []
    rs = self.getTail(rule['Metric'], rule['Of'])

    counter = 0
    if debug: 
        print()
        print(('RULE NAME: '+rule['Name']))
        print('------------------------------------------------------------------------------------------')
        print(f"For {str(rule['For'])}/{str(rule['Of'])} Datapoint on {rule['Metric']} with Threshold {rule['Threshold']} Compare with {rule['Comparison']}")

    for r in rs:
        d = r.strip('\n').split(',')
        try:
            float(d[1])
        except Exception as e:
            d[1] = 0
        
        res['Data'].append({'TS': float(d[0]), 'Value': float(d[1])}) 
        if self.compare(d[1], rule['Threshold'], rule['Comparison']): 
            if debug:
                print(('\t\tThreshold: '+str(d[1])))
            counter = counter + 1
        else:
            if debug:
                print(('\tok: '+str(d[1])))    
        
        statMessage = str(counter)+'/'+str(rule['Of'])+' status versus '+str(rule['For'])+'/'+str(rule['Of'])+' threshold '+rule['Threshold']

        if counter == rule['For']:
            res['Alarm'] = True
            alarm = True
            statMessage = '\t\tThreshold Crossed !!! '+statMessage
        else:
            res['Alarm'] = False
            alarm = False
            statMessage = '\tok... '+statMessage

        if debug:
            print('RESULT -----------------------------------------------------------------------------------')
            print(statMessage)

        self.alarmMetrics.append(res)

        return alarm

    def getStates(self): 
        return [
            'PUBLIC-FAIL',
            'PUBLIC-PASS'
            'BACKUP-FAIL',
            'BACKUP-PASS',
            'FAILOVER-PASS',
            'FAILOVER-FAIL', 
            'UNKNOWN'
        ]

    def getTail(self, metric, lines = 1):
        file = self.getFileName(metric) 
        res = []
        try:
            with open(file, "r") as f:
                res = f.readlines()[-lines:]
                return res[-lines:] if len(res) <= lines else res
        except IOError:
            print('getTail Error! ')
            print('\tMetric:', metric) 
            print('\tFile:', file)
        except Exception as e:
            print(('getTail unexpected error!',str(e)))

        return res

    def getFileName(self, metric):
        return self.metricsPath+self.prefix+metric+self.suffix+self.ext

    def set(self, data, debug=False):
        ts = self.getTs()
        LogSize = 0
        if debug== True: 
            print()
            print('CollectMetrics::set() ................................')
        
        for metric in data:
            file = self.getFileName(metric)
            try:
                logSize = os.path.getsize(file) + logSize 
            except Exception as e:
                print(('New gauge !!! '+str(e))) 
                print(('\tMetric: '+metric)) 
                print(('\tFile: '+file))

        fileMode = 'w' if logSize > self.logSizeLimit else 'a'

        myCheck = {}
        for metric in data:
            file = self.getFileName(metric)
            with open(file, fileMode) as f:
                con = str(ts)+', '+str(data [metric])+"\n" 
                myCheck[metric] = data[metric]
            if debug:
                print(('Write to : '+file)) 
                print(('Content : '+str(con)))
            f.write(con)

        if debug:
            print((json.dumps (myCheck)))

        return True

if __name__ == "__main__":
    PATH = os.path.dirname(os.path.abspath(__file__))
    metricsPath = path+'/logs/'
    rulesFile = path+'/config/alarm-rules.json'

    ts = time.time()
    today= datetime.datetime.fromtimestamp(float(ts)).strftime('%Y-%m-%d')
    utcDate = datetime.datetime. utcfromtimestamp(ts)

    cm = CollectMetrics(metricsPath-metricsPath, rulesFile=rulesFile, suffix=today) 
    rules= cm.getRules()

    res = []

    countRules = str(len(rules))
    print()
    print('RULES('+str(countRules)+') QA -------------------------------------------------------------------')
    print(json.dumps(rules))

    res = cm.hasAlarm()
    if (res.get('Alarm', 0)) > 0:
        print('ALARM ...')

        print()
        print('getAlarmMetrics() QA ----------------------------------------------------------------')
        print((json.dumps (cm.getAlarmMetrics())))

        # getStatistics QA
        rs = [10,10,0,0,10]
        actions = [
            'Minimum'
            'Maximum'
            'Sum',
            'Average',
            'SampleCount',
            #'Precentile',
        ]

print()
print('getStatistics() QA ----------------------------------------------------------------')
print(('Today : '+today))
print('rs: '+str(json.dumps(rs)))
for action in actions:
    print(('getStatistics (rs, '+action+') = '+str(cm.getStatistics (rs, action))))

print('Took : '+str(time.time() - ts))
print(('UTC : '+str(utcDate)))