diff --git a/salmon/apps/monitor/management/commands/run_checks.py b/salmon/apps/monitor/management/commands/run_checks.py index 9b3ddfd..7a3994d 100644 --- a/salmon/apps/monitor/management/commands/run_checks.py +++ b/salmon/apps/monitor/management/commands/run_checks.py @@ -88,14 +88,14 @@ def _handle_result(self, target, func_name, func_opts, result): self.stdout.write( "+ name: {0} -- str(raw_value): {1}".format( name, str(raw_value))) - value = utils.parse_value(raw_value, func_opts) - self.stdout.write(" {0}: {1}".format(name, value)) + values = utils.parse_values(raw_value, func_opts) + self.stdout.write(" {0}: {1}".format(name, values)) minion, _ = models.Minion.objects.get_or_create(name=name) - failed = utils.check_failed(value, func_opts) + failed = utils.check_failed(values, func_opts) self.stdout.write(" Assertion: {1}".format(not failed)) + serialized_values = utils.serialize_values(values) models.Result.objects.create(timestamp=timestamp, - result=value, - result_type=func_opts['type'], + values=serialized_values, check=check, minion=minion, failed=failed) diff --git a/salmon/apps/monitor/migrations/0005_auto__add_field_result_results.py b/salmon/apps/monitor/migrations/0005_auto__add_field_result_results.py new file mode 100644 index 0000000..b3a33ea --- /dev/null +++ b/salmon/apps/monitor/migrations/0005_auto__add_field_result_results.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Result.values' + db.add_column(u'monitor_result', 'values', + self.gf('jsonfield.fields.JSONField')(default={}), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Result.values' + db.delete_column(u'monitor_result', 'values') + + + models = { + u'monitor.check': { + 'Meta': {'object_name': 'Check'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'alert_emails': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'function': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'monitor.minion': { + 'Meta': {'object_name': 'Minion'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + u'monitor.result': { + 'Meta': {'object_name': 'Result'}, + 'check': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['monitor.Check']"}), + 'failed': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'minion': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['monitor.Minion']"}), + 'notified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'result': ('django.db.models.fields.TextField', [], {}), + 'result_type': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'values': ('jsonfield.fields.JSONField', [], {}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}) + } + } + + complete_apps = ['monitor'] diff --git a/salmon/apps/monitor/models.py b/salmon/apps/monitor/models.py index 0c365e4..d711c3c 100644 --- a/salmon/apps/monitor/models.py +++ b/salmon/apps/monitor/models.py @@ -7,6 +7,7 @@ from django.template.loader import render_to_string from django.utils import timezone from django.utils.text import get_valid_filename +from jsonfield import JSONField from . import utils, graph @@ -98,6 +99,7 @@ class Result(models.Model): check = models.ForeignKey('monitor.Check') minion = models.ForeignKey('monitor.Minion') timestamp = models.DateTimeField(default=timezone.now) + values = JSONField() result = models.TextField() result_type = models.CharField(max_length=30) failed = models.NullBooleanField(default=True) @@ -106,44 +108,37 @@ class Result(models.Model): def __unicode__(self): return self.timestamp.isoformat() - @property - def cleaned_result(self): - checker = utils.Checker(cast_to=self.result_type, - raw_value=self.result) - return checker.value + def values_type(self): + """Grab the type from the first item in the values""" + return self.values.values()[0]['type'] - @property - def whisper_filename(self): + def whisper_filename(self, key): """Build a file path to the Whisper database""" - return get_valid_filename("{0}__{1}.wsp".format( - self.minion.name, self.check.name)) - - @property - def floatified_result(self): - cleaned_result = (utils.Checker(cast_to=self.result_type, - raw_value=self.result) - .value) - - if self.result_type == "percent": - return cleaned_result.replace("%", "") - if self.result_type == "string": - return float(not self.failed) - return float(cleaned_result) - - def get_or_create_whisper(self): + name_bits = [self.minion.name, self.check.name, key] + return get_valid_filename("{0}.wsp".format("__".join(name_bits))) + + def get_or_create_whisper(self, key): """ Gets a Whisper DB instance. Creates it if it doesn't exist. """ - return graph.WhisperDatabase(self.whisper_filename) + return graph.WhisperDatabase(self.whisper_filename(key)) - def get_history(self, from_date, to_date=None): + def get_histories(self, from_date, to_date=None): """Loads in historical data from Whisper database""" - return self.get_or_create_whisper().fetch(from_date, to_date) + histories = {} + for key in self.values.keys(): + histories[key] = self.get_or_create_whisper(key).fetch(from_date, + to_date) + return histories + + def save_to_whisper(self): + """Store the value in the whisper database(s)""" + for key, value in self.values.items(): + wsp = self.get_or_create_whisper(key) + wsp.update(self.timestamp, value['float']) def save(self, *args, **kwargs): if not self.pk: - # Store the value in the whisper database - wsp = self.get_or_create_whisper() - wsp.update(self.timestamp, self.floatified_result) + self.save_to_whisper() return super(Result, self).save(*args, **kwargs) diff --git a/salmon/apps/monitor/tests.py b/salmon/apps/monitor/tests.py index 9c3b7aa..d81f155 100644 --- a/salmon/apps/monitor/tests.py +++ b/salmon/apps/monitor/tests.py @@ -48,12 +48,20 @@ def generate_sample_data(point_numbers, interval): now = datetime.now() for i in range(point_numbers): for check in checks: + value1 = randint(1, 100) + value2 = randint(1, 100) Result.objects.create( check=check, minion=minion, timestamp=now-timedelta(minutes=interval*i), - result=str(randint(1, 100)), - result_type="float", + values={'/': {'raw': str(value1), + 'float': float(value1), + 'real': float(value1), + 'type': 'float'}, + '/var': {'raw': str(value2), + 'float': float(value2), + 'real': float(value2), + 'type': 'float'}}, failed=False) return (minion, checks[0], checks[1]) @@ -83,14 +91,19 @@ def test_database_creation(self): def test_database_update(self): now = datetime.now() + key = '' result = Result.objects.create(check=self.active_check, minion=self.minion, timestamp=now, - result="101", - result_type="float", + values={key: { + 'raw': '101', + 'float': 101.0, + 'real': 101, + 'type': 'integer' + }}, failed=False) - history = result.get_history(now-timedelta(minutes=INTERVAL_MIN*20)) - self.assertEqual(len(history), 20) + histories = result.get_histories(now-timedelta(minutes=INTERVAL_MIN*20)) + self.assertEqual(len(histories[key]), 20) class MonitorUrlTest(BaseTestCase): @@ -141,19 +154,24 @@ def setUp(self): minion, created = Minion.objects.get_or_create(name="minion.local") now = datetime.now() for check in [self.check, self.check_with_emails]: + val = randint(1, 100) Result.objects.create( check=check, minion=minion, timestamp=now-timedelta(minutes=INTERVAL_MIN*1), - result=str(randint(1, 100)), - result_type="float", + values={'': {'raw': '101', + 'float': float(val), + 'real': float(val), + 'type': 'float'}}, failed=True) Result.objects.create( check=check, minion=minion, timestamp=now-timedelta(minutes=INTERVAL_MIN*2), - result=str(randint(1, 100)), - result_type="float", + values={'': {'raw': '101', + 'float': float(val), + 'real': float(val), + 'type': 'float'}}, failed=False) def tearDown(self): @@ -204,52 +222,41 @@ def test_salt_proxy_cmd_local(self): class CheckerTest(TestCase): def test_boolean_false_result(self): - cast_to = "boolean" - raw_value = "False" + val = utils.Value(key='', raw='False', cast_to='boolean') assertion_string = "{value} == True" - checker = utils.Checker(cast_to=cast_to, raw_value=raw_value) - self.assertEqual(checker.do_assert(assertion_string), False) + self.assertEqual(val.do_assert(assertion_string), False) def test_boolean_true_result(self): - cast_to = "boolean" - raw_value = "False" + val = utils.Value(key='', raw='False', cast_to='boolean') assertion_string = "{value} == False" - checker = utils.Checker(cast_to=cast_to, raw_value=raw_value) - self.assertEqual(checker.do_assert(assertion_string), True) + self.assertEqual(val.do_assert(assertion_string), True) def test_string_true_result(self): - cast_to = "string" - raw_value = "HTTP/1.1 200 OK" + val = utils.Value(key='', raw='HTTP/1.1 200 OK', cast_to='string') assertion_string = "'{value}' == 'HTTP/1.1 200 OK'" - checker = utils.Checker(cast_to=cast_to, raw_value=raw_value) - self.assertEqual(checker.do_assert(assertion_string), True) + self.assertEqual(val.do_assert(assertion_string), True) class AssertCheckTest(TestCase): def test_failure(self): - failed = utils.check_failed('False', {'type': 'boolean', - 'assert': '{value} == True'}) + value = utils.Value(key='', raw='False', cast_to='boolean') + failed = utils.check_failed([value], {'assert': '{value} == True'}) self.assertTrue(failed) - def test_success(self): - failed = utils.check_failed('1.0', {'type': 'float', - 'assert': '{value} < 5'}) - self.assertFalse(failed) - - def test_no_check(self): - no_check = utils.check_failed('1.0', {'type': 'float'}) - self.assertEqual(no_check, None) - class ParseValueTest(TestCase): def test_with_key(self): - val = utils.parse_value({'e': 'east', 'w': 'west'}, {'key': 'w'}) - self.assertEqual(val, 'west') + val = utils.parse_values({'e': 'east', 'w': 'west'}, + {'key': 'w', 'type': 'string'}) + self.assertEqual(val[0].key, 'w') + self.assertEqual(val[0].raw, 'west') def test_no_key(self): - val = utils.parse_value('north', {}) - self.assertEqual(val, 'north') + val = utils.parse_values('north', {'type': 'string'}) + self.assertEqual(val[0].key, '') + self.assertEqual(val[0].raw, 'north') def test_none(self): - val = utils.parse_value(None, {}) - self.assertEqual(val, "") + val = utils.parse_values(None, {'type': 'float'}) + self.assertEqual(val[0].key, '') + self.assertEqual(val[0].raw, None) diff --git a/salmon/apps/monitor/utils.py b/salmon/apps/monitor/utils.py index 0ca597f..ef83551 100644 --- a/salmon/apps/monitor/utils.py +++ b/salmon/apps/monitor/utils.py @@ -49,6 +49,7 @@ def get_latest_results(minion=None, check_ids=None): latest_results = [] return latest_results + def load_salmon_checks(): """Reads in checks.yaml and returns Python object""" checks_yaml = open(settings.SALMON_CHECKS_PATH).read() @@ -85,60 +86,117 @@ def run(self): logging.exception("Error parsing results.") -def parse_value(raw_value, opts): - value = raw_value +def _traverse_dict(obj, key_string): + """ + Traverse a dictionary using a dotted string. + Example: 'a.b.c' will return obj['a']['b']['c'] + """ + for key in key_string.split('.'): + if 'key' in obj and isinstance(obj, dict): + obj = obj[key] + else: + return None + return obj + + +def parse_values(raw_value, opts): + """Parses Salt return value to find the keys specified""" + + if 'keys' in opts: + results = [] + for key in opts['keys']: + results.append(Value( + key=key, raw=_traverse_dict(raw_value, key), + cast_to=opts['type'])) + return results + if 'key' in opts: - key_tree = opts['key'].split('.') - for key in key_tree: - value = value[key] - # Handle the special case where the value is None - elif value is None: - value = "" - return value + key = opts['key'] + raw_value = _traverse_dict(raw_value, opts['key']) + else: + key = '' + return [Value(key=key, raw=raw_value, cast_to=opts['type'])] -def check_failed(value, opts): +def check_failed(values, opts): if 'assert' not in opts: + [val.floatify() for val in values] return None - checker = Checker(cast_to=opts['type'], raw_value=value) - return not checker.do_assert(opts['assert']) + for value in values: + success = value.do_assert(opts['assert']) + if not success: + return True + return False -class Checker(object): - def __init__(self, cast_to, raw_value): +def serialize_values(values): + """ + Convert a list of `Value` objects into a dictionary + for pushing to the database. + """ + + serialized = {} + for value in values: + serialized[value.key] = value.as_dict() + return serialized + + +class Value(object): + """Everything needed to convert, check, and serialize a value""" + def __init__(self, key, raw, cast_to): + self.key = key + self.raw = raw self.cast_to = cast_to - self.raw_value = raw_value - self.value = self.cast() + self.real = self.cast() + # wait for float until success is known + self.float = None + self.success = None + + def as_dict(self): + return {'real': self.real, 'raw': self.raw, 'float': self.float, + 'type': self.cast_to} def cast(self): - if not hasattr(self, "value"): - self.value = getattr( - self, 'to_{0}'.format(self.cast_to))(self.raw_value) - return self.value + conv_method = getattr(self, 'to_{0}'.format(self.cast_to)) + return conv_method() def do_assert(self, assertion_string): # TODO: try to remove the evil - success = eval(assertion_string.format(value=self.value)) + success = eval(assertion_string.format(value=self.real)) assert isinstance(success, bool) + self.success = success + self.floatify() return success - def to_boolean(self, value): + def floatify(self): + if self.cast_to == 'string': + self.float = float(self.success) + return + try: + self.float = float(self.real) + except ValueError: + self.float = float(0) + + def to_boolean(self): # bool('False') == True - if value == "False": + if self.raw == "False": return False - return bool(value) is True + return bool(self.raw) is True - def to_integer(self, value): - return self.to_float(value) + def to_integer(self): + return self.to_float(self.raw) - def to_percentage(self, value): - return self.to_float(value) + def to_percentage(self): + return self.to_float(self.raw) - def to_percentage_with_sign(self, value): - return self.to_float(value.rstrip('%')) + def to_percentage_with_sign(self): + return self.to_float(self.raw.rstrip('%')) - def to_float(self, value): - return float(value) + def to_float(self, val): + # float(None) blows up + if not val: + return float(0) + return float(val) - def to_string(self, value): - return str(value) + def to_string(self): + return str(self.raw) diff --git a/salmon/apps/monitor/views.py b/salmon/apps/monitor/views.py index 9cd66ae..98ad53d 100644 --- a/salmon/apps/monitor/views.py +++ b/salmon/apps/monitor/views.py @@ -44,15 +44,21 @@ def history(request, name): graphs = [] for result in utils.get_latest_results(minion=minion): - history = result.get_history( + histories = result.get_histories( from_date=from_date, to_date=to_date) # javascript uses milliseconds since epoch - js_data = map(lambda x: (x[0] * 1000, x[1]), history) - if result.result_type.startswith('percentage'): + js_data = [] + for key, history in histories.items(): + js_data.append({ + 'label': key, + 'data': map(lambda x: (x[0] * 1000, x[1]), history), + }) + values_type = result.values_type() + if values_type.startswith('percentage'): graph_type = 'percentage' else: - graph_type = result.result_type + graph_type = values_type graphs.append({ 'name': result.check.name, 'data': json.dumps(js_data), diff --git a/salmon/templates/monitor/history.html b/salmon/templates/monitor/history.html index fa235da..c1ce22c 100644 --- a/salmon/templates/monitor/history.html +++ b/salmon/templates/monitor/history.html @@ -52,8 +52,8 @@