Add the ability to temporarily silence a check, and merge some other changes.

common/static/js/momonitoring.js

@@ -133,6 +133,66 @@ function deleteResource(urlEndpoint) {
 	   });
 }
 
+function setSilencedDescriptionFromField() {
+    if ($("#id_silenced_until").length === 0) {
+        return;
+    }
+
+    // Both of these vars are timestamps (seconds since the epoch).
+    var now = new Date().getTime() / 1000;
+    var silencedUntil = parseInt($("#id_silenced_until").val());
+
+    function round(num) {
+        // Rounds 1.8 to 2, but 1.79 to 1.
+        return Math.floor(num + 0.2);
+    }
+
+    var diffSeconds = round(silencedUntil - now);
+    var diffMinutes = round(diffSeconds / 60);
+    var diffHours   = round(diffMinutes / 60);
+    var diffDays    = round(diffHours / 24);
+
+    var description;
+
+    if (2000000000 <= silencedUntil) {
+        description = "Silenced forever";
+    } else if (silencedUntil < now) {
+        description = "Not silenced";
+    } else if (diffDays >= 2) {
+        description = "Silenced for ~" + diffDays + " day(s)";
+    } else if (diffHours >= 2) {
+        description = "Silenced for ~" + diffHours + " hour(s)";
+    } else if (diffMinutes >= 1) {
+        description = "Silenced for " + diffMinutes + " minute(s)";
+    } else {
+        description = "Silenced for " + diffSeconds + " second(s)";
+    }
+
+    $("#silenced_description").text(description);
+}
+
+function setSilencedFor(secondsFromNow) {
+    var now = new Date().getTime() / 1000; // Timestamp (seconds since the epoch).
+    var silencedUntil = Math.floor(now + secondsFromNow);
+    $("#id_silenced_until").val(silencedUntil);
+    setSilencedDescriptionFromField();
+}
+
+function setSilencedForever() {
+    $("#id_silenced_until").val(2000000000);
+    setSilencedDescriptionFromField();
+}
+
+function setUnSilenced() {
+    $("#id_silenced_until").val(0);
+    setSilencedDescriptionFromField();
+}
+
+function setSilencedForXHours() {
+    var hoursToSilenceFor = parseFloat($("#silence_hours").val());
+    setSilencedFor(hoursToSilenceFor * 60 * 60);
+}
+
 function init(container$) {    
     if(typeof(container$)==='undefined') container$ = $('body');
 
@@ -239,6 +299,18 @@ function init(container$) {
     	});
     });
 
+    setSilencedDescriptionFromField();
+    $('#id_silenced_until',container$).change(setSilencedDescriptionFromField);
+    $('#id_silenced_until',container$).click(setSilencedDescriptionFromField);
+    $('#id_silenced_until',container$).focus(setSilencedDescriptionFromField);
+    $('#id_silenced_until',container$).blur(setSilencedDescriptionFromField);
+    $('#id_silenced_until',container$).keyup(setSilencedDescriptionFromField);
+    $('#id_silenced_until',container$).keydown(setSilencedDescriptionFromField);
+
+    $('#unsilence',container$).click(setUnSilenced);
+    $('#silence_for_x_hours',container$).click(setSilencedForXHours);
+    $('#silence_forever',container$).click(setSilencedForever);
+
     $('form',container$).each(function() {
 	var form$ = $(this);
 	$('input',form$).keypress(function(event) {

main/models/service.py

@@ -77,6 +77,7 @@ def _send_alert_email(self,description):
         if not self.email_contact:
             logging.warning("No email contact for service %s" % self.name)
         try:
+            title = "{} Service Check Failed: {}".format(self.name, description)
             email_msg = get_template("alert_email.txt").render(
                 Context({"description":description,
                          "service_name":self.name,
@@ -87,7 +88,7 @@ def _send_alert_email(self,description):
                          })
                 )
 
-            send_mail("MOMONITOR EVENT TRIGGERED",
+            send_mail(title,
                       email_msg,
                       settings.SERVER_EMAIL,
                       [self.email_contact,],

main/models/service_check.py

@@ -19,14 +19,18 @@ class Meta:
     name = models.CharField(max_length=256)
     description = models.TextField(null=True,blank=True)    
     service = models.ForeignKey(Service,related_name="%(class)s")
-    silenced = models.BooleanField(default=False)
+    silenced_until = models.IntegerField(default=0) # This is a timestamp (seconds since unix epoch).
     alert_type = models.CharField(max_length=64,choices=ALERT_CHOICES,null=True,blank=True)
     frequency = models.CharField(max_length=128,null=True,blank=True)
     failures_before_alert = models.IntegerField(null=True,blank=True)
 
     def __unicode__(self):
         return "%s: %s" % (self.service.name,self.name)
 
+    @property
+    def is_silenced(self):
+        return time.time() < self.silenced_until
+
     @property
     def _redis_key(self):
         return "%s:::%s" % (self.resource_name,self.id)
@@ -70,7 +74,7 @@ def num_failures(self):
         return self._get_state().get("num_failures",0)
 
     def send_alert(self,alert_type=None):
-        if not self.silenced:
+        if not self.is_silenced:
             self.service.send_alert(self.description or self.name,alert_type)
         else:
             logging.info("Triggered alert on %s, but it is silenced" % self.name)

main/templates/main/index.html

@@ -20,7 +20,7 @@ <h1>All Services</h1>
     {% if services %}
     {% for service in services %}
     <tr class="{{service.status|to_bootstrap_rowclass}}">
-      <td width="40%"><a href="{% url main:service service.id %}">{{service.name}} {% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}</a>
+      <td width="40%"><a href="{% url main:service service.id %}">{{service.name}} {% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}</a>
 	<a class="modalize" href="{% url main:modal_form 'service' service.id %}">
 	  <i class="icon-edit"></i>
 	</a>

main/templates/main/modal_forms/codeservicecheck.html

@@ -9,12 +9,17 @@
     {{form.name}}
     <label>Description</label>
     {{form.description}}
-  </div>
-  <div class="pull-right half">
-    <label class="checkbox">{{form.silenced}} Silenced</label>
     <label>Code File</label>
     {{form.code_file}}
   </div>
+  <div class="pull-right half">
+    <label id="silenced_description" style="font-weight: bold;">Silenced or Not Silenced?</label>
+    <label>Silenced until (timestamp)</label>
+    {{form.silenced_until}}
+    <label> <a id="unsilence">UnSilence</a> </label>
+    <label> <a id="silence_for_x_hours">Silence for <input type="text" id="silence_hours" value="0.5" style="width:30px;"> hours</a> </label>
+    <label> <a id="silence_forever">Silence forever</a> </label>
+  </div>
   <div class="clearfix"></div>
 </fieldset>
 <hr />

main/templates/main/modal_forms/compareservicecheck.html

@@ -9,9 +9,6 @@
     {{form.name}}
     <label>Description</label>
     {{form.description}}
-  </div>
-  <div class="pull-right">
-    <label class="checkbox">{{form.silenced}} Silenced</label>
     <label>URL Endpoint</label>
     {{form.endpoint}}
     <label>Serialization</label>
@@ -21,6 +18,14 @@
     <label>Comparator</label>
     {{form.comparator}} {{form.compared_value}}
   </div>
+  <div class="pull-right">
+    <label id="silenced_description" style="font-weight: bold;">Silenced or Not Silenced?</label>
+    <label>Silenced until (timestamp)</label>
+    {{form.silenced_until}}
+    <label> <a id="unsilence">UnSilence</a> </label>
+    <label> <a id="silence_for_x_hours">Silence for <input type="text" id="silence_hours" value="0.5" style="width:30px;"> hours</a> </label>
+    <label> <a id="silence_forever">Silence forever</a> </label>
+  </div>
   <div class="clearfix"></div>
 </fieldset>
 <hr />

main/templates/main/modal_forms/sensuservicecheck.html

@@ -9,12 +9,17 @@
     {{form.name}}
     <label>Description <small class="muted">(optional)</small></label>
     {{form.description}}
-  </div>
-  <div class="pull-right">
-    <label class="checkbox">{{form.silenced}} Silenced</label>
     <label>Sensu Check Name</label>
     {{form.sensu_check_name}}
   </div>
+  <div class="pull-right">
+    <label id="silenced_description" style="font-weight: bold;">Silenced or Not Silenced?</label>
+    <label>Silenced until (timestamp)</label>
+    {{form.silenced_until}}
+    <label> <a id="unsilence">UnSilence</a> </label>
+    <label> <a id="silence_for_x_hours">Silence for <input type="text" id="silence_hours" value="0.5" style="width:30px;"> hours</a> </label>
+    <label> <a id="silence_forever">Silence forever</a> </label>
+  </div>
   <div class="clearfix"></div>
 </fieldset>
 <hr />

main/templates/main/modal_forms/simpleservicecheck.html

@@ -9,14 +9,19 @@
     {{form.name}}
     <label>Description <small class="muted">(optional)</small></label>
     {{form.description}}
-  </div>
-  <div class="pull-right half">
-    <label class="checkbox">{{form.silenced}} Silenced</label>
     <label>URL Endpoint</label>
     {{form.endpoint}}
     <label>Timeout (ms) <small class="muted">(optional)</small></label>
     {{form.timeout}}
   </div>
+  <div class="pull-right half">
+    <label id="silenced_description" style="font-weight: bold;">Silenced or Not Silenced?</label>
+    <label>Silenced until (timestamp)</label>
+    {{form.silenced_until}}
+    <label> <a id="unsilence">UnSilence</a> </label>
+    <label> <a id="silence_for_x_hours">Silence for <input type="text" id="silence_hours" value="0.5" style="width:30px;"> hours</a> </label>
+    <label> <a id="silence_forever">Silence forever</a> </label>
+  </div>
   <div class="clearfix"></div>
 </fieldset>
 <hr />

main/templates/main/modal_forms/umpireservicecheck.html

@@ -13,31 +13,37 @@
     {{form.umpire_check_type}}
     <label>Umpire Range Type</label>
     {{form.umpire_range_type}}
-  </div>
-  <div class="pull-right half">
-    <label class="checkbox">{{form.silenced}} Silenced</label>
     <label>Umpire Metric</label>
     {{form.umpire_metric}}
+  </div>
+  <div class="pull-right half">
+    <label id="silenced_description" style="font-weight: bold;">Silenced or Not Silenced?</label>
+    <label>Silenced until (timestamp)</label>
+    {{form.silenced_until}}
+    <label> <a id="unsilence">UnSilence</a> </label>
+    <label> <a id="silence_for_x_hours">Silence for <input type="text" id="silence_hours" value="0.5" style="width:30px;"> hours</a> </label>
+    <label> <a id="silence_forever">Silence forever</a> </label>
+    <label>-----</label>
     <div class="choicebox" data-name="umpire_check_type">
       <div class="choice" id="static">
-	<label>Umpire Min</label>
-	{{form.umpire_min}}
-	<label>Umpire Max</label>
-	{{form.umpire_max}}
+        <label>Umpire Min</label>
+        {{form.umpire_min}}
+        <label>Umpire Max</label>
+        {{form.umpire_max}}
       </div>
       <div class="choice" id="dynamic">
-	<label>Percent Error (from guessed value)</label>
-	{{form.umpire_percent_error}}
-	<i class="muted">Error range will be computed over time based on past history</i>
+        <label>Percent Error (from guessed value)</label>
+        {{form.umpire_percent_error}}
+        <i class="muted">Error range will be computed over time based on past history</i>
       </div>
     </div>
     <div class="choicebox" data-name="umpire_range_type">
       <div class="choice" id="current">
-	<label>Umpire Range</label>
-	{{form.umpire_range}}
+        <label>Umpire Range</label>
+        {{form.umpire_range}}
       </div>
       <div class="choice" id="day">
-	<i class="muted">Range will be computed from the beginning of the day</i>
+        <i class="muted">Range will be computed from the beginning of the day</i>
       </div>
     </div>
   </div>

main/templates/main/tables/codeservicecheck.html

@@ -19,9 +19,9 @@ <h3><a href="#codetable" class="toggle-hide">Code Checks</a>
   <tbody>
     {% if service_checks.count %}
     {% for check in service_checks|dictsort:"id" %}
-    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.silenced or service.silenced %}muted{% endif %}">
+    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.is_silenced or service.silenced %}muted{% endif %}">
       <td width="30%">
-	{{check.name}} {% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
+	{{check.name}} {% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
 	<a class="modalize" href="{% url main:modal_form check_cls.resource_name check.id %}?sid={{service.id}}">
 	  <i class="icon-edit"></i>
 	</a>

main/templates/main/tables/compareservicecheck.html

@@ -22,10 +22,10 @@ <h3><a href="#comparetable" class="toggle-hide">Compare Checks</a>
   <tbody>
     {% if service_checks.count %}
     {% for check in service_checks.all|dictsort:"id" %}
-    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.silenced or service.silenced %}muted{% endif %}">
+    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.is_silenced or service.silenced %}muted{% endif %}">
       <td width="30%">
 	{{check.name}}
-	{% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
+	{% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
 	<a class="modalize" href="{% url main:modal_form check_cls.resource_name check.id %}?sid={{service.id}}">
 	  <i class="icon-edit"></i>
 	</a>

main/templates/main/tables/sensuservicecheck.html

@@ -20,9 +20,9 @@ <h3><a href="#sensutable" class="toggle-hide">Sensu Checks</a>
   <tbody>
     {% if service_checks.count %}
     {% for check in service_checks.all|dictsort:"id" %}
-    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.silenced or service.silenced %}muted{% endif %}">
+    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.is_silenced or service.silenced %}muted{% endif %}">
       <td width="30%">
-	{{check.name}} {% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
+	{{check.name}} {% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
 	<a class="modalize" href="{% url main:modal_form check_cls.resource_name check.id %}?sid={{service.id}}">
 	  <i class="icon-edit"></i>
 	</a>

main/templates/main/tables/simpleservicecheck.html

@@ -19,9 +19,9 @@ <h3><a href="#simpletable" class="toggle-hide">Simple Checks</a>
   <tbody>
     {% if service_checks.count %}
     {% for check in service_checks|dictsort:"id" %}
-    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.silenced or service.silenced %}muted{% endif %}">
+    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.is_silenced or service.silenced %}muted{% endif %}">
       <td width="30%">
-	{{check.name}} {% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
+	{{check.name}} {% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
 	<a class="modalize" href="{% url main:modal_form check_cls.resource_name check.id %}?sid={{service.id}}">
 	  <i class="icon-edit"></i>
 	</a>

main/templates/main/tables/umpireservicecheck.html

@@ -19,10 +19,10 @@ <h3><a href="#umpiretable" class="toggle-hide">Umpire Checks</a>
   <tbody>
     {% if service_checks.count %}
     {% for check in service_checks|dictsort:"id" %}
-    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.silenced or service.silenced %}muted{% endif %}">
+    <tr class="{{check.status|to_bootstrap_rowclass}} {% if check.is_silenced or service.silenced %}muted{% endif %}">
       <td width="30%">
 	{{check.name}}
-	{% if check.silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
+	{% if check.is_silenced or service.silenced %}<small class="muted">silenced</small>{% endif %}
 	<a class="modalize" href="{% url main:modal_form check_cls.resource_name check.id %}?sid={{service.id}}">
 	  <i class="icon-edit"></i>
 	</a>

main/tests.py

@@ -0,0 +1,33 @@
+from django.utils import unittest
+import mock
+
+from momonitor.main.constants import STATUS_BAD
+from momonitor.main.models import Service, SimpleServiceCheck
+
+
+class ServiceCheckAlertTest(unittest.TestCase):
+    def setUp(self):
+        self.service = Service.objects.create(
+            name='test service',
+            alert_type='pagerduty')
+        self.service_check = SimpleServiceCheck.objects.create(
+            endpoint="http://twitter.com",
+            service=self.service)
+
+        self.service._send_alert_pagerduty = mock.Mock()
+        self.service._send_alert_email = mock.Mock()
+
+    def test_service_check_alert(self):
+        self.service_check.set_state(STATUS_BAD, None)
+
+        self.service._send_alert_pagerduty.assert_called_with('')
+        assert not self.service._send_alert_email.called
+
+    def test_service_check_alert_override(self):
+        self.service_check.alert_type = 'email'
+        self.service_check.save()
+
+        self.service_check.set_state(STATUS_BAD, None)
+
+        assert not self.service._send_alert_pagerduty.called
+        self.service._send_alert_email.assert_called_with('')

mobile/templates/mobile/checks/base.html

@@ -3,11 +3,11 @@
 {% block body %}
 <div data-role="page" data-add-back-btn="true" data-theme="a">
   <div data-role="header" data-position="inline">
-    <h1>Check: <small>{{check.name}} {% if check.silenced or check.service.silenced %}(silenced){% endif %}</small></h1>
+    <h1>Check: <small>{{check.name}} {% if check.is_silenced or check.service.silenced %}(silenced){% endif %}</small></h1>
     <a href="{% url mobile:service check.service.id %}" class="ui-btn-left">Back</a>
     {% if not check.service.silenced %}
     <a href="{% url main:silence check.resource_name check.id %}" class="ajaxify ui-btn-right">
-      {% if check.silenced %}Unsilence{% else %}Silence{% endif %}
+      {% if check.is_silenced %}Unsilence{% else %}Silence{% endif %}
     </a>
     {% endif %}      
   </div>