Ignore silenced

api/config.py

@@ -11,6 +11,7 @@ class Config:
     flask_debug = os.getenv('FLASK_DEBUG', False)
     prometheus_api = os.getenv('PROMETHEUS_API')
     sensu_api = os.getenv('SENSU_API')
+    sensu_silences_api = os.getenv('SENSU_SILENCED_API')
     sensu_key = os.getenv("SENSU_KEY")
 
 

api/monitors/sensu.py

@@ -14,6 +14,19 @@ def convert_severity(status) -> str:
     else:
         return "none"
 
+def get_sensu_silences() -> List[Dict]:
+    headers = {"AUTHORIZATION": f"Key {Config.sensu_key}"}
+    try:
+        request = requests.get(url=Config.sensu_silences_api, headers=headers)
+        data = request.json()
+    except Exception as e:
+        print(f"Fatal: Could not GET Sensu silences API {Config.sensu_silences_api}. Error: {e}")
+        return dead_mans_switch("Sensu checks", Config.sensu_silences_api, e)
+
+    ignored_items = [silenced for silenced in data]
+    ignores = [ignore["metadata"]["name"] for ignore in ignored_items]
+    return ignores
+
 
 def get_sensu_events() -> List[Dict]:
     headers = {"AUTHORIZATION": f"Key {Config.sensu_key}"}
@@ -26,13 +39,15 @@ def get_sensu_events() -> List[Dict]:
 
     not_passing_status = [check for check in data if check["check"]["state"] != "passing"]
 
+    ignores = get_sensu_silences()
+
     events = [{
         'alertname': event["check"]['metadata']['name'],
         'namespace': event["entity"]["metadata"]["name"],
         'severity': convert_severity(event["check"]["status"]),
         'message': truncate_string(event["check"]["output"]),
         'triggered': int(event["check"]["last_ok"]),
         "source": "Sensu"
-    } for event in not_passing_status]
+    } for event in not_passing_status if event["check"]['metadata']['name'] not in ignores]
 
     return events

docker-compose.yaml

@@ -24,6 +24,7 @@ services:
 #         PROMETHEUS_API: "http://prometheus-operator-prometheus.monitoring:9090/api/v1/alerts"
 #        PROMETHEUS_API: "https://aware.dev.sdpaks.equinor.com/api/prometheus"
 #        SENSU_API: "http://mon01.prod.sdp.statoil.no:8080/api/core/v2/namespaces/default/events"
+#        SENSU_SILENCES_API: "http://mon01.prod.sdp.statoil.no:8080/api/core/v2/namespaces/default/silenced"
         IMPORT_URLS: "https://aware.sdpaks.equinor.com/api/exports"
         # IMPORT_URLS: "https://aware.sdpaks.equinor.com/api/exports;https://aware.dev.sdpaks.equinor.com/api/exports"
         SENSU_KEY: ${SENSU_KEY}