diff --git a/api/config.py b/api/config.py index ec56251..08abd59 100644 --- a/api/config.py +++ b/api/config.py @@ -11,6 +11,7 @@ class Config: flask_debug = os.getenv('FLASK_DEBUG', False) prometheus_api = os.getenv('PROMETHEUS_API') sensu_api = os.getenv('SENSU_API') + sensu_silences_api = os.getenv('SENSU_SILENCED_API') sensu_key = os.getenv("SENSU_KEY") diff --git a/api/monitors/sensu.py b/api/monitors/sensu.py index 08558b9..4aa60ae 100644 --- a/api/monitors/sensu.py +++ b/api/monitors/sensu.py @@ -14,6 +14,19 @@ def convert_severity(status) -> str: else: return "none" +def get_sensu_silences() -> List[Dict]: + headers = {"AUTHORIZATION": f"Key {Config.sensu_key}"} + try: + request = requests.get(url=Config.sensu_silences_api, headers=headers) + data = request.json() + except Exception as e: + print(f"Fatal: Could not GET Sensu silences API {Config.sensu_silences_api}. Error: {e}") + return dead_mans_switch("Sensu checks", Config.sensu_silences_api, e) + + ignored_items = [silenced for silenced in data] + ignores = [ignore["metadata"]["name"] for ignore in ignored_items] + return ignores + def get_sensu_events() -> List[Dict]: headers = {"AUTHORIZATION": f"Key {Config.sensu_key}"} @@ -26,6 +39,8 @@ def get_sensu_events() -> List[Dict]: not_passing_status = [check for check in data if check["check"]["state"] != "passing"] + ignores = get_sensu_silences() + events = [{ 'alertname': event["check"]['metadata']['name'], 'namespace': event["entity"]["metadata"]["name"], @@ -33,6 +48,6 @@ def get_sensu_events() -> List[Dict]: 'message': truncate_string(event["check"]["output"]), 'triggered': int(event["check"]["last_ok"]), "source": "Sensu" - } for event in not_passing_status] + } for event in not_passing_status if event["check"]['metadata']['name'] not in ignores] return events diff --git a/docker-compose.yaml b/docker-compose.yaml index 44313b1..40812f9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,6 +24,7 @@ services: # PROMETHEUS_API: "http://prometheus-operator-prometheus.monitoring:9090/api/v1/alerts" # PROMETHEUS_API: "https://aware.dev.sdpaks.equinor.com/api/prometheus" # SENSU_API: "http://mon01.prod.sdp.statoil.no:8080/api/core/v2/namespaces/default/events" +# SENSU_SILENCES_API: "http://mon01.prod.sdp.statoil.no:8080/api/core/v2/namespaces/default/silenced" IMPORT_URLS: "https://aware.sdpaks.equinor.com/api/exports" # IMPORT_URLS: "https://aware.sdpaks.equinor.com/api/exports;https://aware.dev.sdpaks.equinor.com/api/exports" SENSU_KEY: ${SENSU_KEY}