feat(grafana): alert for unsent notifications

README.md

@@ -30,6 +30,10 @@ This project is deployed in accordance to the [DargStack template](https://githu
 
     The observation dashboard's admin user.
 
+ - ### `grafana_discord_webhook`
+
+    The observation dashboard's contact point for Discord.
+
  - ### `jobber_aliases`
 
     The job scheduler's SMTP client mail alias.

src/development/secrets/grafana/webhook_discord.secret

@@ -0,0 +1 @@
+UNSET THIRD PARTY SECRET

src/development/secrets/grafana/webhook_discord.secret.template

@@ -0,0 +1 @@
+https://discord.com/api/webhooks/<id>/<id>

src/development/stack.yml

@@ -4,6 +4,9 @@
 # https://github.com/maevsi/vibetype/
 ---
 secrets:
+  grafana_discord_webhook:
+    # The observation dashboard's contact point for Discord.
+    file: ./secrets/grafana/discord_webhook.secret
   jobber_aliases:
     # The job scheduler's SMTP client mail alias.
     file: ./secrets/jobber/aliases.secret
@@ -161,6 +164,7 @@ services:
       GF_SERVER_ROOT_URL: https://grafana.${STACK_DOMAIN}/
     image: grafana/grafana:12.0.1
     secrets:
+      - grafana_discord_webhook
       - postgres_db
       - postgres_role_service_grafana_password
       - postgres_role_service_grafana_username

src/production/configurations/grafana/dashboards/Infrastructure/operations.json

@@ -0,0 +1,132 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 6,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "grafana-postgresql-datasource",
+        "uid": "PCC52D03280B7034C"
+      },
+      "description": "The amount of notifications that have not been acknowledged yet. Should be zero or very close to it in high volume environments.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "/^count$/",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "12.0.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "grafana-postgresql-datasource",
+            "uid": "PCC52D03280B7034C"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS FALSE;",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "name": "AVG",
+                "parameters": [
+                  {
+                    "name": "email_address",
+                    "type": "functionParameter"
+                  }
+                ],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "table": "vibetype_private.account"
+        }
+      ],
+      "title": "Notifications pending",
+      "type": "stat"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Operations",
+  "uid": "8c7e10e7-fdd5-4670-99d5-240a258ea4f5",
+  "version": 1
+}

src/production/configurations/grafana/provisioning/alerting/alert-notifications-pending.yaml

@@ -0,0 +1,96 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: Critical
+      folder: Infrastructure
+      interval: 10s
+      rules:
+        - uid: een6tmiec4ge8f
+          title: Notifications pending
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PCC52D03280B7034C
+              model:
+                editorMode: code
+                format: table
+                intervalMs: 1000
+                maxDataPoints: 43200
+                rawQuery: true
+                rawSql: SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS NULL OR is_acknowledged IS FALSE
+                refId: A
+                sql:
+                    columns:
+                        - name: COUNT
+                          parameters:
+                            - name: is_acknowledged
+                              type: functionParameter
+                          type: function
+                    groupBy:
+                        - property:
+                            type: string
+                          type: groupBy
+                    limit: 50
+                table: vibetype_private.notification
+            - refId: B
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params: []
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - B
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          noDataState: NoData
+          execErrState: Error
+          for: 1m
+          annotations:
+            summary: There are notifications which are not sent out, or at least not marked as acknowledged.
+          labels: {}
+          isPaused: false
+          notification_settings:
+            receiver: 'Discord'

src/production/configurations/grafana/provisioning/alerting/contact-points.yaml

@@ -0,0 +1,35 @@
+apiVersion: 1
+contactPoints:
+  - orgId: 1
+    name: 'Discord'
+    receivers:
+      - uid: aen6t8xrd6pkwb
+        type: discord
+        settings:
+          message: |-
+            {{ define "__alert_details" }}
+            {{ range . }}
+            ⏰ **Started At:** {{ .StartsAt }}{{ if ne .EndsAt.String "0001-01-01 00:00:00 +0000 UTC" }}
+            🛑 **Ended At:** {{ .EndsAt }}{{ end }}
+
+            📝 **Annotations**
+            {{ range .Annotations.SortedPairs }}- `{{ .Name }}` = {{ .Value }}
+            {{ end }}🏷️ **Labels**
+            {{ range .Labels.SortedPairs }}- `{{ .Name }}` = {{ .Value }}
+            {{ end }}{{ if .GeneratorURL }}
+            🔗 [Alert rule]({{ .GeneratorURL }}){{ end }}{{ if .DashboardURL }}
+            📊 [Dashboard]({{ .DashboardURL }}){{ end }}{{ if .PanelURL }}
+            📈 [Panel]({{ .PanelURL }}){{ end }}{{ if .SilenceURL }}
+            🔕 [Silence this alert]({{ .SilenceURL }}){{ end }}{{ end }}{{ end }}
+            {{ define "default.message_custom" }}
+            {{ if .Alerts.Firing }}
+            ## 🚨 **Firing Alerts**
+            {{ template "__alert_details" .Alerts.Firing }}{{ end }}
+            {{ if .Alerts.Resolved }}
+            ## ✅ **Resolved Alerts**
+            {{ template "__alert_details" .Alerts.Resolved }}{{ end }}{{ end }}
+            {{ template "default.message_custom" . }}
+            <@&1377144332154572831>
+          url: $__file{/run/secrets/grafana_discord_webhook}
+          use_discord_username: false
+        disableResolveMessage: false