diff --git a/README.md b/README.md index df5cc270..b9e6e8c7 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ This project is deployed in accordance to the [DargStack template](https://githu The observation dashboard's admin user. + - ### `grafana_discord_webhook` + + The observation dashboard's contact point for Discord. + - ### `jobber_aliases` The job scheduler's SMTP client mail alias. diff --git a/src/development/secrets/grafana/webhook_discord.secret b/src/development/secrets/grafana/webhook_discord.secret new file mode 100644 index 00000000..6075c4fb --- /dev/null +++ b/src/development/secrets/grafana/webhook_discord.secret @@ -0,0 +1 @@ +UNSET THIRD PARTY SECRET \ No newline at end of file diff --git a/src/development/secrets/grafana/webhook_discord.secret.template b/src/development/secrets/grafana/webhook_discord.secret.template new file mode 100644 index 00000000..3ad15b2a --- /dev/null +++ b/src/development/secrets/grafana/webhook_discord.secret.template @@ -0,0 +1 @@ +https://discord.com/api/webhooks// \ No newline at end of file diff --git a/src/development/stack.yml b/src/development/stack.yml index 3c25c5c0..b2467938 100644 --- a/src/development/stack.yml +++ b/src/development/stack.yml @@ -4,6 +4,9 @@ # https://github.com/maevsi/vibetype/ --- secrets: + grafana_discord_webhook: + # The observation dashboard's contact point for Discord. + file: ./secrets/grafana/discord_webhook.secret jobber_aliases: # The job scheduler's SMTP client mail alias. file: ./secrets/jobber/aliases.secret @@ -161,6 +164,7 @@ services: GF_SERVER_ROOT_URL: https://grafana.${STACK_DOMAIN}/ image: grafana/grafana:12.0.1 secrets: + - grafana_discord_webhook - postgres_db - postgres_role_service_grafana_password - postgres_role_service_grafana_username diff --git a/src/production/configurations/grafana/dashboards/Infrastructure/operations.json b/src/production/configurations/grafana/dashboards/Infrastructure/operations.json new file mode 100644 index 00000000..57a20b24 --- /dev/null +++ b/src/production/configurations/grafana/dashboards/Infrastructure/operations.json @@ -0,0 +1,132 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "description": "The amount of notifications that have not been acknowledged yet. Should be zero or very close to it in high volume environments.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^count$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS FALSE;", + "refId": "A", + "sql": { + "columns": [ + { + "name": "AVG", + "parameters": [ + { + "name": "email_address", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "vibetype_private.account" + } + ], + "title": "Notifications pending", + "type": "stat" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Operations", + "uid": "8c7e10e7-fdd5-4670-99d5-240a258ea4f5", + "version": 1 +} \ No newline at end of file diff --git a/src/production/configurations/grafana/provisioning/alerting/alert-notifications-pending.yaml b/src/production/configurations/grafana/provisioning/alerting/alert-notifications-pending.yaml new file mode 100644 index 00000000..c8258dc9 --- /dev/null +++ b/src/production/configurations/grafana/provisioning/alerting/alert-notifications-pending.yaml @@ -0,0 +1,96 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: Critical + folder: Infrastructure + interval: 10s + rules: + - uid: een6tmiec4ge8f + title: Notifications pending + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PCC52D03280B7034C + model: + editorMode: code + format: table + intervalMs: 1000 + maxDataPoints: 43200 + rawQuery: true + rawSql: SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS NULL OR is_acknowledged IS FALSE + refId: A + sql: + columns: + - name: COUNT + parameters: + - name: is_acknowledged + type: functionParameter + type: function + groupBy: + - property: + type: string + type: groupBy + limit: 50 + table: vibetype_private.notification + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: There are notifications which are not sent out, or at least not marked as acknowledged. + labels: {} + isPaused: false + notification_settings: + receiver: 'Discord' diff --git a/src/production/configurations/grafana/provisioning/alerting/contact-points.yaml b/src/production/configurations/grafana/provisioning/alerting/contact-points.yaml new file mode 100644 index 00000000..278b4f57 --- /dev/null +++ b/src/production/configurations/grafana/provisioning/alerting/contact-points.yaml @@ -0,0 +1,35 @@ +apiVersion: 1 +contactPoints: + - orgId: 1 + name: 'Discord' + receivers: + - uid: aen6t8xrd6pkwb + type: discord + settings: + message: |- + {{ define "__alert_details" }} + {{ range . }} + ⏰ **Started At:** {{ .StartsAt }}{{ if ne .EndsAt.String "0001-01-01 00:00:00 +0000 UTC" }} + 🛑 **Ended At:** {{ .EndsAt }}{{ end }} + + 📝 **Annotations** + {{ range .Annotations.SortedPairs }}- `{{ .Name }}` = {{ .Value }} + {{ end }}🏷️ **Labels** + {{ range .Labels.SortedPairs }}- `{{ .Name }}` = {{ .Value }} + {{ end }}{{ if .GeneratorURL }} + 🔗 [Alert rule]({{ .GeneratorURL }}){{ end }}{{ if .DashboardURL }} + 📊 [Dashboard]({{ .DashboardURL }}){{ end }}{{ if .PanelURL }} + 📈 [Panel]({{ .PanelURL }}){{ end }}{{ if .SilenceURL }} + 🔕 [Silence this alert]({{ .SilenceURL }}){{ end }}{{ end }}{{ end }} + {{ define "default.message_custom" }} + {{ if .Alerts.Firing }} + ## 🚨 **Firing Alerts** + {{ template "__alert_details" .Alerts.Firing }}{{ end }} + {{ if .Alerts.Resolved }} + ## ✅ **Resolved Alerts** + {{ template "__alert_details" .Alerts.Resolved }}{{ end }}{{ end }} + {{ template "default.message_custom" . }} + <@&1377144332154572831> + url: $__file{/run/secrets/grafana_discord_webhook} + use_discord_username: false + disableResolveMessage: false