Skip to content

Commit b09a043

Browse files
dargmueslidargmuesli
authored
Merge pull request #207 from maevsi/feat/grafana/alert
feat(grafana): alert for unsent notifications
2 parents 8070e4a + 4a6b08f commit b09a043

File tree

7 files changed

+273
-0
lines changed

7 files changed

+273
-0
lines changed

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ This project is deployed in accordance to the [DargStack template](https://githu
3030

3131
The observation dashboard's admin user.
3232

33+
- ### `grafana_discord_webhook`
34+
35+
The observation dashboard's contact point for Discord.
36+
3337
- ### `jobber_aliases`
3438

3539
The job scheduler's SMTP client mail alias.

‎src/development/secrets/grafana/webhook_discord.secret‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
UNSET THIRD PARTY SECRET

‎src/development/secrets/grafana/webhook_discord.secret.template‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
https://discord.com/api/webhooks/<id>/<id>

‎src/development/stack.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
# https://github.com/maevsi/vibetype/
55
---
66
secrets:
7+
grafana_discord_webhook:
8+
# The observation dashboard's contact point for Discord.
9+
file: ./secrets/grafana/discord_webhook.secret
710
jobber_aliases:
811
# The job scheduler's SMTP client mail alias.
912
file: ./secrets/jobber/aliases.secret
@@ -161,6 +164,7 @@ services:
161164
GF_SERVER_ROOT_URL: https://grafana.${STACK_DOMAIN}/
162165
image: grafana/grafana:12.0.1
163166
secrets:
167+
- grafana_discord_webhook
164168
- postgres_db
165169
- postgres_role_service_grafana_password
166170
- postgres_role_service_grafana_username

‎src/production/configurations/grafana/dashboards/Infrastructure/operations.json‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
{
2+
"annotations": {
3+
"list": [
4+
{
5+
"builtIn": 1,
6+
"datasource": {
7+
"type": "grafana",
8+
"uid": "-- Grafana --"
9+
},
10+
"enable": true,
11+
"hide": true,
12+
"iconColor": "rgba(0, 211, 255, 1)",
13+
"name": "Annotations & Alerts",
14+
"type": "dashboard"
15+
}
16+
]
17+
},
18+
"editable": true,
19+
"fiscalYearStartMonth": 0,
20+
"graphTooltip": 0,
21+
"id": 6,
22+
"links": [],
23+
"panels": [
24+
{
25+
"datasource": {
26+
"type": "grafana-postgresql-datasource",
27+
"uid": "PCC52D03280B7034C"
28+
},
29+
"description": "The amount of notifications that have not been acknowledged yet. Should be zero or very close to it in high volume environments.",
30+
"fieldConfig": {
31+
"defaults": {
32+
"color": {
33+
"mode": "thresholds"
34+
},
35+
"mappings": [],
36+
"thresholds": {
37+
"mode": "absolute",
38+
"steps": [
39+
{
40+
"color": "green"
41+
},
42+
{
43+
"color": "red",
44+
"value": 1
45+
}
46+
]
47+
}
48+
},
49+
"overrides": []
50+
},
51+
"gridPos": {
52+
"h": 8,
53+
"w": 12,
54+
"x": 0,
55+
"y": 0
56+
},
57+
"id": 1,
58+
"options": {
59+
"colorMode": "value",
60+
"graphMode": "area",
61+
"justifyMode": "auto",
62+
"orientation": "auto",
63+
"percentChangeColorMode": "standard",
64+
"reduceOptions": {
65+
"calcs": [
66+
"lastNotNull"
67+
],
68+
"fields": "/^count$/",
69+
"values": false
70+
},
71+
"showPercentChange": false,
72+
"textMode": "auto",
73+
"wideLayout": true
74+
},
75+
"pluginVersion": "12.0.1",
76+
"targets": [
77+
{
78+
"datasource": {
79+
"type": "grafana-postgresql-datasource",
80+
"uid": "PCC52D03280B7034C"
81+
},
82+
"editorMode": "code",
83+
"format": "table",
84+
"rawQuery": true,
85+
"rawSql": "SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS FALSE;",
86+
"refId": "A",
87+
"sql": {
88+
"columns": [
89+
{
90+
"name": "AVG",
91+
"parameters": [
92+
{
93+
"name": "email_address",
94+
"type": "functionParameter"
95+
}
96+
],
97+
"type": "function"
98+
}
99+
],
100+
"groupBy": [
101+
{
102+
"property": {
103+
"type": "string"
104+
},
105+
"type": "groupBy"
106+
}
107+
],
108+
"limit": 50
109+
},
110+
"table": "vibetype_private.account"
111+
}
112+
],
113+
"title": "Notifications pending",
114+
"type": "stat"
115+
}
116+
],
117+
"preload": false,
118+
"schemaVersion": 41,
119+
"tags": [],
120+
"templating": {
121+
"list": []
122+
},
123+
"time": {
124+
"from": "now-6h",
125+
"to": "now"
126+
},
127+
"timepicker": {},
128+
"timezone": "browser",
129+
"title": "Operations",
130+
"uid": "8c7e10e7-fdd5-4670-99d5-240a258ea4f5",
131+
"version": 1
132+
}

‎src/production/configurations/grafana/provisioning/alerting/alert-notifications-pending.yaml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
apiVersion: 1
2+
groups:
3+
- orgId: 1
4+
name: Critical
5+
folder: Infrastructure
6+
interval: 10s
7+
rules:
8+
- uid: een6tmiec4ge8f
9+
title: Notifications pending
10+
condition: C
11+
data:
12+
- refId: A
13+
relativeTimeRange:
14+
from: 600
15+
to: 0
16+
datasourceUid: PCC52D03280B7034C
17+
model:
18+
editorMode: code
19+
format: table
20+
intervalMs: 1000
21+
maxDataPoints: 43200
22+
rawQuery: true
23+
rawSql: SELECT COUNT(1) FROM vibetype_private.notification WHERE is_acknowledged IS NULL OR is_acknowledged IS FALSE
24+
refId: A
25+
sql:
26+
columns:
27+
- name: COUNT
28+
parameters:
29+
- name: is_acknowledged
30+
type: functionParameter
31+
type: function
32+
groupBy:
33+
- property:
34+
type: string
35+
type: groupBy
36+
limit: 50
37+
table: vibetype_private.notification
38+
- refId: B
39+
datasourceUid: __expr__
40+
model:
41+
conditions:
42+
- evaluator:
43+
params: []
44+
type: gt
45+
operator:
46+
type: and
47+
query:
48+
params:
49+
- B
50+
reducer:
51+
params: []
52+
type: last
53+
type: query
54+
datasource:
55+
type: __expr__
56+
uid: __expr__
57+
expression: A
58+
intervalMs: 1000
59+
maxDataPoints: 43200
60+
reducer: last
61+
refId: B
62+
type: reduce
63+
- refId: C
64+
datasourceUid: __expr__
65+
model:
66+
conditions:
67+
- evaluator:
68+
params:
69+
- 0
70+
type: gt
71+
operator:
72+
type: and
73+
query:
74+
params:
75+
- C
76+
reducer:
77+
params: []
78+
type: last
79+
type: query
80+
datasource:
81+
type: __expr__
82+
uid: __expr__
83+
expression: B
84+
intervalMs: 1000
85+
maxDataPoints: 43200
86+
refId: C
87+
type: threshold
88+
noDataState: NoData
89+
execErrState: Error
90+
for: 1m
91+
annotations:
92+
summary: There are notifications which are not sent out, or at least not marked as acknowledged.
93+
labels: {}
94+
isPaused: false
95+
notification_settings:
96+
receiver: 'Discord'

‎src/production/configurations/grafana/provisioning/alerting/contact-points.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: 1
2+
contactPoints:
3+
- orgId: 1
4+
name: 'Discord'
5+
receivers:
6+
- uid: aen6t8xrd6pkwb
7+
type: discord
8+
settings:
9+
message: |-
10+
{{ define "__alert_details" }}
11+
{{ range . }}
12+
⏰ **Started At:** {{ .StartsAt }}{{ if ne .EndsAt.String "0001-01-01 00:00:00 +0000 UTC" }}
13+
🛑 **Ended At:** {{ .EndsAt }}{{ end }}
14+
15+
📝 **Annotations**
16+
{{ range .Annotations.SortedPairs }}- `{{ .Name }}` = {{ .Value }}
17+
{{ end }}🏷️ **Labels**
18+
{{ range .Labels.SortedPairs }}- `{{ .Name }}` = {{ .Value }}
19+
{{ end }}{{ if .GeneratorURL }}
20+
🔗 [Alert rule]({{ .GeneratorURL }}){{ end }}{{ if .DashboardURL }}
21+
📊 [Dashboard]({{ .DashboardURL }}){{ end }}{{ if .PanelURL }}
22+
📈 [Panel]({{ .PanelURL }}){{ end }}{{ if .SilenceURL }}
23+
🔕 [Silence this alert]({{ .SilenceURL }}){{ end }}{{ end }}{{ end }}
24+
{{ define "default.message_custom" }}
25+
{{ if .Alerts.Firing }}
26+
## 🚨 **Firing Alerts**
27+
{{ template "__alert_details" .Alerts.Firing }}{{ end }}
28+
{{ if .Alerts.Resolved }}
29+
## ✅ **Resolved Alerts**
30+
{{ template "__alert_details" .Alerts.Resolved }}{{ end }}{{ end }}
31+
{{ template "default.message_custom" . }}
32+
<@&1377144332154572831>
33+
url: $__file{/run/secrets/grafana_discord_webhook}
34+
use_discord_username: false
35+
disableResolveMessage: false

0 commit comments

Comments
 (0)