Add 'runbook' Field to Annotations

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+
+## [0.17.0] - 2025-12-03
+### Added
+- `./internal/prometheus/alert_rules.go` add `runbook` field to `extraAnnotations` map.
+- `./examples/_gen/*.yml`, `./internal/app/generate/prometheus_test.go`, `./internal/prometheus/alert_rules_test.go`, `./test/integration/*` configure tests, templates and examples with new `runbook` field.
+- Dependencies updated.
+
 ## [0.16.0] - 2025-07-04
 ### Added
 - `./internal/prometheus/alert_rules.go` add `description` field to `extraAnnotations` map.

examples/_gen/getting-started.yml

@@ -227,6 +227,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: MyServiceHighErrorRate
     expr: |
       (
@@ -251,3 +252,4 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:

examples/_gen/home-wifi.yml

@@ -226,6 +226,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: GoodWifiClientSatisfaction
     expr: |
       (
@@ -249,6 +250,7 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
 - name: sloth-slo-sli-recordings-home-wifi-risk-wifi-client-satisfaction
   rules:
   - record: slo:sli_error:ratio_rate5m
@@ -471,6 +473,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: RiskWifiClientSatisfaction
     expr: |
       (
@@ -494,3 +497,4 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:

examples/_gen/k8s-getting-started.yml

@@ -220,6 +220,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
@@ -244,6 +245,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (3 * 0.0009999999999999432)) without (sloth_window)

examples/_gen/k8s-home-wifi.yml

@@ -224,6 +224,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"} > (14.4 * 0.05)) without (sloth_window)
@@ -247,6 +248,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"} > (3 * 0.05)) without (sloth_window)
@@ -469,6 +471,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
@@ -492,6 +495,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"} > (3 * 0.0009999999999999432)) without (sloth_window)

examples/_gen/k8s-multifile.yml

@@ -220,6 +220,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
@@ -244,6 +245,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (3 * 0.0009999999999999432)) without (sloth_window)
@@ -483,6 +485,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"} > (14.4 * 0.00010000000000005117)) without (sloth_window)
@@ -507,6 +510,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"} > (3 * 0.00010000000000005117)) without (sloth_window)

examples/_gen/kubernetes-apiserver.yml

@@ -221,13 +221,13 @@ groups:
       severity: critical
       sloth_severity: page
     annotations:
-      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
       summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
         rate is over expected.'
       title: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
   - alert: K8sApiserverAvailabilityAlert
     expr: |
       (
@@ -246,13 +246,13 @@ groups:
       severity: warning
       sloth_severity: ticket
     annotations:
-      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
       summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
         rate is over expected.'
       title: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
 - name: sloth-slo-sli-recordings-k8s-apiserver-requests-latency
   rules:
   - record: slo:sli_error:ratio_rate5m
@@ -505,13 +505,13 @@ groups:
       severity: critical
       sloth_severity: page
     annotations:
-      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
       summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
         rate is over expected.'
       title: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
   - alert: K8sApiserverLatencyAlert
     expr: |
       (
@@ -530,10 +530,10 @@ groups:
       severity: warning
       sloth_severity: ticket
     annotations:
-      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
       summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
         rate is over expected.'
       title: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh

examples/_gen/multifile.yml

@@ -227,6 +227,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: MyServiceHighErrorRate
     expr: |
       (
@@ -251,6 +252,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
 
 ---
 # Code generated by Sloth (dev): https://github.com/ostrovok-tech/sloth.
@@ -480,6 +482,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: MyServiceHighErrorRate
     expr: |
       (
@@ -504,3 +507,4 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:

examples/_gen/plugin-getting-started.yml

@@ -234,6 +234,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: MyServiceHighErrorRate
     expr: |
       (
@@ -258,3 +259,4 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:

examples/_gen/plugin-k8s-getting-started.yml

@@ -230,6 +230,7 @@ spec:
           burn rate is too fast.
         description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
           burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
@@ -252,6 +253,7 @@ spec:
           budget burn rate is too fast.
         description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
           budget burn rate is too fast.
+        runbook:
       expr: |
         (
             max(slo:sli_error:ratio_rate2h{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"} > (3 * 0.0009999999999999432)) without (sloth_window)

examples/_gen/raw-home-wifi.yml

@@ -247,6 +247,7 @@ groups:
         burn rate is too fast.
       description: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:
   - alert: WifiClientSatisfaction
     expr: |
       (
@@ -270,3 +271,4 @@ groups:
         burn rate is too fast.
       description: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
         burn rate is too fast.
+      runbook:

internal/app/generate/prometheus_test.go

@@ -363,6 +363,7 @@ or
 										"summary":       "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 										"title":         "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 										"description":   "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+										"runbook":       "",
 									},
 								},
 								{
@@ -388,6 +389,7 @@ or
 										"summary":       "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 										"title":         "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 										"description":   "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+										"runbook":       "",
 									},
 								},
 							},

internal/prometheus/alert_rules.go

@@ -90,6 +90,7 @@ func defaultSLOAlertGenerator(slo SLO, sloAlert AlertMeta, quick, slow alert.MWM
 		"title":       fmt.Sprintf("(%s) {{$labels.%s}} {{$labels.%s}} SLO error budget burn rate is too fast.", severity, sloServiceLabelName, sloNameLabelName),
 		"summary":     fmt.Sprintf("{{$labels.%s}} {{$labels.%s}} SLO error budget burn rate is over expected.", sloServiceLabelName, sloNameLabelName),
 		"description": fmt.Sprintf("(%s) {{$labels.%s}} {{$labels.%s}} SLO error budget burn rate is too fast.", severity, sloServiceLabelName, sloNameLabelName),
+		"runbook":     "",
 	}
 
 	// Add specific labels. We don't add the labels from the rules because we will

internal/prometheus/alert_rules_test.go

@@ -97,6 +97,7 @@ or
 						"summary":      "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 						"title":        "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 						"description":  "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+						"runbook":      "",
 					},
 				},
 				{
@@ -122,6 +123,7 @@ or
 						"summary":      "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 						"title":        "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 						"description":  "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+						"runbook":      "",
 					},
 				},
 			},
@@ -166,6 +168,7 @@ or
 						"summary":      "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 						"title":        "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 						"description":  "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+						"runbook":      "",
 					},
 				},
 			},
@@ -209,6 +212,7 @@ or
 						"summary":      "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 						"title":        "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 						"description":  "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+						"runbook":      "",
 					},
 				},
 			},

test/integration/k8scontroller/exp_base_28d_test.go

@@ -228,6 +228,7 @@ func getBase28DayPromOpPrometheusRule(slothVersion string) *monitoringv1.Prometh
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 						{
@@ -242,6 +243,7 @@ func getBase28DayPromOpPrometheusRule(slothVersion string) *monitoringv1.Prometh
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 					},

test/integration/k8scontroller/exp_base_7d_test.go

@@ -228,6 +228,7 @@ func getBase7DayPromOpPrometheusRule(slothVersion string) *monitoringv1.Promethe
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 						{
@@ -242,6 +243,7 @@ func getBase7DayPromOpPrometheusRule(slothVersion string) *monitoringv1.Promethe
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 					},

test/integration/k8scontroller/exp_base_test.go

@@ -285,6 +285,7 @@ func getBasePromOpPrometheusRule(slothVersion string) *monitoringv1.PrometheusRu
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 						{
@@ -299,6 +300,7 @@ func getBasePromOpPrometheusRule(slothVersion string) *monitoringv1.PrometheusRu
 								"summary":     "{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is over expected.",
 								"title":       "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
 								"description": "(ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn rate is too fast.",
+								"runbook":     "",
 							},
 						},
 					},

test/integration/prometheus/testdata/out-base-28d.yaml.tpl

@@ -194,6 +194,7 @@ groups:
       alert02k1: alert02k2
       description: (page) {{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error
         budget burn rate is too fast.
+      runbook: ""
       summary: '{{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error budget burn
         rate is over expected.'
       title: (page) {{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error budget
@@ -219,6 +220,7 @@ groups:
       alert02k1: alert02k2
       description: (ticket) {{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error
         budget burn rate is too fast.
+      runbook: ""
       summary: '{{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error budget burn
         rate is over expected.'
       title: (ticket) {{"{{$labels.sloth_service}}"}} {{"{{$labels.sloth_slo}}"}} SLO error budget