cvms/docker/prometheus/rules/root.yaml at develop · xrplevm/cvms · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
groups:
  - name: Prometheus
    rules:
      - alert: CVMSExporterDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: 'One of CVMS Service was down.'
          description: |
            The {{ $labels.job }} was down. you should recover this service right now to keep CVMS monitoring service.

      - alert: Found failed evaludation rules in prometheus
        expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: 'Some of rules in CVMS was failed to be evaludated.'
          description: |
            Some of rules in {{ $labels.rule_group }} is not normal. Please check the rules ASAP to monitor normally.

  - name: Root
    rules:
      - alert: FoundUnnormalPackageOps
        expr: changes(cvms_root_processed_ops_total[30m]) < 1
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: 'There are abnormal packages in the CVMS service.'
          description: |
            The {{ $labels.package }} service being collected from {{ $labels.chain }} is not operating correctly.

      - alert: FoundUnhealthyPackage
        expr: cvms_root_health_checker == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'There are abnormal packages in the CVMS service.'
          description: |
            The {{ $labels.package }} service being collected from {{ $labels.chain }} is not operating correctly.

      - alert: FoundSkippedPackage
        expr: cvms_root_skip_counter == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'There are skipped packages in the CVMS service.'
          description: |
            The {{ $labels.package }} service being collected from {{ $labels.instance_name }} has been skipped.

      - alert: IndexerPackageSyncDelayedOver5Minutes
        expr: time() - cvms_root_index_pointer_timestamp{package!~"babylon-checkpoint"} > 300
        labels:
          severity: warning
        annotations:
          summary: 'Indexer package "{{ $labels.package }}" is delayed over 5 minutes'
          description: 'The indexer for package "{{ $labels.package }}" in chain {{ $labels.chain }}-{{ $labels.chain_id }} is out of sync by more than 5 minutes. Please check the indexer status or logs.'