init

Prometheus/Files/prometheus_alertmanager/alert.rules.yml

@@ -0,0 +1,38 @@
+groups:
+  - name: service
+    rules:
+      - alert: service_down
+        expr: up == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.job }} down"
+          description: "{{ $labels.job }} has been down for more than 10 minutes."
+
+  - name: infrastructure
+    rules:
+      - alert: high_load
+        expr: node_load1 > 8
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Instance {{ $labels.job }} under high load"
+          description: "{{ $labels.job }} is under high load."
+
+  - name: disk_space
+    rules:
+      - alert: disk_will_fill
+        expr: predict_linear(node_filesystem_free_bytes{job="host-ps03fd"}[4h], 4 * 3600) < 0
+        for: 5m
+        labels:
+          severity: critical
+
+      - alert: disk_10_percent_free
+        expr: node_exporter:node_filesystem_free_bytes:fs_used_percents >= 90
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.instance }} is low on disk space"
+          description: "{{ $labels.instance }} has only {{ $value }}% free."

Prometheus/Files/prometheus_alertmanager/alertmanager.yml

@@ -0,0 +1,53 @@
+route:
+  receiver: email-self
+
+  group_by: [ alertname, service ]
+
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+
+  routes:
+  - matchers:
+    - service="infrastructure"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+  - matchers:
+    - service="backend"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+  - matchers:
+    - service="frontend"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+inhibit_rules:
+- source_matchers: [ severity="critical" ]
+  target_matchers: [ severity="warning" ]
+  equal: [ alertname, service ]
+
+receivers:
+  - name: pushover-self
+    pushover_configs:
+      - token: agy1n4b4p8chrgatsusxxub2se9x7q
+        user_key: u15he1cutrs9szvdz48tgypgachkh5
+
+  - name: email-self
+    email_configs:
+      - to: alexanderlebens@gmail.com
+        from: alexanderlebens@gmail.com
+        smarthost: smtp.gmail.com:587
+        auth_username: "alexanderlebens@gmail.com"
+        auth_identity: "alexanderlebens@gmail.com"
+        auth_password: "rcqzctyvzzctoesz"