init

Prometheus/Files/prometheus_alertmanager/alert.rules.yml

@@ -0,0 +1,38 @@
+groups:
+  - name: service
+    rules:
+      - alert: service_down
+        expr: up == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.job }} down"
+          description: "{{ $labels.job }} has been down for more than 10 minutes."
+
+  - name: infrastructure
+    rules:
+      - alert: high_load
+        expr: node_load1 > 8
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Instance {{ $labels.job }} under high load"
+          description: "{{ $labels.job }} is under high load."
+
+  - name: disk_space
+    rules:
+      - alert: disk_will_fill
+        expr: predict_linear(node_filesystem_free_bytes{job="host-ps03fd"}[4h], 4 * 3600) < 0
+        for: 5m
+        labels:
+          severity: critical
+
+      - alert: disk_10_percent_free
+        expr: node_exporter:node_filesystem_free_bytes:fs_used_percents >= 90
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.instance }} is low on disk space"
+          description: "{{ $labels.instance }} has only {{ $value }}% free."

Prometheus/Files/prometheus_alertmanager/alertmanager.yml

@@ -0,0 +1,53 @@
+route:
+  receiver: email-self
+
+  group_by: [ alertname, service ]
+
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+
+  routes:
+  - matchers:
+    - service="infrastructure"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+  - matchers:
+    - service="backend"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+  - matchers:
+    - service="frontend"
+    receiver: email-self
+    routes:
+    - matchers:
+      - severity="critical"
+      receiver: 'pushover-self'
+
+inhibit_rules:
+- source_matchers: [ severity="critical" ]
+  target_matchers: [ severity="warning" ]
+  equal: [ alertname, service ]
+
+receivers:
+  - name: pushover-self
+    pushover_configs:
+      - token: agy1n4b4p8chrgatsusxxub2se9x7q
+        user_key: u15he1cutrs9szvdz48tgypgachkh5
+
+  - name: email-self
+    email_configs:
+      - to: alexanderlebens@gmail.com
+        from: alexanderlebens@gmail.com
+        smarthost: smtp.gmail.com:587
+        auth_username: "alexanderlebens@gmail.com"
+        auth_identity: "alexanderlebens@gmail.com"
+        auth_password: "rcqzctyvzzctoesz"

Prometheus/Files/prometheus_config/prometheus.yml

@@ -0,0 +1,313 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - "/alertmanager/alert.rules.yml"
+
+alerting:
+  alertmanagers:
+    - scheme: https
+      static_configs:
+        - targets:
+            - "alertmanager.alexlebens.net"
+
+scrape_configs:
+  - job_name: "prometheus"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["localhost:9090"]
+        labels:
+          service: backend
+
+  - job_name: "alertmanager"
+    scheme: http
+    static_configs:
+      - targets: ["prometheus_alertmanager:9093"]
+        labels:
+          service: backend
+
+  - job_name: "docker-host"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["alexlebens.net:9323"]
+        labels:
+          service: infrastructure
+
+  - job_name: "docker-containers"
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+      - source_labels: [__meta_docker_container_label_prometheus_job]
+        regex: .+
+        action: keep
+      - regex: __meta_docker_container_label_prometheus_(.+)
+        action: labelmap
+        replacement: $1
+
+  - job_name: "windows-vd01wd"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["pd04wd.alexlebens.net:9182"]
+        labels:
+          service: host
+
+  - job_name: "windows-vd01wd-gpu"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["pd04wd.alexlebens.net:9835"]
+        labels:
+          service: host
+
+  - job_name: "host-ps03fd"
+    scrape_interval: 5s
+    scheme: http
+    static_configs:
+      - targets: ["prometheus_node_exporter:9100"]
+        labels:
+          service: host
+
+  - job_name: "folders-ps03fd"
+    scrape_interval: 5m
+    scrape_timeout: 60s
+    scheme: http
+    static_configs:
+      - targets: ["ps03fd.alexlebens.net:9974"]
+        labels:
+          service: host
+    basic_auth:
+      username: ""
+      password: "SimplePassword"
+
+  - job_name: "synology"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["synology.alexlebens.net:9100"]
+        labels:
+          service: device
+
+  - job_name: "pikvm"
+    metrics_path: "/api/export/prometheus/metrics"
+    basic_auth:
+      username: admin
+      password: 35etdgcb#%ETDGCB
+    scheme: https
+    static_configs:
+      - targets: ["pikvm.alexlebens.net"]
+        labels:
+          service: device
+    tls_config:
+      insecure_skip_verify: true # For self-signed certificate
+
+  - job_name: "synology-snmp"
+    scrape_interval: 300s
+    scrape_timeout: 240s
+    scheme: http
+    static_configs:
+      - targets: ["synology.alexlebens.net"]
+        labels:
+          service: device
+    metrics_path: /snmp
+    params:
+      module: [synology]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: prometheus_snmp_exporter:9116
+
+  - job_name: "airgradient-br"
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      - targets: ["airgradientbr.alexlebens.net:9926"]
+        labels:
+          service: device
+
+  - job_name: "traefik"
+    scrape_interval: 5s
+    scheme: https
+    static_configs:
+      - targets: ["traefik.alexlebens.net"]
+        labels:
+          service: backend
+
+  - job_name: "pihole"
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      - targets: ["pihole_exporter:9617"]
+        labels:
+          service: backend
+
+  - job_name: "coredns"
+    scheme: http
+    static_configs:
+      - targets: ["coredns:9153"]
+        labels:
+          service: backend
+
+  - job_name: "cadvisor"
+    scheme: http
+    scrape_interval: 30s
+    scrape_timeout: 20s
+    static_configs:
+      - targets: ["prometheus_cadvisor:8080"]
+        labels:
+          service: infrastructure
+
+  - job_name: "unpoller"
+    scheme: http
+    static_configs:
+      - targets: ["prometheus_unpoller:9130"]
+        labels:
+          service: infrastructure
+
+  - job_name: "grafana"
+    scheme: http
+    static_configs:
+      - targets: ["grafana:3000"]
+        labels:
+          service: backend
+
+  - job_name: "loki"
+    scheme: http
+    static_configs:
+      - targets: ["loki:3100"]
+        labels:
+          service: backend
+
+  - job_name: "promtail"
+    scheme: http
+    static_configs:
+      - targets: ["loki_promtail:9080"]
+        labels:
+          service: backend
+
+  - job_name: "watchtower"
+    scheme: http
+    metrics_path: /v1/metrics
+    bearer_token: token
+    static_configs:
+      - targets: ["watchtower:8080"]
+        labels:
+          service: backend
+
+  - job_name: "qbittorrent"
+    scheme: http
+    static_configs:
+      - targets: ["qbittorrent_exporter:8000"]
+        labels:
+          service: frontend
+
+  - job_name: "speedtest"
+    scheme: http
+    scrape_interval: 1h
+    scrape_timeout: 1m
+    static_configs:
+      - targets: ["speedtest:9798"]
+        labels:
+          service: infrastructure
+
+  - job_name: "authentik"
+    scheme: http
+    static_configs:
+      - targets: ["authentik_server:9300"]
+        labels:
+          service: infrastructure
+
+  - job_name: "uptimekuma"
+    scheme: http
+    static_configs:
+      - targets: ["uptimekuma:3001"]
+        labels:
+          service: backend
+    basic_auth:
+      username: admin
+      password: "24wrsfxv@$WRSFXV"
+
+  - job_name: "plex"
+    scheme: http
+    static_configs:
+      - targets: ["plex_exporter:9594"]
+        labels:
+          service: frontend
+
+  - job_name: "radarr"
+    scheme: http
+    static_configs:
+      - targets: ["radarr_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "radarr_anime"
+    scheme: http
+    static_configs:
+      - targets: ["radarr_anime_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "radarr_art"
+    scheme: http
+    static_configs:
+      - targets: ["radarr_art_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "radarr_documentaries"
+    scheme: http
+    static_configs:
+      - targets: ["radarr_documentaries_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "radarr_standup"
+    scheme: http
+    static_configs:
+      - targets: ["radarr_standup_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "sonarr"
+    scheme: http
+    static_configs:
+      - targets: ["sonarr_exporter:9707"]
+        labels:
+          service: frontend
+
+  - job_name: "sonarr_anime_exporter"
+    scheme: http
+    static_configs:
+      - targets: ["sonarr_anime_exporter:9707"]
+        labels:
+          service: frontend
+
+  - job_name: "sonarr_documentaries_exporter"
+    scheme: http
+    static_configs:
+      - targets: ["sonarr_anime_exporter:9707"]
+        labels:
+          service: frontend
+
+  - job_name: "lidarr"
+    scheme: http
+    static_configs:
+      - targets: ["lidarr_exporter:9709"]
+        labels:
+          service: frontend
+
+  - job_name: "readarr_books"
+    scheme: http
+    static_configs:
+      - targets: ["readarr_books_exporter:9708"]
+        labels:
+          service: frontend
+
+  - job_name: "readarr_audio"
+    scheme: http
+    static_configs:
+      - targets: ["readarr_audio_exporter:9708"]
+        labels:
+          service: frontend