168 lines
9.1 KiB
YAML
168 lines
9.1 KiB
YAML
---
|
|
# Source: kube-prometheus-stack/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: kube-prometheus-stack-etcd
|
|
namespace: kube-prometheus-stack
|
|
labels:
|
|
app: kube-prometheus-stack
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/instance: kube-prometheus-stack
|
|
app.kubernetes.io/version: "79.11.0"
|
|
app.kubernetes.io/part-of: kube-prometheus-stack
|
|
chart: kube-prometheus-stack-79.11.0
|
|
release: "kube-prometheus-stack"
|
|
heritage: "Helm"
|
|
spec:
|
|
groups:
|
|
- name: etcd
|
|
rules:
|
|
- alert: etcdMembersDown
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).'
|
|
summary: etcd cluster members are down.
|
|
expr: |-
|
|
max without (endpoint) (
|
|
sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
|
|
or
|
|
count without (To) (
|
|
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
|
|
)
|
|
)
|
|
> 0
|
|
for: 20m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdInsufficientMembers
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).'
|
|
summary: etcd cluster has insufficient number of members.
|
|
expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance, pod) < ((count(up{job=~".*etcd.*"}) without (instance, pod) + 1) / 2)
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdNoLeader
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.'
|
|
summary: etcd cluster has no leader.
|
|
expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdHighNumberOfLeaderChanges
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
|
|
summary: etcd cluster has high number of leader changes.
|
|
expr: increase((max without (instance, pod) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdHighNumberOfFailedGRPCRequests
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster has high number of failed grpc requests.
|
|
expr: |-
|
|
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
|
|
/
|
|
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
|
|
> 1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdHighNumberOfFailedGRPCRequests
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster has high number of failed grpc requests.
|
|
expr: |-
|
|
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
|
|
/
|
|
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
|
|
> 5
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdGRPCRequestsSlow
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.'
|
|
summary: etcd grpc requests are slow
|
|
expr: |-
|
|
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
|
|
> 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdMemberCommunicationSlow
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster member communication is slow.
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdHighNumberOfFailedProposals
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster has high number of proposal failures.
|
|
expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdHighFsyncDurations
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster 99th percentile fsync durations are too high.
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdHighFsyncDurations
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster 99th percentile fsync durations are too high.
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 1
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdHighCommitDurations
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
|
|
summary: etcd cluster 99th percentile commit durations are too high.
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.25
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdDatabaseQuotaLowSpace
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
|
|
summary: etcd cluster database is running full.
|
|
expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
- alert: etcdExcessiveDatabaseGrowth
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
|
|
summary: etcd cluster database growing very fast.
|
|
expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: etcdDatabaseHighFragmentationRatio
|
|
annotations:
|
|
description: 'etcd cluster "{{ $labels.job }}": database size in use on instance {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
|
|
runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
|
|
summary: etcd database size in use is less than 50% of the actual allocated storage.
|
|
expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|