infrastructure/clusters/cl01tl/helm/rybbit/templates/prometheus-rule.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: clickhouse
  namespace: {{ .Release.Namespace }}
  labels:
    app.kubernetes.io/name: clickhouse
    {{- include "custom.labels" . | nindent 4 }}
spec:
  groups:
    - name: EmbeddedExporter
      rules:
        - alert: ClickHouseNodeDown
          expr: up{job="clickhouse"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse node down (instance {{ `{{ $labels.instance }}` }})
            description: "No metrics received from ClickHouse exporter for over 2 minutes.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseMemoryUsageCritical
          expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse Memory Usage Critical (instance {{ `{{ $labels.instance }}` }})
            description: "Memory usage is critically high, over 90%.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseMemoryUsageWarning
          expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse Memory Usage Warning (instance {{ `{{ $labels.instance }}` }})
            description: "Memory usage is over 80%.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseDiskSpaceLowOnDefault
          expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse Disk Space Low on Default (instance {{ `{{ $labels.instance }}` }})
            description: "Disk space on default is below 20%.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseDiskSpaceCriticalOnDefault
          expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse Disk Space Critical on Default (instance {{ `{{ $labels.instance }}` }})
            description: "Disk space on default disk is critically low, below 10%.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseDiskSpaceLowOnBackups
          expr: ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) > 0
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse Disk Space Low on Backups (instance {{ `{{ $labels.instance }}` }})
            description: "Disk space on backups is below 20%.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseReplicaErrors
          expr: ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse Replica Errors (instance {{ `{{ $labels.instance }}` }})
            description: "Critical replica errors detected, either all replicas are stale or lost.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseNoAvailableReplicas
          expr: ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse No Available Replicas (instance {{ `{{ $labels.instance }}` }})
            description: "No available replicas in ClickHouse.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseNoLiveReplicas
          expr: ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse No Live Replicas (instance {{ `{{ $labels.instance }}` }})
            description: "There are too few live replicas available, risking data loss and service disruption.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseHighTCPConnections
          expr: ClickHouseMetrics_TCPConnection > 400
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse High TCP Connections (instance {{ `{{ $labels.instance }}` }})
            description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseInterserverConnectionIssues
          expr: ClickHouseMetrics_InterserverConnection > 50
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse Interserver Connection Issues (instance {{ `{{ $labels.instance }}` }})
            description: "High number of interserver connections may indicate replication or distributed query handling issues.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseZooKeeperConnectionIssues
          expr: ClickHouseMetrics_ZooKeeperSession != 1
          for: 3m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse ZooKeeper Connection Issues (instance {{ `{{ $labels.instance }}` }})
            description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseAuthenticationFailures
          expr: increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 3
          for: 0m
          labels:
            severity: info
          annotations:
            summary: ClickHouse Authentication Failures (instance {{ `{{ $labels.instance }}` }})
            description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseAccessDeniedErrors
          expr: increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 3
          for: 0m
          labels:
            severity: info
          annotations:
            summary: ClickHouse Access Denied Errors (instance {{ `{{ $labels.instance }}` }})
            description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseRejectedInsertQueries
          expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 2
          for: 1m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse rejected insert queries (instance {{ `{{ $labels.instance }}` }})
            description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseDelayedInsertQueries
          expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 10
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: ClickHouse delayed insert queries (instance {{ `{{ $labels.instance }}` }})
            description: "INSERTs delayed due to high number of active parts.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseZookeeperHardwareException
          expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse zookeeper hardware exception (instance {{ `{{ $labels.instance }}` }})
            description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"
        - alert: ClickHouseDistributedRejectedInserts
          expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 3
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: ClickHouse distributed rejected inserts (instance {{ `{{ $labels.instance }}` }})
            description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n  VALUE = {{ `{{ $value }}` }}\n  LABELS = {{ `{{ $labels }}` }}"