feat: add rules
This commit is contained in:
156
clusters/cl01tl/helm/rybbit/templates/prometheus-rule.yaml
Normal file
156
clusters/cl01tl/helm/rybbit/templates/prometheus-rule.yaml
Normal file
@@ -0,0 +1,156 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: clickhouse
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse
|
||||
{{- include "custom.labels" . | nindent 4 }}
|
||||
spec:
|
||||
groups:
|
||||
- name: EmbeddedExporter
|
||||
rules:
|
||||
- alert: ClickHouseNodeDown
|
||||
expr: up{job="clickhouse"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse node down (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "No metrics received from ClickHouse exporter for over 2 minutes.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseMemoryUsageCritical
|
||||
expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse Memory Usage Critical (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Memory usage is critically high, over 90%.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseMemoryUsageWarning
|
||||
expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse Memory Usage Warning (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Memory usage is over 80%.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseDiskSpaceLowOnDefault
|
||||
expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse Disk Space Low on Default (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Disk space on default is below 20%.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseDiskSpaceCriticalOnDefault
|
||||
expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse Disk Space Critical on Default (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Disk space on default disk is critically low, below 10%.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseDiskSpaceLowOnBackups
|
||||
expr: ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse Disk Space Low on Backups (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Disk space on backups is below 20%.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseReplicaErrors
|
||||
expr: ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse Replica Errors (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Critical replica errors detected, either all replicas are stale or lost.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseNoAvailableReplicas
|
||||
expr: ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse No Available Replicas (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "No available replicas in ClickHouse.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseNoLiveReplicas
|
||||
expr: ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse No Live Replicas (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "There are too few live replicas available, risking data loss and service disruption.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseHighTCPConnections
|
||||
expr: ClickHouseMetrics_TCPConnection > 400
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse High TCP Connections (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseInterserverConnectionIssues
|
||||
expr: ClickHouseMetrics_InterserverConnection > 50
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse Interserver Connection Issues (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "High number of interserver connections may indicate replication or distributed query handling issues.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseZooKeeperConnectionIssues
|
||||
expr: ClickHouseMetrics_ZooKeeperSession != 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse ZooKeeper Connection Issues (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseAuthenticationFailures
|
||||
expr: increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 3
|
||||
for: 0m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: ClickHouse Authentication Failures (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseAccessDeniedErrors
|
||||
expr: increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 3
|
||||
for: 0m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: ClickHouse Access Denied Errors (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseRejectedInsertQueries
|
||||
expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse rejected insert queries (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseDelayedInsertQueries
|
||||
expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: ClickHouse delayed insert queries (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "INSERTs delayed due to high number of active parts.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseZookeeperHardwareException
|
||||
expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse zookeeper hardware exception (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
- alert: ClickHouseDistributedRejectedInserts
|
||||
expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: ClickHouse distributed rejected inserts (instance {{ `{{ $labels.instance }}` }})
|
||||
description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n VALUE = {{ `{{ $value }}` }}\n LABELS = {{ `{{ $labels }}` }}"
|
||||
@@ -150,19 +150,20 @@ rybbit:
|
||||
data:
|
||||
network.xml: |
|
||||
<clickhouse>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
<http_port>8123</http_port>
|
||||
</clickhouse>
|
||||
enable_json.xml: |
|
||||
<clickhouse>
|
||||
<settings>
|
||||
<enable_json_type>1</enable_json_type>
|
||||
</settings>
|
||||
<settings>
|
||||
<enable_json_type>1</enable_json_type>
|
||||
</settings>
|
||||
</clickhouse>
|
||||
logging_rules.xml: |
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<level>warning</level>
|
||||
<console>true</console>
|
||||
<level>warning</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
<query_thread_log remove="remove"/>
|
||||
<query_log remove="remove"/>
|
||||
@@ -185,6 +186,17 @@ rybbit:
|
||||
</default>
|
||||
</profiles>
|
||||
</clickhouse>
|
||||
metrics.xml: |
|
||||
<clickhouse>
|
||||
<prometheus>
|
||||
<endpoint>/metrics</endpoint>
|
||||
<port>9363</port>
|
||||
<metrics>true</metrics>
|
||||
<events>true</events>
|
||||
<asynchronous_metrics>true</asynchronous_metrics>
|
||||
<errors>true</errors>
|
||||
</prometheus>
|
||||
</clickhouse>
|
||||
service:
|
||||
backend:
|
||||
controller: backend
|
||||
@@ -204,6 +216,21 @@ rybbit:
|
||||
http:
|
||||
port: 8123
|
||||
targetPort: 8123
|
||||
metrics:
|
||||
port: 9363
|
||||
targetPort: 9363
|
||||
serviceMonitor:
|
||||
main:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rybbit-clickhouse
|
||||
app.kubernetes.io/instance: rybbit-clickhouse
|
||||
serviceName: '{{ include "bjw-s.common.lib.chart.names.fullname" $ }}'
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
path: /metrics
|
||||
persistence:
|
||||
clickhouse:
|
||||
forceRename: clickhouse-data
|
||||
@@ -238,6 +265,10 @@ rybbit:
|
||||
readOnly: true
|
||||
mountPropagation: None
|
||||
subPath: user_logging.xml
|
||||
- path: /etc/clickhouse-server/config.d/metrics.xml
|
||||
readOnly: true
|
||||
mountPropagation: None
|
||||
subPath: metrics.xml
|
||||
postgres-18-cluster:
|
||||
mode: recovery
|
||||
recovery:
|
||||
|
||||
Reference in New Issue
Block a user