chore: Update manifests after change

clusters/cl01tl/manifests/rybbit/ConfigMap-clickhouse-config.yaml

@@ -11,15 +11,15 @@ metadata:
 data:
   enable_json.xml: |
     <clickhouse>
-        <settings>
-            <enable_json_type>1</enable_json_type>
-        </settings>
+      <settings>
+        <enable_json_type>1</enable_json_type>
+      </settings>
     </clickhouse>
   logging_rules.xml: |
     <clickhouse>
       <logger>
-          <level>warning</level>
-          <console>true</console>
+        <level>warning</level>
+        <console>true</console>
       </logger>
       <query_thread_log remove="remove"/>
       <query_log remove="remove"/>
@@ -32,9 +32,21 @@ data:
       <latency_log remove="remove"/>
       <processors_profile_log remove="remove"/>
     </clickhouse>
+  metrics.xml: |
+    <clickhouse>
+      <prometheus>
+        <endpoint>/metrics</endpoint>
+        <port>9363</port>
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>true</asynchronous_metrics>
+        <errors>true</errors>
+      </prometheus>
+    </clickhouse>
   network.xml: |
     <clickhouse>
-        <listen_host>0.0.0.0</listen_host>
+      <listen_host>0.0.0.0</listen_host>
+      <http_port>8123</http_port>
     </clickhouse>
   user_logging.xml: |
     <clickhouse>

clusters/cl01tl/manifests/rybbit/Deployment-rybbit-backend.yaml

@@ -22,7 +22,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/configMaps: 26a783f1bf2cd5bcd4f724d505a909d6e3bb7b3715f1dfa01a98609864d4ab3e
+        checksum/configMaps: ba4b6ef840b78e95c76ebd6d10e0cc8536bd139037769a25c40c76f1eceb029d
       labels:
         app.kubernetes.io/controller: backend
         app.kubernetes.io/instance: rybbit

clusters/cl01tl/manifests/rybbit/Deployment-rybbit-clickhouse.yaml

@@ -22,7 +22,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/configMaps: 26a783f1bf2cd5bcd4f724d505a909d6e3bb7b3715f1dfa01a98609864d4ab3e
+        checksum/configMaps: ba4b6ef840b78e95c76ebd6d10e0cc8536bd139037769a25c40c76f1eceb029d
       labels:
         app.kubernetes.io/controller: clickhouse
         app.kubernetes.io/instance: rybbit
@@ -87,6 +87,11 @@ spec:
               name: clickhouse-config
               readOnly: true
               subPath: user_logging.xml
+            - mountPath: /etc/clickhouse-server/config.d/metrics.xml
+              mountPropagation: None
+              name: clickhouse-config
+              readOnly: true
+              subPath: metrics.xml
       volumes:
         - name: clickhouse
           persistentVolumeClaim:

clusters/cl01tl/manifests/rybbit/Deployment-rybbit-client.yaml

@@ -22,7 +22,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/configMaps: 26a783f1bf2cd5bcd4f724d505a909d6e3bb7b3715f1dfa01a98609864d4ab3e
+        checksum/configMaps: ba4b6ef840b78e95c76ebd6d10e0cc8536bd139037769a25c40c76f1eceb029d
       labels:
         app.kubernetes.io/controller: client
         app.kubernetes.io/instance: rybbit

clusters/cl01tl/manifests/rybbit/PrometheusRule-clickhouse.yaml

@@ -0,0 +1,157 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: clickhouse
+  namespace: rybbit
+  labels:
+    app.kubernetes.io/name: clickhouse
+    app.kubernetes.io/instance: rybbit
+    app.kubernetes.io/part-of: rybbit
+spec:
+  groups:
+    - name: EmbeddedExporter
+      rules:
+        - alert: ClickHouseNodeDown
+          expr: up{job="clickhouse"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse node down (instance {{ $labels.instance }})
+            description: "No metrics received from ClickHouse exporter for over 2 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseMemoryUsageCritical
+          expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse Memory Usage Critical (instance {{ $labels.instance }})
+            description: "Memory usage is critically high, over 90%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseMemoryUsageWarning
+          expr: ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse Memory Usage Warning (instance {{ $labels.instance }})
+            description: "Memory usage is over 80%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseDiskSpaceLowOnDefault
+          expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse Disk Space Low on Default (instance {{ $labels.instance }})
+            description: "Disk space on default is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseDiskSpaceCriticalOnDefault
+          expr: ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse Disk Space Critical on Default (instance {{ $labels.instance }})
+            description: "Disk space on default disk is critically low, below 10%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseDiskSpaceLowOnBackups
+          expr: ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) > 0
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse Disk Space Low on Backups (instance {{ $labels.instance }})
+            description: "Disk space on backups is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseReplicaErrors
+          expr: ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse Replica Errors (instance {{ $labels.instance }})
+            description: "Critical replica errors detected, either all replicas are stale or lost.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseNoAvailableReplicas
+          expr: ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse No Available Replicas (instance {{ $labels.instance }})
+            description: "No available replicas in ClickHouse.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseNoLiveReplicas
+          expr: ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse No Live Replicas (instance {{ $labels.instance }})
+            description: "There are too few live replicas available, risking data loss and service disruption.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseHighTCPConnections
+          expr: ClickHouseMetrics_TCPConnection > 400
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse High TCP Connections (instance {{ $labels.instance }})
+            description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseInterserverConnectionIssues
+          expr: ClickHouseMetrics_InterserverConnection > 50
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse Interserver Connection Issues (instance {{ $labels.instance }})
+            description: "High number of interserver connections may indicate replication or distributed query handling issues.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseZooKeeperConnectionIssues
+          expr: ClickHouseMetrics_ZooKeeperSession != 1
+          for: 3m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse ZooKeeper Connection Issues (instance {{ $labels.instance }})
+            description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseAuthenticationFailures
+          expr: increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 3
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: ClickHouse Authentication Failures (instance {{ $labels.instance }})
+            description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseAccessDeniedErrors
+          expr: increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 3
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }})
+            description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseRejectedInsertQueries
+          expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 2
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse rejected insert queries (instance {{ $labels.instance }})
+            description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseDelayedInsertQueries
+          expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 10
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: ClickHouse delayed insert queries (instance {{ $labels.instance }})
+            description: "INSERTs delayed due to high number of active parts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseZookeeperHardwareException
+          expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse zookeeper hardware exception (instance {{ $labels.instance }})
+            description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: ClickHouseDistributedRejectedInserts
+          expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 3
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: ClickHouse distributed rejected inserts (instance {{ $labels.instance }})
+            description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

clusters/cl01tl/manifests/rybbit/Service-rybbit-clickhouse.yaml

@@ -16,6 +16,10 @@ spec:
       targetPort: 8123
       protocol: TCP
       name: http
+    - port: 9363
+      targetPort: 9363
+      protocol: TCP
+      name: metrics
   selector:
     app.kubernetes.io/controller: clickhouse
     app.kubernetes.io/instance: rybbit

clusters/cl01tl/manifests/rybbit/ServiceMonitor-rybbit.yaml

@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: rybbit
+  labels:
+    app.kubernetes.io/instance: rybbit
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: rybbit
+    helm.sh/chart: rybbit-4.6.2
+  namespace: rybbit
+spec:
+  jobLabel: rybbit
+  namespaceSelector:
+    matchNames:
+      - rybbit
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: rybbit-clickhouse
+      app.kubernetes.io/name: rybbit-clickhouse
+  endpoints:
+    - interval: 30s
+      path: /metrics
+      port: metrics
+      scrapeTimeout: 10s