From 574cd86064c3775ee4f5a9db92dfbb1ba03fbcac Mon Sep 17 00:00:00 2001 From: Alex Lebens Date: Thu, 15 May 2025 14:02:27 -0500 Subject: [PATCH] update dashboard --- dashboards/system/loki.json | 235 +++++++++++++++++++----------------- 1 file changed, 122 insertions(+), 113 deletions(-) diff --git a/dashboards/system/loki.json b/dashboards/system/loki.json index 9d31829..e9e4467 100644 --- a/dashboards/system/loki.json +++ b/dashboards/system/loki.json @@ -20,7 +20,7 @@ "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 90, + "id": 17, "links": [], "panels": [ { @@ -42,17 +42,19 @@ "groupBy": [], "groupMode": "default", "maxItems": 20, + "showInactiveAlerts": false, "sortOrder": 1, "stateFilter": { "error": true, "firing": true, "noData": false, "normal": false, - "pending": true + "pending": true, + "recovering": true }, "viewMode": "list" }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "title": "Summary of alerts status", "type": "alertlist" }, @@ -69,7 +71,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total number of messages logged by Loki itself", "fieldConfig": { @@ -117,8 +119,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -208,16 +209,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(log_messages_total{app=\"loki\",level=~\"error|warn\"}[1m])) by (level)", @@ -232,7 +234,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "log_messages_total", "format": "time_series", @@ -248,8 +250,8 @@ }, { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": {}, @@ -264,6 +266,7 @@ "id": 511, "options": { "dedupStrategy": "none", + "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, @@ -272,16 +275,17 @@ "sortOrder": "Descending", "wrapLogMessage": false }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "{app=\"loki\"} | logfmt | level=\"warn\" or level=\"error\"", + "expr": "{app=\"loki\"}", "queryType": "range", + "range": true, "refId": "A" } ], @@ -291,7 +295,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total number of messages logged by Promtail", "fieldConfig": { @@ -339,8 +343,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -430,16 +433,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(log_messages_total{app_kubernetes_io_name=\"promtail\",level=~\"error|warn\"}[1m])) by (level)\n", "format": "time_series", @@ -455,8 +459,8 @@ }, { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": {}, @@ -471,6 +475,7 @@ "id": 586, "options": { "dedupStrategy": "none", + "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, @@ -479,16 +484,16 @@ "sortOrder": "Descending", "wrapLogMessage": false }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "{app=\"promtail\"} | logfmt | level=\"warn\" or level=\"error\"", - "queryType": "range", + "expr": "{app=\"promtail\"}", + "range": true, "refId": "A" } ], @@ -498,7 +503,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -544,8 +549,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -574,16 +578,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(loki_distributor_ingester_append_failures_total[1m])*60", "instant": false, @@ -598,7 +603,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Number of log entries dropped because failed to be sent to the Loki ingester after all retries.", "fieldConfig": { @@ -645,8 +650,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -675,16 +679,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(promtail_dropped_entries_total[1m])*60", "instant": false, @@ -713,7 +718,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "The total number of streams created per tenant.\nThis should not increase after startup.", "fieldConfig": { @@ -761,8 +766,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -778,7 +782,7 @@ "h": 6, "w": 24, "x": 0, - "y": 281 + "y": 511 }, "id": 78, "options": { @@ -790,16 +794,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "loki_ingester_memory_streams", "interval": "", @@ -813,7 +818,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "The total number of log entries received per tenant (not necessarily of lines, as an entry can have more than one line of text).", "fieldConfig": { @@ -861,8 +866,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -878,7 +882,7 @@ "h": 6, "w": 24, "x": 0, - "y": 533 + "y": 517 }, "id": 56, "options": { @@ -890,16 +894,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(loki_distributor_lines_received_total[1m])", "interval": "", @@ -913,7 +918,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "The total number of uncompressed bytes received per tenant.\n", "fieldConfig": { @@ -961,8 +966,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -978,7 +982,7 @@ "h": 6, "w": 24, "x": 0, - "y": 539 + "y": 523 }, "id": 137, "options": { @@ -990,16 +994,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(loki_distributor_bytes_received_total[1m]))", "interval": "", @@ -1027,7 +1032,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of actual usage over configured limit", "fieldConfig": { @@ -1076,8 +1081,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -1093,7 +1097,7 @@ "h": 6, "w": 24, "x": 0, - "y": 282 + "y": 776 }, "id": 247, "options": { @@ -1106,16 +1110,17 @@ "width": 200 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "100 * max by(pod,container) (container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}) / on (pod,container) group_left kube_pod_container_resource_limits_memory_bytes{pod=~\"loki.*\"}", @@ -1132,7 +1137,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1179,8 +1184,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1246,7 +1250,7 @@ "h": 6, "w": 24, "x": 0, - "y": 452 + "y": 782 }, "id": 25, "options": { @@ -1259,16 +1263,17 @@ "width": 200 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}", "interval": "10s", @@ -1281,7 +1286,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_memory_bytes{pod=~\"loki.*\"})", "interval": "", @@ -1291,7 +1296,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_memory_bytes{pod=~\"loki.*\"})", "interval": "", @@ -1305,7 +1310,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of actual usage over configured limit", "fieldConfig": { @@ -1354,8 +1359,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -1371,7 +1375,7 @@ "h": 6, "w": 24, "x": 0, - "y": 458 + "y": 788 }, "id": 772, "options": { @@ -1384,16 +1388,17 @@ "width": 200 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "100* max by(pod,container) (container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"})\n/\non (pod,container) (kube_pod_container_resource_limits_memory_bytes{pod=~\"promtail.*\"})", "hide": false, @@ -1408,7 +1413,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1455,8 +1460,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1522,7 +1526,7 @@ "h": 6, "w": 24, "x": 0, - "y": 464 + "y": 794 }, "id": 1103, "options": { @@ -1535,16 +1539,17 @@ "width": 200 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}", "interval": "10s", @@ -1557,7 +1562,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_memory_bytes{pod=~\"promtail.*\"})", "interval": "", @@ -1567,7 +1572,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_memory_bytes{pod=~\"promtail.*\"})", "interval": "", @@ -1595,7 +1600,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "percentage of configured limit", "fieldConfig": { @@ -1644,8 +1649,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -1661,7 +1665,7 @@ "h": 6, "w": 24, "x": 0, - "y": 283 + "y": 965 }, "id": 347, "options": { @@ -1673,16 +1677,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "100* rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",pod=~\"loki.*\"}[1m])\n/\non (pod,container) kube_pod_container_resource_limits_cpu_cores{container=\"loki\"}", "interval": "10s", @@ -1699,7 +1704,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -1747,8 +1752,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1814,7 +1818,7 @@ "h": 6, "w": 24, "x": 0, - "y": 289 + "y": 971 }, "id": 6, "options": { @@ -1826,16 +1830,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}[1m])", "interval": "10s", @@ -1848,7 +1853,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_cpu_cores{pod=~\"loki.*\"})", "interval": "", @@ -1858,7 +1863,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_cpu_cores{pod=~\"loki.*\"})", "hide": false, @@ -1873,7 +1878,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "percentage of configured limit", "fieldConfig": { @@ -1922,8 +1927,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -1939,7 +1943,7 @@ "h": 6, "w": 24, "x": 0, - "y": 295 + "y": 977 }, "id": 910, "options": { @@ -1951,16 +1955,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "100 * rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}[1m])\n/\non (pod,container) kube_pod_container_resource_limits_cpu_cores{pod=~\"promtail.*\"}", "interval": "10s", @@ -1977,7 +1982,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -2025,8 +2030,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2092,7 +2096,7 @@ "h": 6, "w": 24, "x": 0, - "y": 301 + "y": 983 }, "id": 1104, "options": { @@ -2104,16 +2108,17 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}[1m])", "interval": "10s", @@ -2126,7 +2131,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_cpu_cores{pod=~\"promtail.*\"})", "interval": "", @@ -2136,7 +2141,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_cpu_cores{pod=~\"promtail.*\"})", "hide": false, @@ -2155,11 +2160,15 @@ ], "preload": true, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": ["system", "monitoring"], "templating": { "list": [ { + "current": { + "text": "Prometheus", + "value": "kube-prometheus-stack" + }, "includeAll": false, "label": "Datasource", "name": "DS_PROMETHEUS", @@ -2179,5 +2188,5 @@ "timezone": "browser", "title": "Loki", "uid": "e6a0b019f34e9468223bc1669d5a9366", - "weekStart": "" + "version": 2 }