{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": false, "iconColor": "rgba(0, 211, 255, 1)", "limit": 100, "name": "Annotations & Alerts", "showIn": 0, "type": "dashboard" } ] }, "editable": false, "graphTooltip": 0, "panels": [ { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 12, "w": 24, "x": 0, "y": 0 }, "id": 987, "options": { "alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": false, "groupBy": [], "groupMode": "default", "maxItems": 20, "showInactiveAlerts": false, "sortOrder": 1, "stateFilter": { "error": true, "firing": true, "noData": false, "normal": false, "pending": true, "recovering": true }, "viewMode": "list" }, "pluginVersion": "12.0.0", "title": "Summary of alerts status", "type": "alertlist" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 435, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of messages logged by Loki itself", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "debug" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C0D8FF", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [ { "id": "color", "value": { "fixedColor": "#5794F2", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "warn" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FF9830", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 9, "x": 0, "y": 13 }, "id": 10, "interval": "1m", "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(log_messages_total{app=\"loki\",level=~\"error|warn\"}[1m])) by (level)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "log_messages_total", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "B" } ], "title": "Number of error/warning messages logged by Loki itself", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 5, "w": 15, "x": 9, "y": 13 }, "id": 511, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": false, "sortOrder": "Descending", "wrapLogMessage": false }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "{app=\"loki\"}", "queryType": "range", "range": true, "refId": "A" } ], "title": "Error/warning messages logged by Loki itself", "type": "logs" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of messages logged by Promtail", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "debug" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C0D8FF", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [ { "id": "color", "value": { "fixedColor": "#5794F2", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "warn" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FF9830", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 9, "x": 0, "y": 18 }, "id": 106, "interval": "1m", "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(log_messages_total{app_kubernetes_io_name=\"promtail\",level=~\"error|warn\"}[1m])) by (level)\n", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}}", "refId": "A" } ], "title": "Error/warning messages logged by Promtail", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 5, "w": 15, "x": 9, "y": 18 }, "id": 586, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": false, "sortOrder": "Descending", "wrapLogMessage": false }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "{app=\"promtail\"}", "range": true, "refId": "A" } ], "title": "Error/warning messages logged by Promtail", "type": "logs" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0.01 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 23 }, "id": 676, "interval": "1m", "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(loki_distributor_ingester_append_failures_total[1m])*60", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Loki (distributor) - failed batch appends sent to ingesters", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of log entries dropped because failed to be sent to the Loki ingester after all retries.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0.01 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 28 }, "id": 677, "interval": "1m", "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(promtail_dropped_entries_total[1m])*60", "instant": false, "interval": "", "legendFormat": "pod={{kubernetes_pod_name}}, instance={{instance}}", "refId": "A" } ], "title": "Promtail - dropped log entries", "type": "timeseries" } ], "title": "Problems in the Loki Stack", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 47, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The total number of streams created per tenant.\nThis should not increase after startup.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 5000 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 511 }, "id": 78, "options": { "alertThreshold": true, "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "loki_ingester_memory_streams", "interval": "", "legendFormat": "Streams", "refId": "A" } ], "title": "Number of Streams in Loki", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The total number of log entries received per tenant (not necessarily of lines, as an entry can have more than one line of text).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 517 }, "id": 56, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(loki_distributor_lines_received_total[1m])", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Loki (distributor) - log entries received per second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The total number of uncompressed bytes received per tenant.\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 1000000 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 523 }, "id": 137, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(loki_distributor_bytes_received_total[1m]))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Bytes received in ingester per second", "type": "timeseries" } ], "title": "Logging activity", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 18, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of actual usage over configured limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 776 }, "id": 247, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 200 }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "100 * max by(pod,container) (container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}) / on (pod,container) group_left kube_pod_container_resource_limits_memory_bytes{pod=~\"loki.*\"}", "hide": false, "interval": "", "legendFormat": "Usage (pod={{pod}}, container={{container}}, name={{name}})", "range": true, "refId": "A" } ], "title": "Loki memory usage (% of Kubernetes memory limit)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.56)", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } } ] }, { "matcher": { "id": "byName", "options": "Request" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.78)", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Usage" }, "properties": [ { "id": "custom.fillOpacity", "value": 10 } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 782 }, "id": 25, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 200 }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}", "interval": "10s", "intervalFactor": 1, "legendFormat": "Usage (pod={{pod}}, container={{container}})", "metric": "container_memory_usage:sort_desc", "refId": "A", "step": 10 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_memory_bytes{pod=~\"loki.*\"})", "interval": "", "legendFormat": "Limit", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_memory_bytes{pod=~\"loki.*\"})", "interval": "", "legendFormat": "Request", "refId": "C" } ], "title": "Loki memory usage vs Kubernetes limit and request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of actual usage over configured limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 788 }, "id": 772, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 200 }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100* max by(pod,container) (container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"})\n/\non (pod,container) (kube_pod_container_resource_limits_memory_bytes{pod=~\"promtail.*\"})", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Promtail memory usage (% of Kubernetes memory limit)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.56)", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } } ] }, { "matcher": { "id": "byName", "options": "Request" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.78)", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Usage" }, "properties": [ { "id": "custom.fillOpacity", "value": 10 } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 794 }, "id": 1103, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 200 }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "container_memory_working_set_bytes{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}", "interval": "10s", "intervalFactor": 1, "legendFormat": "Usage (pod={{pod}}, container={{container}})", "metric": "container_memory_usage:sort_desc", "refId": "A", "step": 10 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_memory_bytes{pod=~\"promtail.*\"})", "interval": "", "legendFormat": "Limit", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_memory_bytes{pod=~\"promtail.*\"})", "interval": "", "legendFormat": "Request", "refId": "C" } ], "title": "Promtail memory usage vs Kubernetes limit and request", "type": "timeseries" } ], "title": "Memory usage", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 8, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "percentage of configured limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 965 }, "id": 347, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100* rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\",pod=~\"loki.*\"}[1m])\n/\non (pod,container) kube_pod_container_resource_limits_cpu_cores{container=\"loki\"}", "interval": "10s", "intervalFactor": 1, "legendFormat": "", "metric": "container_cpu", "refId": "A", "step": 10 } ], "title": "Loki CPU usage (% of Kubernetes CPU limit)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU Limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.46)", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } } ] }, { "matcher": { "id": "byName", "options": "CPU Request" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.54)", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Usage" }, "properties": [ { "id": "custom.fillOpacity", "value": 10 } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 971 }, "id": 6, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"loki.*\"}[1m])", "interval": "10s", "intervalFactor": 1, "legendFormat": "CPU Usage (pod={{pod}}, container={{container}})", "metric": "container_cpu", "refId": "A", "step": 10 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_cpu_cores{pod=~\"loki.*\"})", "interval": "", "legendFormat": "CPU Limit", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_cpu_cores{pod=~\"loki.*\"})", "hide": false, "interval": "", "legendFormat": "CPU Request", "refId": "C" } ], "title": "Loki CPU usage vs Kubernetes limit and request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "percentage of configured limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 977 }, "id": 910, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100 * rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}[1m])\n/\non (pod,container) kube_pod_container_resource_limits_cpu_cores{pod=~\"promtail.*\"}", "interval": "10s", "intervalFactor": 1, "legendFormat": "", "metric": "container_cpu", "refId": "A", "step": 10 } ], "title": "Promtail CPU usage (% of Kubernetes limit)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU Limit" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.46)", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } } ] }, { "matcher": { "id": "byName", "options": "CPU Request" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(255, 255, 255, 0.54)", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Usage" }, "properties": [ { "id": "custom.fillOpacity", "value": 10 } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 983 }, "id": 1104, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",pod=~\"promtail.*\"}[1m])", "interval": "10s", "intervalFactor": 1, "legendFormat": "CPU Usage (pod={{pod}}, container={{container}})", "metric": "container_cpu", "refId": "A", "step": 10 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_limits_cpu_cores{pod=~\"promtail.*\"})", "interval": "", "legendFormat": "CPU Limit", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(kube_pod_container_resource_requests_cpu_cores{pod=~\"promtail.*\"})", "hide": false, "interval": "", "legendFormat": "CPU Request", "refId": "C" } ], "title": "Promtail CPU usage vs Kubernetes limit and request", "type": "timeseries" } ], "title": "CPU Usage", "type": "row" } ], "preload": true, "tags": ["system", "monitoring"], "templating": { "list": [ { "current": { "text": "Prometheus", "value": "kube-prometheus-stack" }, "includeAll": false, "label": "Datasource", "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" } ] }, "time": { "from": "now-24h", "to": "now" }, "timezone": "browser", "title": "Loki", "uid": "e6a0b019f34e9468223bc1669d5a9366", "version": 10 }