From af81ab8daf09c879d5b8ec186d8d0693aa8a5ac6 Mon Sep 17 00:00:00 2001
From: Alex Lebens <alexanderlebens@gmail.com>
Date: Sat, 27 Dec 2025 22:56:15 -0600
Subject: [PATCH] update loki config

---
 clusters/cl01tl/helm/loki/values.yaml | 59 ++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 11 deletions(-)

diff --git a/clusters/cl01tl/helm/loki/values.yaml b/clusters/cl01tl/helm/loki/values.yaml
index 0da6e20f8..b056a7c8c 100644
--- a/clusters/cl01tl/helm/loki/values.yaml
+++ b/clusters/cl01tl/helm/loki/values.yaml
@@ -68,32 +68,69 @@ alloy:
   configMap:
     create: true
     content: |
+      logging {
+        level = "debug"
+        format = "logfmt"
+      }
       discovery.kubernetes "pods" {
         role = "pod"
       }
-
-      discovery.relabel "pod_logs" {
+      discovery.relabel "pods" {
         targets = discovery.kubernetes.pods.targets
+
         rule {
           source_labels = ["__meta_kubernetes_namespace"]
-          target_label  = "namespace"
+          target_label = "namespace"
+          action = "replace"
         }
+
         rule {
-          source_labels = ["__meta_kubernetes_pod_name"]
-          target_label  = "pod"
+          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+          target_label = "app"
+          action = "replace"
         }
+
         rule {
           source_labels = ["__meta_kubernetes_pod_container_name"]
-          target_label  = "container"
+          target_label = "container"
+          action = "replace"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          target_label = "pod"
+          action = "replace"
         }
       }
-
-      loki.source.kubernetes "pod_logs" {
-        targets    = discovery.relabel.pod_logs.output
-        forward_to = [loki.write.loki_endpoint.receiver]
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pods.output
+        forward_to = [loki.process.process.receiver]
       }
+      loki.process "process" {
+        forward_to = [loki.write.loki.receiver]
 
-      loki.write "loki_endpoint" {
+        stage.drop {
+          older_than          = "1h"
+          drop_counter_reason = "too old"
+        }
+        stage.match {
+          selector = "{instance=~\".*\"}"
+          stage.json {
+            expressions = {
+              level = "\"level\"",
+            }
+          }
+          stage.labels {
+            values = {
+              level = "level",
+            }
+          }
+        }
+        stage.label_drop {
+          values = [ "service_name" ]
+        }
+      }
+      loki.write "loki" {
         endpoint {
           url = "http://loki-gateway.loki.svc.cluster.local:80/loki/api/v1/push"
         }